From 3a8d1441dc8ec94f9db754fa5f3b84270ca8d895 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 25 Mar 2026 15:56:36 +0100 Subject: [PATCH 01/48] build: openwrt 25.12.1 --- build.conf.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.conf.example b/build.conf.example index 0f05f1e39..0c57aac9e 100644 --- a/build.conf.example +++ b/build.conf.example @@ -1,4 +1,4 @@ -OWRT_VERSION=v24.10.5 -NETHSECURITY_VERSION=8.7.2 +OWRT_VERSION=v25.12.1 +NETHSECURITY_VERSION=8.8.0 TARGET=x86_64 REPO_CHANNEL=dev From 4204845e4efce04a9410822d5f6b68c55c9281f0 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 25 Mar 2026 15:57:11 +0100 Subject: [PATCH 02/48] chore: updated build container --- builder/Containerfile | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/builder/Containerfile b/builder/Containerfile index 351dda0ef..52068a443 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -1,35 +1,39 @@ # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # -# 2025-06-30 -FROM debian:12.11 AS base -ARG SOURCE_DATE_EPOCH=1751241600 +FROM debian:13.1 AS base RUN apt-get update \ && apt-get install --yes --no-install-recommends --no-install-suggests \ + # openwrt build dependencies https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem#debianubuntumint bison \ build-essential \ - ca-certificates \ clang \ - cmake \ - curl \ file \ flex \ g++ \ + g++-multilib \ gawk \ gcc-multilib \ gettext \ git \ libncurses5-dev \ libssl-dev \ - python3-distutils \ + python3-setuptools \ rsync \ - sudo \ + swig \ unzip \ - vim \ wget \ - zlib1g-dev + zlib1g-dev \ + # other dependencies + ca-certificates \ + cmake \ + curl \ + less \ + quilt \ + sudo \ + vim FROM base AS usign_build RUN git clone --depth 1 https://git.openwrt.org/project/usign.git /tmp/usign \ @@ -40,7 +44,14 @@ RUN git clone --depth 1 https://git.openwrt.org/project/usign.git /tmp/usign \ FROM base AS builder RUN groupadd -g 1000 'buildbot' \ && useradd -m -s '/bin/bash' -u 1000 -g 1000 'buildbot' \ - && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot + && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot \ + && echo "QUILT_DIFF_ARGS=--no-timestamps --no-index -p ab --color=auto" \ + "QUILT_REFRESH_ARGS=--no-timestamps --no-index -p ab" \ + "QUILT_SERIES_ARGS=--color=auto" \ + "QUILT_PATCH_OPTS=--unified" \ + "QUILT_DIFF_OPTS=-p" \ + "EDITOR=vim" \ + > /home/buildbot/.quiltrc USER buildbot WORKDIR /home/buildbot # OpenWRT repo From 98579a6b3b08722c9a609a59a1d7a09e9921ce99 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 25 Mar 2026 15:58:31 +0100 Subject: [PATCH 03/48] build(ppp): upstream patched the package --- patches/package/100-pppd-edit-bcopy.patch | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 patches/package/100-pppd-edit-bcopy.patch diff --git a/patches/package/100-pppd-edit-bcopy.patch b/patches/package/100-pppd-edit-bcopy.patch deleted file mode 100644 index 2874da9fa..000000000 --- a/patches/package/100-pppd-edit-bcopy.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/network/services/ppp/patches/200-use-memmove-for-bcopy.patch b/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -new file mode 100644 -index 0000000000..1632eb27b8 ---- /dev/null -+++ b/package/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -@@ -0,0 +1,11 @@ -+--- a/pppd/pppd-private.h -++++ b/pppd/pppd-private.h -+@@ -523,7 +523,7 @@ int parse_dotted_ip(char *, u_int32_t *) -+ #define TIMEOUT(r, f, t) ppp_timeout((r), (f), (t), 0) -+ #define UNTIMEOUT(r, f) ppp_untimeout((r), (f)) -+ -+-#define BCOPY(s, d, l) memcpy(d, s, l) -++#define BCOPY(s, d, l) memmove(d, s, l) -+ #define BZERO(s, n) memset(s, 0, n) -+ #define BCMP(s1, s2, l) memcmp(s1, s2, l) -+ From d09af010cfcbe71d2324b93db2d649754c7b1f69 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 15 Apr 2026 14:25:59 +0200 Subject: [PATCH 04/48] build: openwrt 25.12.2 --- build.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.conf.example b/build.conf.example index 0c57aac9e..24f9afec7 100644 --- a/build.conf.example +++ b/build.conf.example @@ -1,4 +1,4 @@ -OWRT_VERSION=v25.12.1 +OWRT_VERSION=v25.12.2 NETHSECURITY_VERSION=8.8.0 TARGET=x86_64 REPO_CHANNEL=dev From d02b25fcbed280da1cd0a91c1fac5c2782ac97c0 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 15 Apr 2026 14:26:15 +0200 Subject: [PATCH 05/48] build(netifyd): updated binaries --- .gitignore | 1 + packages/netifyd/Makefile | 26 +++++++------- .../etc/netifyd/profiles.d/00-default.conf | 1 + scripts/Readme.md | 19 ++++++++-- scripts/netifyd-packages.sh | 36 ++++++++++--------- 5 files changed, 52 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index f54bf6bc0..142d683b2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ build-logs build.conf netify-flow-actions netify-agent-stats-plugin +scripts/netifyd-apks diff --git a/packages/netifyd/Makefile b/packages/netifyd/Makefile index 54a598df9..291baa833 100644 --- a/packages/netifyd/Makefile +++ b/packages/netifyd/Makefile @@ -8,8 +8,8 @@ PKG_MAINTAINER:=Darryl Sokoloski PKG_LICENSE:=Unlicensed # Base URL for downloads -NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(NETIFYD_VERSION)/$(ARCH) -DL_DIR:=$(DL_DIR)/netifyd-$(NETIFYD_VERSION)-$(ARCH) +NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(PKG_VERSION)/25.12.2/$(ARCH) +DL_DIR:=$(DL_DIR)/netifyd-$(PKG_VERSION)-$(ARCH) include $(INCLUDE_DIR)/package.mk @@ -66,7 +66,7 @@ define Download/libnetifyd URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetifyd.so.4.0.0 FILE:=libnetifyd.so.4.0.0 - HASH:=a7bef78717e200eef177da8ee94557a565828ce796d3fd1a1f078e3f55959dd5 + HASH:=ffbbe5078a2b6575db4c478cf1b4d257f9b7241797c84aa1bcbca17ee1b23f3b endef $(eval $(call Download,libnetifyd)) @@ -74,7 +74,7 @@ define Download/libnetify-plm URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-plm.so.1.0.0 FILE:=libnetify-plm.so.1.0.0 - HASH:=b5c8994dc5f497ef1ccf8aae3685c1541702031726bc9f2819489cacfcb67e8f + HASH:=4320235873539f561238c92094702e74b75b8939a301b694255a5512fa036a00 endef $(eval $(call Download,libnetify-plm)) @@ -82,7 +82,7 @@ define Download/libnetify-proc-aggregator URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-aggregator.so.0.0.0 FILE:=libnetify-proc-aggregator.so.0.0.0 - HASH:=c9e5981fd4410776a1808902fbe284f40882823fc4c711f54e14421c80e94c6b + HASH:=736acb4c22d891da66694eee5507ea70679885f0747dec7a07eec8c926dd76fc endef $(eval $(call Download,libnetify-proc-aggregator)) @@ -90,7 +90,7 @@ define Download/libnetify-proc-core URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-core.so.0.0.0 FILE:=libnetify-proc-core.so.0.0.0 - HASH:=725415498d05fc29695a6f1fe14aed4c6b5967ff0d7d4703d06b80489c852222 + HASH:=8c9a1f26a498a6d1c88d76e6aa0367749779ca5f44ea3fa614bc1814387258ed endef $(eval $(call Download,libnetify-proc-core)) @@ -98,7 +98,7 @@ define Download/libnetify-proc-dev-discovery URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-dev-discovery.so.0.0.0 FILE:=libnetify-proc-dev-discovery.so.0.0.0 - HASH:=c25c4a497925cc0cf9d48f353de76fd2b5c9aa223390c87f2163feb1b0d61efb + HASH:=756c01e22c3724311eb6952ca6d580c728592771948207b93bab9b9795c38d1c endef $(eval $(call Download,libnetify-proc-dev-discovery)) @@ -106,7 +106,7 @@ define Download/libnetify-proc-flow-actions URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-flow-actions.so.0.0.0 FILE:=libnetify-proc-flow-actions.so.0.0.0 - HASH:=06ed65a3ec5d840edff008aaebbe294e4b0f7545ab295b287f6ed286bf28b7d7 + HASH:=1d6f03ead8760e314f98f5fe083515b19bf9c60e720d36bc03f70fa60a9b0d2b endef $(eval $(call Download,libnetify-proc-flow-actions)) @@ -114,7 +114,7 @@ define Download/libnetify-sink-http URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-http.so.0.0.0 FILE:=libnetify-sink-http.so.0.0.0 - HASH:=aaaadb9179f3df215f08d29f3a46db314d3990adafa5f3c4abeff305f4bb9583 + HASH:=95d0d6fa2b47b3764318a2f8680441d126c94bb39c84440b3e056767e330991d endef $(eval $(call Download,libnetify-sink-http)) @@ -122,7 +122,7 @@ define Download/libnetify-sink-log URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-log.so.0.0.0 FILE:=libnetify-sink-log.so.0.0.0 - HASH:=26d136562f7416ce4fe7c70d8023ae3f578a2d2dff4239791c65ad4289d28765 + HASH:=f9cafc30e150109dc3ab5e57eb203d51525a4ad0241fe76724206edf73c575b1 endef $(eval $(call Download,libnetify-sink-log)) @@ -130,7 +130,7 @@ define Download/libnetify-sink-socket URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-socket.so.0.0.0 FILE:=libnetify-sink-socket.so.0.0.0 - HASH:=8772335da50b702edd2a1412c98c82e32058dee2158db6e874ba3964ac66590a + HASH:=7ed9fe8d4d952ccbedca1c2be505fe288080f19c5c60d4db46932078c97d5364 endef $(eval $(call Download,libnetify-sink-socket)) @@ -138,7 +138,7 @@ define Download/libnetify-sink-sqlite URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-sqlite.so.0.0.0 FILE:=libnetify-sink-sqlite.so.0.0.0 - HASH:=6c2879cb8c9da36d592c85b9f8b4042339f0322fe2ebf335a5d15d06562ec65b + HASH:=0de0ef72cbd092fedd5cc278a4b57977891650aa10712f5ae0e6381a5a02fa52 endef $(eval $(call Download,libnetify-sink-sqlite)) @@ -146,7 +146,7 @@ define Download/netifyd URL:=$(NETIFYD_BASE_URL)/usr/sbin URL_FILE:=netifyd FILE:=netifyd - HASH:=02a8647d779f7dc4506fe9dd10295a06f6f1cb7b701edbfe791e6f319c4d7fd3 + HASH:=a1d8f40f87ba58876c7652dc0f82e699d6f8139793b8ede9d33cea043a693c72 endef $(eval $(call Download,netifyd)) diff --git a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf index 61266c775..687f9d690 100644 --- a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf +++ b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf @@ -158,5 +158,6 @@ all = include [netlink] # Set the Netlink buffer size buffer_size = 32768 +bridge_pvid_discovery = no # vim: set ft=dosini : diff --git a/scripts/Readme.md b/scripts/Readme.md index f7b16b6cb..32fa85a37 100644 --- a/scripts/Readme.md +++ b/scripts/Readme.md @@ -99,7 +99,14 @@ This script retrieves open issues labeled "testing" from the NethServer/nethsecu ## netifyd-packages.sh -This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpacks them and puts the files inside the `netifyd` package. This script is useful when an update of `netifyd` requires changes in the integration meta package. +This script extracts Netify `.apk` packages from the `netifyd-apks` directory, unpacks them, and merges the files for analysis and integration. This script is useful when an update of `netifyd` requires changes in the integration meta package. + +### Prerequisites + +- **apk-tools**: The Alpine package extraction tool must be installed on your system. + - On Fedora/RHEL: `dnf install apk-tools` + - On Debian/Ubuntu: `apt-get install apk-tools` + - On Alpine Linux: Already included ### Usage @@ -107,4 +114,12 @@ This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpack ./netifyd-packages.sh ``` -All process will be handled by the script, there will be an output that lists the files that were being copied in the process, this output needs to be copied and pasted inside the `packages/netifyd/Makefile` file, inside the `define Package/netifyd/install` section. +The script will process all `.apk` files in the `netifyd-apks/{arch}/` directories and extract their contents into `netifyd-apks/tmp/{arch}/netifyd/`. The merged files can then be copied into the `packages/netifyd/` directory as needed. + +### How It Works + +1. Iterates over each architecture subdirectory in `netifyd-apks/` +2. For each `.apk` file found, extracts its contents to a temporary location +3. Merges all extracted files into a single `netifyd` output directory per architecture +4. Skips metadata files (`.PKGINFO`, install scripts, etc.) during extraction +5. Outputs the merged file structure in `netifyd-apks/tmp/{arch}/netifyd/` diff --git a/scripts/netifyd-packages.sh b/scripts/netifyd-packages.sh index 8425d380c..d3261af1c 100755 --- a/scripts/netifyd-packages.sh +++ b/scripts/netifyd-packages.sh @@ -1,28 +1,32 @@ #!/bin/bash -# This is a helper script to extract all Netify IPK packages -# into netifyd-ipks/tmp/{arch} directories for analysis and integration. +# This is a helper script to extract all Netify APK packages +# into netifyd-apks/tmp/{arch} directories for analysis and integration. # # Usage: ./netifyd-packages.sh # -# Input: netifyd-ipks/{arch}/*.ipk -# Output: netifyd-ipks/tmp/{arch}/netifyd/ — merged contents of all IPKs for that arch -# netifyd-ipks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) +# Input: netifyd-apks/{arch}/*.apk +# Output: netifyd-apks/tmp/{arch}/netifyd/ — merged contents of all APKs for that arch +# netifyd-apks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IPKS_DIR="${SCRIPT_DIR}/netifyd-ipks" -TMP_DIR="${IPKS_DIR}/tmp" +APKS_DIR="${SCRIPT_DIR}/netifyd-apks" +TMP_DIR="${APKS_DIR}/tmp" -if [ ! -d "${IPKS_DIR}" ]; then - echo "ERROR: IPKs directory not found: ${IPKS_DIR}" >&2 +if [ ! -d "${APKS_DIR}" ]; then + echo "ERROR: APKs directory not found: ${APKS_DIR}" >&2 exit 1 fi # Iterate over each arch subdirectory -for arch_dir in "${IPKS_DIR}"/*/; do +for arch_dir in "${APKS_DIR}"/*/; do [ -d "${arch_dir}" ] || continue + + # Skip the tmp directory + [ "$(basename "${arch_dir}")" = "tmp" ] && continue + arch="$(basename "${arch_dir}")" echo "==> Processing arch: ${arch}" @@ -31,13 +35,13 @@ for arch_dir in "${IPKS_DIR}"/*/; do rm -rf "${TMP_DIR:?}/${arch}" mkdir -p "${output_dir}" - # Extract each IPK into its own per-package subdirectory, then merge - for ipk_file in "${arch_dir}"*.ipk; do - [ -f "${ipk_file}" ] || continue + # Extract each APK into its own per-package subdirectory, then merge + for apk_file in "${arch_dir}"*.apk; do + [ -f "${apk_file}" ] || continue # Derive package name: strip version and arch suffix - # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.ipk -> netify-plm - filename="$(basename "${ipk_file}" .ipk)" + # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.apk -> netify-plm + filename="$(basename "${apk_file}" .apk)" pkg_name="${filename%%_*}" # Extract to a temporary directory first to avoid conflicts @@ -45,7 +49,7 @@ for arch_dir in "${IPKS_DIR}"/*/; do mkdir -p "${pkg_extract_dir}" echo " Extracting ${filename} -> ${pkg_name}/" - tar -xf "${ipk_file}" ./data.tar.gz -O | tar -xzf - -C "${pkg_extract_dir}" + apk extract --allow-untrusted --destination "${pkg_extract_dir}" "${apk_file}" # Merge extracted files into the single netifyd output directory cp -a "${pkg_extract_dir}/." "${output_dir}/" From 5cbde211b5c907faaced03c283d5eba2173961a3 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 16 Apr 2026 10:03:05 +0200 Subject: [PATCH 06/48] fix: updated syntax due to python update --- packages/ns-api/files/ns.dashboard | 4 ++-- packages/ns-api/files/ns.nathelpers | 4 ++-- packages/ns-api/files/ns.ovpnrw | 6 ++---- packages/ns-api/files/ns.redirects | 2 +- packages/ns-migration/files/scripts/openvpn | 3 +-- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/ns-api/files/ns.dashboard b/packages/ns-api/files/ns.dashboard index 449fec46e..1e3954eed 100644 --- a/packages/ns-api/files/ns.dashboard +++ b/packages/ns-api/files/ns.dashboard @@ -154,7 +154,7 @@ def check_adblock(): if not adb_enabled: return "disabled" pa = subprocess.run(["service", "adblock", "status"], check=False, capture_output=True, text=True) - if adb_enabled and re.search('adblock_status\s+:\s+enabled', pa.stdout): + if adb_enabled and re.search(r'adblock_status\s+:\s+enabled', pa.stdout): return "ok" else: return "error" @@ -165,7 +165,7 @@ def check_banip(): if not bip_enabled: return "disabled" pa = subprocess.run(["service", "banip", "status"], check=False, capture_output=True, text=True) - if bip_enabled and re.search('status\s+:\s+active', pa.stdout): + if bip_enabled and re.search(r'status\s+:\s+active', pa.stdout): return "ok" def check_ts_ip(): diff --git a/packages/ns-api/files/ns.nathelpers b/packages/ns-api/files/ns.nathelpers index c674078ac..15afe300e 100755 --- a/packages/ns-api/files/ns.nathelpers +++ b/packages/ns-api/files/ns.nathelpers @@ -58,12 +58,12 @@ DEFAULT_PARAMS = { def get_nat_helper_names(): nat_helpers = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers = proc.stdout.splitlines() nat_helpers_extra = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers_extra = proc.stdout.splitlines() return nat_helpers + nat_helpers_extra diff --git a/packages/ns-api/files/ns.ovpnrw b/packages/ns-api/files/ns.ovpnrw index 147b7e86e..a43e24e10 100755 --- a/packages/ns-api/files/ns.ovpnrw +++ b/packages/ns-api/files/ns.ovpnrw @@ -63,8 +63,7 @@ def add_tap_to_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def remove_tap_from_bridge(u, bridge, interface): if bridge is None or interface is None: @@ -85,8 +84,7 @@ def remove_tap_from_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def get_ip_and_mask(bridge): u = EUci() diff --git a/packages/ns-api/files/ns.redirects b/packages/ns-api/files/ns.redirects index 8356c065c..4af2f2837 100755 --- a/packages/ns-api/files/ns.redirects +++ b/packages/ns-api/files/ns.redirects @@ -21,7 +21,7 @@ def get_services(): line = line.strip() if not line: continue - tmp = re.split("\s+", line) + tmp = re.split(r"\s+", line) port = tmp[1][0:tmp[1].index("/")] services[port] = tmp[0] return services diff --git a/packages/ns-migration/files/scripts/openvpn b/packages/ns-migration/files/scripts/openvpn index 06908f81a..09cf6a1b2 100755 --- a/packages/ns-migration/files/scripts/openvpn +++ b/packages/ns-migration/files/scripts/openvpn @@ -62,8 +62,7 @@ def add_tap_to_bridge(u, bridge, interface): u.commit("network") except: pass - finally: - return + return iname="ns_roadwarrior1" From 4a2484f265fd6812402f29d021b7cd892d338681 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 5 May 2026 09:45:15 +0200 Subject: [PATCH 07/48] chore: updated checkmk version --- packages/checkmk-agent/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/checkmk-agent/Makefile b/packages/checkmk-agent/Makefile index 8f33df9f8..30053f9f0 100644 --- a/packages/checkmk-agent/Makefile +++ b/packages/checkmk-agent/Makefile @@ -6,7 +6,9 @@ include $(TOPDIR)/rules.mk PKG_NAME:=checkmk-agent -PKG_VERSION:=2.4.0p24 +# renovate: datasource=github-tags depName=Checkmk/checkmk +CHECKMK_UPSTREAM_VERSION:=2.5.0 +PKG_VERSION:=$(subst p,_p,$(CHECKMK_UPSTREAM_VERSION)) PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/checkmk-agent-$(PKG_VERSION) @@ -32,7 +34,7 @@ endef # Download the Check_MK agent binary define Download/checkmk-agent-binary - URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(PKG_VERSION)/agents + URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(CHECKMK_UPSTREAM_VERSION)/agents URL_FILE:=check_mk_agent.openwrt FILE:=check_mk_agent.openwrt HASH:=skip From 0363541399c6cddac22383fcf006acab6fc799fb Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 29 Apr 2026 15:30:37 +0200 Subject: [PATCH 08/48] build: added suffix support --- .github/workflows/build-image.yml | 20 +++++++++---------- .gitignore | 1 - build-nethsec.sh | 12 ++++++++++-- build.conf.example => build.conf.defaults | 0 builder/Containerfile | 1 + builder/configure-build.sh | 9 ++++++++- docs/build/index.md | 24 +++++++++++++++++++---- 7 files changed, 49 insertions(+), 18 deletions(-) rename build.conf.example => build.conf.defaults (100%) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index eae1f39af..c066d047d 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -11,7 +11,7 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' tags: - '*' @@ -22,7 +22,7 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' jobs: @@ -42,32 +42,32 @@ jobs: - name: Generate build variables id: build_vars run: | - # export OWRT_VERSION from build.conf.example - echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export OWRT_VERSION from build.conf.defaults + echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export TARGET from build.conf.example - echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export TARGET from build.conf.defaults + echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT # export NETHSECURITY_VERSION from build - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT # When pushing a tag, set REPO_CHANNEL to stable if [[ "${{ github.ref }}" == refs/tags/* ]]; then echo "REPO_CHANNEL=stable" >> $GITHUB_OUTPUT # save NETHSECURITY_VERSION to env - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT # When pushing to main branch, set REPO_CHANNEL to dev elif [[ "${{ github.ref }}" == refs/heads/main ]]; then echo "REPO_CHANNEL=dev" >> $GITHUB_OUTPUT # save NETHSECURITY_VERSION to env and append -dev to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-dev+$(git rev-parse --short HEAD).$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT + echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)-dev+$(git rev-parse --short HEAD).$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT # Otherwise, get the branch name of the PR pushing if REPO_CHANNEL is not set elif [[ "${{ github.event_name }}" == 'pull_request' && ! -v REPO_CHANNEL ]]; then echo "REPO_CHANNEL=${{ github.head_ref }}" >> $GITHUB_OUTPUT # save NETHSECURITY_VERSION to env and append last commit hash to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-${{ github.head_ref }}+$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)-${{ github.head_ref }}+$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT fi - name: Build the image env: diff --git a/.gitignore b/.gitignore index 142d683b2..b597d98fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ key-build* -build.conf.override /bin build-logs build.conf diff --git a/build-nethsec.sh b/build-nethsec.sh index 9e570ea92..225177e94 100755 --- a/build-nethsec.sh +++ b/build-nethsec.sh @@ -7,10 +7,16 @@ set -e -# Source build files if it exists +# Source versioned defaults first set -o allexport +if [ -f build.conf.defaults ]; then + echo "Loading build.conf.defaults..." + . ./build.conf.defaults +fi + +# Source local overrides second (can override anything) if [ -f build.conf ]; then - echo "Loading build.conf file..." + echo "Loading build.conf (local overrides)..." . ./build.conf fi set +o allexport @@ -20,6 +26,7 @@ OWRT_VERSION=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} NETHSECURITY_VERSION=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION environment variable} REPO_CHANNEL=${REPO_CHANNEL:-dev} TARGET=${TARGET:-x86_64} +BUILD_SEMVER_SUFFIX=${BUILD_SEMVER_SUFFIX:-} if [ -f "./key-build" ] && [ -f "./key-build.pub" ]; then USIGN_PRIV_KEY="$(cat ./key-build)" @@ -38,6 +45,7 @@ podman build \ --build-arg REPO_CHANNEL="$REPO_CHANNEL" \ --build-arg TARGET="$TARGET" \ --build-arg NETHSECURITY_VERSION="$NETHSECURITY_VERSION" \ + --build-arg BUILD_SEMVER_SUFFIX="$BUILD_SEMVER_SUFFIX" \ . set +e diff --git a/build.conf.example b/build.conf.defaults similarity index 100% rename from build.conf.example rename to build.conf.defaults diff --git a/builder/Containerfile b/builder/Containerfile index 52068a443..027cd44ac 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -74,6 +74,7 @@ COPY --chown=buildbot:buildbot config config ARG REPO_CHANNEL ARG TARGET ARG NETHSECURITY_VERSION +ARG BUILD_SEMVER_SUFFIX COPY --chmod=777 builder/configure-build.sh /usr/local/bin/configure-build RUN /usr/local/bin/configure-build COPY --chmod=777 builder/entrypoint.sh /usr/local/bin/entrypoint.sh diff --git a/builder/configure-build.sh b/builder/configure-build.sh index 8f1cf189a..f21b34fe1 100644 --- a/builder/configure-build.sh +++ b/builder/configure-build.sh @@ -12,6 +12,13 @@ nethsecurity_version=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION enviro repo_channel=${REPO_CHANNEL:?Missing REPO_CHANNEL environment variable} target=${TARGET:?Missing TARGET environment variable} owrt_version=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} +build_semver_suffix=${BUILD_SEMVER_SUFFIX:-} + +if [ -n "$build_semver_suffix" ]; then + image_version="${nethsecurity_version}${build_semver_suffix}" +else + image_version="${nethsecurity_version}" +fi # For each file inside the config directory, cat the content into a .config file for file in config/*.conf; do @@ -31,7 +38,7 @@ CONFIG_VERSION_DIST="NethSecurity" CONFIG_VERSION_HOME_URL="https://github.com/nethserver/nethsecurity" CONFIG_VERSION_MANUFACTURER="Nethesis" CONFIG_VERSION_MANUFACTURER_URL="https://www.nethesis.it" -CONFIG_VERSION_NUMBER="${nethsecurity_version}" +CONFIG_VERSION_NUMBER="${image_version}" CONFIG_VERSION_CODE="${owrt_version}" CONFIG_VERSION_PRODUCT="NethSecurity" CONFIG_VERSION_REPO="https://updates.nethsecurity.nethserver.org/${repo_channel}/${nethsecurity_version}" diff --git a/docs/build/index.md b/docs/build/index.md index 1afbbc4dc..f0992ff5c 100644 --- a/docs/build/index.md +++ b/docs/build/index.md @@ -39,7 +39,15 @@ By default, the CI will build the `x86_64` target. To build a different target, To build locally, it's recommended to populate the `build.conf` file with the options you want to use for the build. This file is ignored by Git and should not be committed to the repository. -You can use the `build.conf.example` file as a starting point. Refer to [Environment variables](#environment-variables) for more details on the available options. +The `build.conf.defaults` file contains the versioned defaults and is always tracked by Git. + +You can create a local `build.conf` override that inherits from `build.conf.defaults`: +```bash +cp build.conf.defaults build.conf +# Edit build.conf to override any variables as needed +``` + +Refer to [Environment variables](#environment-variables) for more details on the available options. To build images locally on your machine, make sure these minimum requirements are met: @@ -65,14 +73,21 @@ During the start-up, the container will download netifyd plugins if configuratio ### Environment variables -The `build-nethsec.sh` script behavior can be changed by giving the following environment variables or setting them inside the `build.conf` file: +The `build-nethsec.sh` script behavior can be changed by setting environment variables or by populating the `build.conf` file (git-ignored, local overrides only). + +**Variable loading order:** +1. `build.conf.defaults` (versioned, always loaded first — contains canonical defaults) +2. `build.conf` (git-ignored, optional — can override any variable) +3. Environment variables set before calling `./build-nethsec.sh` (highest priority) + +**Available variables:** - `OWRT_VERSION`: specify the OpenWrt version to build, it can be either a TAG or a branch in the [GitHub OpenWRT repo](https://github.com/openwrt/openwrt); **required** - `NETHSECURITY_VERSION`: specify what to call the NethSecurity image; **required** - `TARGET`: specify the target to build; if not set default is `x86_64` - `REPO_CHANNEL`: specify the channel to publish the image to; if not set default is `dev` +- `BUILD_SEMVER_SUFFIX`: optional semver suffix appended to the image version only (not the distfeed URL). Use pre-release format (`-rc.1`, `-beta.2`) or metadata format (`+hotfix.1`, `+testing`) or both (`-rc.1+fix.1`). - `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: see [package signing section](#package-signing) - with the given keys The `USIGN_PUB_KEY`, `USIGN_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but for [security reasons](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#accessing-secrets) @@ -133,7 +148,8 @@ Development version example: ## Upstream version change -To change the OpenWrt version used by NethSecurity, you can just replace the `OWRT_VERSION` variable inside the `build.conf.example` file with the new OpenWrt version. +To change the OpenWrt version used by NethSecurity, update the `OWRT_VERSION` variable inside the `build.conf.defaults` file (versioned, always tracked by Git). +This ensures all developers and CI get the same default version. ## Release new image checklist From 9b9cb7af27fc22b5471ef0eba4dc6c7e7398c5c7 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 29 Apr 2026 16:05:51 +0200 Subject: [PATCH 09/48] updated variable generation --- .github/workflows/build-image.yml | 41 ++++++++++++++++--------------- tools/cleanup/cleanup.py | 33 ++++++++++++++++++------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index c066d047d..4cb59ad75 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -13,8 +13,6 @@ on: - 'patches/**' - 'build.conf.defaults' - 'build-nethsec.sh' - tags: - - '*' pull_request: paths: - 'builder/**' @@ -48,26 +46,23 @@ jobs: # export TARGET from build.conf.defaults echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export NETHSECURITY_VERSION from build - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - - # When pushing a tag, set REPO_CHANNEL to stable - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - echo "REPO_CHANNEL=stable" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT + # export base NETHSECURITY_VERSION from build.conf.defaults + BASE_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults) + COMMIT_HASH=$(git rev-parse --short HEAD) + TIMESTAMP=$(date +'%Y%m%d%H%M%S') # When pushing to main branch, set REPO_CHANNEL to dev - elif [[ "${{ github.ref }}" == refs/heads/main ]]; then + if [[ "${{ github.ref }}" == refs/heads/main ]]; then echo "REPO_CHANNEL=dev" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append -dev to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)-dev+$(git rev-parse --short HEAD).$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT + # Append -dev.. for easy sorting + echo "NETHSECURITY_VERSION=${BASE_VERSION}-dev.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT - # Otherwise, get the branch name of the PR pushing if REPO_CHANNEL is not set - elif [[ "${{ github.event_name }}" == 'pull_request' && ! -v REPO_CHANNEL ]]; then - echo "REPO_CHANNEL=${{ github.head_ref }}" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append last commit hash to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults)-${{ github.head_ref }}+$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + # Otherwise, get the branch name for branch channel builds + elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then + BRANCH_NAME="${{ github.head_ref }}" + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + # Append -.. for easy sorting + echo "NETHSECURITY_VERSION=${BASE_VERSION}-${BRANCH_NAME}.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT fi - name: Build the image env: @@ -78,7 +73,7 @@ jobs: run: ./build-nethsec.sh - name: Update latest_release file run: | - # Create release file pointing to 8-VERSION + # Create release file with the built version echo "${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" > latest_release echo "::notice title='Image published':: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" - uses: actions/upload-artifact@v7 @@ -117,8 +112,14 @@ jobs: RCLONE_CONFIG_REPO_ACCESS_KEY_ID: ${{ secrets.DO_SPACE_ACCESS_KEY }} RCLONE_CONFIG_REPO_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACE_SECRET_KEY }} run: | + # Sync binaries to channel/version/ rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} --progress --create-empty-src-dirs - rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress --create-empty-src-dirs + + # Place latest_release inside the version directory + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}/ --progress + + # Also place latest_release at channel root as fallback + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress tools: name: 'Run tools' diff --git a/tools/cleanup/cleanup.py b/tools/cleanup/cleanup.py index 4c413e494..2b6b562e5 100755 --- a/tools/cleanup/cleanup.py +++ b/tools/cleanup/cleanup.py @@ -2,9 +2,9 @@ # # Cleanup old development builds from DigitalOcean Spaces: -# - Keep all tagged releases -# - Keep at least 3 versions of each sub release +# - Keep the latest 5 versions of each channel # +# Version format: 8.7.2-dev.. or 8.7.2-branch.. import os import boto3 @@ -27,20 +27,35 @@ parsed_files = [] for file in files: file_name = file.lstrip(f'{prefix}/').rstrip('/') - version_parsed = VersionInfo.parse(file_name) - if version_parsed.build is None: + try: + version_parsed = VersionInfo.parse(file_name) + except ValueError: + print(f'Skipping {file_name} - not a valid semver version.') + continue + + # Check if it's a development build (has prerelease segment) + if version_parsed.prerelease is None: print(f'Skipping {file_name} as it is not a development build.') continue - build_split = version_parsed.build.split('.') + + # Extract timestamp from prerelease segment: "dev.." or "branch.." + prerelease_parts = version_parsed.prerelease.split('.') + if len(prerelease_parts) < 3: + print(f'Skipping {file_name} - prerelease segment does not contain timestamp.') + continue + + timestamp = prerelease_parts[1] # middle part is the timestamp parsed_files.append({ - 'timestamp': build_split[1], - 'file': file + 'timestamp': timestamp, + 'file': file, + 'version': file_name }) -# keep only the latest 5 dev builds +# Keep only the latest 5 dev builds, sorted by timestamp (descending) to_delete = sorted(parsed_files, key=lambda k: k['timestamp'], reverse=True)[min_versions:] for d in to_delete: - print(f"Deleting {d['file']} ...") + print(f"Deleting {d['version']} ...") objects_to_delete = s3_client.list_objects(Bucket=bucket_name, Prefix=d['file']) delete_keys = {'Objects': [{'Key': k['Key']} for k in objects_to_delete.get('Contents', [])]} s3_client.delete_objects(Bucket=bucket_name, Delete=delete_keys) + From 56b7ee0a51c0327f73d9232e160c7cf93019290b Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 5 May 2026 10:26:39 +0200 Subject: [PATCH 10/48] chore: updated python semver package --- packages/ns-api/files/ns.update | 2 +- packages/python-semver/Makefile | 4 ++-- tools/cleanup/cleanup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ns-api/files/ns.update b/packages/ns-api/files/ns.update index b1fef000d..9121fdffd 100755 --- a/packages/ns-api/files/ns.update +++ b/packages/ns-api/files/ns.update @@ -86,7 +86,7 @@ def check_system_update(): response = requests.get(f"{url}/latest_release", headers={"Accept": "application/json"}, timeout=5) response.raise_for_status() version = response.text.strip() - if semver.compare(version, current_version) > 0: + if semver.Version.compare(version, current_version) > 0: data["lastVersion"] = f'NethSecurity {version}' except requests.exceptions.ConnectionError: return utils.generic_error("connection_error") diff --git a/packages/python-semver/Makefile b/packages/python-semver/Makefile index 9132e194e..4affdbf63 100644 --- a/packages/python-semver/Makefile +++ b/packages/python-semver/Makefile @@ -5,11 +5,11 @@ include $(TOPDIR)/rules.mk PKG_NAME:=python-semver -PKG_VERSION:=2.13.0 +PKG_VERSION:=3.0.4 PKG_RELEASE:=1 PYPI_NAME:=semver -PKG_HASH:=fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f +PKG_HASH:=afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602 PKG_MAINTAINER:=Tommaso Bailetti PKG_LICENSE:=BSD-3-Clause diff --git a/tools/cleanup/cleanup.py b/tools/cleanup/cleanup.py index 2b6b562e5..caa163bf4 100755 --- a/tools/cleanup/cleanup.py +++ b/tools/cleanup/cleanup.py @@ -8,7 +8,7 @@ import os import boto3 -from semver import VersionInfo +from semver import Version as VersionInfo region = "ams3" bucket_name = "nethsecurity" From dd032542dd10d93a05a3d2f282219bd0323cc294 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 5 May 2026 12:49:11 +0200 Subject: [PATCH 11/48] fixed release path --- .github/workflows/build-image.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 4cb59ad75..100884e44 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -48,6 +48,7 @@ jobs: # export base NETHSECURITY_VERSION from build.conf.defaults BASE_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults) + echo "BASE_VERSION=${BASE_VERSION}" >> $GITHUB_OUTPUT COMMIT_HASH=$(git rev-parse --short HEAD) TIMESTAMP=$(date +'%Y%m%d%H%M%S') @@ -113,10 +114,10 @@ jobs: RCLONE_CONFIG_REPO_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACE_SECRET_KEY }} run: | # Sync binaries to channel/version/ - rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} --progress --create-empty-src-dirs + rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.BASE_VERSION }} --progress --create-empty-src-dirs # Place latest_release inside the version directory - rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}/ --progress + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.BASE_VERSION }}/ --progress # Also place latest_release at channel root as fallback rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress From b45bef6a3588a6e1506433081c441e612ef3da76 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 14:52:24 +0200 Subject: [PATCH 12/48] feat: added victoria-metrics package --- packages/victoria-metrics/Makefile | 75 +++++++++++++++++++ .../files/victoria-metrics.conf | 4 + .../files/victoria-metrics.initd | 42 +++++++++++ 3 files changed, 121 insertions(+) create mode 100644 packages/victoria-metrics/Makefile create mode 100644 packages/victoria-metrics/files/victoria-metrics.conf create mode 100644 packages/victoria-metrics/files/victoria-metrics.initd diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile new file mode 100644 index 000000000..f9c364f6e --- /dev/null +++ b/packages/victoria-metrics/Makefile @@ -0,0 +1,75 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=victoria-metrics +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaMetrics +PKG_VERSION:=1.110.1 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaMetrics/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaMetrics-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaMetrics +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics \ + github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert +GO_PKG_GCFLAGS:= \ + -trimpath \ + -buildvcs=false +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-metrics + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Metrics + URL:=https://github.com/VictoriaMetrics/VictoriaMetrics + DEPENDS:=$(GO_ARCH_DEPENDS) +endef + +define Package/victoria-metrics/description + VictoriaMetrics time series database / single-node server. +endef + +define Package/victoria-metrics/conffiles +/etc/config/victoria-metrics +endef + +define Package/victoria-metrics/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-metrics.initd $(1)/etc/init.d/victoria-metrics + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-metrics.conf $(1)/etc/config/victoria-metrics +endef + +define Package/victoria-metrics/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-metrics restart +exit 0 +endef + +$(eval $(call GoBinPackage,victoria-metrics)) +$(eval $(call BuildPackage,victoria-metrics)) \ No newline at end of file diff --git a/packages/victoria-metrics/files/victoria-metrics.conf b/packages/victoria-metrics/files/victoria-metrics.conf new file mode 100644 index 000000000..5b6813a89 --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.conf @@ -0,0 +1,4 @@ +config victoriametrics 'main' + option storage_path '/var/lib/victoriametrics' + option retention_period '1d' + option http_listen_addr '127.0.0.1:8428' diff --git a/packages/victoria-metrics/files/victoria-metrics.initd b/packages/victoria-metrics/files/victoria-metrics.initd new file mode 100644 index 000000000..1a86e55ec --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.initd @@ -0,0 +1,42 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-metrics" + +start_service() { + config_load victoria-metrics + local storage_path retention_period http_listen_addr + config_get storage_path main storage_path /var/lib/victoriametrics + config_get retention_period main retention_period 1d + config_get http_listen_addr main http_listen_addr 127.0.0.1:8428 + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retentionPeriod="$retention_period" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-metrics +} + +reload_service() +{ + stop + start +} From 86357bd802cd8bc1349336ce87752dbac4875b24 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 15:02:36 +0200 Subject: [PATCH 13/48] feat: added victoria-logs --- packages/victoria-logs/Makefile | 74 +++++++++++++++++++ packages/victoria-logs/files/35_victoria-logs | 14 ++++ .../files/rsyslog-victoria-logs.conf | 18 +++++ .../victoria-logs/files/victoria-logs.conf | 4 + .../victoria-logs/files/victoria-logs.initd | 43 +++++++++++ 5 files changed, 153 insertions(+) create mode 100644 packages/victoria-logs/Makefile create mode 100644 packages/victoria-logs/files/35_victoria-logs create mode 100644 packages/victoria-logs/files/rsyslog-victoria-logs.conf create mode 100644 packages/victoria-logs/files/victoria-logs.conf create mode 100644 packages/victoria-logs/files/victoria-logs.initd diff --git a/packages/victoria-logs/Makefile b/packages/victoria-logs/Makefile new file mode 100644 index 000000000..7e80d30e3 --- /dev/null +++ b/packages/victoria-logs/Makefile @@ -0,0 +1,74 @@ +include $(TOPDIR)/rules.mk + +PKG_NAME:=victoria-logs +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaLogs +PKG_VERSION:=1.14.0-victorialogs +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaLogs/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaLogs-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaMetrics +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-logs \ + github.com/VictoriaMetrics/VictoriaMetrics/app/vlogscli +GO_PKG_GCFLAGS:= \ + -trimpath \ + -buildvcs=false +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-logs + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Logs + URL:=https://github.com/VictoriaMetrics/VictoriaLogs + DEPENDS:=$(GO_ARCH_DEPENDS) +rsyslog +endef + +define Package/victoria-logs/description + VictoriaLogs — fast and easy-to-use database for logs. +endef + +define Package/victoria-logs/conffiles +/etc/config/victoria-logs +endef + +define Package/victoria-logs/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-logs.initd $(1)/etc/init.d/victoria-logs + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-logs.conf $(1)/etc/config/victoria-logs + $(INSTALL_DIR) $(1)/etc/rsyslog.d + $(INSTALL_CONF) ./files/rsyslog-victoria-logs.conf $(1)/etc/rsyslog.d/victoria-logs.conf + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/35_victoria-logs $(1)/etc/uci-defaults/35_victoria-logs +endef + +define Package/victoria-logs/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-logs restart +exit 0 +endef + +$(eval $(call GoPackage,victoria-logs)) +$(eval $(call BuildPackage,victoria-logs)) diff --git a/packages/victoria-logs/files/35_victoria-logs b/packages/victoria-logs/files/35_victoria-logs new file mode 100644 index 000000000..079e1a4eb --- /dev/null +++ b/packages/victoria-logs/files/35_victoria-logs @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +RSYSLOG_CONF="/etc/rsyslog.d/victoria-logs.conf" + +# Register the victoria-logs rsyslog drop-in in rsyslog UCI includes +if ! uci -q get rsyslog.syslog.includes | grep -qF "${RSYSLOG_CONF}"; then + uci add_list rsyslog.syslog.includes="${RSYSLOG_CONF}" + uci commit rsyslog +fi diff --git a/packages/victoria-logs/files/rsyslog-victoria-logs.conf b/packages/victoria-logs/files/rsyslog-victoria-logs.conf new file mode 100644 index 000000000..c2a66de3b --- /dev/null +++ b/packages/victoria-logs/files/rsyslog-victoria-logs.conf @@ -0,0 +1,18 @@ +# Rsyslog configuration for VictoriaLogs + +ruleset(name="victoria-logs") { + *.* action( + type="omfwd" + target="127.0.0.1" + port="5514" + protocol="tcp" + TCP_Framing="octet-counted" + Template="RSYSLOG_SyslogProtocol23Format" + + action.resumeRetryCount="-1" + queue.type="linkedList" + queue.size="10000" + ) +} + +*.* call victoria-logs diff --git a/packages/victoria-logs/files/victoria-logs.conf b/packages/victoria-logs/files/victoria-logs.conf new file mode 100644 index 000000000..84aba8dbe --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.conf @@ -0,0 +1,4 @@ +config victorialogs 'main' + option storage_path '/var/lib/victoria-logs' + option max_disk_usage '50MB' + option http_listen_addr '127.0.0.1:9428' diff --git a/packages/victoria-logs/files/victoria-logs.initd b/packages/victoria-logs/files/victoria-logs.initd new file mode 100644 index 000000000..2fc7d4908 --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.initd @@ -0,0 +1,43 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2023 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-logs" + +start_service() { + config_load victoria-logs + local storage_path max_disk_usage http_listen_addr + config_get storage_path main storage_path /var/lib/victoria-logs + config_get max_disk_usage main max_disk_usage 50MB + config_get http_listen_addr main http_listen_addr 127.0.0.1:9428 + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retention.maxDiskSpaceUsageBytes="$max_disk_usage" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_append_param command -syslog.listenAddr.tcp=127.0.0.1:5514 + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-logs +} + +reload_service() +{ + stop + start +} From 6d2461f16a81218645a924c86297e5344a1144a7 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 15:41:05 +0200 Subject: [PATCH 14/48] feat: added telegraf package --- packages/telegraf/Makefile | 90 ++++++++++ packages/telegraf/files/telegraf-config | 168 ++++++++++++++++++ packages/telegraf/files/telegraf.conf | 5 + .../telegraf/files/telegraf.conf.d/os.conf | 166 +++++++++++++++++ packages/telegraf/files/telegraf.initd | 36 ++++ 5 files changed, 465 insertions(+) create mode 100644 packages/telegraf/Makefile create mode 100644 packages/telegraf/files/telegraf-config create mode 100644 packages/telegraf/files/telegraf.conf create mode 100644 packages/telegraf/files/telegraf.conf.d/os.conf create mode 100644 packages/telegraf/files/telegraf.initd diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile new file mode 100644 index 000000000..b44767a7e --- /dev/null +++ b/packages/telegraf/Makefile @@ -0,0 +1,90 @@ +# +# Copyright (C) 2025 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=telegraf +# renovate: datasource=github-tags depName=influxdata/telegraf +PKG_VERSION:=1.33.2 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/influxdata/telegraf/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=telegraf-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=MIT + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_BUILD_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_PKG_LDFLAGS_X:=github.com/influxdata/telegraf/internal.Version=$(PKG_VERSION) +GO_PKG_TAGS:= \ + custom \ + inputs.bond \ + inputs.cpu \ + inputs.disk \ + inputs.ethtool \ + inputs.mem \ + inputs.net \ + inputs.netstat \ + inputs.nstat \ + inputs.processes \ + inputs.sensors \ + inputs.system \ + outputs.influxdb \ + outputs.prometheus_client + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/telegraf + SECTION:=base + CATEGORY:=NethServer + TITLE:=Telegraf + URL:=https://github.com/influxdata/telegraf + DEPENDS:= \ + $(GO_ARCH_DEPENDS) \ + +lm-sensors \ + +victoria-metrics \ + +python3-uci \ + +python3-jinja2 +endef + +define Package/telegraf/description + Telegraf is an agent for collecting, processing, aggregating, and writing metrics. +endef + +define Package/telegraf/conffiles +/etc/config/telegraf +endef + +define Package/telegraf/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/telegraf.initd $(1)/etc/init.d/telegraf + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/telegraf.conf $(1)/etc/config/telegraf + $(INSTALL_DIR) $(1)/etc/telegraf + $(INSTALL_DATA) ./files/telegraf.conf $(1)/etc/telegraf.conf + $(INSTALL_DIR) $(1)/etc/telegraf.conf.d + $(INSTALL_DATA) ./files/telegraf.conf.d/os.conf $(1)/etc/telegraf.conf.d/os.conf + $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_BIN) ./files/telegraf-config $(1)/usr/sbin/telegraf-config +endef + +define Package/telegraf/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/telegraf restart +exit 0 +endef + +$(eval $(call GoBinPackage,telegraf)) +$(eval $(call BuildPackage,telegraf)) diff --git a/packages/telegraf/files/telegraf-config b/packages/telegraf/files/telegraf-config new file mode 100644 index 000000000..7b5dfb69b --- /dev/null +++ b/packages/telegraf/files/telegraf-config @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2025 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import subprocess +import glob +from jinja2 import Environment, BaseLoader +from euci import EUci + +PROMETHEUS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. +# Do not edit manually — changes will be overwritten. + +# Prometheus client output +{% if enabled == '1' -%} +[[outputs.prometheus_client]] + ## Address to listen on. + ## ex: + ## listen = ":9273" + ## listen = "vsock://:9273" + listen = "{{ listen_addr }}" + + ## Maximum duration before timing out read of the request + # read_timeout = "10s" + ## Maximum duration before timing out write of the response + # write_timeout = "10s" + + ## Metric version controls the mapping from Prometheus metrics into Telegraf metrics. + ## See "Metric Format Configuration" in plugins/inputs/prometheus/README.md for details. + ## Valid options: 1, 2 + # metric_version = 1 + + ## Use HTTP Basic Authentication. +{%- if basic_auth_username and basic_auth_password %} + basic_username = "{{ basic_auth_username }}" + basic_password = "{{ basic_auth_password }}" +{%- else %} + # basic_username = "Foo" + # basic_password = "Bar" +{%- endif %} + + ## If set, the IP Ranges which are allowed to access metrics. + ## ex: ip_range = ["192.168.0.0/24", "192.168.1.0/30"] + # ip_range = [] + + ## Path to publish the metrics on. + # path = "/metrics" + + ## Expiration interval for each metric. 0 == no expiration + # expiration_interval = "60s" + + ## Collectors to enable, valid entries are "gocollector" and "process". + ## If unset, both are enabled. + # collectors_exclude = ["gocollector", "process"] + + ## Send string metrics as Prometheus labels. + ## Unless set to false all string metrics will be sent as labels. + # string_as_label = true + + ## If set, enable TLS with the given certificate. + # tls_cert = "/etc/ssl/telegraf.crt" + # tls_key = "/etc/ssl/telegraf.key" + + ## Set one or more allowed client CA certificate file names to + ## enable mutually authenticated TLS connections + # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] + + ## Export metric collection time. + # export_timestamp = false + + ## Specify the metric type explicitly. + ## This overrides the metric-type of the Telegraf metric. Globbing is allowed. + # [outputs.prometheus_client.metric_types] + # counter = [] + # gauge = [] +{%- endif %} +""" + +SENSORS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. +# Do not edit manually — changes will be overwritten. + +# Monitor sensors, requires lm-sensors package +# This plugin ONLY supports Linux +{% if sensors_available -%} +[[inputs.sensors]] + ## Remove numbers from field names. + ## If true, a field name like 'temp1_input' will be changed to 'temp_input'. + # remove_numbers = true + + ## Timeout is the maximum amount of time that the sensors command can run. + # timeout = "5s" + [inputs.sensors.tags] + influxdb_db = "os-metrics" +{%- endif %} +""" + + +def generate_prometheus_config(): + """Read UCI config and render Prometheus client output section.""" + e_uci = EUci() + + try: + enabled = e_uci.get('telegraf', 'output_prometheus', 'enabled', dtype=str, default='0') + listen_addr = e_uci.get('telegraf', 'output_prometheus', 'listen_addr', dtype=str, default=':9273') + basic_auth_username = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_username', dtype=str, default='') + basic_auth_password = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_password', dtype=str, default='') + except Exception as e: + print(f"Error reading UCI config: {e}") + return False + + # Render template + template = Environment(loader=BaseLoader()).from_string(PROMETHEUS_TEMPLATE) + rendered = template.render( + enabled=enabled, + listen_addr=listen_addr, + basic_auth_username=basic_auth_username, + basic_auth_password=basic_auth_password + ) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/prometheus.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +def sensors_available(): + """Check if sensors command is available on the system.""" + try: + result = subprocess.run(['which', 'sensors'], capture_output=True, timeout=2) + return result.returncode == 0 + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + +def generate_sensors_config(): + """Render sensors input section based on system availability.""" + has_sensors = sensors_available() + + # Render template + template = Environment(loader=BaseLoader()).from_string(SENSORS_TEMPLATE) + rendered = template.render(sensors_available=has_sensors) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/sensors.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +if __name__ == '__main__': + generate_prometheus_config() + generate_sensors_config() + exit(0) diff --git a/packages/telegraf/files/telegraf.conf b/packages/telegraf/files/telegraf.conf new file mode 100644 index 000000000..d1225d452 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf @@ -0,0 +1,5 @@ +config output_prometheus 'output_prometheus' + option enabled '0' + option listen_addr ':9273' + option basic_auth_username '' + option basic_auth_password '' diff --git a/packages/telegraf/files/telegraf.conf.d/os.conf b/packages/telegraf/files/telegraf.conf.d/os.conf new file mode 100644 index 000000000..7a14c9a13 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/os.conf @@ -0,0 +1,166 @@ +# OS and system metrics collection +# Includes CPU, memory, disk, network, and kernel statistics +# All metrics from this section are tagged with influxdb_db=os-metrics + +# Read metrics about cpu usage +[[inputs.cpu]] + ## Whether to report per-cpu stats or not + percpu = true + ## Whether to report total system cpu stats or not + totalcpu = true + ## If true, collect raw CPU time metrics + collect_cpu_time = false + ## If true, compute and report the sum of all non-idle CPU states + ## NOTE: The resulting 'time_active' field INCLUDES 'iowait'! + report_active = false + ## If true and the info is available then add core_id and physical_id tags + core_tags = false + [inputs.cpu.tags] + influxdb_db = "os-metrics" + + +# Read metrics about disk usage by mount point +[[inputs.disk]] + ## By default stats will be gathered for all mount points. + ## Set mount_points will restrict the stats to only the specified mount points. + # mount_points = ["/"] + + ## Ignore mount points by filesystem type. + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + + ## Ignore mount points by mount options. + ## The 'mount' command reports options of all mounts in parathesis. + ## Bind mounts can be ignored with the special 'bind' option. + # ignore_mount_opts = [] + [inputs.disk.tags] + influxdb_db = "os-metrics" + + +# Read metrics about memory usage +[[inputs.mem]] + # no configuration + [inputs.mem.tags] + influxdb_db = "os-metrics" + + +# Get the number of processes and group them by status +# This plugin ONLY supports non-Windows +[[inputs.processes]] + ## Use sudo to run ps command on *BSD systems. Linux systems will read + ## /proc, so this does not apply there. + # use_sudo = false + [inputs.processes.tags] + influxdb_db = "os-metrics" + + +# Read metrics about system load & uptime +[[inputs.system]] + # no configuration + [inputs.system.tags] + influxdb_db = "os-metrics" + + +# Collect bond interface status, slaves statuses and failures count +[[inputs.bond]] + ## Sets 'proc' directory path + ## If not specified, then default is /proc + # host_proc = "/proc" + + ## Sets 'sys' directory path + ## If not specified, then default is /sys + # host_sys = "/sys" + + ## By default, telegraf gather stats for all bond interfaces + ## Setting interfaces will restrict the stats to the specified + ## bond interfaces. + # bond_interfaces = ["bond0"] + + ## Tries to collect additional bond details from /sys/class/net/{bond} + ## currently only useful for LACP (mode 4) bonds + # collect_sys_details = false + [inputs.bond.tags] + influxdb_db = "os-metrics" + + +# Returns ethtool statistics for given interfaces +# This plugin ONLY supports Linux +[[inputs.ethtool]] + ## List of interfaces to pull metrics for + # interface_include = ["eth0"] + + ## List of interfaces to ignore when pulling metrics. + interface_exclude = ["wg*", "ipsec*", "tun*"] + + ## Plugin behavior for downed interfaces + ## Available choices: + ## - expose: collect & report metrics for down interfaces + ## - skip: ignore interfaces that are marked down + # down_interfaces = "expose" + + ## Reading statistics from interfaces in additional namespaces is also + ## supported, so long as the namespaces are named (have a symlink in + ## /var/run/netns). The telegraf process will also need the CAP_SYS_ADMIN + ## permission. + ## By default, only the current namespace will be used. For additional + ## namespace support, at least one of `namespace_include` and + ## `namespace_exclude` must be provided. + ## To include all namespaces, set `namespace_include` to `["*"]`. + ## The initial namespace (if anonymous) can be specified with the empty + ## string (""). + + ## List of namespaces to pull metrics for + # namespace_include = [] + + ## List of namespace to ignore when pulling metrics. + # namespace_exclude = [] + + ## Some drivers declare statistics with extra whitespace, different spacing, + ## and mix cases. This list, when enabled, can be used to clean the keys. + ## Here are the current possible normalizations: + ## * snakecase: converts fooBarBaz to foo_bar_baz + ## * trim: removes leading and trailing whitespace + ## * lower: changes all capitalized letters to lowercase + ## * underscore: replaces spaces with underscores + # normalize_keys = ["snakecase", "trim", "lower", "underscore"] + [inputs.ethtool.tags] + influxdb_db = "os-metrics" + + +# Gather metrics about network interfaces +[[inputs.net]] + ## By default, telegraf gathers stats from any up interface (excluding loopback) + ## Setting interfaces will tell it to gather these explicit interfaces, + ## regardless of status. When specifying an interface, glob-style + ## patterns are also supported. + # interfaces = ["eth*", "enp0s[0-1]", "lo"] + + ## On linux systems telegraf also collects protocol stats. + ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics. + ## + ## DEPRECATION NOTICE: A value of 'false' is deprecated and discouraged! + ## Please set this to `true` and use the 'inputs.nstat' + ## plugin instead. + # ignore_protocol_stats = false + [inputs.net.tags] + influxdb_db = "os-metrics" + + +# Read TCP metrics such as established, time wait and sockets counts. +[[inputs.netstat]] + # no configuration + [inputs.netstat.tags] + influxdb_db = "os-metrics" + + +# Collect kernel snmp counters and network interface statistics +[[inputs.nstat]] + ## file paths for proc files. If empty default paths will be used: + ## /proc/net/netstat, /proc/net/snmp, /proc/net/snmp6 + ## These can also be overridden with env variables, see README. + proc_net_netstat = "/proc/net/netstat" + proc_net_snmp = "/proc/net/snmp" + proc_net_snmp6 = "/proc/net/snmp6" + ## dump metrics with 0 values too + dump_zeros = true + [inputs.nstat.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.initd b/packages/telegraf/files/telegraf.initd new file mode 100644 index 000000000..d847757ca --- /dev/null +++ b/packages/telegraf/files/telegraf.initd @@ -0,0 +1,36 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2025 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/telegraf" + +start_service() { + /usr/sbin/telegraf-config + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command --watch-config notify + procd_append_param command --config /etc/telegraf.conf + procd_append_param command --config-directory /etc/telegraf.conf.d + procd_close_instance +} + +reload_service() +{ + /usr/sbin/telegraf-config +} + +service_triggers() +{ + procd_add_reload_trigger telegraf +} From 7c07910645041da19dac7eb5eb5b42a443c52bdb Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 15:43:35 +0200 Subject: [PATCH 15/48] fix: updated spdx --- packages/telegraf/Makefile | 2 +- packages/telegraf/files/telegraf-config | 2 +- packages/telegraf/files/telegraf.initd | 2 +- packages/victoria-logs/Makefile | 5 +++++ packages/victoria-logs/files/victoria-logs.initd | 2 +- packages/victoria-metrics/files/victoria-metrics.initd | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index b44767a7e..db67c2686 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -1,5 +1,5 @@ # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # diff --git a/packages/telegraf/files/telegraf-config b/packages/telegraf/files/telegraf-config index 7b5dfb69b..536d0a53d 100644 --- a/packages/telegraf/files/telegraf-config +++ b/packages/telegraf/files/telegraf-config @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # diff --git a/packages/telegraf/files/telegraf.initd b/packages/telegraf/files/telegraf.initd index d847757ca..49197475e 100644 --- a/packages/telegraf/files/telegraf.initd +++ b/packages/telegraf/files/telegraf.initd @@ -1,7 +1,7 @@ #!/bin/sh /etc/rc.common # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # diff --git a/packages/victoria-logs/Makefile b/packages/victoria-logs/Makefile index 7e80d30e3..74002a88f 100644 --- a/packages/victoria-logs/Makefile +++ b/packages/victoria-logs/Makefile @@ -1,5 +1,10 @@ include $(TOPDIR)/rules.mk +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + PKG_NAME:=victoria-logs # renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaLogs PKG_VERSION:=1.14.0-victorialogs diff --git a/packages/victoria-logs/files/victoria-logs.initd b/packages/victoria-logs/files/victoria-logs.initd index 2fc7d4908..820ae7877 100644 --- a/packages/victoria-logs/files/victoria-logs.initd +++ b/packages/victoria-logs/files/victoria-logs.initd @@ -1,7 +1,7 @@ #!/bin/sh /etc/rc.common # -# Copyright (C) 2023 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # diff --git a/packages/victoria-metrics/files/victoria-metrics.initd b/packages/victoria-metrics/files/victoria-metrics.initd index 1a86e55ec..fd2cee851 100644 --- a/packages/victoria-metrics/files/victoria-metrics.initd +++ b/packages/victoria-metrics/files/victoria-metrics.initd @@ -1,7 +1,7 @@ #!/bin/sh /etc/rc.common # -# Copyright (C) 2023 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # From 3961fa795a04b0ee40f5598b4cfd3b7f7fa21d4b Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 15:47:17 +0200 Subject: [PATCH 16/48] feat: compiling packages --- config/monitoring.conf | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 config/monitoring.conf diff --git a/config/monitoring.conf b/config/monitoring.conf new file mode 100644 index 000000000..f5e4fa381 --- /dev/null +++ b/config/monitoring.conf @@ -0,0 +1,3 @@ +CONFIG_PACKAGE_victoria-metrics=y +CONFIG_PACKAGE_victoria-logs=m +CONFIG_PACKAGE_telegraf=y From 31609a4a2952c0b62736f65dd974690054aa63ea Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 16:07:34 +0200 Subject: [PATCH 17/48] fix: accidentally overwrote telegraf config --- packages/telegraf/Makefile | 2 +- packages/telegraf/files/telegraf.conf | 35 +++++++++++++++++++++++---- packages/telegraf/files/telegraf.uci | 5 ++++ 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 packages/telegraf/files/telegraf.uci diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index db67c2686..398b45716 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -71,7 +71,7 @@ define Package/telegraf/install $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_BIN) ./files/telegraf.initd $(1)/etc/init.d/telegraf $(INSTALL_DIR) $(1)/etc/config - $(INSTALL_CONF) ./files/telegraf.conf $(1)/etc/config/telegraf + $(INSTALL_CONF) ./files/telegraf.uci $(1)/etc/config/telegraf $(INSTALL_DIR) $(1)/etc/telegraf $(INSTALL_DATA) ./files/telegraf.conf $(1)/etc/telegraf.conf $(INSTALL_DIR) $(1)/etc/telegraf.conf.d diff --git a/packages/telegraf/files/telegraf.conf b/packages/telegraf/files/telegraf.conf index d1225d452..fcbe12c40 100644 --- a/packages/telegraf/files/telegraf.conf +++ b/packages/telegraf/files/telegraf.conf @@ -1,5 +1,30 @@ -config output_prometheus 'output_prometheus' - option enabled '0' - option listen_addr ':9273' - option basic_auth_username '' - option basic_auth_password '' +# Telegraf Configuration +# Managed by NethSecurity — do not edit manually. + +[global_tags] + +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "0s" + omit_hostname = true + +############################################################################### +# OUTPUT PLUGINS # +############################################################################### + +[[outputs.influxdb]] + urls = ["http://127.0.0.1:8428"] + database = "nethsecurity" + database_tag = "influxdb_db" + exclude_database_tag = true + content_encoding = "gzip" + +############################################################################### +# INPUT PLUGINS # +############################################################################### diff --git a/packages/telegraf/files/telegraf.uci b/packages/telegraf/files/telegraf.uci new file mode 100644 index 000000000..d1225d452 --- /dev/null +++ b/packages/telegraf/files/telegraf.uci @@ -0,0 +1,5 @@ +config output_prometheus 'output_prometheus' + option enabled '0' + option listen_addr ':9273' + option basic_auth_username '' + option basic_auth_password '' From 8acfb3f9d95bc9d77ae7a5048253c8b406ad4331 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 16:23:27 +0200 Subject: [PATCH 18/48] fix: addressed deprecation warnings and removed not useful comments --- packages/telegraf/files/telegraf.conf | 10 +- .../telegraf/files/telegraf.conf.d/os.conf | 116 +----------------- 2 files changed, 7 insertions(+), 119 deletions(-) diff --git a/packages/telegraf/files/telegraf.conf b/packages/telegraf/files/telegraf.conf index fcbe12c40..90f4b4061 100644 --- a/packages/telegraf/files/telegraf.conf +++ b/packages/telegraf/files/telegraf.conf @@ -13,18 +13,12 @@ flush_jitter = "0s" precision = "0s" omit_hostname = true + skip_processors_after_aggregators = true -############################################################################### -# OUTPUT PLUGINS # -############################################################################### - +# Victoria metrics output plugin configuration [[outputs.influxdb]] urls = ["http://127.0.0.1:8428"] database = "nethsecurity" database_tag = "influxdb_db" exclude_database_tag = true content_encoding = "gzip" - -############################################################################### -# INPUT PLUGINS # -############################################################################### diff --git a/packages/telegraf/files/telegraf.conf.d/os.conf b/packages/telegraf/files/telegraf.conf.d/os.conf index 7a14c9a13..b66b55661 100644 --- a/packages/telegraf/files/telegraf.conf.d/os.conf +++ b/packages/telegraf/files/telegraf.conf.d/os.conf @@ -2,165 +2,59 @@ # Includes CPU, memory, disk, network, and kernel statistics # All metrics from this section are tagged with influxdb_db=os-metrics -# Read metrics about cpu usage [[inputs.cpu]] - ## Whether to report per-cpu stats or not percpu = true - ## Whether to report total system cpu stats or not totalcpu = true - ## If true, collect raw CPU time metrics collect_cpu_time = false - ## If true, compute and report the sum of all non-idle CPU states - ## NOTE: The resulting 'time_active' field INCLUDES 'iowait'! report_active = false - ## If true and the info is available then add core_id and physical_id tags core_tags = false [inputs.cpu.tags] influxdb_db = "os-metrics" - -# Read metrics about disk usage by mount point [[inputs.disk]] - ## By default stats will be gathered for all mount points. - ## Set mount_points will restrict the stats to only the specified mount points. - # mount_points = ["/"] - - ## Ignore mount points by filesystem type. ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] - - ## Ignore mount points by mount options. - ## The 'mount' command reports options of all mounts in parathesis. - ## Bind mounts can be ignored with the special 'bind' option. - # ignore_mount_opts = [] [inputs.disk.tags] influxdb_db = "os-metrics" - -# Read metrics about memory usage [[inputs.mem]] - # no configuration [inputs.mem.tags] influxdb_db = "os-metrics" - -# Get the number of processes and group them by status -# This plugin ONLY supports non-Windows [[inputs.processes]] - ## Use sudo to run ps command on *BSD systems. Linux systems will read - ## /proc, so this does not apply there. - # use_sudo = false [inputs.processes.tags] influxdb_db = "os-metrics" - -# Read metrics about system load & uptime [[inputs.system]] - # no configuration [inputs.system.tags] influxdb_db = "os-metrics" - -# Collect bond interface status, slaves statuses and failures count [[inputs.bond]] - ## Sets 'proc' directory path - ## If not specified, then default is /proc - # host_proc = "/proc" - - ## Sets 'sys' directory path - ## If not specified, then default is /sys - # host_sys = "/sys" - - ## By default, telegraf gather stats for all bond interfaces - ## Setting interfaces will restrict the stats to the specified - ## bond interfaces. - # bond_interfaces = ["bond0"] - - ## Tries to collect additional bond details from /sys/class/net/{bond} - ## currently only useful for LACP (mode 4) bonds - # collect_sys_details = false [inputs.bond.tags] influxdb_db = "os-metrics" - -# Returns ethtool statistics for given interfaces -# This plugin ONLY supports Linux [[inputs.ethtool]] - ## List of interfaces to pull metrics for + ## List of interfaces to include (default: all up interfaces) # interface_include = ["eth0"] - - ## List of interfaces to ignore when pulling metrics. + ## List of interfaces to ignore (default: none) interface_exclude = ["wg*", "ipsec*", "tun*"] - ## Plugin behavior for downed interfaces - ## Available choices: - ## - expose: collect & report metrics for down interfaces - ## - skip: ignore interfaces that are marked down - # down_interfaces = "expose" - - ## Reading statistics from interfaces in additional namespaces is also - ## supported, so long as the namespaces are named (have a symlink in - ## /var/run/netns). The telegraf process will also need the CAP_SYS_ADMIN - ## permission. - ## By default, only the current namespace will be used. For additional - ## namespace support, at least one of `namespace_include` and - ## `namespace_exclude` must be provided. - ## To include all namespaces, set `namespace_include` to `["*"]`. - ## The initial namespace (if anonymous) can be specified with the empty - ## string (""). - - ## List of namespaces to pull metrics for - # namespace_include = [] - - ## List of namespace to ignore when pulling metrics. - # namespace_exclude = [] - - ## Some drivers declare statistics with extra whitespace, different spacing, - ## and mix cases. This list, when enabled, can be used to clean the keys. - ## Here are the current possible normalizations: - ## * snakecase: converts fooBarBaz to foo_bar_baz - ## * trim: removes leading and trailing whitespace - ## * lower: changes all capitalized letters to lowercase - ## * underscore: replaces spaces with underscores - # normalize_keys = ["snakecase", "trim", "lower", "underscore"] [inputs.ethtool.tags] influxdb_db = "os-metrics" - -# Gather metrics about network interfaces [[inputs.net]] - ## By default, telegraf gathers stats from any up interface (excluding loopback) - ## Setting interfaces will tell it to gather these explicit interfaces, - ## regardless of status. When specifying an interface, glob-style - ## patterns are also supported. - # interfaces = ["eth*", "enp0s[0-1]", "lo"] - - ## On linux systems telegraf also collects protocol stats. - ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics. - ## - ## DEPRECATION NOTICE: A value of 'false' is deprecated and discouraged! - ## Please set this to `true` and use the 'inputs.nstat' - ## plugin instead. - # ignore_protocol_stats = false + ## Skip protocol stats, use inputs.nstat instead (fixes deprecation warning) + ignore_protocol_stats = true [inputs.net.tags] influxdb_db = "os-metrics" - -# Read TCP metrics such as established, time wait and sockets counts. [[inputs.netstat]] - # no configuration [inputs.netstat.tags] influxdb_db = "os-metrics" - -# Collect kernel snmp counters and network interface statistics [[inputs.nstat]] - ## file paths for proc files. If empty default paths will be used: - ## /proc/net/netstat, /proc/net/snmp, /proc/net/snmp6 - ## These can also be overridden with env variables, see README. proc_net_netstat = "/proc/net/netstat" proc_net_snmp = "/proc/net/snmp" proc_net_snmp6 = "/proc/net/snmp6" - ## dump metrics with 0 values too - dump_zeros = true + dump_zeros = true [inputs.nstat.tags] influxdb_db = "os-metrics" From cdde65d5702c6a882633e077b31aad1dfeec4689 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 30 Apr 2026 17:24:16 +0200 Subject: [PATCH 19/48] fix: adjusted config for sensors and ethtool --- packages/telegraf/files/telegraf-config | 4 ++-- packages/telegraf/files/telegraf.conf.d/os.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/telegraf/files/telegraf-config b/packages/telegraf/files/telegraf-config index 536d0a53d..235476d86 100644 --- a/packages/telegraf/files/telegraf-config +++ b/packages/telegraf/files/telegraf-config @@ -133,9 +133,9 @@ def generate_prometheus_config(): def sensors_available(): - """Check if sensors command is available on the system.""" + """Check if sensors command works by running it with -A -u flags.""" try: - result = subprocess.run(['which', 'sensors'], capture_output=True, timeout=2) + result = subprocess.run(['sensors', '-A', '-u'], capture_output=True, timeout=5) return result.returncode == 0 except (subprocess.TimeoutExpired, FileNotFoundError): return False diff --git a/packages/telegraf/files/telegraf.conf.d/os.conf b/packages/telegraf/files/telegraf.conf.d/os.conf index b66b55661..04f84000e 100644 --- a/packages/telegraf/files/telegraf.conf.d/os.conf +++ b/packages/telegraf/files/telegraf.conf.d/os.conf @@ -36,7 +36,7 @@ ## List of interfaces to include (default: all up interfaces) # interface_include = ["eth0"] ## List of interfaces to ignore (default: none) - interface_exclude = ["wg*", "ipsec*", "tun*"] + interface_exclude = ["wg*", "ipsec*", "tun*", "br*"] [inputs.ethtool.tags] influxdb_db = "os-metrics" From 038c6ce0a02840632e8fb5fab3691880cd07dbc1 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 5 May 2026 14:22:10 +0200 Subject: [PATCH 20/48] chore: updated monitoring tools --- packages/telegraf/Makefile | 2 +- packages/victoria-logs/Makefile | 11 ++++------- packages/victoria-metrics/Makefile | 5 +---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index 398b45716..e63822acb 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=telegraf # renovate: datasource=github-tags depName=influxdata/telegraf -PKG_VERSION:=1.33.2 +PKG_VERSION:=1.38.3 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz diff --git a/packages/victoria-logs/Makefile b/packages/victoria-logs/Makefile index 74002a88f..f41d99302 100644 --- a/packages/victoria-logs/Makefile +++ b/packages/victoria-logs/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=victoria-logs # renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaLogs -PKG_VERSION:=1.14.0-victorialogs +PKG_VERSION:=1.50.0 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz @@ -23,12 +23,9 @@ PKG_BUILD_DEPENDS:=golang/host PKG_BUILD_PARALLEL:=1 PKG_BUILD_FLAGS:=no-mips16 -GO_PKG:=github.com/VictoriaMetrics/VictoriaMetrics -GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-logs \ - github.com/VictoriaMetrics/VictoriaMetrics/app/vlogscli -GO_PKG_GCFLAGS:= \ - -trimpath \ - -buildvcs=false +GO_PKG:=github.com/VictoriaMetrics/VictoriaLogs +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaLogs/app/victoria-logs \ + github.com/VictoriaMetrics/VictoriaLogs/app/vlogscli GO_PKG_LDFLAGS:= \ -extldflags \ -static diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile index f9c364f6e..356b71d58 100644 --- a/packages/victoria-metrics/Makefile +++ b/packages/victoria-metrics/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=victoria-metrics # renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaMetrics -PKG_VERSION:=1.110.1 +PKG_VERSION:=1.142.0 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz @@ -26,9 +26,6 @@ PKG_BUILD_FLAGS:=no-mips16 GO_PKG:=github.com/VictoriaMetrics/VictoriaMetrics GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics \ github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert -GO_PKG_GCFLAGS:= \ - -trimpath \ - -buildvcs=false GO_PKG_LDFLAGS:= \ -extldflags \ -static From 2a48303c3e90f1633c269d98fc93e05d06ac8b76 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 5 May 2026 14:36:38 +0200 Subject: [PATCH 21/48] chore: downgraded version to keep compatibility with go --- packages/victoria-logs/Makefile | 2 +- packages/victoria-metrics/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/victoria-logs/Makefile b/packages/victoria-logs/Makefile index f41d99302..813fe68a3 100644 --- a/packages/victoria-logs/Makefile +++ b/packages/victoria-logs/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=victoria-logs # renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaLogs -PKG_VERSION:=1.50.0 +PKG_VERSION:=1.49.0 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile index 356b71d58..5fe85f5a8 100644 --- a/packages/victoria-metrics/Makefile +++ b/packages/victoria-metrics/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=victoria-metrics # renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaMetrics -PKG_VERSION:=1.142.0 +PKG_VERSION:=1.139.0 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz From ac58e2496dcccf93c67df7a1841e76eb9959e9e7 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 5 May 2026 15:11:00 +0200 Subject: [PATCH 22/48] chore: added skill to update packages --- .../skills/openwrt-package-update/SKILL.md | 211 +++++++++++++++++ .../openwrt-package-update/assets/.gitignore | 2 + .../references/REFERENCE.md | 219 ++++++++++++++++++ .../scripts/fetch-upstream.sh | 142 ++++++++++++ AGENTS.md | 1 + 5 files changed, 575 insertions(+) create mode 100644 .agents/skills/openwrt-package-update/SKILL.md create mode 100644 .agents/skills/openwrt-package-update/assets/.gitignore create mode 100644 .agents/skills/openwrt-package-update/references/REFERENCE.md create mode 100755 .agents/skills/openwrt-package-update/scripts/fetch-upstream.sh diff --git a/.agents/skills/openwrt-package-update/SKILL.md b/.agents/skills/openwrt-package-update/SKILL.md new file mode 100644 index 000000000..9ecbdb217 --- /dev/null +++ b/.agents/skills/openwrt-package-update/SKILL.md @@ -0,0 +1,211 @@ +--- +name: openwrt-package-update +description: Update a forked OpenWrt upstream package to a newer upstream version while preserving local customizations. Use when updating packages like adblock, mwan3, or other non-ns- prefixed upstream packages to newer releases, and when comparing local forks against the canonical OpenWrt packages feed to identify and merge upstream improvements. +license: GPL-2.0-only +compatibility: Requires git, curl, and bash. Works with OpenWrt release tags (v24.10.x, v24.11.x, etc.) and HEAD of the packages feed. +metadata: + domain: nethsecurity-packages + type: upstream-package-update +--- + +## What I do + +I help you update forked OpenWrt upstream packages (non-`ns-*` packages) to newer upstream versions while carefully preserving intentional local customizations. I: + +1. Fetch the canonical OpenWrt `feeds.conf.default` for a given release tag or the latest HEAD +2. Extract the pinned commit hash for the packages feed +3. Clone or update a local copy of the `openwrt/packages` GitHub mirror +4. Locate the package in the upstream feed +5. Generate a detailed diff showing all differences between upstream and your local fork +6. Guide you through analyzing the diff to identify: + - Pure upstream improvements to apply + - Your intentional customizations to preserve + - Conflicting changes requiring manual adaptation + +## When to use me + +Use this skill when you are: +- **Updating an existing forked package** to a newer upstream release (e.g., adblock 4.1.5 → 4.5.5) +- **Merging upstream improvements** while keeping local patches and configuration +- **Comparing against multiple OpenWrt releases** to understand what changed between versions +- **Evaluating the scope of local customization** before deciding whether to continue forking or contribute upstream +- **Refreshing patches and adapting them** to newer upstream versions + +Do **not** use this skill for: +- Creating entirely new `ns-*` packages (use `openwrt-package` skill instead) +- Patches that should be contributed upstream (this skill is for local forks only) +- Non-OpenWrt packages or custom software + +## Workflow + +### Step 1: Identify the target package and version + +Ask the user: +- **Which package are you updating?** (e.g., `adblock`, `mwan3`, `keepalived`) +- **Which OpenWrt version?** Suggest either: + - A specific release tag (e.g., `v24.10.5`) — extracts the pinned hash from that release's `feeds.conf.default` + - `HEAD` — uses the latest upstream version + +### Step 2: Run the diff script + +From the skill root directory, invoke: + +```bash +scripts/fetch-upstream.sh +``` + +Examples: +```bash +scripts/fetch-upstream.sh v24.10.5 adblock ../../../packages/adblock +scripts/fetch-upstream.sh HEAD mwan3 /absolute/path/to/packages/mwan3 +``` + +The script will: +- Fetch the feeds.conf.default for the target tag +- Clone the packages feed to `assets/packages/` (gitignored) +- Output a detailed `diff -ru` comparing upstream vs local + +### Step 3: Analyze the diff + +Review the diff output and classify each change: + +#### Category A: Pure Upstream Changes (Safe to apply) +These are bug fixes or features from upstream with no local modification: +- Upstream script improvements, new functions, or bugfixes +- Version bumps in dependencies +- Documentation updates + +**Action:** Copy the upstream version directly into the local package. + +#### Category B: Intentional Local Customizations (Always preserve) +These are deliberate modifications specific to NethSecurity: +- UCI configuration defaults hardcoded for NethSecurity +- Removal of upstream features not needed in NethSecurity +- Integration with ns-api or other NethSecurity components +- Build system tweaks (PKG_RELEASE bumps for local patches) + +**Action:** Keep these changes as-is. They should remain in the diff even after updating. + +#### Category C: Upstream Changes That Conflict With Local Customizations (Needs manual work) +These require careful adaptation: +- Upstream modified the same function that you customized +- New upstream feature overlaps with your local workaround +- Upstream changed configuration options you've modified + +**Action:** +- Read both versions carefully +- Decide whether upstream's approach is better (merge it) +- Or adapt upstream's changes to work with your customization +- Test thoroughly after merging + +### Step 4: Update the Makefile + +Update version information in the package Makefile: + +- **`PKG_VERSION`**: Set to the upstream version (e.g., `4.1.5` → `4.5.5`) +- **`PKG_RELEASE`**: Reset to `1` if the upstream version changed, or increment if only applying new local patches + +Example transitions: +```makefile +# Before: 4.1.5 with 9 local releases +PKG_VERSION:=4.1.5 +PKG_RELEASE:=9 + +# After updating to 4.5.5: +PKG_VERSION:=4.5.5 +PKG_RELEASE:=1 + +# Or if staying at 4.1.5 but adding a new local patch: +PKG_VERSION:=4.1.5 +PKG_RELEASE:=10 +``` + +### Step 5: Apply the changes + +For each file in the diff: + +1. **If it's Category A (safe upstream change)**: Copy the upstream version directly +2. **If it's Category B (local customization)**: Leave unchanged +3. **If it's Category C (conflict)**: Manually edit to merge both approaches + +Common files to check: +- `Makefile` — version and release +- `files/*.sh` — main scripts (often have conflicts) +- `files/*.conf` — configuration templates +- `files/*.init` — OpenWrt init scripts +- `files/README.md` — documentation + +### Step 6: Re-run the diff for verification + +After making changes, re-run: + +```bash +scripts/fetch-upstream.sh +``` + +The diff should now only show your **intentional** customizations. Review it one more time: +- Do all remaining differences make sense? +- Are there any accidental changes that slipped in? +- Should any new upstream code be adopted? + +### Step 7: Update documentation (if significant changes) + +If the package's behavior changes, update the package's `README.md` or comments to document: +- The version it's based on (e.g., "Based on upstream adblock 4.5.5") +- A summary of local customizations (e.g., "Adds automatic DNS reload on UCI changes") +- Any compatibility notes with the controller or UI + +## Common Scenarios + +### Scenario: Small upstream bugfix, large local customization + +**Situation:** adblock 4.1.5→4.1.6 (just a bugfix), but you've heavily customized the script for NethSecurity. + +**Steps:** +1. Run diff with `v24.10.5` tag +2. See the bugfix hunk in `files/adblock.sh` +3. Merge the bugfix into your customized version +4. Update Makefile: `PKG_VERSION:=4.1.6 PKG_RELEASE:=1` +5. Re-run diff — should show only your customizations remain + +### Scenario: Multiple version jumps (e.g., 4.1.5 → 4.5.5) + +**Situation:** You skipped several releases and want to jump to the latest. + +**Steps:** +1. Run diff with `HEAD` (latest packages feed) +2. Review all changes — likely many new features and bugfixes +3. Categorize each: adopt, preserve, or merge +4. Update Makefile: `PKG_VERSION:=4.5.5 PKG_RELEASE:=1` +5. Test extensively — more changes = higher risk +6. Consider running against intermediate versions to ease the transition + +### Scenario: Package has local patches in `patches/feeds/packages/` + +**Situation:** You have patches for the upstream package (in `patches/feeds/packages/`), and the upstream has evolved. + +**Steps:** +1. Run diff to see what's different +2. Review whether your patches are still needed (upstream might have fixed the issue) +3. If needed, regenerate the patches from the upstream source: + ```bash + cd assets/packages/ + git diff HEAD^ HEAD > /tmp/new-patch.diff + ``` +4. Compare with your current patch; update if necessary +5. Test the patched version in a build + +## Edge Cases + +- **Package renamed upstream**: The script won't find it. Manually verify it still exists in the feed under a new name. +- **Package removed from upstream**: You're now the sole maintainer. Update documentation and consider contributing upstream. +- **Multiple packages with same name**: The script finds the first one. Use `find` manually to disambiguate. +- **Large binary files**: The diff may be very large. Use `diff -r --exclude="*.bin" ...` if needed. + +## Reference Files + +See [REFERENCE.md](references/REFERENCE.md) for: +- Shell script explanation of `fetch-upstream.sh` +- OpenWrt feeds.conf.default format +- Git workflow for managing local patches +- Diff interpretation guide diff --git a/.agents/skills/openwrt-package-update/assets/.gitignore b/.agents/skills/openwrt-package-update/assets/.gitignore new file mode 100644 index 000000000..c96a04f00 --- /dev/null +++ b/.agents/skills/openwrt-package-update/assets/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/.agents/skills/openwrt-package-update/references/REFERENCE.md b/.agents/skills/openwrt-package-update/references/REFERENCE.md new file mode 100644 index 000000000..277ac78aa --- /dev/null +++ b/.agents/skills/openwrt-package-update/references/REFERENCE.md @@ -0,0 +1,219 @@ +# Reference: openwrt-package-update Skill + +## fetch-upstream.sh Script Details + +### Purpose +The `fetch-upstream.sh` script automates fetching upstream package code and generating a diff for manual review. + +### Workflow + +1. **Argument parsing**: Takes ``, ``, `` +2. **Fetch feeds.conf.default**: If a tag is provided, curl the feeds.conf.default from that OpenWrt release +3. **Extract hash**: Parse the `src-git packages` line to find the pinned commit hash after `^` +4. **Clone/update feed**: Clone the GitHub mirror of openwrt/packages or update an existing clone +5. **Checkout hash**: `git checkout` the extracted hash (or HEAD if no hash provided) +6. **Locate package**: `find` the package directory (up to 3 levels deep) +7. **Generate diff**: Run `diff -ru` between upstream and local + +### Output Format + +``` +=== Step 1: Fetching feeds.conf.default from OpenWrt === +... +=== Step 2: Cloning/updating packages feed to === +... +=== Step 3: Locating package '' in upstream feed === +... +=== Step 4: Comparing upstream vs local package === +... +--- Diff (upstream vs local) --- +[diff output] +... +=== Diff Summary === +... +``` + +### Exit Codes +- `0`: Success +- `1`: Missing arguments, network error, or package not found + +### Asset Management + +The script clones the packages feed into `.agents/skills/openwrt-package-update/assets/packages/` which is added to `.gitignore`. This folder: +- Grows over time as more packages are cloned +- Can be manually deleted to save space (script will re-clone on next run) +- Is not committed to git +- Is safe to delete between sessions + +### Network Requirements + +The script needs: +- `curl` to fetch feeds.conf.default from GitHub +- `git` to clone and checkout the packages repository +- Network access to github.com (both GitHub CDN and Git protocol) + +Firewall/corporate proxy rules should allow: +- `https://raw.githubusercontent.com/openwrt/openwrt/...` +- `https://github.com/openwrt/packages.git` (or git:// protocol) + +## OpenWrt feeds.conf.default Format + +The feeds.conf.default file lists the official feeds: + +``` +src-git packages https://git.openwrt.org/feed/packages.git^ +src-git luci https://git.openwrt.org/project/luci.git^ +... +``` + +Each line format: +- `src-git` — source control type (Git) +- `` — feed name (e.g., `packages`, `luci`) +- `` — Git repository URL +- `^` — (optional) pinned commit hash + +The hash ensures reproducible builds by locking to a specific point in time. + +## Diff Interpretation Guide + +When reviewing the script's diff output: + +### Common diff symbols: +- `---` — upstream file +- `+++` — local file +- `@@` — hunk header showing line numbers +- `-` (red) — line removed in local (upstream has it, you removed it) +- `+` (green) — line added in local (you added this, upstream doesn't have it) + +### Example: Local customization + +```diff +--- a/files/adblock.sh ++++ b/files/adblock.sh +@@ -10,6 +10,8 @@ + UPSTREAM_VERSION="4.1.5" + + # Initialize DNS backend ++# NethSecurity: Force dnsmasq backend ++DNS_BACKEND="dnsmasq" + init_dns_backend() { +``` + +Here: +- Upstream has no hardcoded `DNS_BACKEND` +- Local (your fork) forces `dnsmasq` +- This is a **local customization** to preserve + +### Example: Pure upstream change + +```diff +--- a/files/adblock.sh ++++ b/files/adblock.sh +@@ -234,6 +234,8 @@ + log "Error: Invalid source URL" + return 1 + fi ++ # Upstream fix: sanitize URL before processing ++ sanitize_url "$source_url" + process_source "$source_url" +``` + +Here: +- Upstream added a sanitization step +- Neither the old nor new version is in your local file +- This is a **pure upstream improvement** to adopt + +## Git Workflow for Patches + +If your package has patches in `patches/feeds/packages/`: + +### Understand the current patch +```bash +cd assets/packages/ +git log --oneline -5 # See recent changes +git show HEAD # See the latest change +cat /path/to/your/patches/feeds/packages/*-.patch # Your patch +``` + +### Check if patch still applies +```bash +cd assets/packages/ +git apply --check /path/to/your/patch.patch +``` + +### Regenerate patch if needed +```bash +cd assets/packages/ +# Make your changes to files +git diff > /tmp/regenerated.patch +# Copy to repo +cp /tmp/regenerated.patch /path/to/patches/feeds/packages/ +``` + +## Decision Tree for Categorizing Diffs + +``` +For each hunk in the diff: + +1. Is this hunk in a file you modified locally? + NO → Category A (pure upstream, safe to apply) + YES → go to 2 + +2. Does this hunk match a specific customization documented in README.md or comments? + YES → Category B (intentional customization, preserve) + NO → go to 3 + +3. Does this hunk conflict with your customization (same lines touched)? + YES → Category C (conflict, needs manual merge) + NO → Category B (you customized different parts, keep both) +``` + +## Example: Updating adblock 4.1.5 → 4.5.5 + +**Preparation:** +```bash +scripts/fetch-upstream.sh HEAD adblock ../../../packages/adblock +``` + +**Review output:** +- Makefile version bumped from 4.1.5 to 4.5.5 (upstream) +- New categories in `adblock.categories` (upstream) +- New sources in `adblock.sources` (upstream) +- Custom DNS backend forcing in `adblock.sh` (local) +- Custom email notification logic in `adblock.sh` (local) + +**Categorization:** +- Makefile version: Category A (pure upstream) → adopt +- New categories: Category A → adopt +- New sources: Category A → adopt +- Custom DNS backend: Category B → preserve +- Custom email: Category B → preserve + +**Actions:** +1. Copy upstream Makefile, update locally to preserve DNS backend forcing +2. Copy upstream categories file +3. Copy upstream sources file +4. Merge adblock.sh: take upstream as base, reapply the two local customizations +5. Update Makefile PKG_VERSION=4.5.5, PKG_RELEASE=1 +6. Re-run script to verify remaining diffs are just your customizations + +## Troubleshooting + +### Script fails to clone the feed +**Cause:** Network connectivity or git configuration issue +**Solution:** Verify git works: `git clone https://github.com/openwrt/packages.git /tmp/test` + +### Script can't find the package +**Cause:** Package name doesn't match directory name, or doesn't exist in that release +**Solution:** Manually browse the assets folder: `ls assets/packages/net/ | grep ` + +### Diff is huge (thousands of lines) +**Cause:** You're comparing very different versions +**Solution:** +- Try an intermediate version first +- Use `diff -r --exclude-from=.diffignore` to skip binary files +- Split review into categories (Makefile, scripts, configs) + +### Cloned repo is getting too large +**Cause:** Multiple runs accumulate git history +**Solution:** Delete the assets folder: `rm -rf assets/packages/`, script will re-clone on next run diff --git a/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh b/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh new file mode 100755 index 000000000..ab7cb2761 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# +# Fetch upstream OpenWrt package and compare with local fork. +# +# Usage: ./fetch-upstream.sh +# +# Arguments: +# openwrt-tag - OpenWrt release tag (e.g., v24.10.5) or literal 'HEAD' for latest +# package-name - Name of the package (e.g., 'adblock', 'mwan3') +# local-package-dir - Path to the local forked package (relative or absolute) +# +# Examples: +# ./fetch-upstream.sh v24.10.5 adblock ../../../packages/adblock +# ./fetch-upstream.sh HEAD mwan3 /absolute/path/to/packages/mwan3 + +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ASSETS_DIR="$SCRIPT_DIR/assets/packages" + +OPENWRT_TAG="$1" +PACKAGE_NAME="$2" +LOCAL_PKG_DIR="$3" + +# Validate arguments +if [[ -z "$OPENWRT_TAG" || -z "$PACKAGE_NAME" || -z "$LOCAL_PKG_DIR" ]]; then + echo "ERROR: Missing arguments" + echo "Usage: $0 " + exit 1 +fi + +# Resolve local package directory to absolute path +if [[ ! "$LOCAL_PKG_DIR" = /* ]]; then + LOCAL_PKG_DIR="$(cd "$LOCAL_PKG_DIR" 2>/dev/null && pwd)" || { + echo "ERROR: Local package directory does not exist: $LOCAL_PKG_DIR" + exit 1 + } +else + if [[ ! -d "$LOCAL_PKG_DIR" ]]; then + echo "ERROR: Local package directory does not exist: $LOCAL_PKG_DIR" + exit 1 + fi +fi + +FEEDS_URL="https://raw.githubusercontent.com/openwrt/openwrt" +PACKAGES_REPO="https://github.com/openwrt/packages.git" +TARGET_HASH="" + +# Step 1: Determine the packages feed hash +echo "=== Step 1: Fetching feeds.conf.default from OpenWrt $OPENWRT_TAG ===" + +if [[ "$OPENWRT_TAG" == "HEAD" ]]; then + echo "Using HEAD of packages feed (no pinned commit hash)" + TARGET_HASH="HEAD" +else + FEEDS_CONF_URL="$FEEDS_URL/$OPENWRT_TAG/feeds.conf.default" + echo "Fetching: $FEEDS_CONF_URL" + + FEEDS_CONF=$(curl -sS "$FEEDS_CONF_URL" 2>&1) || { + echo "ERROR: Failed to fetch feeds.conf.default from $FEEDS_CONF_URL" + echo "$FEEDS_CONF" + exit 1 + } + + # Extract the packages feed line and parse the hash after '^' + PACKAGES_LINE=$(echo "$FEEDS_CONF" | grep "^src-git packages") + if [[ -z "$PACKAGES_LINE" ]]; then + echo "ERROR: Could not find 'src-git packages' line in feeds.conf.default" + exit 1 + fi + + # Hash is after the '^' character if present + TARGET_HASH=$(echo "$PACKAGES_LINE" | sed -n 's/.*\^\([a-f0-9]*\).*/\1/p') + + if [[ -z "$TARGET_HASH" ]]; then + echo "WARNING: No pinned commit hash found in feeds.conf.default" + echo " Line: $PACKAGES_LINE" + echo " Using HEAD instead" + TARGET_HASH="HEAD" + else + echo "Found packages feed hash: $TARGET_HASH" + fi +fi + +# Step 2: Clone or update the packages feed +echo "" +echo "=== Step 2: Cloning/updating packages feed to $ASSETS_DIR ===" + +if [[ -d "$ASSETS_DIR/.git" ]]; then + echo "Repository already exists, fetching latest..." + cd "$ASSETS_DIR" + git fetch origin "$TARGET_HASH" 2>&1 | grep -v "^From https" | grep -v "^ " || true +else + echo "Cloning packages feed..." + mkdir -p "$ASSETS_DIR" + git clone --depth 1 "$PACKAGES_REPO" "$ASSETS_DIR" 2>&1 | grep -E "(Cloning|fatal)" || true + cd "$ASSETS_DIR" +fi + +# Checkout the target hash/branch +echo "Checking out $TARGET_HASH..." +git checkout "$TARGET_HASH" 2>&1 | grep -v "^Already on" || true + +# Step 3: Find the package directory in the upstream feed +echo "" +echo "=== Step 3: Locating package '$PACKAGE_NAME' in upstream feed ===" + +UPSTREAM_PKG_DIR=$(find "$ASSETS_DIR" -maxdepth 3 -type d -name "$PACKAGE_NAME" 2>/dev/null | head -1) + +if [[ -z "$UPSTREAM_PKG_DIR" ]]; then + echo "ERROR: Package '$PACKAGE_NAME' not found in upstream feed" + echo "Searched in: $ASSETS_DIR" + exit 1 +fi + +echo "Found upstream package at: $UPSTREAM_PKG_DIR" + +# Step 4: Compare the directories +echo "" +echo "=== Step 4: Comparing upstream vs local package ===" +echo "" +echo "Upstream: $UPSTREAM_PKG_DIR" +echo "Local: $LOCAL_PKG_DIR" +echo "" +echo "--- Diff (upstream vs local) ---" +echo "" + +diff -ru "$UPSTREAM_PKG_DIR" "$LOCAL_PKG_DIR" || true + +echo "" +echo "=== Diff Summary ===" +echo "If no changes are shown above, the local package matches upstream." +echo "" +echo "To update the local package:" +echo " 1. Review the diff above carefully" +echo " 2. Copy relevant files from: $UPSTREAM_PKG_DIR" +echo " 3. Preserve intentional local customizations" +echo " 4. Update PKG_VERSION and PKG_RELEASE in Makefile" +echo " 5. Re-run this script to verify changes" diff --git a/AGENTS.md b/AGENTS.md index 6425615b1..6f3be5b70 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ This project has domain-specific skills available. You MUST activate the relevan - `ns-api` — **ACTIVATE** when writing, modifying, or reviewing Python RPCD API scripts. Triggers: creating or updating `ns.*` RPCD API endpoints, handling UCI configuration changes, managing pre/post-commit hooks, defining ACL permissions, documenting methods in OpenAPI 3.1.0, or when user mentions ns-api, API endpoints, hooks, or references `/usr/libexec/rpcd/ns.` files. Covers stdin/stdout JSON protocol, error handling, naming conventions, code style, and spec file updates. - `openwrt-package` — **ACTIVATE** when creating or modifying OpenWrt `ns-*` packages. Triggers: building new packages for NethSecurity, managing package dependencies, patching upstream feeds, modifying Makefiles, or when user mentions Makefile, package structure, config fragments, or upstream patches. Covers naming conventions, required Makefile fields, architecture selection, external version management, and patch workflows. +- `openwrt-package-update` — **ACTIVATE** when updating forked OpenWrt upstream packages to newer versions. Triggers: updating packages like adblock, mwan3, or other non-ns- prefixed upstream packages, comparing local forks against the canonical OpenWrt packages feed, or when user mentions upstream package updates and merging improvements. Covers version updates, preserving local customizations, and identifying upstream improvements. - `python-nethsecurity` — **ACTIVATE** when writing or modifying Python scripts for NethSecurity packages. Triggers: creating new Python scripts, configuring package build systems, writing utilities, or when user mentions Python code in packages/ns-* or references Python scripts in the NethSecurity package tree. Covers shebang, license headers, extension handling, ruff compliance, available modules, and UCI commit conventions. --- From 4c368ffd8316fe023d8a59812b147c7600167d40 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 5 May 2026 16:46:56 +0200 Subject: [PATCH 23/48] ci: fixed versioning for image build --- .github/workflows/build-image.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 100884e44..b9de9862e 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -48,22 +48,26 @@ jobs: # export base NETHSECURITY_VERSION from build.conf.defaults BASE_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults) - echo "BASE_VERSION=${BASE_VERSION}" >> $GITHUB_OUTPUT COMMIT_HASH=$(git rev-parse --short HEAD) TIMESTAMP=$(date +'%Y%m%d%H%M%S') + echo "NETHSECURITY_VERSION=${BASE_VERSION}" >> $GITHUB_OUTPUT # When pushing to main branch, set REPO_CHANNEL to dev if [[ "${{ github.ref }}" == refs/heads/main ]]; then echo "REPO_CHANNEL=dev" >> $GITHUB_OUTPUT - # Append -dev.. for easy sorting - echo "NETHSECURITY_VERSION=${BASE_VERSION}-dev.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-dev.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT - # Otherwise, get the branch name for branch channel builds + # For pull requests and workflow_dispatch, add timestamp suffix elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then BRANCH_NAME="${{ github.head_ref }}" echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT - # Append -.. for easy sorting - echo "NETHSECURITY_VERSION=${BASE_VERSION}-${BRANCH_NAME}.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-${BRANCH_NAME}.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + + elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then + BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's|refs/heads/||') + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + fi - name: Build the image env: @@ -71,12 +75,14 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} TARGET: ${{ steps.build_vars.outputs.TARGET }} + BUILD_SEMVER_SUFFIX: ${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }} run: ./build-nethsec.sh - name: Update latest_release file run: | - # Create release file with the built version - echo "${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" > latest_release - echo "::notice title='Image published':: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" + # Create release file with the full display version (base + suffix) + FULL_VERSION="${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }}" + echo "${FULL_VERSION}" > latest_release + echo "::notice title='Image published':: ${FULL_VERSION}" - uses: actions/upload-artifact@v7 name: Upload image with: @@ -114,10 +120,10 @@ jobs: RCLONE_CONFIG_REPO_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACE_SECRET_KEY }} run: | # Sync binaries to channel/version/ - rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.BASE_VERSION }} --progress --create-empty-src-dirs + rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} --progress --create-empty-src-dirs # Place latest_release inside the version directory - rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.BASE_VERSION }}/ --progress + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}/ --progress # Also place latest_release at channel root as fallback rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress From 888a53e56607e66a4e360d855819f37fffa68e8a Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 12 May 2026 09:13:21 +0200 Subject: [PATCH 24/48] chore: removed ipcalc fork --- files/bin/ipcalc.sh | 142 -------------------------------------------- 1 file changed, 142 deletions(-) delete mode 100755 files/bin/ipcalc.sh diff --git a/files/bin/ipcalc.sh b/files/bin/ipcalc.sh deleted file mode 100755 index e8c7a07df..000000000 --- a/files/bin/ipcalc.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh - -. /lib/functions/ipv4.sh - -PROG="$(basename "$0")" - -# wrapper to convert an integer to an address, unless we're using -# decimal output format. -# hook for library function -_ip2str() { - local var="$1" n="$2" - assert_uint32 "$n" || exit 1 - - if [ "$decimal" -ne 0 ]; then - export -- "$var=$n" - elif [ "$hexadecimal" -ne 0 ]; then - export -- "$var=$(printf "%x" "$n")" - else - ip2str "$@" - fi -} - -usage() { - echo "Usage: $PROG [ -d | -x ] address/prefix [ start limit ]" >&2 - exit 1 -} - -decimal=0 -hexadecimal=0 -if [ "$1" = "-d" ]; then - decimal=1 - shift -elif [ "$1" = "-x" ]; then - hexadecimal=1 - shift -fi - -if [ $# -eq 0 ]; then - usage -fi - -case "$1" in -*/*.*) - # data is n.n.n.n/m.m.m.m format, like on a Cisco router - str2ip ipaddr "${1%/*}" || exit 1 - str2ip netmask "${1#*/}" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift - ;; -*/*) - # more modern prefix notation of n.n.n.n/p - str2ip ipaddr "${1%/*}" || exit 1 - prefix="${1#*/}" - assert_uint32 "$prefix" || exit 1 - if [ "$prefix" -gt 32 ]; then - printf "Prefix out of range (%s)\n" "$prefix" >&2 - exit 1 - fi - prefix2netmask netmask "$prefix" || exit 1 - shift - ;; -*) - # address and netmask as two separate arguments - str2ip ipaddr "$1" || exit 1 - str2ip netmask "$2" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift 2 - ;; -esac - -# we either have no arguments left, or we have a range start and length -if [ $# -ne 0 ] && [ $# -ne 2 ]; then - usage -fi - -# complement of the netmask, i.e. the hostmask -hostmask=$((netmask ^ 0xffffffff)) -network=$((ipaddr & netmask)) -broadcast=$((network | hostmask)) -count=$((hostmask + 1)) - -_ip2str IP "$ipaddr" -_ip2str NETMASK "$netmask" -_ip2str NETWORK "$network" - -echo "IP=$IP" -echo "NETMASK=$NETMASK" -# don't include this-network or broadcast addresses -if [ "$prefix" -le 30 ]; then - _ip2str BROADCAST "$broadcast" - echo "BROADCAST=$BROADCAST" -fi -echo "NETWORK=$NETWORK" -echo "PREFIX=$prefix" -echo "COUNT=$count" - -# if there's no range, we're done -[ $# -eq 0 ] && exit 0 -[ -z "$1$2" ] && exit 0 - -if [ "$prefix" -le 30 ]; then - lower=$((network + 1)) -else - lower="$network" -fi - -start="$1" -assert_uint32 "$start" || exit 1 -start=$((network | (start & hostmask))) -[ "$start" -lt "$lower" ] && start="$lower" -[ "$start" -eq "$ipaddr" ] && start=$((start + 1)) - -if [ "$prefix" -le 30 ]; then - upper=$(((network | hostmask) - 1)) -elif [ "$prefix" -eq 31 ]; then - upper=$((network | hostmask)) -else - upper="$network" -fi - -range="$2" -assert_uint32 "$range" || exit 1 -end=$((start + range - 1)) -[ "$end" -gt "$upper" ] && end="$upper" -[ "$end" -eq "$ipaddr" ] && end=$((end - 1)) - -if [ "$start" -gt "$end" ]; then - echo "network ($NETWORK/$prefix) too small" >&2 - exit 1 -fi - -_ip2str START "$start" -_ip2str END "$end" - -if [ "$start" -le "$ipaddr" ] && [ "$ipaddr" -le "$end" ]; then - echo "warning: address $IP inside range $START..$END" >&2 -fi - -echo "START=$START" -echo "END=$END" - -exit 0 From b328bec9d8b0eb388156b2713e9ac4dd47bcbddf Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 12 May 2026 09:13:34 +0200 Subject: [PATCH 25/48] chore: removed jinja2 fork --- packages/python-jinja2/Makefile | 50 --------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 packages/python-jinja2/Makefile diff --git a/packages/python-jinja2/Makefile b/packages/python-jinja2/Makefile deleted file mode 100644 index f29df71ec..000000000 --- a/packages/python-jinja2/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# This is free software, licensed under the GNU General Public License v2. -# See /LICENSE for more information. -# - -include $(TOPDIR)/rules.mk - -PKG_NAME:=python-jinja2 -PKG_VERSION:=3.1.2 -PKG_RELEASE:=2 - -PYPI_NAME:=Jinja2 -PYTHON3_PKG_WHEEL_NAME:=jinja2 -PKG_HASH:=31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 - -PKG_MAINTAINER:=Michal Vasilek -PKG_LICENSE:=BSD-3-Clause -PKG_LICENSE_FILES:=LICENSE.rst -PKG_CPE_ID:=cpe:/a:pocoo:jinja2 -HOST_BUILD_DEPENDS:= python-markupsafe/host - -include $(TOPDIR)/feeds/packages/lang/python/pypi.mk -include $(INCLUDE_DIR)/package.mk -include $(INCLUDE_DIR)/host-build.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-package.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-host-build.mk - -define Package/python3-jinja2 - SECTION:=lang - CATEGORY:=Languages - SUBMENU:=Python - TITLE:=Very fast and expressive template engine - URL:=https://palletsprojects.com/p/jinja/ - DEPENDS:= \ - +python3-light \ - +python3-asyncio \ - +python3-logging \ - +python3-urllib \ - +python3-markupsafe -endef - -define Package/python3-jinja2/description -Jinja2 is a full featured template engine for Python. It has full -unicode support, an optional integrated sandboxed execution -environment, widely used and BSD licensed. -endef - -$(eval $(call Py3Package,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2-src)) -$(eval $(call HostBuild)) \ No newline at end of file From 4ad0868860b952bf6d10506da505f8b5fb1067b3 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 12 May 2026 18:01:33 +0200 Subject: [PATCH 26/48] build: updated signing method for APKs (#1659) --- .github/workflows/build-image.yml | 4 ++-- .gitignore | 3 ++- build-nethsec.sh | 11 +++++------ builder/Containerfile | 30 ++++++++++++------------------ builder/entrypoint.sh | 9 +++------ docs/build/index.md | 28 ++++++++++++++-------------- 6 files changed, 38 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index b9de9862e..c52f863b8 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -31,8 +31,8 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} env: - USIGN_PUB_KEY: ${{ secrets.USIGN_PUB_KEY }} - USIGN_PRIV_KEY: ${{ secrets.USIGN_PRIV_KEY }} + APK_PUB_KEY: ${{ secrets.APK_PUB_KEY }} + APK_PRIV_KEY: ${{ secrets.APK_PRIV_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} steps: diff --git a/.gitignore b/.gitignore index b597d98fc..40665e2e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -key-build* +private-key.pem +public-key.pem /bin build-logs build.conf diff --git a/build-nethsec.sh b/build-nethsec.sh index 225177e94..000cb2100 100755 --- a/build-nethsec.sh +++ b/build-nethsec.sh @@ -28,9 +28,9 @@ REPO_CHANNEL=${REPO_CHANNEL:-dev} TARGET=${TARGET:-x86_64} BUILD_SEMVER_SUFFIX=${BUILD_SEMVER_SUFFIX:-} -if [ -f "./key-build" ] && [ -f "./key-build.pub" ]; then - USIGN_PRIV_KEY="$(cat ./key-build)" - USIGN_PUB_KEY="$(cat ./key-build.pub)" +if [ -f "./private-key.pem" ] && [ -f "./public-key.pem" ]; then + APK_PRIV_KEY="$(cat ./private-key.pem)" + APK_PUB_KEY="$(cat ./public-key.pem)" fi @@ -39,7 +39,6 @@ podman build \ --layers \ --file builder/Containerfile \ --tag nethsecurity-next \ - --target builder \ --jobs 0 \ --build-arg OWRT_VERSION="$OWRT_VERSION" \ --build-arg REPO_CHANNEL="$REPO_CHANNEL" \ @@ -52,8 +51,8 @@ set +e status=0 podman run \ - --env USIGN_PRIV_KEY="$USIGN_PRIV_KEY" \ - --env USIGN_PUB_KEY="$USIGN_PUB_KEY" \ + --env APK_PRIV_KEY="$APK_PRIV_KEY" \ + --env APK_PUB_KEY="$APK_PUB_KEY" \ --name nethsecurity-builder \ --interactive \ --tty \ diff --git a/builder/Containerfile b/builder/Containerfile index 027cd44ac..da0532d5f 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -35,13 +35,6 @@ RUN apt-get update \ sudo \ vim -FROM base AS usign_build -RUN git clone --depth 1 https://git.openwrt.org/project/usign.git /tmp/usign \ - && cd /tmp/usign \ - && cmake . \ - && make - -FROM base AS builder RUN groupadd -g 1000 'buildbot' \ && useradd -m -s '/bin/bash' -u 1000 -g 1000 'buildbot' \ && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot \ @@ -58,7 +51,9 @@ WORKDIR /home/buildbot ARG OWRT_VERSION RUN git clone --branch "${OWRT_VERSION}" https://github.com/openwrt/openwrt.git WORKDIR /home/buildbot/openwrt -RUN sed -i '/telephony/d' feeds.conf.default +RUN sed -i '/telephony/d' feeds.conf.default \ + && sed -i '/routing/d' feeds.conf.default \ + && sed -i '/video/d' feeds.conf.default RUN ./scripts/feeds update -a COPY --chmod=777 builder/apply-patches.sh /usr/local/bin/apply-patches COPY --chown=buildbot:buildbot patches patches @@ -79,15 +74,14 @@ COPY --chmod=777 builder/configure-build.sh /usr/local/bin/configure-build RUN /usr/local/bin/configure-build COPY --chmod=777 builder/entrypoint.sh /usr/local/bin/entrypoint.sh ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] -COPY --from=usign_build /tmp/usign/usign /usr/local/bin/usign RUN mkdir -p \ - .ccache \ - build_dir \ - dl \ - download \ - staging_dir + .ccache \ + build_dir \ + dl \ + download \ + staging_dir VOLUME "/home/buildbot/openwrt/.ccache" \ - "/home/buildbot/openwrt/build_dir" \ - "/home/buildbot/openwrt/dl" \ - "/home/buildbot/openwrt/download" \ - "/home/buildbot/openwrt/staging_dir" + "/home/buildbot/openwrt/build_dir" \ + "/home/buildbot/openwrt/dl" \ + "/home/buildbot/openwrt/download" \ + "/home/buildbot/openwrt/staging_dir" diff --git a/builder/entrypoint.sh b/builder/entrypoint.sh index ff3b2a27f..09f4f5409 100644 --- a/builder/entrypoint.sh +++ b/builder/entrypoint.sh @@ -7,12 +7,9 @@ set -e -if [ -n "$USIGN_PUB_KEY" ] && [ -n "$USIGN_PRIV_KEY" ]; then - echo "$USIGN_PUB_KEY" > /home/buildbot/openwrt/key-build.pub - echo "$USIGN_PRIV_KEY" > /home/buildbot/openwrt/key-build -else - echo "No signing keys found. Generating dummy keys..." - usign -G -s ./key-build -p ./key-build.pub -c "Local build key" +if [ -n "$APK_PRIV_KEY" ] && [ -n "$APK_PUB_KEY" ]; then + echo "$APK_PRIV_KEY" > /home/buildbot/openwrt/private-key.pem + echo "$APK_PUB_KEY" > /home/buildbot/openwrt/public-key.pem fi # if command $1 is a file or a executable, run it diff --git a/docs/build/index.md b/docs/build/index.md index f0992ff5c..a0c4e9ea9 100644 --- a/docs/build/index.md +++ b/docs/build/index.md @@ -87,16 +87,16 @@ The `build-nethsec.sh` script behavior can be changed by setting environment var - `TARGET`: specify the target to build; if not set default is `x86_64` - `REPO_CHANNEL`: specify the channel to publish the image to; if not set default is `dev` - `BUILD_SEMVER_SUFFIX`: optional semver suffix appended to the image version only (not the distfeed URL). Use pre-release format (`-rc.1`, `-beta.2`) or metadata format (`+hotfix.1`, `+testing`) or both (`-rc.1+fix.1`). -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: see [package signing section](#package-signing) +- `APK_PUB_KEY` and `APK_PRIV_KEY`: see [package signing section](#package-signing) -The `USIGN_PUB_KEY`, `USIGN_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but +The `APK_PUB_KEY`, `APK_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but for [security reasons](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#accessing-secrets) they are not accessible when building pull requests from forks. ### Build locally for a release If you need to build some packages locally for a release, make sure the following environment variables are set: -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info +- `APK_PUB_KEY` and `APK_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info Then execute the build as described in the [Build locally](#build-locally) section. @@ -272,26 +272,26 @@ To replace an upstream package just create a new package with the same name insi ### Package signing -All packages are signed with the following public key generated with [OpenBSD signify](nethsecurity-pub.key). +Packages are signed using an EC prime256v1 key pair (PEM format) via the APK package manager. -Public key fingerprint: `7640d16662de3b89` - -Public key content: +To generate a new signing key pair: ``` -untrusted comment: NethSecurity sign key -RWR2QNFmYt47ieK7g/zEPwgk+MN8bHsA2vFnPThSpnLZ48L7sh6wxB/f +openssl ecparam -name prime256v1 -genkey -noout -out private-key.pem +openssl ec -in private-key.pem -pubout -out public-key.pem ``` -To sign the packages, just execute the `build-nethsec.sh` script with the following environment variables: -- `USIGN_PUB_KEY` -- `USIGN_PRIV_KEY` +To sign the packages, execute the `build-nethsec.sh` script with the following environment variables: +- `APK_PUB_KEY` +- `APK_PRIV_KEY` Usage example: ``` -USIGN_PUB_KEY=$(cat nethsecurity-pub.key) USIGN_PRIV_KEY=$(cat nethsecurity-priv.key) ./build-nethsec.sh +APK_PUB_KEY=$(cat public-key.pem) APK_PRIV_KEY=$(cat private-key.pem) ./build-nethsec.sh ``` -Or you can have the keys as two files named `key-build` and `key-build.pub` in the root of the repository. They will be automatically used by the build script. +Or you can place the keys as two files named `private-key.pem` and `public-key.pem` in the root of the repository. They will be automatically used by the build script. + +If no keys are provided, OpenWrt will auto-generate a throwaway key pair at build time. Builds executed inside CI will sign the packages with the correct key. From 55f1f69f55e7b67b61e3ae1b4ca88564b0e44dc6 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 12 May 2026 18:01:50 +0200 Subject: [PATCH 27/48] build: compiling avahi mdns (#1657) --- config/avahi.conf | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/avahi.conf diff --git a/config/avahi.conf b/config/avahi.conf new file mode 100644 index 000000000..c06fb1e5c --- /dev/null +++ b/config/avahi.conf @@ -0,0 +1 @@ +CONFIG_PACKAGE_avahi-nodbus-daemon=m From 85836dda973dc6a35254f19e1ab0ccffcfd0cc4a Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 12 May 2026 18:02:33 +0200 Subject: [PATCH 28/48] build: updated debian version and openwrt version --- build.conf.defaults | 2 +- builder/Containerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.conf.defaults b/build.conf.defaults index 24f9afec7..15e0198cd 100644 --- a/build.conf.defaults +++ b/build.conf.defaults @@ -1,4 +1,4 @@ -OWRT_VERSION=v25.12.2 +OWRT_VERSION=v25.12.3 NETHSECURITY_VERSION=8.8.0 TARGET=x86_64 REPO_CHANNEL=dev diff --git a/builder/Containerfile b/builder/Containerfile index da0532d5f..7d44137ae 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -FROM debian:13.1 AS base +FROM debian:13.4 AS base RUN apt-get update \ && apt-get install --yes --no-install-recommends --no-install-suggests \ # openwrt build dependencies https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem#debianubuntumint From c8949e3588ca9ddeffe9788425c9c274b59e6fe5 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 14 May 2026 15:13:00 +0200 Subject: [PATCH 29/48] chore: move back history to persistend dir For the sake of god, avoid to loose the history at every reboot --- files/etc/profile.d/busybox-history-file.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 files/etc/profile.d/busybox-history-file.sh diff --git a/files/etc/profile.d/busybox-history-file.sh b/files/etc/profile.d/busybox-history-file.sh new file mode 100644 index 000000000..9f2c9ab5f --- /dev/null +++ b/files/etc/profile.d/busybox-history-file.sh @@ -0,0 +1 @@ +export HISTFILE=~/.bash_history From 9b953c9a887f4a1ba36dbb1bf7a58ec964f0675e Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Fri, 15 May 2026 09:31:24 +0200 Subject: [PATCH 30/48] build: fixed issue where env variables were not honoured --- build-nethsec.sh | 8 +++++++- builder/Containerfile | 4 +--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build-nethsec.sh b/build-nethsec.sh index 000cb2100..426025776 100755 --- a/build-nethsec.sh +++ b/build-nethsec.sh @@ -7,6 +7,9 @@ set -e +# Snapshot current environment so it can be restored with highest precedence +_env_snapshot=$(export -p) + # Source versioned defaults first set -o allexport if [ -f build.conf.defaults ]; then @@ -14,13 +17,16 @@ if [ -f build.conf.defaults ]; then . ./build.conf.defaults fi -# Source local overrides second (can override anything) +# Source local overrides second (can override defaults) if [ -f build.conf ]; then echo "Loading build.conf (local overrides)..." . ./build.conf fi set +o allexport +# Re-apply original environment variables so they take final precedence over config files +eval "$_env_snapshot" + # Check required environment variables OWRT_VERSION=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} NETHSECURITY_VERSION=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION environment variable} diff --git a/builder/Containerfile b/builder/Containerfile index 7d44137ae..a8e05b581 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -51,9 +51,7 @@ WORKDIR /home/buildbot ARG OWRT_VERSION RUN git clone --branch "${OWRT_VERSION}" https://github.com/openwrt/openwrt.git WORKDIR /home/buildbot/openwrt -RUN sed -i '/telephony/d' feeds.conf.default \ - && sed -i '/routing/d' feeds.conf.default \ - && sed -i '/video/d' feeds.conf.default +RUN sed -i '/telephony/d' feeds.conf.default RUN ./scripts/feeds update -a COPY --chmod=777 builder/apply-patches.sh /usr/local/bin/apply-patches COPY --chown=buildbot:buildbot patches patches From c43bdcdef603fe90b1ebaa7100cbd51b5287d05d Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Fri, 15 May 2026 09:33:49 +0200 Subject: [PATCH 31/48] build: bumped to 24.12.4 --- build.conf.defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.conf.defaults b/build.conf.defaults index 15e0198cd..eabd4d1d1 100644 --- a/build.conf.defaults +++ b/build.conf.defaults @@ -1,4 +1,4 @@ -OWRT_VERSION=v25.12.3 +OWRT_VERSION=v25.12.4 NETHSECURITY_VERSION=8.8.0 TARGET=x86_64 REPO_CHANNEL=dev From 33e6707286fa7601aa0cb667f6dac0e6d785dad5 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 22 Apr 2026 15:35:26 +0200 Subject: [PATCH 32/48] feat(victoria): add vmalert alerting Replace Netdata alerting with vmalert: - add vmalert init script (vmalert.initd) to start/stop vmalert service - add vmalert UCI configuration file (vmalert.conf) with datasource settings - add comprehensive alert rules - update Makefile to install vmalert configuration and rules - add detailed documentation of vmalert setup and metrics mapping - support for Mimir integration when configured via ns-plug - add ns-plug-alert-proxy that listens on 127.0.0.1:9095 and receives notifications from vmalert: the proxy verify if an alert is firing or resolved Then it translates selected alerts to the legacy portal format and forwards them to my.nethesis.it or my.nethserver.com - if Mimir credentials are present in ns-plug UCI config, the Mimir alertmanager endpoint is added as a second notifier alongside the proxy - port to Victoria Metrics also alert about non-encrypted backup - add telegraf-mwan Python script that reads /var/run/mwan3/iface_state/ to collect WAN interface connectivity state. - add telegraf-services Python script that queries ubus to collect the running state of all procd-managed services. Outputs JSON for Assisted-by: Copilot:Sonnet4.6 --- packages/ns-plug/Makefile | 13 +- packages/ns-plug/README.md | 37 ++-- packages/ns-plug/files/20_ns-plug | 1 - .../ns-plug/files/backup-encryption-alert | 36 ---- packages/ns-plug/files/ns-plug-alert-proxy | 176 ++++++++++++++++++ .../ns-plug/files/ns-plug-alert-proxy.init | 19 ++ packages/ns-storage/Makefile | 2 - packages/ns-storage/README.md | 4 + .../ns-storage/files/ns-storage-alert.init | 29 --- packages/ns-storage/files/storage-alarm | 27 --- packages/telegraf/Makefile | 11 ++ packages/telegraf/README.md | 145 +++++++++++++++ .../telegraf/files/telegraf-backup-encryption | 17 ++ packages/telegraf/files/telegraf-mwan | 54 ++++++ packages/telegraf/files/telegraf-services | 110 +++++++++++ .../telegraf/files/telegraf-storage-status | 21 +++ .../files/telegraf.conf.d/backup.conf | 16 ++ .../telegraf/files/telegraf.conf.d/mwan.conf | 20 ++ .../files/telegraf.conf.d/services.conf | 20 ++ .../files/telegraf.conf.d/storage.conf | 16 ++ packages/victoria-metrics/Makefile | 6 + packages/victoria-metrics/README.md | 138 ++++++++++++++ .../files/vmalert-rules/backup.yaml | 20 ++ .../files/vmalert-rules/host.yaml | 96 ++++++++++ .../files/vmalert-rules/mwan.yaml | 25 +++ .../files/vmalert-rules/services.yaml | 22 +++ .../files/vmalert-rules/storage.yaml | 18 ++ packages/victoria-metrics/files/vmalert.conf | 3 + packages/victoria-metrics/files/vmalert.initd | 69 +++++++ 29 files changed, 1047 insertions(+), 124 deletions(-) delete mode 100644 packages/ns-plug/files/backup-encryption-alert create mode 100644 packages/ns-plug/files/ns-plug-alert-proxy create mode 100644 packages/ns-plug/files/ns-plug-alert-proxy.init delete mode 100644 packages/ns-storage/files/ns-storage-alert.init delete mode 100644 packages/ns-storage/files/storage-alarm create mode 100644 packages/telegraf/README.md create mode 100644 packages/telegraf/files/telegraf-backup-encryption create mode 100644 packages/telegraf/files/telegraf-mwan create mode 100644 packages/telegraf/files/telegraf-services create mode 100644 packages/telegraf/files/telegraf-storage-status create mode 100644 packages/telegraf/files/telegraf.conf.d/backup.conf create mode 100644 packages/telegraf/files/telegraf.conf.d/mwan.conf create mode 100644 packages/telegraf/files/telegraf.conf.d/services.conf create mode 100644 packages/telegraf/files/telegraf.conf.d/storage.conf create mode 100644 packages/victoria-metrics/README.md create mode 100644 packages/victoria-metrics/files/vmalert-rules/backup.yaml create mode 100644 packages/victoria-metrics/files/vmalert-rules/host.yaml create mode 100644 packages/victoria-metrics/files/vmalert-rules/mwan.yaml create mode 100644 packages/victoria-metrics/files/vmalert-rules/services.yaml create mode 100644 packages/victoria-metrics/files/vmalert-rules/storage.yaml create mode 100644 packages/victoria-metrics/files/vmalert.conf create mode 100644 packages/victoria-metrics/files/vmalert.initd diff --git a/packages/ns-plug/Makefile b/packages/ns-plug/Makefile index b1715109b..5b602cb44 100644 --- a/packages/ns-plug/Makefile +++ b/packages/ns-plug/Makefile @@ -44,6 +44,8 @@ if [ -z "$${IPKG_INSTROOT}" ]; then /etc/init.d/cron restart /usr/libexec/ns-plug/40_ns-plug_mwan_hooks /etc/init.d/ns-plug restart + /etc/init.d/ns-plug-alert-proxy enable + /etc/init.d/ns-plug-alert-proxy restart fi exit 0 endef @@ -55,6 +57,8 @@ if [ -z "$${IPKG_INSTROOT}" ]; then crontab -l | grep -v "/usr/sbin/send-inventory" | sort | uniq | crontab - crontab -l | grep -v "/usr/sbin/send-heartbeat" | sort | uniq | crontab - sed -i '/\/usr\/libexec\/ns-plug\/mwan-hooks/d' /etc/mwan3.user + /etc/init.d/ns-plug-alert-proxy stop + /etc/init.d/ns-plug-alert-proxy disable fi exit 0 endef @@ -68,12 +72,13 @@ define Package/ns-plug/install $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_DIR) $(1)/etc/config $(INSTALL_DIR) $(1)/etc/uci-defaults - $(INSTALL_DIR) $(1)/etc/netdata $(INSTALL_DIR) $(1)/lib/upgrade/keep.d $(INSTALL_DIR) $(1)/usr/libexec/ns-plug $(INSTALL_DIR) $(1)/usr/libexec/mwan-hooks $(INSTALL_BIN) ./files/ns-plug.init $(1)/etc/init.d/ns-plug + $(INSTALL_BIN) ./files/ns-plug-alert-proxy.init $(1)/etc/init.d/ns-plug-alert-proxy $(INSTALL_BIN) ./files/ns-plug $(1)/usr/sbin/ns-plug + $(INSTALL_BIN) ./files/ns-plug-alert-proxy $(1)/usr/sbin/ns-plug-alert-proxy $(INSTALL_BIN) ./files/distfeed-setup $(1)/usr/sbin/distfeed-setup $(INSTALL_BIN) ./files/remote-backup $(1)/usr/sbin $(INSTALL_BIN) ./files/send-backup $(1)/usr/sbin @@ -89,20 +94,14 @@ define Package/ns-plug/install $(INSTALL_BIN) ./files/ns-push-reports $(1)/usr/bin $(INSTALL_BIN) ./files/ns-controller-push-info $(1)/usr/sbin $(INSTALL_BIN) ./files/20_ns-plug $(1)/etc/uci-defaults - $(INSTALL_BIN) ./files/30_ns-plug_alerts $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/40_ns-plug_automatic_updates $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/40_ns-plug_automatic_updates $(1)/usr/libexec/ns-plug $(INSTALL_BIN) ./files/40_ns-plug_mwan_hooks $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/40_ns-plug_mwan_hooks $(1)/usr/libexec/ns-plug - $(INSTALL_BIN) ./files/netadata_enable_alerts $(1)/usr/share/ns-plug/hooks/register/70netadata_enable_alerts - $(INSTALL_BIN) ./files/netadata_disable_alerts $(1)/usr/share/ns-plug/hooks/unregister/70netadata_disable_alerts $(INSTALL_BIN) ./files/enable_automatic_updates $(1)/usr/share/ns-plug/hooks/register/60enable_automatic_updates $(INSTALL_BIN) ./files/disable_automatic_updates $(1)/usr/share/ns-plug/hooks/unregister/60disable_automatic_updates $(INSTALL_CONF) ./files/config $(1)/etc/config/ns-plug $(INSTALL_CONF) files/ns-plug.keep $(1)/lib/upgrade/keep.d/ns-plug - $(INSTALL_CONF) files/health_alarm_notify.conf $(1)/etc/netdata - $(INSTALL_BIN) ./files/send-mwan-alert $(1)/usr/libexec/mwan-hooks - $(INSTALL_BIN) ./files/backup-encryption-alert $(1)/usr/libexec $(INSTALL_BIN) ./files/mwan-hooks $(1)/usr/libexec/ns-plug $(INSTALL_BIN) ./files/ns-plug-rsyslog-fixup.uci-default $(1)/etc/uci-defaults/rsyslog-fixup endef diff --git a/packages/ns-plug/README.md b/packages/ns-plug/README.md index 263ea7a7b..670169c90 100644 --- a/packages/ns-plug/README.md +++ b/packages/ns-plug/README.md @@ -110,13 +110,9 @@ the given passphrase: only the encrypted backup will be sent to the remote serve To disable the encryption, just delete the file `/etc/backup.pass`. -If the backup is not encrypted, an alert will be sent to the remote portal (my.nethesis.it or my.nethserver.com). -Unencrypted backups are deprecated and will be removed in the future. -The alert can be disabled using this command: -``` -uci set ns-plug.config.backup_alert_disabled=1 -uci commit ns-plug -``` +Non-encrypted backups are not sent to the remote server for security reasons. +If the backup is not encrypted, an alert will be sent to the remote portal (my.nethesis.it or my.nethserver.com) +so the user can be aware of the risk and take action to secure the backup. ### Restore @@ -141,20 +137,17 @@ Alerts are also logged to `/var/log/messages` and are visible within the netdata Only the following alerts are sent to the remote system: -- disk space occupation -- WAN down events - -When an alert is resolved, netdata will also send a clear command to remote server. +| Alert | Condition | Legacy alert_id | +|---|---|---| +| `WanDown` | WAN interface offline for 2m | `wan::down` | +| `DiskSpaceCritical` | Disk usage > 90% for 2m | `df:root:percent_bytes:free` or `df:boot:percent_bytes:free` | +| `BackupEncryptionDisabled` | Backup passphrase missing | `backup:config:notencrypted` | +| `StorageStatus` | Storage status is error | `storage:status` | -### MultiWAN alerts +All other alert are silently dropped by the proxy. +If the machine is not registered, all alerts are silently dropped. -MultiWAN alerts are managed using `/etc/mwan3.user` script. - -When a WAN changes its status, all executable scripts inside the `/usr/libexec/mwan-hooks/` directory will be executed. -If the machine has a valid subscription, the `send-mwan-alert` script will send an alert to my.nethesis.it and my.nethserver.com monitoring portals. -Sent alerts are logged to `/var/log/messages`, example: -``` -Jul 31 12:40:42 NethSec mwan3-alert: Sending alert wan:wanb:down with status FAILURE -... -Jul 31 12:41:04 NethSec mwan3-alert: Sending alert wan:wanb:down with status OK -``` +The proxy starts automatically at boot regardless of registration state. +Firing/resolved state is determined from the Alertmanager-standard `endsAt` field: +if `endsAt` is in the future (or zero/missing) a **FAILURE** is sent; if `endsAt` is in +the past an **OK** is sent. diff --git a/packages/ns-plug/files/20_ns-plug b/packages/ns-plug/files/20_ns-plug index b87f6a92f..4026f4031 100644 --- a/packages/ns-plug/files/20_ns-plug +++ b/packages/ns-plug/files/20_ns-plug @@ -2,7 +2,6 @@ # setup cron jobs for remote servers crontab -l | grep -q '/usr/sbin/send-backup' || echo '02 2 * * * sleep $(( RANDOM % 1800 )); /usr/sbin/send-backup' >> /etc/crontabs/root -crontab -l | grep -q '/usr/libexec/backup-encryption-alert' || echo '02 3 * * * sleep $(( RANDOM % 1800 )); /usr/libexec/backup-encryption-alert' >> /etc/crontabs/root crontab -l | grep -q '/usr/sbin/send-heartbeat' || echo '*/10 * * * * sleep $(( RANDOM % 60 )); /usr/sbin/send-heartbeat' >> /etc/crontabs/root crontab -l | grep -q '/usr/sbin/send-inventory' || echo '05 3 * * * sleep $(( RANDOM % 1800 )); /usr/sbin/send-inventory' >> /etc/crontabs/root diff --git a/packages/ns-plug/files/backup-encryption-alert b/packages/ns-plug/files/backup-encryption-alert deleted file mode 100644 index 1249f5479..000000000 --- a/packages/ns-plug/files/backup-encryption-alert +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2025 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Send a backup alert if backup is not encrypted - -lk=$(uci -q get ns-plug.config.system_id) -secret=$(uci -q get ns-plug.config.secret) -url=$(uci -q get ns-plug.config.alerts_url)"alerts/store" - -# Do not send alert if system_id or secret is not set -if [ -z "$lk" ] || [ -z "$secret" ]; then - exit 0 -fi - -# Check if alert is enabled -if [ "$(uci -q get ns-plug.config.backup_alert_disabled)" = "1" ]; then - exit 0 -fi - -# Send the alert -if [ ! -f "/etc/backup.pass" ]; then - status="FAILURE" -else - status="OK" -fi - -alert_id="backup:config:notencrypted" -logger -t backup-alert "Sending alert ${alert_id} with status ${status}" -payload='{"lk": "'$lk'", "alert_id": "'$alert_id'", "status": "'$status'"}' - -/usr/bin/curl -m 30 --retry 3 -L -s \ - --header "Authorization: token ${secret}" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-raw "${payload}" ${url} > /dev/null diff --git a/packages/ns-plug/files/ns-plug-alert-proxy b/packages/ns-plug/files/ns-plug-alert-proxy new file mode 100644 index 000000000..f50f70d4b --- /dev/null +++ b/packages/ns-plug/files/ns-plug-alert-proxy @@ -0,0 +1,176 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Alert proxy: receives Alertmanager-like notifications from vmalert and +forwards selected alerts to the legacy my.nethesis.it / my.nethserver.com +monitoring portals. + +Only the following alerts are forwarded: + - WanDown → wan::down + - DiskSpaceCritical → df:root:percent_bytes:free (path=/) + df:boot:percent_bytes:free (path=/boot) + - BackupEncryptionDisabled → backup:config:notencrypted + - StorageStatus → storage:status + +All other alerts are silently dropped. +If the machine is not registered (no system_id/secret in UCI), all alerts +are silently dropped. + +Firing/resolved state is determined from the Alertmanager-standard endsAt +field: if endsAt is in the future (or zero/missing) the alert is FAILURE; +if endsAt is in the past the alert is OK. +""" + +import json +import re +import sys +import time +import urllib.request +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn +from euci import EUci + +LISTEN_ADDR = "127.0.0.1" +LISTEN_PORT = 9095 + +_DISK_PATH_MAP = { + "/": "df:root:percent_bytes:free", + "/boot": "df:boot:percent_bytes:free", +} + +_ZERO_TIME = "0001-01-01T00:00:00Z" +# vmalert uses nanosecond precision; strip to microseconds for Python parsing +_NANO_RE = re.compile(r"(\.\d{6})\d+(Z|[+-]\d{2}:\d{2})$") + + +def _is_firing(alert): + """Return True if the alert is currently firing based on endsAt.""" + ends_at_str = alert.get("endsAt", "") + if not ends_at_str or ends_at_str == _ZERO_TIME: + return True + ends_at_str = _NANO_RE.sub(r"\1\2", ends_at_str) + ends_at_str = ends_at_str.replace("Z", "+00:00") + try: + ends_at = datetime.fromisoformat(ends_at_str) + return ends_at > datetime.now(timezone.utc) + except Exception: + return True + + +def _map_alert_id(alert_name, labels): + """Return the legacy alert_id string, or None if the alert is not mapped.""" + if alert_name == "WanDown": + iface = labels.get("interface", "unknown") + return f"wan:{iface}:down" + if alert_name == "DiskSpaceCritical": + path = labels.get("path", "") + return _DISK_PATH_MAP.get(path) + if alert_name == "BackupEncryptionDisabled": + return "backup:config:notencrypted" + if alert_name == "StorageStatus": + return "storage:status" + return None + + +def _send_alert(system_id, secret, alerts_url, alert_id, status, retry=3): + url = alerts_url.rstrip("/") + "/alerts/store" + payload = json.dumps( + {"lk": system_id, "alert_id": alert_id, "status": status} + ).encode() + req = urllib.request.Request( + url, + data=payload, + method="POST", + headers={ + "Authorization": f"token {secret}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + print(f"Alert sent: {alert_id} {status} → {resp.status}", file=sys.stderr) + except Exception as ex: + if retry > 0: + print( + f"Alert send failed: {alert_id} {ex} — retrying in 20s", file=sys.stderr + ) + time.sleep(20) + _send_alert(system_id, secret, alerts_url, alert_id, status, retry - 1) + else: + print(f"Alert send aborted: {alert_id} {ex}", file=sys.stderr) + + +class _AlertHandler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + # Suppress access log + pass + + def do_GET(self): + self.send_response(200) + self.end_headers() + + def do_POST(self): + if self.system_id is None or self.secret is None or self.alerts_url is None: + # Just drop the alert if not configured + self.send_response(200) + self.end_headers() + return + try: + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + data = json.loads(body) + except Exception as ex: + self.send_response(400) + self.end_headers() + self.wfile.write(str(ex).encode()) + return + + if type(data) is list: + alerts = data + else: + alerts = data.get("alerts", []) + for alert in alerts: + labels = alert.get("labels", {}) + alert_name = labels.get("alertname", "") + legacy_status = "FAILURE" if _is_firing(alert) else "OK" + + alert_id = _map_alert_id(alert_name, labels) + if not alert_id: + print( + f"Alert dropped (no mapping): {alert_name} {labels}", + file=sys.stderr, + ) + continue + + _send_alert(self.system_id, self.secret, self.alerts_url, alert_id, legacy_status) + + self.send_response(200) + self.end_headers() + + def __init__(self, *args, **kwargs): + uci = EUci() + self.system_id = uci.get("ns-plug", "config", "system_id", default=None) + self.secret = uci.get("ns-plug", "config", "secret", default=None) + self.alerts_url = uci.get("ns-plug", "config", "alerts_url", default=None) + super().__init__(*args, **kwargs) + + +class _ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + +def main(): + server = _ThreadingHTTPServer((LISTEN_ADDR, LISTEN_PORT), _AlertHandler) + print(f"alert-proxy listening on {LISTEN_ADDR}:{LISTEN_PORT}", file=sys.stderr) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/packages/ns-plug/files/ns-plug-alert-proxy.init b/packages/ns-plug/files/ns-plug-alert-proxy.init new file mode 100644 index 000000000..31b5fa430 --- /dev/null +++ b/packages/ns-plug/files/ns-plug-alert-proxy.init @@ -0,0 +1,19 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +START=95 +STOP=4 +USE_PROCD=1 + +start_service() { + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param command '/usr/sbin/ns-plug-alert-proxy' + procd_set_param respawn 3600 5 0 + procd_close_instance +} diff --git a/packages/ns-storage/Makefile b/packages/ns-storage/Makefile index 27938cfd1..8bd79ac3b 100644 --- a/packages/ns-storage/Makefile +++ b/packages/ns-storage/Makefile @@ -70,8 +70,6 @@ define Package/ns-storage/install $(INSTALL_BIN) ./files/32-ns-storage-convert-uuid.uci-default $(1)/etc/uci-defaults/32-ns-storage-convert-uuid $(INSTALL_CONF) ./files/data.conf $(1)/etc/logrotate.d $(INSTALL_BIN) ./files/storage-status $(1)/usr/sbin - $(INSTALL_BIN) ./files/storage-alarm $(1)/usr/libexec - $(INSTALL_BIN) ./files/ns-storage-alert.init $(1)/etc/init.d/ns-storage-alert $(INSTALL_BIN) ./files/ns-storage-check.init $(1)/etc/init.d/ns-storage-check endef diff --git a/packages/ns-storage/README.md b/packages/ns-storage/README.md index b9554a8b2..ba92773c3 100644 --- a/packages/ns-storage/README.md +++ b/packages/ns-storage/README.md @@ -69,6 +69,10 @@ the system as follow: - rsyslog will write logs also inside `/mnt/data/logs/messages` file - logrotate will rotate `/mnt/data/logs/messages` once a week (see `/etc/logrotate/data.conf` for more info) +### Storage status alert + +The storage health check is exported to Telegraf by `/usr/libexec/telegraf-storage-status` and evaluated by vmalert as `StorageStatus`. + ## Data sync customization Every night the cron will run a script named `sync-data` to sync data from in-memory diff --git a/packages/ns-storage/files/ns-storage-alert.init b/packages/ns-storage/files/ns-storage-alert.init deleted file mode 100644 index 26a259041..000000000 --- a/packages/ns-storage/files/ns-storage-alert.init +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh /etc/rc.common - -# -# Copyright (C) 2025 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -START=99 -USE_PROCD=1 - -start_service() -{ - procd_open_instance - procd_set_param stdout 1 - procd_set_param stderr 1 - procd_set_param command '/usr/libexec/storage-alarm' - procd_close_instance -} - -reload_service() -{ - start -} - -service_triggers() -{ - procd_add_reload_trigger fstab ns-plug - procd_add_reload_mount_trigger /mnt/data -} diff --git a/packages/ns-storage/files/storage-alarm b/packages/ns-storage/files/storage-alarm deleted file mode 100644 index adaccedad..000000000 --- a/packages/ns-storage/files/storage-alarm +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -# -# Copyright (C) 2025 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -system_id=$(uci -q get ns-plug.config.system_id) -system_secret=$(uci -q get ns-plug.config.secret) -if [ -z "$system_id" ] || [ -z "$system_secret" ]; then - # not subscription - exit 0 -fi -url="$(uci -q get ns-plug.config.alerts_url)/alerts/store" - -storage_status=$(storage-status) -status="OK" -if [ "$storage_status" = "error" ]; then - status="FAILURE" -fi - -/usr/bin/curl -m 180 --retry 10 -L -s \ - --header "Authorization: token $system_secret" \ - --header "Content-Type: application/json" \ - --header "Accept: application/json" \ - --data-binary "{\"lk\": \"$system_id\", \"alert_id\": \"storage:status\", \"status\": \"$status\"}" \ - "$url" > /dev/null diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index e63822acb..1f4175a2d 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -76,8 +76,19 @@ define Package/telegraf/install $(INSTALL_DATA) ./files/telegraf.conf $(1)/etc/telegraf.conf $(INSTALL_DIR) $(1)/etc/telegraf.conf.d $(INSTALL_DATA) ./files/telegraf.conf.d/os.conf $(1)/etc/telegraf.conf.d/os.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/backup.conf $(1)/etc/telegraf.conf.d/backup.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/storage.conf $(1)/etc/telegraf.conf.d/storage.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/services.conf $(1)/etc/telegraf.conf.d/services.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/mwan.conf $(1)/etc/telegraf.conf.d/mwan.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/ping.conf $(1)/etc/telegraf.conf.d/ping.conf $(INSTALL_DIR) $(1)/usr/sbin $(INSTALL_BIN) ./files/telegraf-config $(1)/usr/sbin/telegraf-config + $(INSTALL_DIR) $(1)/usr/libexec + $(INSTALL_BIN) ./files/telegraf-services $(1)/usr/libexec/telegraf-services + $(INSTALL_BIN) ./files/telegraf-backup-encryption $(1)/usr/libexec/telegraf-backup-encryption + $(INSTALL_BIN) ./files/telegraf-storage-status $(1)/usr/libexec/telegraf-storage-status + $(INSTALL_BIN) ./files/telegraf-services $(1)/usr/libexec/telegraf-services + $(INSTALL_BIN) ./files/telegraf-mwan $(1)/usr/libexec/telegraf-mwan endef define Package/telegraf/postinst diff --git a/packages/telegraf/README.md b/packages/telegraf/README.md new file mode 100644 index 000000000..b6cb55a73 --- /dev/null +++ b/packages/telegraf/README.md @@ -0,0 +1,145 @@ +# Telegraf + +## Overview + +Telegraf is the metrics collection agent that gathers host and service metrics and forwards them to Victoria Metrics for storage, alerting, and visualization. + +## Architecture + +``` +/usr/libexec/telegraf-services ← service status via ubus +/var/run/mwan3/iface_state/ ← WAN interface status via mwan3 state files +/proc filesystem ← CPU, memory, disk, network + │ + ▼ + Telegraf (inputs.exec, inputs.cpu, inputs.mem, …) + │ + ▼ +Victoria Metrics (http://127.0.0.1:8428) + │ + └─▶ vmalert (alert rules evaluation) +``` + +## Configuration Files + +| Path | Description | +|------|-------------| +| `/etc/telegraf.conf` | Main Telegraf agent config and InfluxDB output | +| `/etc/telegraf.conf.d/*.conf` | Additional Telegraf input configurations for plugins | + + +## Collected Metrics + +To see the list of metrics collected by Telegraf, use: +```bash +/usr/bin/telegraf --config /etc/telegraf.conf --config-directory /etc/telegraf.conf.d --test +``` + +### Service Health Monitoring (services.conf) + +**How it works**: Every 60 seconds, Telegraf executes `/usr/libexec/telegraf-services`, which queries procd via `ubus call service list`, filters the fixed monitored service whitelist, and converts the matching configured instances to metrics. + +**Metric format**: +``` +procd_service_running{service="nginx", instance="instance1"} = 1 (running) or 0 (down) +procd_service_pid{service="nginx", instance="instance1"} = process_id +procd_service_exit_code{service="nginx", instance="instance1"} = last_exit_code +``` + +Only these services are monitored: + +```text +banip +conntrackd +cron +dedalo +dedalo_users_auth +dnsmasq +dropbear +keepalived +mwan3 +netifyd +nginx +ns-api-server +ns-clm +ns-flashstart +ns-flows +ns-plug +ns-plug-alert-proxy +ns-stats +ns-ui +odhcpd +openvpn +qosify +rpcd +rsyslog +snort +swanctl +sysntpd +telegraf +victoria-metrics +vmalert +``` + +Services with no instances are skipped. + +Notable skipped services: +- `adblock`: excluded because ubus info is not reliable (always shows 1 instance even when disabled) + +##### Querying Service Status + +```bash +# All services and their running state +curl -s 'http://127.0.0.1:8428/api/v1/query?query=procd_service_running' + +# Run collection script manually to preview output +/usr/libexec/telegraf-services +``` + +### Multi-WAN Monitoring (mwan.conf) + +**How it works**: Every 60 seconds, Telegraf executes `/usr/libexec/telegraf-mwan`, which reads `/var/run/mwan3/iface_state/` to determine each WAN interface's online/offline state (maintained by mwan3 in real-time). + +**Metric format**: +``` +mwan_interface_online{interface="wan"} = 1 (online) or 0 (offline) +``` + +#### Querying WAN Status + +```bash +# All WAN interfaces and current state +curl -s 'http://127.0.0.1:8428/api/v1/query?query=mwan_interface_online' + +# Run collection script manually +/usr/libexec/telegraf-mwan +``` + +### Storage Status Monitoring (storage.conf) + +**How it works**: Every 60 seconds, Telegraf executes `/usr/libexec/telegraf-storage-status`, which runs `storage-status` and exports the current storage health as a metric. + +**Metric format**: +``` +storage_status_error = 1 (error) or 0 (ok / not configured) +``` + +## Advanced Configuration + +To add custom metrics or modify collection intervals, edit the `/etc/telegraf.conf.d/` files following [Telegraf documentation](https://docs.influxdata.com/telegraf/). Common customizations: + +- Modify collection intervals: change `interval` in main config +- Add new input plugins: append `[[inputs.plugin_name]]` sections + +After changes, restart Telegraf: +```bash +/etc/init.d/telegraf restart +``` + +## References + +- [Telegraf documentation](https://docs.influxdata.com/telegraf/) +- [Telegraf exec plugin](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec) +- [OpenWrt procd init scripts](https://openwrt.org/docs/guide-developer/procd-init-scripts) +- [OpenWrt ubus reference](https://openwrt.org/docs/techref/ubus) +- [Victoria Metrics integration](../victoria-metrics/README.md) diff --git a/packages/telegraf/files/telegraf-backup-encryption b/packages/telegraf/files/telegraf-backup-encryption new file mode 100644 index 000000000..a0026c6f3 --- /dev/null +++ b/packages/telegraf/files/telegraf-backup-encryption @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Export backup encryption state for Telegraf. +# +# The metric is 1 when /etc/backup.pass exists and is non-empty, otherwise 0. + +if [ -s /etc/backup.pass ]; then + encrypted=1 +else + encrypted=0 +fi + +printf '[{"encrypted":%s}]\n' "$encrypted" diff --git a/packages/telegraf/files/telegraf-mwan b/packages/telegraf/files/telegraf-mwan new file mode 100644 index 000000000..ab6d12837 --- /dev/null +++ b/packages/telegraf/files/telegraf-mwan @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# +# Collect mwan3 WAN interface status from /var/run/mwan3/iface_state/. +# +# Each file in that directory is named after an mwan3 interface and contains +# a single word: "online" or "offline". The directory is managed by mwan3 +# and only exists when the daemon is running. +# +# Output metric: mwan_interface +# Tags: interface +# Fields: online (int 0/1) +# +# Prints a JSON array to stdout, consumed by telegraf inputs.exec with +# data_format = "json_v2" (parsers.json_v2 build tag). + +import json +import os + +IFACE_STATE_DIR = "/var/run/mwan3/iface_state" + + +def build_records(): + """Return one record per mwan3 interface found in the state directory.""" + if not os.path.isdir(IFACE_STATE_DIR): + return [] + + records = [] + for name in sorted(os.listdir(IFACE_STATE_DIR)): + path = os.path.join(IFACE_STATE_DIR, name) + if not os.path.isfile(path): + continue + try: + status = open(path).read().strip() + except OSError: + continue + records.append( + { + "interface": name, + "online": 1 if status == "online" else 0, + } + ) + return records + + +def main(): + records = build_records() + print(json.dumps(records)) + + +if __name__ == "__main__": + main() diff --git a/packages/telegraf/files/telegraf-services b/packages/telegraf/files/telegraf-services new file mode 100644 index 000000000..932049020 --- /dev/null +++ b/packages/telegraf/files/telegraf-services @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# +# Collect procd service status via ubus. +# +# Monitored services: fixed whitelist of NethSecurity services. +# Services with no instances are ignored because they are disabled or not +# configured. +# +# Usage: +# /usr/libexec/telegraf-services +# +# Output metric: procd_service +# Tags: service, instance +# Fields: running (int 0/1), pid (int), exit_code (int) +# +# Prints a JSON array to stdout, consumed by telegraf inputs.exec with +# data_format = "json_v2" (parsers.json_v2 build tag). +# +import json +import subprocess +import sys +MONITORED_SERVICES = { + "banip", + "conntrackd", + "cron", + "dedalo", + "dedalo_users_auth", + "dnsmasq", + "dropbear", + "keepalived", + "mwan3", + "netifyd", + "nginx", + "ns-api-server", + "ns-clm", + "ns-flashstart", + "ns-flows", + "ns-plug", + "ns-plug-alert-proxy", + "ns-stats", + "ns-ui", + "odhcpd", + "openvpn", + "qosify", + "rpcd", + "rsyslog", + "snort", + "swanctl", + "sysntpd", + "telegraf", + "victoria-metrics", + "vmalert", +} + +# Excluded service: adblock + +def get_service_list(): + result = subprocess.run( + ["ubus", "call", "service", "list"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode != 0: + print(f"Error calling ubus: {result.stderr}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout) + + +def sanitize_tag(value): + # InfluxDB line protocol: tag values must not contain commas, spaces or equals + return value.replace(",", "_").replace(" ", "_").replace("=", "_") + + +def build_records(data): + """Return a list of dicts, one per configured monitored service instance.""" + records = [] + + for svc_name in sorted(MONITORED_SERVICES): + svc_body = data.get(svc_name) + instances = (svc_body or {}).get("instances") or {} + if not instances: + continue + + for inst_name, inst in instances.items(): + records.append( + { + "service": sanitize_tag(svc_name), + "instance": sanitize_tag(inst_name), + "running": 1 if inst.get("running", False) else 0, + "pid": inst.get("pid", 0), + "exit_code": inst.get("exit_code", 0), + } + ) + return records + + +def main(): + data = get_service_list() + records = build_records(data) + + # JSON array — consumed by telegraf inputs.exec with data_format=json_v2 + print(json.dumps(records)) + + +if __name__ == "__main__": + main() diff --git a/packages/telegraf/files/telegraf-storage-status b/packages/telegraf/files/telegraf-storage-status new file mode 100644 index 000000000..f7dd52113 --- /dev/null +++ b/packages/telegraf/files/telegraf-storage-status @@ -0,0 +1,21 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Export the storage health state for Telegraf. +# +# The metric is 1 when storage-status reports "error", otherwise 0. + +if storage_status=$(/usr/sbin/storage-status 2>/dev/null); then + if [ "$storage_status" = "error" ]; then + error=1 + else + error=0 + fi +else + error=1 +fi + +printf '[{"error":%s}]\n' "$error" diff --git a/packages/telegraf/files/telegraf.conf.d/backup.conf b/packages/telegraf/files/telegraf.conf.d/backup.conf new file mode 100644 index 000000000..65e4fdf1e --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/backup.conf @@ -0,0 +1,16 @@ +# Backup encryption status monitoring +# Reports whether /etc/backup.pass is set so vmalert can alert on unencrypted backups. + +[[inputs.exec]] + name_override = "backup_encryption" + commands = ["/usr/libexec/telegraf-backup-encryption"] + interval = "60s" + timeout = "5s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.conf.d/mwan.conf b/packages/telegraf/files/telegraf.conf.d/mwan.conf new file mode 100644 index 000000000..8658cfdcf --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/mwan.conf @@ -0,0 +1,20 @@ +# mwan3 WAN interface status monitoring +# Reads /var/run/mwan3/iface_state/ — one file per interface, content is +# "online" or "offline". No-ops silently when mwan3 is not running. +# +# Uses parsers.json_v2 — available in the default NethSecurity Telegraf build. + +[[inputs.exec]] + name_override = "mwan_interface" + commands = ["/usr/libexec/telegraf-mwan"] + interval = "60s" + timeout = "10s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + tags = ["interface"] + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.conf.d/services.conf b/packages/telegraf/files/telegraf.conf.d/services.conf new file mode 100644 index 000000000..04c5f98c6 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/services.conf @@ -0,0 +1,20 @@ +# Procd service status monitoring +# Collects running state for a fixed whitelist of persistent services. +# Services with no instances are ignored by the collector. +# +# Uses parsers.json_v2 — available in the default NethSecurity Telegraf build. + +[[inputs.exec]] + name_override = "procd_service" + commands = ["/usr/libexec/telegraf-services"] + interval = "60s" + timeout = "10s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + tags = ["service", "instance"] + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.conf.d/storage.conf b/packages/telegraf/files/telegraf.conf.d/storage.conf new file mode 100644 index 000000000..10e6efee2 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/storage.conf @@ -0,0 +1,16 @@ +# Storage status monitoring +# Reports whether the persistent data storage is mounted so vmalert can alert on storage:status. + +[[inputs.exec]] + name_override = "storage_status" + commands = ["/usr/libexec/telegraf-storage-status"] + interval = "60s" + timeout = "5s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile index 5fe85f5a8..6510e5a0a 100644 --- a/packages/victoria-metrics/Makefile +++ b/packages/victoria-metrics/Makefile @@ -52,19 +52,25 @@ endef define Package/victoria-metrics/conffiles /etc/config/victoria-metrics +/etc/config/vmalert endef define Package/victoria-metrics/install $(call GoPackage/Package/Install/Bin,$(1)) $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_BIN) ./files/victoria-metrics.initd $(1)/etc/init.d/victoria-metrics + $(INSTALL_BIN) ./files/vmalert.initd $(1)/etc/init.d/vmalert $(INSTALL_DIR) $(1)/etc/config $(INSTALL_DATA) ./files/victoria-metrics.conf $(1)/etc/config/victoria-metrics + $(INSTALL_DATA) ./files/vmalert.conf $(1)/etc/config/vmalert + $(INSTALL_DIR) $(1)/etc/vmalert/rules + $(INSTALL_DATA) ./files/vmalert-rules/*.yaml $(1)/etc/vmalert/rules/ endef define Package/victoria-metrics/postinst #!/bin/sh [ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-metrics restart +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/vmalert restart exit 0 endef diff --git a/packages/victoria-metrics/README.md b/packages/victoria-metrics/README.md new file mode 100644 index 000000000..d878bf4f1 --- /dev/null +++ b/packages/victoria-metrics/README.md @@ -0,0 +1,138 @@ +# Victoria Metrics + +## Overview + +This package provides **Victoria Metrics** and **vmalert** for time-series metrics storage and alerting in NethSecurity. Metrics are collected by Telegraf, stored in Victoria Metrics, and evaluated by vmalert according to alert rules. + +**Key Components:** +- **victoria-metrics**: Time-series database on port 8428 +- **vmalert**: Alert rule evaluator on port 8081 +- **Telegraf integration**: Host metrics, service health, WAN status, storage status +- **Mimir integration**: Optional centralized alerting (via ns-plug) + +## Quick Start + +### View Active Alerts + +```bash +# List all firing and pending alerts +curl http://127.0.0.1:8082/api/v1/alerts | jq + +# Get a specific alert status +curl 'http://127.0.0.1:8082/api/v1/rules?type=alert' | \ + jq '.data.groups[].rules[] | select(.name == "HighCpuUsage") | {name, state, lastEvaluation}' +``` + +### List Available Metrics + +```bash +# All metrics currently being stored +curl -s 'http://127.0.0.1:8428/api/v1/label/__name__/values' | jq -r '.data[]' | sort +``` + +## Configuration + +Configuration is located at `/etc/config/victoria-metrics`: + +``` +config victoriametrics 'main' + option storage_path '/var/lib/victoriametrics' + option retention_period '1y' + option http_listen_addr '127.0.0.1:8428' +``` + +**Options:** +- `storage_path`: Where to store metrics data +- `retention_period`: How long to keep metrics (`1d`, `7d`, `30d`, `1y`, etc.) +- `http_listen_addr`: Address and port for the HTTP server + +### Accessing the Web UI + +By default the server is accessible only on localhost for security. +The service also exposes a Web UI on port 8428 for browsing metrics and testing queries. + +To access the Web UI, you can change the `http_listen_addr` to `0.0.0.0:8428` to allow external access, but this is not recommended for production environments without proper security measures. +A safer approach is to use SSH port forwarding: +```bash +ssh -L 8428:127.0.0.1:8428 root@remote_host +``` + +Then open `http://127.0.0.1:8428` in your web browser to see all exposed endpoints. +The UI to query metrics is available at `http://127.0.0.1:8428/vmui + +## Alerting Rules + +All alert rules are defined as YAML files in `/etc/vmalert/rules/*.yaml`. Each file corresponds to a specific monitoring category. + +Some alerts implement a two-tier severity model with `warning` and `critical` levels and are designed to suppress lower-severity alerts when higher-severity ones are firing. + +Warning alerts use `unless` clauses to suppress them when their critical counterpart is already firing, reducing noise. For example, `HighCpuUsage` warning is silenced when `CriticalCpuUsage` is firing. + +See rule files for specific thresholds and suppression logic. + +An alert can be in one of three states: + +1. **Pending**: Condition is true but hasn't met the required `for` duration +2. **Firing**: Condition has been true for at least the `for` duration +3. **Resolved**: Condition is no longer true + +Example: An alert with `for: 5m` takes 5 minutes to transition from pending → firing. + +### Custom Alert Rules + +To add custom alerts, create a new YAML file in `/etc/vmalert/rules/`. +Example `my_alerts.yaml`: + +```yaml +groups: + - name: "my_alerts" + interval: "30s" + rules: + - alert: MyAlert + expr: 'metric_name > threshold' + for: "5m" + labels: + severity: "warning" + service: "my_service" + annotations: + summary_en: "Alert summary" + summary_it: "Riepilogo avviso" + description_en: "Value is {{ $value }}" +``` + +Then restart vmalert: +```bash +/etc/init.d/vmalert restart +``` + +## Mimir Integration (ns-plug) + +Mimir is a multi-tenant Prometheus-compatible long-term storage and alerting system used by nextgen [my](https://github.com/NethServer/my/) monitoring +platform. +When Mimir is configured via ns-plug, vmalert automatically forwards alerts. No manual vmalert configuration needed. + +**Enable Mimir forwarding:** +```bash +uci set ns-plug.config.my_url='https://mimir.example.com' +uci set ns-plug.config.my_system_key='your_api_key' +uci set ns-plug.config.my_system_secret='your_api_secret' +uci commit ns-plug +/etc/init.d/vmalert restart +``` + +**Disable (alert-proxy only mode):** +```bash +uci delete ns-plug.config.my_url +uci delete ns-plug.config.my_system_key +uci delete ns-plug.config.my_system_secret +uci commit ns-plug +/etc/init.d/vmalert restart +``` + +## References + +- [Victoria Metrics vmalert docs](https://docs.victoriametrics.com/vmalert/) +- [MetricsQL documentation](https://docs.victoriametrics.com/metricsql/) +- [Prometheus alerting rules](https://samber.github.io/awesome-prometheus-alerts/) +- [vmalert documentation](https://docs.victoriametrics.com/vmalert/) +- [Telegraf metrics collection](../telegraf/README.md) diff --git a/packages/victoria-metrics/files/vmalert-rules/backup.yaml b/packages/victoria-metrics/files/vmalert-rules/backup.yaml new file mode 100644 index 000000000..30800186e --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/backup.yaml @@ -0,0 +1,20 @@ +# Victoria Metrics Alert Rules for backup encryption monitoring +# +# Monitors whether /etc/backup.pass is present and non-empty via the +# backup_encryption_encrypted metric collected by Telegraf. + +groups: + - name: "backup" + interval: "60s" + rules: + - alert: BackupEncryptionDisabled + expr: 'backup_encryption_encrypted == 0' + for: "2m" + labels: + severity: "warning" + service: "backup" + annotations: + summary_en: "Backup encryption is disabled" + summary_it: "La cifratura dei backup e disattivata" + description_en: "The backup passphrase file /etc/backup.pass is missing or empty. If the firewall has a subscription, the backup will not be sent to the remote storage server." + description_it: "Il file della passphrase dei backup /etc/backup.pass manca o e vuoto. Se il firewall ha una subscription, i backup non verranno inviati al server di archiviazione remoto." diff --git a/packages/victoria-metrics/files/vmalert-rules/host.yaml b/packages/victoria-metrics/files/vmalert-rules/host.yaml new file mode 100644 index 000000000..b01dae22f --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/host.yaml @@ -0,0 +1,96 @@ +# Victoria Metrics Alert Rules for Host and Hardware Monitoring +# +# Based on: https://samber.github.io/awesome-prometheus-alerts/rules/basic-resource-monitoring/host-and-hardware/ +# Adapted for Telegraf metrics names + +groups: + - name: "host_and_hardware" + interval: "30s" + rules: + # CPU Monitoring + - alert: HighCpuUsage + expr: 'round(100 - avg(cpu_usage_idle), 0.1) > 70 unless round(100 - avg(cpu_usage_idle), 0.1) > 85' + for: "5m" + labels: + severity: "info" + service: "host" + annotations: + summary_en: "High CPU usage detected" + summary_it: "Utilizzo elevato di CPU rilevato" + description_en: "CPU usage is {{ $value }}%" + description_it: "Utilizzo della CPU è {{ $value }}%" + + - alert: CriticalCpuUsage + expr: 'round(100 - avg(cpu_usage_idle), 0.1) > 85' + for: "2m" + labels: + severity: "warning" + service: "host" + annotations: + summary_en: "Critical CPU usage detected" + summary_it: "Utilizzo critico di CPU rilevato" + description_en: "CPU usage is {{ $value }}%" + description_it: "Utilizzo della CPU è {{ $value }}%" + + # Memory Monitoring + - alert: HighMemoryUsage + expr: 'round((mem_used / mem_total) * 100, 0.1) > 80 unless round((mem_used / mem_total) * 100, 0.1) > 90' + for: "5m" + labels: + severity: "info" + service: "host" + annotations: + summary_en: "High memory usage detected" + summary_it: "Utilizzo elevato di memoria rilevato" + description_en: "Memory usage is {{ $value }}%" + description_it: "Utilizzo della memoria è {{ $value }}%" + + - alert: CriticalMemoryUsage + expr: 'round((mem_used / mem_total) * 100, 0.1) > 90' + for: "2m" + labels: + severity: "warning" + service: "host" + annotations: + summary_en: "Critical memory usage detected" + summary_it: "Utilizzo critico di memoria rilevato" + description_en: "Memory usage is {{ $value }}%" + description_it: "Utilizzo della memoria è {{ $value }}%" + + # Disk Space Monitoring + - alert: DiskSpaceWarning + expr: 'round((disk_used / disk_total) * 100, 0.1) > 80 unless round((disk_used / disk_total) * 100, 0.1) > 90' + for: "5m" + labels: + severity: "warning" + service: "storage" + annotations: + summary_en: "Disk space low on {{ $labels.path }}" + summary_it: "Spazio disco in esaurimento su {{ $labels.path }}" + description_en: "Disk usage is {{ $value }}% on {{ $labels.path }}" + description_it: "Utilizzo del disco è {{ $value }}% su {{ $labels.path }}" + + - alert: DiskSpaceCritical + expr: 'round((disk_used / disk_total) * 100, 0.1) > 90' + for: "2m" + labels: + severity: "critical" + service: "storage" + annotations: + summary_en: "Disk space critical on {{ $labels.path }}" + summary_it: "Spazio disco critico su {{ $labels.path }}" + description_en: "Disk usage is {{ $value }}% on {{ $labels.path }}" + description_it: "Utilizzo del disco è {{ $value }}% su {{ $labels.path }}" + + # System Load Monitoring + - alert: HighSystemLoad + expr: 'system_load1 / system_n_cpus > 2' + for: "5m" + labels: + severity: "warning" + service: "host" + annotations: + summary_en: "High system load detected" + summary_it: "Carico di sistema elevato rilevato" + description_en: "System load is {{ $value }}" + description_it: "Carico di sistema è {{ $value }}" diff --git a/packages/victoria-metrics/files/vmalert-rules/mwan.yaml b/packages/victoria-metrics/files/vmalert-rules/mwan.yaml new file mode 100644 index 000000000..a4da2b96d --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/mwan.yaml @@ -0,0 +1,25 @@ +# Victoria Metrics Alert Rules for mwan3 WAN Monitoring +# +# Monitors WAN interface connectivity via the mwan_interface_online metric +# collected by /usr/libexec/telegraf-mwan. +# +# The metric is sourced from /var/run/mwan3/iface_state/ which mwan3 +# writes as "online" or "offline" based on its tracking probes. +# Only interfaces present in that directory are monitored — interfaces +# not managed by mwan3 are not included. + +groups: + - name: "mwan" + interval: "60s" + rules: + - alert: WanDown + expr: 'mwan_interface_online == 0' + for: "2m" + labels: + severity: "critical" + service: "network" + annotations: + summary_en: "WAN interface {{ $labels.interface }} is offline" + summary_it: "L'interfaccia WAN {{ $labels.interface }} non è raggiungibile" + description_en: "WAN interface {{ $labels.interface }} is down. Internet connectivity lost." + description_it: "L'interfaccia WAN {{ $labels.interface }} non è raggiungibile. Connettività Internet persa." diff --git a/packages/victoria-metrics/files/vmalert-rules/services.yaml b/packages/victoria-metrics/files/vmalert-rules/services.yaml new file mode 100644 index 000000000..f87b1c38c --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/services.yaml @@ -0,0 +1,22 @@ +# Victoria Metrics Alert Rules for Service Monitoring +# +# Monitors configured procd-managed services via the procd_service_* metrics +# collected by /usr/libexec/telegraf-services. +# +# Services with no instances are ignored by the collector. + +groups: + - name: "services" + interval: "60s" + rules: + - alert: ServiceDown + expr: 'procd_service_running == 0 and procd_service_exit_code != 0' + for: "2m" + labels: + severity: "critical" + alertgroup: "services" + annotations: + summary_en: "Service {{ $labels.service }} is down" + summary_it: "Il servizio {{ $labels.service }} non è attivo" + description_en: "Service {{ $labels.service }} (instance {{ $labels.instance }}) has been down for more than 2 minutes" + description_it: "Il servizio {{ $labels.service }} (istanza {{ $labels.instance }}) non è attivo da più di 2 minuti" diff --git a/packages/victoria-metrics/files/vmalert-rules/storage.yaml b/packages/victoria-metrics/files/vmalert-rules/storage.yaml new file mode 100644 index 000000000..fdcca20cf --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/storage.yaml @@ -0,0 +1,18 @@ +# Storage status monitoring +# +# The alert is driven by the Telegraf storage_status_error metric. + +groups: + - name: "storage" + interval: "30s" + rules: + - alert: StorageStatus + expr: 'storage_status_error == 1' + labels: + severity: "critical" + service: "storage" + annotations: + summary_en: "Storage is in error state" + summary_it: "Lo storage è in stato di errore" + description_en: "The configured data storage is not mounted or is otherwise in error." + description_it: "Lo storage dati configurato non è montato o è in errore." diff --git a/packages/victoria-metrics/files/vmalert.conf b/packages/victoria-metrics/files/vmalert.conf new file mode 100644 index 000000000..c40cc9c37 --- /dev/null +++ b/packages/victoria-metrics/files/vmalert.conf @@ -0,0 +1,3 @@ +config main 'main' + option datasource_url 'http://localhost:8428' + option http_listen_addr '127.0.0.1:8082' diff --git a/packages/victoria-metrics/files/vmalert.initd b/packages/victoria-metrics/files/vmalert.initd new file mode 100644 index 000000000..5ff4efd58 --- /dev/null +++ b/packages/victoria-metrics/files/vmalert.initd @@ -0,0 +1,69 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +START=95 +STOP=5 +USE_PROCD=1 + +PROG="/usr/bin/vmalert" +RULE_DIR="/etc/vmalert/rules" + +start_service() { + config_load vmalert 2>/dev/null || true + + local datasource_url http_listen_addr + config_get datasource_url main datasource_url "http://localhost:8428" + config_get http_listen_addr main http_listen_addr "127.0.0.1:8081" + + # Check if Mimir integration is configured in ns-plug + local mimir_url mimir_key mimir_secret notifier_url + config_load ns-plug 2>/dev/null && { + config_get mimir_url config my_url "" + config_get mimir_key config my_system_key "" + config_get mimir_secret config my_system_secret "" + } + + # If all Mimir credentials are present, configure alert forwarding to Mimir + if [ -n "$mimir_url" ] && [ -n "$mimir_key" ] && [ -n "$mimir_secret" ]; then + notifier_url="${mimir_url%/}/collect/api/services/mimir/alertmanager" + else + notifier_url="" + fi + + procd_open_instance + procd_set_param command $PROG + procd_append_param command -rule="$RULE_DIR/*.yaml" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_append_param command -datasource.url="$datasource_url" + procd_append_param command -remoteRead.url="$datasource_url" + procd_append_param command -remoteWrite.url="$datasource_url" + procd_append_param command -evaluationInterval=30s + + # Always notify the local alert-proxy (handles unregistered machines gracefully) + procd_append_param command -notifier.url="http://127.0.0.1:9095" + + # Also forward to Mimir if credentials are configured + if [ -n "$notifier_url" ]; then + procd_append_param command -notifier.url="$notifier_url" + procd_append_param command -notifier.basicAuth.username="$mimir_key" + procd_append_param command -notifier.basicAuth.password="$mimir_secret" + fi + + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 5 + procd_close_instance +} + +reload_service() { + stop + start +} + +service_triggers() { + procd_add_reload_trigger vmalert +} From 166b9f207985a9b87a3411cca372cb07506cedec Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 23 Apr 2026 16:33:23 +0200 Subject: [PATCH 33/48] feat(api): replace ns.netdata with ns.telegraf Changes: - migrate ping monitoring from netdata's fping plugin to telegraf's native ping input plugin - expose metrics to the UI The ping plugin uses native method (method="native") which sends ICMP packets directly without external ping command, requiring CAP_NET_RAW capability or root privileges. Metrics are tagged with influxdb_db="ping-metrics" for proper InfluxDB database routing. Assited-by: Copilot:Sonnet4.6 --- packages/ns-api/Makefile | 6 +- packages/ns-api/README.md | 129 +++++- packages/ns-api/files/ns.dashboard | 29 +- packages/ns-api/files/ns.netdata | 66 --- packages/ns-api/files/ns.netdata.json | 13 - packages/ns-api/files/ns.report | 87 ++-- packages/ns-api/files/ns.telegraf | 383 ++++++++++++++++++ packages/ns-api/files/ns.telegraf.json | 13 + packages/ns-api/openapi.yml | 77 ++++ packages/telegraf/Makefile | 33 +- .../telegraf/files/telegraf.conf.d/ping.conf | 26 ++ 11 files changed, 725 insertions(+), 137 deletions(-) delete mode 100755 packages/ns-api/files/ns.netdata delete mode 100644 packages/ns-api/files/ns.netdata.json create mode 100755 packages/ns-api/files/ns.telegraf create mode 100644 packages/ns-api/files/ns.telegraf.json create mode 100644 packages/telegraf/files/telegraf.conf.d/ping.conf diff --git a/packages/ns-api/Makefile b/packages/ns-api/Makefile index 6f09ec1cc..03e3336a5 100644 --- a/packages/ns-api/Makefile +++ b/packages/ns-api/Makefile @@ -118,8 +118,10 @@ define Package/ns-api/install $(INSTALL_DATA) ./files/ns.mwan.json $(1)/usr/share/rpcd/acl.d/ $(INSTALL_BIN) ./files/ns.dpi $(1)/usr/libexec/rpcd/ $(INSTALL_DATA) ./files/ns.dpi.json $(1)/usr/share/rpcd/acl.d/ - $(INSTALL_BIN) ./files/ns.netdata $(1)/usr/libexec/rpcd/ - $(INSTALL_DATA) ./files/ns.netdata.json $(1)/usr/share/rpcd/acl.d/ + $(INSTALL_BIN) ./files/ns.telegraf $(1)/usr/libexec/rpcd/ + $(INSTALL_DATA) ./files/ns.telegraf.json $(1)/usr/share/rpcd/acl.d/ + $(LN) ns.telegraf $(1)/usr/libexec/rpcd/ns.netdata + $(LN) ns.telegraf.json $(1)/usr/share/rpcd/acl.d/ns.netdata.json $(INSTALL_BIN) ./files/ns.storage $(1)/usr/libexec/rpcd/ $(INSTALL_DATA) ./files/ns.storage.json $(1)/usr/share/rpcd/acl.d/ $(INSTALL_BIN) ./files/ns.account $(1)/usr/libexec/rpcd/ diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 6c2e38672..f3b84620e 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -150,6 +150,124 @@ Response: } ``` +## ns.telegraf + +Read and update Telegraf ping monitoring targets, query historical metrics stored in VictoriaMetrics, and list the current alerts evaluated by vmalert. + +### get-configuration + +Get the current list of hosts monitored by the Telegraf ping input: +``` +api-cli ns.telegraf get-configuration +``` + +Output example: +```json +{ + "hosts": [ + "1.1.1.1", + "google.com" + ] +} +``` + +### set-hosts + +Set the list of hosts monitored by the Telegraf ping input and restart Telegraf: +``` +api-cli ns.telegraf set-hosts --data '{"hosts": ["1.1.1.1", "8.8.8.8"]}' +``` + +Parameters: +- `hosts`: array of hostnames or IP addresses to monitor + +Output example: +```json +{ + "success": true +} +``` + +### metrics-history + +Return historical system and network metrics collected by Telegraf and stored in VictoriaMetrics: +``` +api-cli ns.telegraf metrics-history --data '{"start": 1746607800, "end": 1746608400, "step": 60}' +``` + +Parameters: +- `start`: start of the time range as Unix timestamp +- `end`: end of the time range as Unix timestamp +- `step`: sampling interval in seconds + +Output example: +```json +{ + "connections": { + "labels": [1746608100], + "datasets": [{ "label": "Connections", "data": [123] }] + }, + "traffic": {}, + "cpu": { + "labels": [1746608100], + "datasets": [{ "label": "CPU (%)", "data": [14.2] }] + }, + "load": { + "labels": [1746608100], + "datasets": [ + { "label": "1m", "data": [0.12] }, + { "label": "5m", "data": [0.08] }, + { "label": "15m", "data": [0.05] } + ] + }, + "diskio": { "labels": [], "datasets": [] }, + "disk": { "labels": [], "datasets": [] }, + "processes": { "labels": [], "datasets": [] }, + "memory": { "labels": [], "datasets": [] }, + "packets": { "labels": [], "datasets": [] }, + "latency_quality": {} +} +``` + +### list-alerts + +List the current pending and firing alerts evaluated by vmalert: +``` +api-cli ns.telegraf list-alerts +``` + +Output example: +```json +{ + "alerts": [ + { + "state": "firing", + "name": "BackupEncryptionDisabled", + "value": "0", + "labels": { + "alertgroup": "backup", + "alertname": "BackupEncryptionDisabled", + "severity": "warning", + "service": "backup" + }, + "annotations": { + "summary_en": "Backup encryption is disabled", + "summary_it": "La cifratura dei backup e disattivata", + "description_en": "The backup passphrase file /etc/backup.pass is missing or empty.", + "description_it": "Il file della passphrase dei backup /etc/backup.pass manca o e vuoto." + }, + "activeAt": "2026-05-07T09:18:00Z", + "expression": "backup_encryption_encrypted == 0", + "source": "http://NethSec:8082/vmalert/alert?group_id=10212661952842894290&alert_id=4214684507782533109" + } + ] +} +``` + +Possible errors: +- `cannot_retrieve_alerts` +- `invalid_alerts_response` + ## ns.firewall ### list-forward-rules @@ -2436,7 +2554,10 @@ Response example: ### traffic-interface -Return an array of point describing the network traffic in the last hour: +Return an array of points describing the network traffic in the last hour. +Data is sourced from Victoria Metrics using `net_bytes_recv` and `net_bytes_sent` Telegraf counters, +converted to kb/s (kilobits per second). Labels are Unix timestamps in descending order (newest first), +with one point every 20 seconds (~180 points total). ``` api-cli ns.dashboard interface-traffic --data '{"interface": "eth0"}' ``` @@ -7932,7 +8053,7 @@ Output example: ### latency-and-quality-report -Report latency metrics (minimum, maximum and average) and connectivy quality data (packet delivery rate) for every host configured in Netdata fping configuration file, located at `/etc/netdata/fping.conf`. +Report latency metrics (minimum, maximum and average) and connectivity quality data (packet loss percentage) for every host configured in the Telegraf ping plugin configuration file, located at `/etc/telegraf.conf.d/ping.conf`. Usage example: ``` api-cli ns.report latency-and-quality-report @@ -7982,7 +8103,7 @@ Output example: ], [ 1731485262, - 99.8152174 + 100 ], [ 1731484894, @@ -8032,7 +8153,7 @@ Output example: ], [ 1731485262, - 99.8152174 + 100 ], [ 1731484894, diff --git a/packages/ns-api/files/ns.dashboard b/packages/ns-api/files/ns.dashboard index 1e3954eed..2a98ddae7 100644 --- a/packages/ns-api/files/ns.dashboard +++ b/packages/ns-api/files/ns.dashboard @@ -12,6 +12,8 @@ import os import sys import json import subprocess +import time +import urllib.parse import urllib.request from euci import EUci from nethsec import utils, ovpn @@ -274,17 +276,24 @@ def system_info(): def interface_traffic(interface): ret = {"labels": [], "data": []} - # retrieve from netdata the traffic for the last hour - url = f'http://127.0.0.1:19999/api/v1/data?chart=net.{interface}&after=-3600&points=180&options=abs' - try: - with urllib.request.urlopen(url, timeout=10) as fu: - data = json.loads(fu.read()) - except: - return ret + vm_url = "http://127.0.0.1:8428/api/v1/query_range" + now = int(time.time()) + one_hour_ago = now - 3600 + + def vm_query(expr): + params = urllib.parse.urlencode({"query": expr, "start": one_hour_ago, "end": now, "step": 20}) + with urllib.request.urlopen(f"{vm_url}?{params}", timeout=5) as resp: + data = json.loads(resp.read()) + result = data.get("data", {}).get("result", []) + return result[0].get("values", []) if result else [] - for record in data["data"]: - ret["labels"].append(record[0]) - ret["data"].append([record[1], record[2]]) + try: + recv = vm_query(f'rate(net_bytes_recv{{interface="{interface}"}}[20s]) * 8 / 1000') + sent = vm_query(f'rate(net_bytes_sent{{interface="{interface}"}}[20s]) * 8 / 1000') + ret["labels"] = [int(ts) for ts, _ in reversed(recv)] + ret["data"] = [[float(r), float(s)] for (_, r), (_, s) in zip(reversed(recv), reversed(sent))] + except Exception: + pass return ret diff --git a/packages/ns-api/files/ns.netdata b/packages/ns-api/files/ns.netdata deleted file mode 100755 index bb2309181..000000000 --- a/packages/ns-api/files/ns.netdata +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/python3 - -# -# Copyright (C) 2023 Nethesi3 S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Read and set fping configuration for netdata - -import os -import sys -import json -import subprocess -import configparser - -fping_conf_file = "/etc/netdata/fping.conf" -netdata_conf_file = "/etc/netdata/netdata.conf" - -def get_config(): - hosts = [] - # create a simpligied fping.conf if not exists - # the file must contain only one line: hosts="" - if not os.path.exists(fping_conf_file): - with open(fping_conf_file, 'w') as fp: - fp.write('hosts=""\n') - # parse the simplified config file - try: - with open(fping_conf_file, 'r') as fp: - line = fp.readline() - line = line[7:-2] - hosts = line.split(" ") - except: - pass - return {"hosts": hosts} - -def set_config(config): - # Enable and disable fping plugin on netdata - nparser = configparser.ConfigParser() - nparser.read(netdata_conf_file) - if len(config['hosts']) > 0: - nparser['plugins']['fping'] = 'yes' - else: - nparser['plugins']['fping'] = 'no' - with open(netdata_conf_file, 'w') as fpc: - nparser.write(fpc) - - try: - with open(fping_conf_file, 'w') as fp: - hosts = " ".join(config['hosts']) - fp.write(f'hosts="{hosts}"\n') - subprocess.run(["/etc/init.d/netdata", "restart"], check=True) - return {"success": True} - except: - return {"success": False} - -cmd = sys.argv[1] - -if cmd == 'list': - print(json.dumps({"get-configuration": {}, "set-hosts": {"hosts": ["1.1.1.1", "google.com"]}})) -else: - action = sys.argv[2] - if action == "get-configuration": - print(json.dumps(get_config())) - elif action == "set-hosts": - args = json.loads(sys.stdin.read()) - print(json.dumps(set_config(args))) diff --git a/packages/ns-api/files/ns.netdata.json b/packages/ns-api/files/ns.netdata.json deleted file mode 100644 index 5764ef6d4..000000000 --- a/packages/ns-api/files/ns.netdata.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "netdata-manager": { - "description": "Read and set netdata configuration", - "write": {}, - "read": { - "ubus": { - "ns.netdata": [ - "*" - ] - } - } - } -} diff --git a/packages/ns-api/files/ns.report b/packages/ns-api/files/ns.report index 1eca1eca1..e188c25a3 100755 --- a/packages/ns-api/files/ns.report +++ b/packages/ns-api/files/ns.report @@ -15,6 +15,7 @@ import subprocess from datetime import datetime from collections import defaultdict from nethsec import utils +import urllib.parse import urllib.request from euci import EUci @@ -324,41 +325,69 @@ def ovpnrw_bytes_by_hour_and_user(instance, day, user): return {"hours": hours_bytes} -def get_fping_hosts(): - # read fping hosts from /etc/netdata/fping.conf - try: - with open("/etc/netdata/fping.conf", 'r') as fp: - line = fp.readline() - line = line[7:-2] - hosts = line.split(" ") - return hosts - except: - return [] - - -def get_netdata_chart_data(chart_name): - ret = {"labels": [], "data": []} - # retrieve chart data from netdata - url = f'http://127.0.0.1:19999/api/v1/data?chart={chart_name}&after=-3600&points=180&options=abs' +def get_ping_hosts(): + # read ping hosts from telegraf configuration + ping_conf_file = "/etc/telegraf.conf.d/ping.conf" + hosts = [] + if os.path.exists(ping_conf_file): + try: + with open(ping_conf_file, 'r') as fp: + content = fp.read() + # Find the urls line in TOML format: urls = ["host1", "host2"] + match = re.search(r'urls\s*=\s*(\[[^\]]*\])', content) + if match: + urls_str = match.group(1) + # Parse JSON array + hosts = json.loads(urls_str) + except Exception: + pass + return hosts + + +def get_victoria_metrics_ping_data(host): + """ + Query Victoria Metrics for ping metrics. + Returns: {"latency": {"labels": [...], "data": [...]}, "quality": {"labels": [...], "data": [...]}} + """ + ret_latency = {"labels": ["time", "minimum", "maximum", "average"], "data": []} + ret_quality = {"labels": ["time", "returned"], "data": []} + + vm_url = "http://127.0.0.1:8428/api/v1/query_range" + now = int(time.time()) + one_hour_ago = now - 3600 + timeout = 5 + + def vm_query(metric_expr): + params = urllib.parse.urlencode({'query': metric_expr, 'start': one_hour_ago, 'end': now, 'step': 20}) + with urllib.request.urlopen(f"{vm_url}?{params}", timeout=timeout) as resp: + data = json.loads(resp.read()) + result = data.get('data', {}).get('result', []) + return result[0].get('values', []) if result else [] + try: - with urllib.request.urlopen(url, timeout=10) as fu: - data = json.loads(fu.read()) - except: - return ret - return data + min_values = vm_query(f'ping_minimum_response_ms{{url="{host}"}}') + max_values = vm_query(f'ping_maximum_response_ms{{url="{host}"}}') + avg_values = vm_query(f'ping_average_response_ms{{url="{host}"}}') + + ret_latency["data"] = [ + [int(ts), float(mn), float(mx), float(av)] + for (ts, mn), (_, mx), (_, av) in zip(min_values, max_values, avg_values) + ] + + loss_values = vm_query(f'100 - ping_percent_packet_loss{{url="{host}"}} or 100 - ping_percent_reply_loss{{url="{host}"}}') + ret_quality["data"] = [[int(ts), float(val)] for ts, val in loss_values] + + except Exception as e: + print(f"Error querying Victoria Metrics for {host}: {str(e)}", file=sys.stderr) + + return {"latency": ret_latency, "quality": ret_quality} def latency_and_quality_report(): - hosts = get_fping_hosts() + hosts = get_ping_hosts() ret = {} for host in hosts: - host_replaced = host.replace('.', '_') - latency_chart_data = get_netdata_chart_data(f'fping.{host_replaced}_latency') - quality_chart_data = get_netdata_chart_data(f'fping.{host_replaced}_quality') - ret[host] = { - "latency": latency_chart_data, - "quality": quality_chart_data - } + ret[host] = get_victoria_metrics_ping_data(host) return ret diff --git a/packages/ns-api/files/ns.telegraf b/packages/ns-api/files/ns.telegraf new file mode 100755 index 000000000..0a4dd5192 --- /dev/null +++ b/packages/ns-api/files/ns.telegraf @@ -0,0 +1,383 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Read and set ping configuration for telegraf, and expose metrics history from VictoriaMetrics + +import os +import sys +import json +import subprocess +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from euci import EUci + +ping_conf_file = "/etc/telegraf.conf.d/ping.conf" + + +def _read_ping_hosts(): + """Read the list of monitored ping hosts from the telegraf config file.""" + if not os.path.exists(ping_conf_file): + return [] + try: + with open(ping_conf_file) as fp: + match = re.search(r"urls\s*=\s*(\[[^\]]*\])", fp.read()) + if match: + return json.loads(match.group(1)) + except Exception: + pass + return [] + + +def get_config(): + return {"hosts": _read_ping_hosts()} + + +def set_config(config): + try: + # Ensure directory exists + os.makedirs(os.path.dirname(ping_conf_file), exist_ok=True) + + # Create the telegraf ping configuration + with open(ping_conf_file, "w") as fp: + fp.write("# Ping input plugin configuration\n") + fp.write("[[inputs.ping]]\n") + if len(config["hosts"]) > 0: + # Format hosts as TOML array + hosts_str = json.dumps(config["hosts"]) + fp.write(f" urls = {hosts_str}\n") + fp.write(' method = "native"\n') + fp.write(" count = 5\n") + fp.write(" ping_interval = 1.0\n") + fp.write(" deadline = 10\n") + fp.write(" [inputs.ping.tags]\n") + fp.write(' influxdb_db = "ping-metrics"\n') + else: + # Write empty config to disable + fp.write(" urls = []\n") + + # Restart telegraf service + subprocess.run(["/etc/init.d/telegraf", "restart"], check=True) + return {"success": True} + except Exception as e: + return {"success": False, "error": str(e)} + + +VM_URL = "http://127.0.0.1:8428/api/v1/query_range" +VM_ALERTS_URL = "http://127.0.0.1:8082/api/v1/alerts" + + +def _get_interface_to_zone_map(): + """Map physical interface names to firewall zone names and devices. + Returns: {"eth1": {"zone": "wan", "device": "eth1"}, "br-lan": {"zone": "lan", "device": "br-lan"}} + """ + interface_map = {} + try: + u = EUci() + zones = u.get("firewall") + networks = u.get("network") + + if not zones or not networks: + return interface_map + + # First, build a map of network names to devices + network_to_device = {} + for net_name, net_config in networks.items(): + if isinstance(net_config, dict): + device = net_config.get("device") or net_config.get("ifname") + if device: + network_to_device[net_name] = device + + # Now map zones to networks and then to devices + for zone_name, zone_config in zones.items(): + if isinstance(zone_config, dict) and "name" in zone_config: + zone_label = zone_config.get("name", zone_name) + networks_list = zone_config.get("network", ()) + + # Convert to list if needed + if isinstance(networks_list, str): + networks_list = [networks_list] + elif not isinstance(networks_list, (list, tuple)): + networks_list = [] + + for network in networks_list: + network = str(network).strip() + if network in network_to_device: + device = network_to_device[network] + interface_map[device] = {"zone": zone_label, "device": device} + except Exception: + pass + + return interface_map + + +def _vm_query(expr, start, end, step): + """Execute a single PromQL range query against VictoriaMetrics.""" + params = urllib.parse.urlencode( + {"query": expr, "start": start, "end": end, "step": step} + ) + with urllib.request.urlopen(f"{VM_URL}?{params}", timeout=10) as resp: + data = json.loads(resp.read()) + return data.get("data", {}).get("result", []) + + +def _single_series(results): + """Extract timestamps and values from a single-series result.""" + if not results: + return {"labels": [], "data": []} + values = results[0].get("values", []) + return { + "labels": [int(ts) for ts, _ in values], + "data": [float(v) for _, v in values], + } + + +def list_alerts(): + try: + with urllib.request.urlopen(VM_ALERTS_URL, timeout=10) as resp: + data = json.loads(resp.read()) + except (TimeoutError, urllib.error.URLError): + return {"error": "cannot_retrieve_alerts"} + except json.JSONDecodeError: + return {"error": "invalid_alerts_response"} + + alerts = data.get("data", {}).get("alerts") + if not isinstance(alerts, list): + return {"error": "invalid_alerts_response"} + + return {"alerts": alerts} + + +def metrics_history(args): + now = int(time.time()) + start = int(args.get("start", now - 86400)) + end = int(args.get("end", now)) + step = int(args.get("step", 300)) + + def q(expr): + try: + return _vm_query(expr, start, end, step) + except Exception: + return [] + + def single(expr): + return _single_series(q(expr)) + + # Connections (conntrack) + s = single("conntrack_ip_conntrack_count") + connections = { + "labels": s["labels"], + "datasets": [{"label": "Connections", "data": s["data"]}], + } + + # Traffic per interface – exclude loopback and intermediate functional blocks (ifb*) + # Organize traffic by interface with zone name labels + interface_map = _get_interface_to_zone_map() + recv_results = q('rate(net_bytes_recv{interface!~"lo|ifb.*"}[5m])') + sent_results = q('rate(net_bytes_sent{interface!~"lo|ifb.*"}[5m])') + + traffic_labels: list = [] + for r in recv_results + sent_results: + if r.get("values"): + traffic_labels = [int(ts) for ts, _ in r["values"]] + break + + # Build traffic data organized by interface + traffic_by_interface = {} + for r in recv_results: + iface = r.get("metric", {}).get("interface", "unknown") + iface_info = interface_map.get(iface, {"zone": iface, "device": iface}) + zone_key = f"{iface_info['zone']}|{iface_info['device']}" # Use composite key + if zone_key not in traffic_by_interface: + traffic_by_interface[zone_key] = { + "labels": traffic_labels, + "datasets": [], + "zone": iface_info["zone"], + "device": iface_info["device"], + } + traffic_by_interface[zone_key]["datasets"].append( + {"label": "Download", "data": [float(v) for _, v in r.get("values", [])]} + ) + + for r in sent_results: + iface = r.get("metric", {}).get("interface", "unknown") + iface_info = interface_map.get(iface, {"zone": iface, "device": iface}) + zone_key = f"{iface_info['zone']}|{iface_info['device']}" + if zone_key not in traffic_by_interface: + traffic_by_interface[zone_key] = { + "labels": traffic_labels, + "datasets": [], + "zone": iface_info["zone"], + "device": iface_info["device"], + } + traffic_by_interface[zone_key]["datasets"].append( + {"label": "Upload", "data": [float(v) for _, v in r.get("values", [])]} + ) + + # CPU usage (%) + s = single('100 - (avg(cpu_usage_idle{cpu="cpu-total"}))') + cpu = {"labels": s["labels"], "datasets": [{"label": "CPU (%)", "data": s["data"]}]} + + # System load (1m / 5m / 15m) + s1 = single("system_load1") + s5 = single("system_load5") + s15 = single("system_load15") + load = { + "labels": s1["labels"], + "datasets": [ + {"label": "1m", "data": s1["data"]}, + {"label": "5m", "data": s5["data"]}, + {"label": "15m", "data": s15["data"]}, + ], + } + + # Disk I/O – sum across all non-loop block devices + s_read = single('sum(rate(diskio_read_bytes{name!~"loop.*"}[5m]))') + s_write = single('sum(rate(diskio_write_bytes{name!~"loop.*"}[5m]))') + diskio = { + "labels": s_read["labels"], + "datasets": [ + {"label": "Read", "data": s_read["data"]}, + {"label": "Write", "data": s_write["data"]}, + ], + } + + # Disk usage per real partition (exclude virtual filesystems) + disk_results = q( + 'disk_used_percent{fstype!~"tmpfs|cgroup2|devtmpfs|sysfs|proc|overlay|squashfs"}' + ) + if disk_results: + disk_labels = [int(ts) for ts, _ in disk_results[0].get("values", [])] + disk_datasets = [ + { + "label": r.get("metric", {}).get("path", "unknown"), + "data": [float(v) for _, v in r.get("values", [])], + } + for r in disk_results + ] + disk = {"labels": disk_labels, "datasets": disk_datasets} + else: + disk = {"labels": [], "datasets": []} + + # Total processes + s = single("processes_total") + processes = { + "labels": s["labels"], + "datasets": [{"label": "Processes", "data": s["data"]}], + } + + # RAM usage: Used and Free (MB) + s_used = single("mem_used / 1048576") + s_free = single("mem_free / 1048576") + memory = { + "labels": s_used["labels"], + "datasets": [ + {"label": "Used", "data": s_used["data"]}, + {"label": "Free", "data": s_free["data"]}, + ], + } + + # Packets Rx/Tx – sum across all interfaces + s_rx = single("sum(rate(net_packets_recv[5m]))") + s_tx = single("sum(rate(net_packets_sent[5m]))") + packets = { + "labels": s_rx["labels"], + "datasets": [ + {"label": "Rx", "data": s_rx["data"]}, + {"label": "Tx", "data": s_tx["data"]}, + ], + } + + # Latency and quality reports for monitored hosts + latency_quality = latency_and_quality_report(start, end, step) + + return { + "connections": connections, + "traffic": traffic_by_interface, + "cpu": cpu, + "load": load, + "diskio": diskio, + "disk": disk, + "processes": processes, + "memory": memory, + "packets": packets, + "latency_quality": latency_quality, + } + + +def latency_and_quality_report(start, end, step): + """Fetch latency and packet loss data for monitored hosts.""" + + def q_values(expr): + try: + result = _vm_query(expr, start, end, step) + return result[0].get("values", []) if result else [] + except Exception: + return [] + + ret = {} + for host in _read_ping_hosts(): + latency_data = {"labels": [], "datasets": []} + quality_data = {"labels": [], "datasets": []} + + try: + min_values = q_values(f'ping_minimum_response_ms{{url="{host}"}}') + max_values = q_values(f'ping_maximum_response_ms{{url="{host}"}}') + avg_values = q_values(f'ping_average_response_ms{{url="{host}"}}') + loss_values = q_values( + f'100 - ping_percent_packet_loss{{url="{host}"}} or 100 - ping_percent_reply_loss{{url="{host}"}}' + ) + + if min_values and max_values and avg_values: + labels = [int(ts) for ts, _ in min_values] + latency_data["labels"] = labels + latency_data["datasets"] = [ + {"label": "Min", "data": [float(v) for _, v in min_values]}, + {"label": "Avg", "data": [float(v) for _, v in avg_values]}, + {"label": "Max", "data": [float(v) for _, v in max_values]}, + ] + + if loss_values: + quality_data["labels"] = [int(ts) for ts, _ in loss_values] + quality_data["datasets"] = [ + {"label": "Delivery %", "data": [float(v) for _, v in loss_values]}, + ] + except Exception: + pass + + ret[host] = {"latency": latency_data, "quality": quality_data} + + return ret + + +cmd = sys.argv[1] + +if cmd == "list": + print( + json.dumps( + { + "get-configuration": {}, + "set-hosts": {"hosts": ["1.1.1.1", "google.com"]}, + "metrics-history": {"start": 0, "end": 0, "step": 300}, + "list-alerts": {}, + } + ) + ) +else: + action = sys.argv[2] + if action == "get-configuration": + print(json.dumps(get_config())) + elif action == "set-hosts": + args = json.loads(sys.stdin.read()) + print(json.dumps(set_config(args))) + elif action == "metrics-history": + args = json.loads(sys.stdin.read()) + print(json.dumps(metrics_history(args))) + elif action == "list-alerts": + print(json.dumps(list_alerts())) diff --git a/packages/ns-api/files/ns.telegraf.json b/packages/ns-api/files/ns.telegraf.json new file mode 100644 index 000000000..972119685 --- /dev/null +++ b/packages/ns-api/files/ns.telegraf.json @@ -0,0 +1,13 @@ +{ + "telegraf-manager": { + "description": "Read and set telegraf ping monitor configuration", + "write": {}, + "read": { + "ubus": { + "ns.telegraf": [ + "*" + ] + } + } + } +} diff --git a/packages/ns-api/openapi.yml b/packages/ns-api/openapi.yml index c5372c489..ece5198e0 100644 --- a/packages/ns-api/openapi.yml +++ b/packages/ns-api/openapi.yml @@ -188,3 +188,80 @@ paths: example: success - $ref: "#/components/schemas/ValidationError" - $ref: "#/components/schemas/Error" + POST /ubus/ns.telegraf/list-alerts: + post: + summary: List current monitoring alerts + operationId: ns.telegraf.list-alerts + tags: + - telegraf + responses: + "200": + description: Current pending and firing alerts evaluated by vmalert + content: + application/json: + schema: + oneOf: + - type: object + required: + - alerts + properties: + alerts: + type: array + items: + type: object + required: + - state + - name + - labels + - annotations + - activeAt + properties: + state: + type: string + description: Alert state reported by vmalert + example: firing + name: + type: string + description: Alert name + example: BackupEncryptionDisabled + value: + type: string + description: Evaluated expression value + example: "0" + labels: + type: object + description: Alert labels emitted by vmalert + additionalProperties: + type: string + annotations: + type: object + description: Localized summaries and descriptions emitted by vmalert + additionalProperties: + type: string + activeAt: + type: string + format: date-time + description: Time when the alert became active + example: "2026-05-07T09:18:00Z" + id: + type: string + description: Alert instance identifier + rule_id: + type: string + description: Alert rule identifier + group_id: + type: string + description: Alert group identifier + expression: + type: string + description: Alert rule expression + source: + type: string + description: vmalert source URL for the alert + restored: + type: boolean + description: True when the alert is being restored + stabilizing: + type: boolean + description: True when the alert is stabilizing + - $ref: "#/components/schemas/Error" diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index 1f4175a2d..3129a4ff9 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -28,19 +28,26 @@ GO_BUILD_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) GO_PKG_LDFLAGS_X:=github.com/influxdata/telegraf/internal.Version=$(PKG_VERSION) GO_PKG_TAGS:= \ custom \ - inputs.bond \ - inputs.cpu \ - inputs.disk \ - inputs.ethtool \ - inputs.mem \ - inputs.net \ - inputs.netstat \ - inputs.nstat \ - inputs.processes \ - inputs.sensors \ - inputs.system \ - outputs.influxdb \ - outputs.prometheus_client + inputs.bond \ + inputs.cpu \ + inputs.disk \ + inputs.ethtool \ + inputs.exec \ + inputs.file \ + inputs.http_listener_v2 \ + inputs.mem \ + inputs.net \ + inputs.netstat \ + inputs.nftables \ + inputs.nstat \ + inputs.ping \ + inputs.processes \ + inputs.sensors \ + inputs.system \ + inputs.tail \ + outputs.influxdb \ + parsers.grok \ + parsers.json_v2 include $(INCLUDE_DIR)/package.mk include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk diff --git a/packages/telegraf/files/telegraf.conf.d/ping.conf b/packages/telegraf/files/telegraf.conf.d/ping.conf new file mode 100644 index 000000000..40c7c8087 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/ping.conf @@ -0,0 +1,26 @@ +# Ping input plugin - monitors ICMP ping to configured hosts +# Uses native method for better performance and no external dependencies + +[[inputs.ping]] + # Hosts to send ping packets to + urls = [] + + # Method: "native" for improved compatibility and performance + # Uses privileged raw ICMP sockets (requires CAP_NET_RAW or root) + method = "native" + + # Number of ping packets to send per interval + count = 1 + + # Time to wait between sending ping packets (seconds) + ping_interval = 1.0 + + # Total ping deadline (seconds) + deadline = 10 + + # Data size for ping packets (bytes) + size = 56 + + # Tags for metric routing + [inputs.ping.tags] + influxdb_db = "ping-metrics" From 5ace2ebc6e7e4a3fbfbfc8e5b1be0b0ed0f9b834 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 Apr 2026 16:34:12 +0200 Subject: [PATCH 34/48] feat(telegraf): add missing plugins These plugins are required to replace all Netdata features --- packages/telegraf/Makefile | 8 +++----- .../telegraf/files/telegraf.conf.d/os.conf | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index 3129a4ff9..c00cd7384 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -29,24 +29,23 @@ GO_PKG_LDFLAGS_X:=github.com/influxdata/telegraf/internal.Version=$(PKG_VERSION) GO_PKG_TAGS:= \ custom \ inputs.bond \ + inputs.conntrack \ inputs.cpu \ inputs.disk \ + inputs.diskio \ inputs.ethtool \ inputs.exec \ inputs.file \ - inputs.http_listener_v2 \ inputs.mem \ inputs.net \ inputs.netstat \ - inputs.nftables \ inputs.nstat \ inputs.ping \ inputs.processes \ inputs.sensors \ inputs.system \ - inputs.tail \ outputs.influxdb \ - parsers.grok \ + outputs.prometheus_client \ parsers.json_v2 include $(INCLUDE_DIR)/package.mk @@ -94,7 +93,6 @@ define Package/telegraf/install $(INSTALL_BIN) ./files/telegraf-services $(1)/usr/libexec/telegraf-services $(INSTALL_BIN) ./files/telegraf-backup-encryption $(1)/usr/libexec/telegraf-backup-encryption $(INSTALL_BIN) ./files/telegraf-storage-status $(1)/usr/libexec/telegraf-storage-status - $(INSTALL_BIN) ./files/telegraf-services $(1)/usr/libexec/telegraf-services $(INSTALL_BIN) ./files/telegraf-mwan $(1)/usr/libexec/telegraf-mwan endef diff --git a/packages/telegraf/files/telegraf.conf.d/os.conf b/packages/telegraf/files/telegraf.conf.d/os.conf index 04f84000e..be6368093 100644 --- a/packages/telegraf/files/telegraf.conf.d/os.conf +++ b/packages/telegraf/files/telegraf.conf.d/os.conf @@ -58,3 +58,23 @@ dump_zeros = true [inputs.nstat.tags] influxdb_db = "os-metrics" + + +# Read metrics about disk I/O +[[inputs.diskio]] + ## By default, telegraf will gather stats for all devices. + ## Setting devices will restrict the stats to the specified devices. + # devices = ["sda", "sdb", "vd*"] + + ## Uncomment the following line if you need disk serial numbers. + # skip_serial_number = false + [inputs.diskio.tags] + influxdb_db = "os-metrics" + + +# Collect metrics from the nf_conntrack kernel module +[[inputs.conntrack]] + files = ["nf_conntrack_count", "nf_conntrack_max"] + dirs = ["/proc/sys/net/netfilter", "/proc/sys/net/netfilter"] + [inputs.conntrack.tags] + influxdb_db = "os-metrics" From d4d179f6ea4816024cd86c3d85ed7f13c909e770 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 5 May 2026 11:25:08 +0200 Subject: [PATCH 35/48] refactor(ns-plug): remove netdata Netdata has been replaced by Victoria Metrics. --- config/netdata.conf | 2 - packages/ns-api/Makefile | 3 - packages/ns-api/README.md | 33 -------- .../files/post-commit/restart-netdata.py | 14 ---- packages/ns-plug/Makefile | 1 + packages/ns-plug/README.md | 15 ++-- packages/ns-plug/files/30_ns-plug_alerts | 64 ---------------- .../99_ns-plug-netdata-cleanup.uci-default | 13 ++++ .../ns-plug/files/health_alarm_notify.conf | 75 ------------------- .../ns-plug/files/netadata_disable_alerts | 5 -- packages/ns-plug/files/netadata_enable_alerts | 5 -- packages/ns-plug/files/send-mwan-alert | 49 ------------ .../100-netdata-export-promethus.patch | 12 --- 13 files changed, 23 insertions(+), 268 deletions(-) delete mode 100644 config/netdata.conf delete mode 100755 packages/ns-api/files/post-commit/restart-netdata.py delete mode 100644 packages/ns-plug/files/30_ns-plug_alerts create mode 100644 packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default delete mode 100644 packages/ns-plug/files/health_alarm_notify.conf delete mode 100644 packages/ns-plug/files/netadata_disable_alerts delete mode 100644 packages/ns-plug/files/netadata_enable_alerts delete mode 100644 packages/ns-plug/files/send-mwan-alert delete mode 100644 patches/feeds/packages/100-netdata-export-promethus.patch diff --git a/config/netdata.conf b/config/netdata.conf deleted file mode 100644 index e4636c07f..000000000 --- a/config/netdata.conf +++ /dev/null @@ -1,2 +0,0 @@ -CONFIG_PACKAGE_netdata=y -CONFIG_PACKAGE_fping=y diff --git a/packages/ns-api/Makefile b/packages/ns-api/Makefile index 03e3336a5..1ce3a07c5 100644 --- a/packages/ns-api/Makefile +++ b/packages/ns-api/Makefile @@ -120,8 +120,6 @@ define Package/ns-api/install $(INSTALL_DATA) ./files/ns.dpi.json $(1)/usr/share/rpcd/acl.d/ $(INSTALL_BIN) ./files/ns.telegraf $(1)/usr/libexec/rpcd/ $(INSTALL_DATA) ./files/ns.telegraf.json $(1)/usr/share/rpcd/acl.d/ - $(LN) ns.telegraf $(1)/usr/libexec/rpcd/ns.netdata - $(LN) ns.telegraf.json $(1)/usr/share/rpcd/acl.d/ns.netdata.json $(INSTALL_BIN) ./files/ns.storage $(1)/usr/libexec/rpcd/ $(INSTALL_DATA) ./files/ns.storage.json $(1)/usr/share/rpcd/acl.d/ $(INSTALL_BIN) ./files/ns.account $(1)/usr/libexec/rpcd/ @@ -188,7 +186,6 @@ define Package/ns-api/install $(INSTALL_CONF) ./files/config/ns-api $(1)/etc/config/ns-api $(INSTALL_CONF) ./files/config/ns-wizard $(1)/etc/config/ns-wizard $(INSTALL_CONF) ./files/templates $(1)/etc/config/ - $(INSTALL_BIN) ./files/post-commit/restart-netdata.py $(1)/usr/libexec/ns-api/post-commit/ $(INSTALL_BIN) ./files/pre-commit/fix-redirect-reflections.py $(1)/usr/libexec/ns-api/pre-commit $(INSTALL_BIN) ./files/pre-commit/update-objects.py $(1)/usr/libexec/ns-api/pre-commit $(INSTALL_BIN) ./files/post-commit/reload-ipsets.py $(1)/usr/libexec/ns-api/post-commit diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index f3b84620e..3f19bdaff 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -4625,39 +4625,6 @@ Error response example: {"error": "restart_failed"} ``` -## ns.netdata - -Configure netdata reporting daemon. - -### get-configuration - -Get current netdata configuration: -``` -api-cli ns.netdata get-configuration -``` - -Response example: -```json -{ - "hosts": [ - "1.2.3.4", - "google.it" - ] -} -``` - -### set-hosts - -Configure hosts to be monitored by fping: -``` -api-cli ns.netdata set-hosts --data '{"hosts": ["1.1.1.1", "google.com"]}' -``` - -Response example: -```json -{"result": "success"} -``` - ## ns.factoryreset ### reset diff --git a/packages/ns-api/files/post-commit/restart-netdata.py b/packages/ns-api/files/post-commit/restart-netdata.py deleted file mode 100755 index ae6a7017b..000000000 --- a/packages/ns-api/files/post-commit/restart-netdata.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python - -# -# Copyright (C) 2024 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# This script restarts netdata is a WAN has changed to update the multiwan chart. - -import subprocess - -# The changes variable is already within the scope from the caller -if 'mwan3' in changes: - subprocess.run(["/etc/init.d/netdata", "restart"]) diff --git a/packages/ns-plug/Makefile b/packages/ns-plug/Makefile index 5b602cb44..020a61948 100644 --- a/packages/ns-plug/Makefile +++ b/packages/ns-plug/Makefile @@ -104,6 +104,7 @@ define Package/ns-plug/install $(INSTALL_CONF) files/ns-plug.keep $(1)/lib/upgrade/keep.d/ns-plug $(INSTALL_BIN) ./files/mwan-hooks $(1)/usr/libexec/ns-plug $(INSTALL_BIN) ./files/ns-plug-rsyslog-fixup.uci-default $(1)/etc/uci-defaults/rsyslog-fixup + $(INSTALL_BIN) ./files/99_ns-plug-netdata-cleanup.uci-default $(1)/etc/uci-defaults/99_ns-plug-netdata-cleanup endef $(eval $(call BuildPackage,ns-plug)) diff --git a/packages/ns-plug/README.md b/packages/ns-plug/README.md index 670169c90..eebc96abb 100644 --- a/packages/ns-plug/README.md +++ b/packages/ns-plug/README.md @@ -129,13 +129,12 @@ remote-backup download $(remote-backup list | jq -r .[0].file) - | gpg --batch - ## Alerts -All system alerts, except MultiWAN ones, are handled by netdata, including those from the multiwan monitoring. -Alerts are disabled by default and enabled only if the machine has a valid subscription. -In this case, alerts are automatically sent to the remote server (either my.nethesis.it or my.nethserver.com) using a -custom sender (`/etc/netdata/health_alarm_notify.conf`). -Alerts are also logged to `/var/log/messages` and are visible within the netdata UI. +System alerts are handled by **vmalert** (Victoria Metrics alert evaluation engine) which evaluates +alert rules against metrics collected by telegraf. -Only the following alerts are sent to the remote system: +When an alert fires or resolves, vmalert sends an Alertmanager-format webhook to `ns-plug-alert-proxy` +running on `127.0.0.1:9095`. The proxy forwards the following alerts to the registered monitoring +portal (my.nethesis.it or my.nethserver.com): | Alert | Condition | Legacy alert_id | |---|---|---| @@ -151,3 +150,7 @@ The proxy starts automatically at boot regardless of registration state. Firing/resolved state is determined from the Alertmanager-standard `endsAt` field: if `endsAt` is in the future (or zero/missing) a **FAILURE** is sent; if `endsAt` is in the past an **OK** is sent. +A FAILURE is sent when the alert starts firing and an OK is sent when it resolves. + +If Mimir credentials are configured in ns-plug UCI (`my_url`, `my_system_key`, `my_system_secret`), +vmalert also forwards all alerts to the Mimir alertmanager for cloud-side processing. diff --git a/packages/ns-plug/files/30_ns-plug_alerts b/packages/ns-plug/files/30_ns-plug_alerts deleted file mode 100644 index dfb62a848..000000000 --- a/packages/ns-plug/files/30_ns-plug_alerts +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh - -# Custom disk alerts -disks_f="/etc/netdata/health.d/disks.conf" -if [ ! -f "$disks_f" ]; then - cat << EOF > "$disks_f" -template: disk_space_usage - on: disk.space - class: Utilization - type: System -component: Disk - os: linux freebsd - hosts: * - families: !/dev !/dev/* !/run !/run/* !/overlay * - calc: \$used * 100 / (\$avail + \$used) - units: % - every: 1m - warn: \$this > ((\$status >= \$WARNING ) ? (80) : (90)) - crit: \$this > ((\$status == \$CRITICAL) ? (90) : (98)) - delay: up 1m down 15m multiplier 1.5 max 1h - info: disk $family space utilization - to: sysadmin -EOF -fi - -# Disable unwanted alerts -files="cpu disks entropy ipc load memory net netfilter processes ram softnet tcp_conn tcp_listen tcp_mem tcp_orphans tcp_resets timex udp_errors" -for f in $files -do - file="/etc/netdata/health.d/${f}.conf" - if [ ! -f $file ]; then - > $file - fi -done - -# Enable mwan chart -sed -i 's/python.d = no/python.d = yes/' /etc/netdata/netdata.conf -python_f="/etc/netdata/python.d.conf" -if [ ! -f "$python_f" ]; then - cat << EOF > "$python_f" -enabled: yes -gc_run: yes -gc_interval: 300 -apache_cache: no -chrony: no -example: no -go_expvar: no -gunicorn_log: no -hpssa: no -logind: no -nginx_log: no -EOF -fi - -# Create mwan alert -cat << EOF > /etc/netdata/health.d/mwan.conf -template: wan_status - on: mwan.score -lookup: min -1m foreach * - every: 1m - warn: \$this < 5 - crit: \$this <= 1 - info: The score of the WAN, 0 means down -EOF diff --git a/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default b/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default new file mode 100644 index 000000000..df1a859c5 --- /dev/null +++ b/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default @@ -0,0 +1,13 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Remove bundled netdata configuration files on first boot. +rm -rf /etc/netdata 2>/dev/null || true + +# Remove the legacy backup-encryption alert cron entry if it is still present. +if crontab -l 2>/dev/null | grep -q '/usr/libexec/backup-encryption-alert'; then + crontab -l 2>/dev/null | grep -v '/usr/libexec/backup-encryption-alert' | sort | uniq | crontab - +fi diff --git a/packages/ns-plug/files/health_alarm_notify.conf b/packages/ns-plug/files/health_alarm_notify.conf deleted file mode 100644 index 00852cfa9..000000000 --- a/packages/ns-plug/files/health_alarm_notify.conf +++ /dev/null @@ -1,75 +0,0 @@ -# Configuration for alarm notifications - -SEND_EMAIL="NO" -SEND_DYNATRACE="NO" -SEND_STACKPULSE="NO" -SEND_OPSGENIE="NO" -SEND_HANGOUTS="NO" -SEND_PUSHOVER="NO" -SEND_PUSHBULLET="NO" -SEND_TWILIO="NO" -SEND_MESSAGEBIRD="NO" -SEND_KAVENEGAR="NO" -SEND_TELEGRAM="NO" -SEND_SLACK="NO" -SEND_MSTEAMS="NO" -SEND_ROCKETCHAT="NO" -SEND_ALERTA="NO" -SEND_FLOCK="NO" -SEND_DISCORD="NO" -SEND_HIPCHAT="NO" -SEND_KAFKA="NO" -SEND_PD="NO" -SEND_FLEEP="NO" -SEND_IRC="NO" -SEND_SYSLOG="NO" -SEND_PROWL="NO" -SEND_AWSSNS="NO" -SEND_SMS="NO" -SEND_MATRIX="NO" - -# Enable only syslog and custom notification -use_fqdn='YES' -SEND_SYSLOG="YES" -SYSLOG_FACILITY='' -DEFAULT_RECIPIENT_SYSLOG="sysadmin" -SEND_CUSTOM="YES" -DEFAULT_RECIPIENT_CUSTOM="sysadmin" - -# Always generate clear events -clear_alarm_always='YES' - -# Send alerts to my.nethesis.it or my.nethserver.com -custom_sender() { - lk=$(uci -q get ns-plug.config.system_id) - secret=$(uci -q get ns-plug.config.secret) - url=$(uci -q get ns-plug.config.alerts_url)"alerts/store" - alert_id=${name} - if [ "${status}" == "CRITICAL" ]; then - status="FAILURE" - elif [ "${status}" == "CLEAR" ]; then - status="OK" - fi - - # map to old alerts, when possible - if [ "${chart}" == "disk_space._overlay" ] || [ "${chart}" == "disk_space._" ]; then - alert_id="df:root:percent_bytes:free" - elif [ "${chart}" == "disk_space._boot" ]; then - alert_id="df:boot:percent_bytes:free" - else - alert_id="${name}:${chart}" - fi - payload='{"lk": "'$lk'", "alert_id": "'$alert_id'", "status": "'$status'"}' - - # send only if the machine is registered - if [ -z "${lk}" ] || [ -z "${secret}" ]; then - return - fi - - # send to remote server - if [ "${status}" == "FAILURE" ] || [ "${status}" == "OK" ]; then - /usr/bin/curl -m 180 --retry 3 -L -s \ - --header "Authorization: token ${secret}" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-raw "${payload}" ${url} - fi -} diff --git a/packages/ns-plug/files/netadata_disable_alerts b/packages/ns-plug/files/netadata_disable_alerts deleted file mode 100644 index b21473515..000000000 --- a/packages/ns-plug/files/netadata_disable_alerts +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# Disable netdata alerts -sed -i 's/enabled = yes/enabled = no/' /etc/netdata/netdata.conf -/etc/init.d/netdata restart diff --git a/packages/ns-plug/files/netadata_enable_alerts b/packages/ns-plug/files/netadata_enable_alerts deleted file mode 100644 index cf066e58a..000000000 --- a/packages/ns-plug/files/netadata_enable_alerts +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# Enable netdata alerts -sed -i 's/enabled = no/enabled = yes/' /etc/netdata/netdata.conf -/etc/init.d/netdata restart diff --git a/packages/ns-plug/files/send-mwan-alert b/packages/ns-plug/files/send-mwan-alert deleted file mode 100644 index 1a74ec96c..000000000 --- a/packages/ns-plug/files/send-mwan-alert +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2024 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Send WAN alert to monitoring portal - -lk=$(uci -q get ns-plug.config.system_id) -secret=$(uci -q get ns-plug.config.secret) -url=$(uci -q get ns-plug.config.alerts_url)"alerts/store" -pidfile="/tmp/mwan3.$INTERFACE" - -# Do not send alert if system_id or secret is not set -if [ -z "$lk" ] || [ -z "$secret" ]; then - exit 0 -fi - -# Ignore ifup and ifdown events, they both triggers connected and disconnected events -if [ "${ACTION}" == "connected" ]; then - pid=$(cat "$pidfile" 2>/dev/null) - # If a wan is connected within 30 seconds from disconnect, assume it's a restart - # and kill the alert sending process - # mwan3 restart should complete within 30 seconds - if [ -n "$pid" ]; then - kill -s SIGHUP "$pid" - rm "$pidfile" - exit 0 - fi - status="OK" -elif [ "${ACTION}" == "disconnected" ]; then - echo $$ > "$pidfile" - # Delay alert by 30 seconds, so that it can be canceled - sleep 30 - rm "$pidfile" - status="FAILURE" -fi - -# Exit if status is not set -if [ -z "$status" ]; then - exit 0 -fi - -alert_id="wan:${INTERFACE}:down" -logger -t mwan3-alert "Sending alert ${alert_id} with status ${status}" -payload='{"lk": "'$lk'", "alert_id": "'$alert_id'", "status": "'$status'"}' -/usr/bin/curl -m 30 --retry 3 -L -s \ - --header "Authorization: token ${secret}" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-raw "${payload}" ${url} diff --git a/patches/feeds/packages/100-netdata-export-promethus.patch b/patches/feeds/packages/100-netdata-export-promethus.patch deleted file mode 100644 index 4db53cf7c..000000000 --- a/patches/feeds/packages/100-netdata-export-promethus.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/admin/netdata/Makefile b/admin/netdata/Makefile -index e471f27b2..def44a874 100644 ---- a/admin/netdata/Makefile -+++ b/admin/netdata/Makefile -@@ -62,7 +62,6 @@ CONFIGURE_ARGS += \ - --disable-plugin-freeipmi \ - --disable-plugin-cups \ - --disable-plugin-xenstat \ -- --disable-backend-prometheus-remote-write \ - --disable-unit-tests \ - --disable-ml \ - --disable-cloud From f6d4b84c8b52afe7f7cef8dc849d09e807852e2c Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 7 May 2026 16:16:40 +0200 Subject: [PATCH 36/48] feat(telegraf): using uci conf for ping config, added migration path from netdata --- packages/ns-api/files/ns.telegraf | 77 +++++----- packages/telegraf/Makefile | 4 +- packages/telegraf/files/telegraf-config | 138 +++++++++--------- .../telegraf/files/telegraf.conf.d/ping.conf | 26 ---- packages/telegraf/files/telegraf.uci | 6 + .../uci-defaults/99-telegraf-migrate-netdata | 36 +++++ 6 files changed, 156 insertions(+), 131 deletions(-) delete mode 100644 packages/telegraf/files/telegraf.conf.d/ping.conf create mode 100644 packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata diff --git a/packages/ns-api/files/ns.telegraf b/packages/ns-api/files/ns.telegraf index 0a4dd5192..cd18a11cc 100755 --- a/packages/ns-api/files/ns.telegraf +++ b/packages/ns-api/files/ns.telegraf @@ -13,26 +13,36 @@ import json import subprocess import re import time +import ipaddress import urllib.error import urllib.parse import urllib.request from euci import EUci +from nethsec.utils import validation_error -ping_conf_file = "/etc/telegraf.conf.d/ping.conf" +def _is_valid_ip_or_hostname(host): + """Validate if a string is a valid IPv4, IPv6, or hostname.""" + if not isinstance(host, str) or not host.strip(): + return False -def _read_ping_hosts(): - """Read the list of monitored ping hosts from the telegraf config file.""" - if not os.path.exists(ping_conf_file): - return [] + # Try to parse as IP address (IPv4 or IPv6) try: - with open(ping_conf_file) as fp: - match = re.search(r"urls\s*=\s*(\[[^\]]*\])", fp.read()) - if match: - return json.loads(match.group(1)) - except Exception: + ipaddress.ip_address(host) + return True + except ValueError: pass - return [] + + # Hostname pattern (alphanumeric, dots, hyphens) + hostname_pattern = r'^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?)(\.([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?))*$' + return bool(re.match(hostname_pattern, host)) + + +def _read_ping_hosts(): + """Read the list of monitored ping hosts from UCI config.""" + e_uci = EUci() + pings = e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[]) + return pings def get_config(): @@ -40,33 +50,24 @@ def get_config(): def set_config(config): - try: - # Ensure directory exists - os.makedirs(os.path.dirname(ping_conf_file), exist_ok=True) - - # Create the telegraf ping configuration - with open(ping_conf_file, "w") as fp: - fp.write("# Ping input plugin configuration\n") - fp.write("[[inputs.ping]]\n") - if len(config["hosts"]) > 0: - # Format hosts as TOML array - hosts_str = json.dumps(config["hosts"]) - fp.write(f" urls = {hosts_str}\n") - fp.write(' method = "native"\n') - fp.write(" count = 5\n") - fp.write(" ping_interval = 1.0\n") - fp.write(" deadline = 10\n") - fp.write(" [inputs.ping.tags]\n") - fp.write(' influxdb_db = "ping-metrics"\n') - else: - # Write empty config to disable - fp.write(" urls = []\n") - - # Restart telegraf service - subprocess.run(["/etc/init.d/telegraf", "restart"], check=True) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} + if 'hosts' not in config: + return validation_error('hosts', 'required') + + hosts = config.get('hosts') + if not isinstance(hosts, list): + return validation_error('hosts', 'invalid') + + # Validate each host with per-index error reporting + for idx, host in enumerate(hosts): + if not _is_valid_ip_or_hostname(host): + return validation_error(f'hosts.{idx}', 'invalid', value=host) + + e_uci = EUci() + current = set(e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[])) + if current != set(hosts): + e_uci.set('telegraf', 'internet', 'pings', hosts) + e_uci.save('telegraf') + return {"success": True} VM_URL = "http://127.0.0.1:8428/api/v1/query_range" diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index c00cd7384..b2e7ffa91 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -34,6 +34,7 @@ GO_PKG_TAGS:= \ inputs.disk \ inputs.diskio \ inputs.ethtool \ + inputs.dns_query \ inputs.exec \ inputs.file \ inputs.mem \ @@ -86,7 +87,8 @@ define Package/telegraf/install $(INSTALL_DATA) ./files/telegraf.conf.d/storage.conf $(1)/etc/telegraf.conf.d/storage.conf $(INSTALL_DATA) ./files/telegraf.conf.d/services.conf $(1)/etc/telegraf.conf.d/services.conf $(INSTALL_DATA) ./files/telegraf.conf.d/mwan.conf $(1)/etc/telegraf.conf.d/mwan.conf - $(INSTALL_DATA) ./files/telegraf.conf.d/ping.conf $(1)/etc/telegraf.conf.d/ping.conf + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/uci-defaults/99-telegraf-migrate-netdata $(1)/etc/uci-defaults/99-telegraf-migrate-netdata $(INSTALL_DIR) $(1)/usr/sbin $(INSTALL_BIN) ./files/telegraf-config $(1)/usr/sbin/telegraf-config $(INSTALL_DIR) $(1)/usr/libexec diff --git a/packages/telegraf/files/telegraf-config b/packages/telegraf/files/telegraf-config index 235476d86..7cc2b0d0c 100644 --- a/packages/telegraf/files/telegraf-config +++ b/packages/telegraf/files/telegraf-config @@ -10,87 +10,45 @@ import glob from jinja2 import Environment, BaseLoader from euci import EUci -PROMETHEUS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. -# Do not edit manually — changes will be overwritten. - -# Prometheus client output +PROMETHEUS_TEMPLATE = """# Prometheus client output {% if enabled == '1' -%} [[outputs.prometheus_client]] - ## Address to listen on. - ## ex: - ## listen = ":9273" - ## listen = "vsock://:9273" listen = "{{ listen_addr }}" - - ## Maximum duration before timing out read of the request - # read_timeout = "10s" - ## Maximum duration before timing out write of the response - # write_timeout = "10s" - - ## Metric version controls the mapping from Prometheus metrics into Telegraf metrics. - ## See "Metric Format Configuration" in plugins/inputs/prometheus/README.md for details. - ## Valid options: 1, 2 - # metric_version = 1 - - ## Use HTTP Basic Authentication. {%- if basic_auth_username and basic_auth_password %} basic_username = "{{ basic_auth_username }}" basic_password = "{{ basic_auth_password }}" -{%- else %} - # basic_username = "Foo" - # basic_password = "Bar" {%- endif %} +{%- endif %} +""" - ## If set, the IP Ranges which are allowed to access metrics. - ## ex: ip_range = ["192.168.0.0/24", "192.168.1.0/30"] - # ip_range = [] - - ## Path to publish the metrics on. - # path = "/metrics" - - ## Expiration interval for each metric. 0 == no expiration - # expiration_interval = "60s" - - ## Collectors to enable, valid entries are "gocollector" and "process". - ## If unset, both are enabled. - # collectors_exclude = ["gocollector", "process"] - - ## Send string metrics as Prometheus labels. - ## Unless set to false all string metrics will be sent as labels. - # string_as_label = true - - ## If set, enable TLS with the given certificate. - # tls_cert = "/etc/ssl/telegraf.crt" - # tls_key = "/etc/ssl/telegraf.key" - - ## Set one or more allowed client CA certificate file names to - ## enable mutually authenticated TLS connections - # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] +PING_TEMPLATE = """# Ping input plugin - monitors ICMP ping to configured hosts +{% if pings -%} +[[inputs.ping]] + urls = [{{ pings|map('tojson')|join(', ') }}] + method = "native" + count = 1 + ping_interval = 1.0 + deadline = 10 + + [inputs.ping.tags] + influxdb_db = "ping-metrics" +{%- endif %} +""" - ## Export metric collection time. - # export_timestamp = false +DNS_TEMPLATE = """# DNS Query input plugin - monitors DNS resolution +{% if dns_domains -%} +[[inputs.dns_query]] + domains = [{{ dns_domains|map('tojson')|join(', ') }}] + servers = ["127.0.0.1"] - ## Specify the metric type explicitly. - ## This overrides the metric-type of the Telegraf metric. Globbing is allowed. - # [outputs.prometheus_client.metric_types] - # counter = [] - # gauge = [] + [inputs.dns_query.tags] + influxdb_db = "ping-metrics" {%- endif %} """ -SENSORS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. -# Do not edit manually — changes will be overwritten. - -# Monitor sensors, requires lm-sensors package -# This plugin ONLY supports Linux +SENSORS_TEMPLATE = """# Monitor sensors (requires lm-sensors package) {% if sensors_available -%} [[inputs.sensors]] - ## Remove numbers from field names. - ## If true, a field name like 'temp1_input' will be changed to 'temp_input'. - # remove_numbers = true - - ## Timeout is the maximum amount of time that the sensors command can run. - # timeout = "5s" [inputs.sensors.tags] influxdb_db = "os-metrics" {%- endif %} @@ -132,6 +90,52 @@ def generate_prometheus_config(): return False +def generate_ping_config(): + """Read UCI config and render ping input section.""" + e_uci = EUci() + try: + pings = e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[]) + except Exception: + pings = [] + + template = Environment(loader=BaseLoader()).from_string(PING_TEMPLATE) + rendered = template.render(pings=pings) + + config_file = '/etc/telegraf.conf.d/ping.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +def generate_dns_config(): + """Read UCI config and render DNS query input section.""" + e_uci = EUci() + try: + dns_domains = e_uci.get('telegraf', 'internet', 'dns', dtype=str, list=True, default=[]) + except Exception: + dns_domains = [] + + template = Environment(loader=BaseLoader()).from_string(DNS_TEMPLATE) + rendered = template.render(dns_domains=dns_domains) + + config_file = '/etc/telegraf.conf.d/dns.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + def sensors_available(): """Check if sensors command works by running it with -A -u flags.""" try: @@ -165,4 +169,6 @@ def generate_sensors_config(): if __name__ == '__main__': generate_prometheus_config() generate_sensors_config() + generate_ping_config() + generate_dns_config() exit(0) diff --git a/packages/telegraf/files/telegraf.conf.d/ping.conf b/packages/telegraf/files/telegraf.conf.d/ping.conf deleted file mode 100644 index 40c7c8087..000000000 --- a/packages/telegraf/files/telegraf.conf.d/ping.conf +++ /dev/null @@ -1,26 +0,0 @@ -# Ping input plugin - monitors ICMP ping to configured hosts -# Uses native method for better performance and no external dependencies - -[[inputs.ping]] - # Hosts to send ping packets to - urls = [] - - # Method: "native" for improved compatibility and performance - # Uses privileged raw ICMP sockets (requires CAP_NET_RAW or root) - method = "native" - - # Number of ping packets to send per interval - count = 1 - - # Time to wait between sending ping packets (seconds) - ping_interval = 1.0 - - # Total ping deadline (seconds) - deadline = 10 - - # Data size for ping packets (bytes) - size = 56 - - # Tags for metric routing - [inputs.ping.tags] - influxdb_db = "ping-metrics" diff --git a/packages/telegraf/files/telegraf.uci b/packages/telegraf/files/telegraf.uci index d1225d452..b7073f50d 100644 --- a/packages/telegraf/files/telegraf.uci +++ b/packages/telegraf/files/telegraf.uci @@ -3,3 +3,9 @@ config output_prometheus 'output_prometheus' option listen_addr ':9273' option basic_auth_username '' option basic_auth_password '' + +config internet 'internet' + list pings '8.8.8.8' + list pings '1.1.1.1' + list dns 'google.com' + list dns 'cloudflare.com' diff --git a/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata b/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata new file mode 100644 index 000000000..5fa351f41 --- /dev/null +++ b/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Migrate fping hosts from the legacy netdata configuration to the telegraf +# UCI config (telegraf.internet.pings). +# +# The old fping.conf format is a single line: +# hosts="1.1.1.1 8.8.8.8 google.com" + +FPING_CONF="/etc/netdata/fping.conf" + +[ -f "$FPING_CONF" ] || exit 0 + +# Extract the hosts string: strip 'hosts="..."' +hosts_line=$(grep '^hosts=' "$FPING_CONF" | head -n1) +[ -n "$hosts_line" ] || exit 0 + +# Strip key and surrounding quotes +hosts_val="${hosts_line#hosts=}" +hosts_val="${hosts_val#\"}" +hosts_val="${hosts_val%\"}" +[ -n "$hosts_val" ] || exit 0 + +# Add each host to telegraf.internet.pings if not already present +uci -q get telegraf.internet > /dev/null || uci set telegraf.internet='internet' + +for host in $hosts_val; do + # Skip if already in the list + uci -q get telegraf.internet.pings | grep -qw "$host" && continue + uci add_list telegraf.internet.pings="$host" +done + +uci commit telegraf From 0b9d664947a12f3b0bc008e0d18ed6b684c8dd68 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 7 May 2026 16:21:45 +0200 Subject: [PATCH 37/48] chore(ns-ui): version bump --- packages/ns-ui/Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 171b73bde..aba307639 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -10,10 +10,12 @@ PKG_NAME:=ns-ui PKG_VERSION:=2.20.1 PKG_RELEASE:=1 -PKG_SOURCE:=nethsecurity-ui-$(PKG_VERSION).tar.gz -PKG_BUILD_DIR=$(BUILD_DIR)/nethsecurity-ui-$(PKG_VERSION) -PKG_SOURCE_URL:=https://codeload.github.com/nethserver/nethsecurity-ui/tar.gz/$(PKG_VERSION)? -PKG_HASH:=skip +PKG_SOURCE_PROTO:=git +PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git +PKG_SOURCE_VERSION:=efba51830602aa2c879ec41081208b7a33f93351 +PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) +PKG_MIRROR_HASH:=skip PKG_MAINTAINER:=Giacomo Sanchietti PKG_LICENSE:=GPL-3.0-only From 4ba60a9582951d3201293e4e98e3c73aecfb8714 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 7 May 2026 16:58:04 +0200 Subject: [PATCH 38/48] fix(telegraf): moving pruning of netdata config to telegraf migration --- packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default | 3 --- .../telegraf/files/uci-defaults/99-telegraf-migrate-netdata | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default b/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default index df1a859c5..e76fe4356 100644 --- a/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default +++ b/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default @@ -4,9 +4,6 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Remove bundled netdata configuration files on first boot. -rm -rf /etc/netdata 2>/dev/null || true - # Remove the legacy backup-encryption alert cron entry if it is still present. if crontab -l 2>/dev/null | grep -q '/usr/libexec/backup-encryption-alert'; then crontab -l 2>/dev/null | grep -v '/usr/libexec/backup-encryption-alert' | sort | uniq | crontab - diff --git a/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata b/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata index 5fa351f41..2d5921b82 100644 --- a/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata +++ b/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata @@ -34,3 +34,6 @@ for host in $hosts_val; do done uci commit telegraf + +# Remove bundled netdata configuration files since they are no longer used. +rm -rf /etc/netdata 2>/dev/null || true From 7c94fad4aec3d7037c6c8accb494373c7205c315 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 13 May 2026 14:27:29 +0200 Subject: [PATCH 39/48] fix(ns-api): using uci config for pings --- packages/ns-api/files/ns.report | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/packages/ns-api/files/ns.report b/packages/ns-api/files/ns.report index e188c25a3..47dc1c315 100755 --- a/packages/ns-api/files/ns.report +++ b/packages/ns-api/files/ns.report @@ -325,25 +325,6 @@ def ovpnrw_bytes_by_hour_and_user(instance, day, user): return {"hours": hours_bytes} -def get_ping_hosts(): - # read ping hosts from telegraf configuration - ping_conf_file = "/etc/telegraf.conf.d/ping.conf" - hosts = [] - if os.path.exists(ping_conf_file): - try: - with open(ping_conf_file, 'r') as fp: - content = fp.read() - # Find the urls line in TOML format: urls = ["host1", "host2"] - match = re.search(r'urls\s*=\s*(\[[^\]]*\])', content) - if match: - urls_str = match.group(1) - # Parse JSON array - hosts = json.loads(urls_str) - except Exception: - pass - return hosts - - def get_victoria_metrics_ping_data(host): """ Query Victoria Metrics for ping metrics. @@ -384,7 +365,8 @@ def get_victoria_metrics_ping_data(host): def latency_and_quality_report(): - hosts = get_ping_hosts() + e_uci = EUci() + hosts = e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[]) ret = {} for host in hosts: ret[host] = get_victoria_metrics_ping_data(host) From 84e46f3523d68a0a9d1b7cb6f51c4ab54387c69a Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Wed, 13 May 2026 14:35:35 +0200 Subject: [PATCH 40/48] fix(ns-plug): added service triggers for alert-proxy --- packages/ns-plug/files/ns-plug-alert-proxy.init | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/ns-plug/files/ns-plug-alert-proxy.init b/packages/ns-plug/files/ns-plug-alert-proxy.init index 31b5fa430..38ce01012 100644 --- a/packages/ns-plug/files/ns-plug-alert-proxy.init +++ b/packages/ns-plug/files/ns-plug-alert-proxy.init @@ -17,3 +17,14 @@ start_service() { procd_set_param respawn 3600 5 0 procd_close_instance } + +service_triggers() +{ + procd_add_reload_trigger "ns-plug" +} + +reload_service() +{ + stop + start +} From 13705332ee760133b9e96d7adac6a7c1841f340b Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Fri, 15 May 2026 11:42:31 +0200 Subject: [PATCH 41/48] feat(victoria-metrics): added storage configuration --- packages/victoria-metrics/README.md | 10 +++---- .../files/victoria-metrics.conf | 2 -- .../files/victoria-metrics.initd | 27 +++++++++++++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/packages/victoria-metrics/README.md b/packages/victoria-metrics/README.md index d878bf4f1..cd7c11728 100644 --- a/packages/victoria-metrics/README.md +++ b/packages/victoria-metrics/README.md @@ -36,16 +36,16 @@ Configuration is located at `/etc/config/victoria-metrics`: ``` config victoriametrics 'main' - option storage_path '/var/lib/victoriametrics' - option retention_period '1y' option http_listen_addr '127.0.0.1:8428' ``` -**Options:** -- `storage_path`: Where to store metrics data -- `retention_period`: How long to keep metrics (`1d`, `7d`, `30d`, `1y`, etc.) +**Required options:** - `http_listen_addr`: Address and port for the HTTP server +**Optional options:** +- `storage_path`: Where to store metrics data (default: `/var/lib/victoriametrics`, auto-detects `/mnt/data/victoriametrics` if available) +- `retention_period`: How long to keep metrics (`1d`, `7d`, `30d`, `1y`, etc.) (default: `7d`, auto-detects `1y` if not set) + ### Accessing the Web UI By default the server is accessible only on localhost for security. diff --git a/packages/victoria-metrics/files/victoria-metrics.conf b/packages/victoria-metrics/files/victoria-metrics.conf index 5b6813a89..bf3541fe8 100644 --- a/packages/victoria-metrics/files/victoria-metrics.conf +++ b/packages/victoria-metrics/files/victoria-metrics.conf @@ -1,4 +1,2 @@ config victoriametrics 'main' - option storage_path '/var/lib/victoriametrics' - option retention_period '1d' option http_listen_addr '127.0.0.1:8428' diff --git a/packages/victoria-metrics/files/victoria-metrics.initd b/packages/victoria-metrics/files/victoria-metrics.initd index fd2cee851..148804cab 100644 --- a/packages/victoria-metrics/files/victoria-metrics.initd +++ b/packages/victoria-metrics/files/victoria-metrics.initd @@ -15,10 +15,33 @@ PROG="/usr/bin/victoria-metrics" start_service() { config_load victoria-metrics local storage_path retention_period http_listen_addr - config_get storage_path main storage_path /var/lib/victoriametrics - config_get retention_period main retention_period 1d + config_get storage_path main storage_path + config_get retention_period main retention_period config_get http_listen_addr main http_listen_addr 127.0.0.1:8428 + # Detect if external storage is mounted + local disk_mount + config_load fstab + config_get disk_mount ns_data target + + # Auto-detect storage_path if not customized + if [ -z "$storage_path" ]; then + if [ -n "$disk_mount" ]; then + storage_path="$disk_mount/victoria-metrics-data" + else + storage_path="/var/lib/victoria-metrics-data" + fi + fi + + # Set retention_period default based on storage availability + if [ -z "$retention_period" ]; then + if [ -n "$disk_mount" ]; then + retention_period="1y" + else + retention_period="7d" + fi + fi + procd_open_instance procd_set_param stdout 1 procd_set_param stderr 1 From b01b738966e170b6a2b3580f49c0af194d9f2b5a Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Fri, 15 May 2026 11:42:49 +0200 Subject: [PATCH 42/48] chore(ns-ui): version bump --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index aba307639..ee852564b 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=efba51830602aa2c879ec41081208b7a33f93351 +PKG_SOURCE_VERSION:=6c362f1eb30c8ef6b034d4127885d928e6640553 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From e406a1b8e7eade7032b579e6a4cbb12461eeae43 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Mon, 11 May 2026 14:40:54 +0200 Subject: [PATCH 43/48] chore(skill): improve OpenWrt package update --- .../skills/openwrt-package-update/SKILL.md | 231 +++--------------- .../openwrt-package-update/assets/.gitignore | 1 - .../references/REFERENCE.md | 219 ----------------- .../references/WORKFLOW.md | 88 +++++++ .../scripts/diff-package.sh | 38 +++ .../scripts/fetch-upstream.sh | 142 ----------- .../scripts/get-target-commit.sh | 44 ++++ .../scripts/setup-package.sh | 162 ++++++++++++ AGENTS.md | 2 +- 9 files changed, 373 insertions(+), 554 deletions(-) delete mode 100644 .agents/skills/openwrt-package-update/references/REFERENCE.md create mode 100644 .agents/skills/openwrt-package-update/references/WORKFLOW.md create mode 100755 .agents/skills/openwrt-package-update/scripts/diff-package.sh delete mode 100755 .agents/skills/openwrt-package-update/scripts/fetch-upstream.sh create mode 100755 .agents/skills/openwrt-package-update/scripts/get-target-commit.sh create mode 100755 .agents/skills/openwrt-package-update/scripts/setup-package.sh diff --git a/.agents/skills/openwrt-package-update/SKILL.md b/.agents/skills/openwrt-package-update/SKILL.md index 9ecbdb217..7f9164525 100644 --- a/.agents/skills/openwrt-package-update/SKILL.md +++ b/.agents/skills/openwrt-package-update/SKILL.md @@ -1,211 +1,60 @@ --- name: openwrt-package-update -description: Update a forked OpenWrt upstream package to a newer upstream version while preserving local customizations. Use when updating packages like adblock, mwan3, or other non-ns- prefixed upstream packages to newer releases, and when comparing local forks against the canonical OpenWrt packages feed to identify and merge upstream improvements. -license: GPL-2.0-only -compatibility: Requires git, curl, and bash. Works with OpenWrt release tags (v24.10.x, v24.11.x, etc.) and HEAD of the packages feed. +description: > + Use when updating any forked OpenWrt package in a NethSecurity workspace from + the upstream openwrt/packages feed. Covers the full update cycle: version bump, + merging upstream file changes, updating dependent ns-api handlers and migration + scripts, UCI overlay defaults, and correlated documentation. Triggers on updating + a package by name (adblock, mwan3, banip, rsyslog, snort3, keepalived, + python-jinja2, python-semver, and similar forks), syncing with upstream, or any + request to align a local package fork with openwrt/packages — even if upstream + is not mentioned explicitly. +compatibility: Requires git and curl. Works in a NethSecurity workspace with build.conf.defaults present. metadata: domain: nethsecurity-packages - type: upstream-package-update + type: package-update +allowed-tools: Bash(git:*) Bash(curl:*) Bash(diff:*) Bash(tar:*) Bash(grep:*) Bash(sed:*) Bash(awk:*) Read Write --- ## What I do -I help you update forked OpenWrt upstream packages (non-`ns-*` packages) to newer upstream versions while carefully preserving intentional local customizations. I: +Update a forked upstream OpenWrt package from the openwrt/packages feed. Auto-discovers the package path, compares the current version against the upstream target, extracts snapshots for comparison, applies upstream changes, propagates impact to ns-api handlers and migration scripts, and updates correlated documentation. -1. Fetch the canonical OpenWrt `feeds.conf.default` for a given release tag or the latest HEAD -2. Extract the pinned commit hash for the packages feed -3. Clone or update a local copy of the `openwrt/packages` GitHub mirror -4. Locate the package in the upstream feed -5. Generate a detailed diff showing all differences between upstream and your local fork -6. Guide you through analyzing the diff to identify: - - Pure upstream improvements to apply - - Your intentional customizations to preserve - - Conflicting changes requiring manual adaptation +## Quick start -## When to use me +1. **Setup**: `bash scripts/setup-package.sh ` — auto-discovers upstream path, finds version commits, extracts old/new snapshots into `assets/_old/` and `assets/_new/` +2. **Compare**: `bash scripts/diff-package.sh ` — shows what changed upstream +3. **Merge**: Read both snapshots, apply upstream changes to local files in `packages//`; update PKG_VERSION and PKG_RELEASE in the Makefile +4. **Cross-package**: Grep for any changed config keys or function names in ns-api, migration scripts, and files/ overlay; update as needed +5. **Documentation**: Update `docs/` and `packages/ns-api/README.md` if user-facing behavior changed +6. **Verify**: Run `ruff check` on any changed Python files; build the package inside the container -Use this skill when you are: -- **Updating an existing forked package** to a newer upstream release (e.g., adblock 4.1.5 → 4.5.5) -- **Merging upstream improvements** while keeping local patches and configuration -- **Comparing against multiple OpenWrt releases** to understand what changed between versions -- **Evaluating the scope of local customization** before deciding whether to continue forking or contribute upstream -- **Refreshing patches and adapting them** to newer upstream versions +For per-step detail on any of the above, read [references/WORKFLOW.md](references/WORKFLOW.md). -Do **not** use this skill for: -- Creating entirely new `ns-*` packages (use `openwrt-package` skill instead) -- Patches that should be contributed upstream (this skill is for local forks only) -- Non-OpenWrt packages or custom software +## How it works -## Workflow +The skill uses three helper scripts: -### Step 1: Identify the target package and version +- **get-target-commit.sh** — reads `build.conf.defaults` for OWRT_VERSION, fetches `feeds.conf.default` from that tag, extracts the openwrt/packages commit hash +- **setup-package.sh** — auto-discovers package upstream path, uses git pickaxe to find the commit matching current PKG_VERSION then PKG_RELEASE, clones/fetches openwrt/packages bare repo into `assets/.openwrt-packages-repo/` (persistent cache), extracts both versions into named snapshots +- **diff-package.sh** — convenience wrapper: `diff -rN assets/_old/ assets/_new/` -Ask the user: -- **Which package are you updating?** (e.g., `adblock`, `mwan3`, `keepalived`) -- **Which OpenWrt version?** Suggest either: - - A specific release tag (e.g., `v24.10.5`) — extracts the pinned hash from that release's `feeds.conf.default` - - `HEAD` — uses the latest upstream version +## Snapshot structure -### Step 2: Run the diff script - -From the skill root directory, invoke: - -```bash -scripts/fetch-upstream.sh -``` - -Examples: -```bash -scripts/fetch-upstream.sh v24.10.5 adblock ../../../packages/adblock -scripts/fetch-upstream.sh HEAD mwan3 /absolute/path/to/packages/mwan3 +Each snapshot has the exact upstream package files at root level: ``` - -The script will: -- Fetch the feeds.conf.default for the target tag -- Clone the packages feed to `assets/packages/` (gitignored) -- Output a detailed `diff -ru` comparing upstream vs local - -### Step 3: Analyze the diff - -Review the diff output and classify each change: - -#### Category A: Pure Upstream Changes (Safe to apply) -These are bug fixes or features from upstream with no local modification: -- Upstream script improvements, new functions, or bugfixes -- Version bumps in dependencies -- Documentation updates - -**Action:** Copy the upstream version directly into the local package. - -#### Category B: Intentional Local Customizations (Always preserve) -These are deliberate modifications specific to NethSecurity: -- UCI configuration defaults hardcoded for NethSecurity -- Removal of upstream features not needed in NethSecurity -- Integration with ns-api or other NethSecurity components -- Build system tweaks (PKG_RELEASE bumps for local patches) - -**Action:** Keep these changes as-is. They should remain in the diff even after updating. - -#### Category C: Upstream Changes That Conflict With Local Customizations (Needs manual work) -These require careful adaptation: -- Upstream modified the same function that you customized -- New upstream feature overlaps with your local workaround -- Upstream changed configuration options you've modified - -**Action:** -- Read both versions carefully -- Decide whether upstream's approach is better (merge it) -- Or adapt upstream's changes to work with your customization -- Test thoroughly after merging - -### Step 4: Update the Makefile - -Update version information in the package Makefile: - -- **`PKG_VERSION`**: Set to the upstream version (e.g., `4.1.5` → `4.5.5`) -- **`PKG_RELEASE`**: Reset to `1` if the upstream version changed, or increment if only applying new local patches - -Example transitions: -```makefile -# Before: 4.1.5 with 9 local releases -PKG_VERSION:=4.1.5 -PKG_RELEASE:=9 - -# After updating to 4.5.5: -PKG_VERSION:=4.5.5 -PKG_RELEASE:=1 - -# Or if staying at 4.1.5 but adding a new local patch: -PKG_VERSION:=4.1.5 -PKG_RELEASE:=10 -``` - -### Step 5: Apply the changes - -For each file in the diff: - -1. **If it's Category A (safe upstream change)**: Copy the upstream version directly -2. **If it's Category B (local customization)**: Leave unchanged -3. **If it's Category C (conflict)**: Manually edit to merge both approaches - -Common files to check: -- `Makefile` — version and release -- `files/*.sh` — main scripts (often have conflicts) -- `files/*.conf` — configuration templates -- `files/*.init` — OpenWrt init scripts -- `files/README.md` — documentation - -### Step 6: Re-run the diff for verification - -After making changes, re-run: - -```bash -scripts/fetch-upstream.sh +assets/adblock_old/ +├── Makefile +├── files/ +│ ├── adblock.sh +│ ├── adblock.init +│ ├── adblock.conf +│ └── ... ``` -The diff should now only show your **intentional** customizations. Review it one more time: -- Do all remaining differences make sense? -- Are there any accidental changes that slipped in? -- Should any new upstream code be adopted? - -### Step 7: Update documentation (if significant changes) - -If the package's behavior changes, update the package's `README.md` or comments to document: -- The version it's based on (e.g., "Based on upstream adblock 4.5.5") -- A summary of local customizations (e.g., "Adds automatic DNS reload on UCI changes") -- Any compatibility notes with the controller or UI - -## Common Scenarios - -### Scenario: Small upstream bugfix, large local customization - -**Situation:** adblock 4.1.5→4.1.6 (just a bugfix), but you've heavily customized the script for NethSecurity. - -**Steps:** -1. Run diff with `v24.10.5` tag -2. See the bugfix hunk in `files/adblock.sh` -3. Merge the bugfix into your customized version -4. Update Makefile: `PKG_VERSION:=4.1.6 PKG_RELEASE:=1` -5. Re-run diff — should show only your customizations remain - -### Scenario: Multiple version jumps (e.g., 4.1.5 → 4.5.5) - -**Situation:** You skipped several releases and want to jump to the latest. - -**Steps:** -1. Run diff with `HEAD` (latest packages feed) -2. Review all changes — likely many new features and bugfixes -3. Categorize each: adopt, preserve, or merge -4. Update Makefile: `PKG_VERSION:=4.5.5 PKG_RELEASE:=1` -5. Test extensively — more changes = higher risk -6. Consider running against intermediate versions to ease the transition - -### Scenario: Package has local patches in `patches/feeds/packages/` - -**Situation:** You have patches for the upstream package (in `patches/feeds/packages/`), and the upstream has evolved. - -**Steps:** -1. Run diff to see what's different -2. Review whether your patches are still needed (upstream might have fixed the issue) -3. If needed, regenerate the patches from the upstream source: - ```bash - cd assets/packages/ - git diff HEAD^ HEAD > /tmp/new-patch.diff - ``` -4. Compare with your current patch; update if necessary -5. Test the patched version in a build - -## Edge Cases - -- **Package renamed upstream**: The script won't find it. Manually verify it still exists in the feed under a new name. -- **Package removed from upstream**: You're now the sole maintainer. Update documentation and consider contributing upstream. -- **Multiple packages with same name**: The script finds the first one. Use `find` manually to disambiguate. -- **Large binary files**: The diff may be very large. Use `diff -r --exclude="*.bin" ...` if needed. - -## Reference Files +## Edge cases -See [REFERENCE.md](references/REFERENCE.md) for: -- Shell script explanation of `fetch-upstream.sh` -- OpenWrt feeds.conf.default format -- Git workflow for managing local patches -- Diff interpretation guide +- **mwan3**: Deep fork — apply upstream changes conservatively, one by one +- **snort3**: Has local patches — verify patches still apply after update +- **Version not found**: If script fails, local version is very old; check `assets/.openwrt-packages-repo/` git history +- **Empty diff**: Versions match; only update PKG_VERSION/PKG_RELEASE if tracking newer release diff --git a/.agents/skills/openwrt-package-update/assets/.gitignore b/.agents/skills/openwrt-package-update/assets/.gitignore index c96a04f00..72e8ffc0d 100644 --- a/.agents/skills/openwrt-package-update/assets/.gitignore +++ b/.agents/skills/openwrt-package-update/assets/.gitignore @@ -1,2 +1 @@ * -!.gitignore \ No newline at end of file diff --git a/.agents/skills/openwrt-package-update/references/REFERENCE.md b/.agents/skills/openwrt-package-update/references/REFERENCE.md deleted file mode 100644 index 277ac78aa..000000000 --- a/.agents/skills/openwrt-package-update/references/REFERENCE.md +++ /dev/null @@ -1,219 +0,0 @@ -# Reference: openwrt-package-update Skill - -## fetch-upstream.sh Script Details - -### Purpose -The `fetch-upstream.sh` script automates fetching upstream package code and generating a diff for manual review. - -### Workflow - -1. **Argument parsing**: Takes ``, ``, `` -2. **Fetch feeds.conf.default**: If a tag is provided, curl the feeds.conf.default from that OpenWrt release -3. **Extract hash**: Parse the `src-git packages` line to find the pinned commit hash after `^` -4. **Clone/update feed**: Clone the GitHub mirror of openwrt/packages or update an existing clone -5. **Checkout hash**: `git checkout` the extracted hash (or HEAD if no hash provided) -6. **Locate package**: `find` the package directory (up to 3 levels deep) -7. **Generate diff**: Run `diff -ru` between upstream and local - -### Output Format - -``` -=== Step 1: Fetching feeds.conf.default from OpenWrt === -... -=== Step 2: Cloning/updating packages feed to === -... -=== Step 3: Locating package '' in upstream feed === -... -=== Step 4: Comparing upstream vs local package === -... ---- Diff (upstream vs local) --- -[diff output] -... -=== Diff Summary === -... -``` - -### Exit Codes -- `0`: Success -- `1`: Missing arguments, network error, or package not found - -### Asset Management - -The script clones the packages feed into `.agents/skills/openwrt-package-update/assets/packages/` which is added to `.gitignore`. This folder: -- Grows over time as more packages are cloned -- Can be manually deleted to save space (script will re-clone on next run) -- Is not committed to git -- Is safe to delete between sessions - -### Network Requirements - -The script needs: -- `curl` to fetch feeds.conf.default from GitHub -- `git` to clone and checkout the packages repository -- Network access to github.com (both GitHub CDN and Git protocol) - -Firewall/corporate proxy rules should allow: -- `https://raw.githubusercontent.com/openwrt/openwrt/...` -- `https://github.com/openwrt/packages.git` (or git:// protocol) - -## OpenWrt feeds.conf.default Format - -The feeds.conf.default file lists the official feeds: - -``` -src-git packages https://git.openwrt.org/feed/packages.git^ -src-git luci https://git.openwrt.org/project/luci.git^ -... -``` - -Each line format: -- `src-git` — source control type (Git) -- `` — feed name (e.g., `packages`, `luci`) -- `` — Git repository URL -- `^` — (optional) pinned commit hash - -The hash ensures reproducible builds by locking to a specific point in time. - -## Diff Interpretation Guide - -When reviewing the script's diff output: - -### Common diff symbols: -- `---` — upstream file -- `+++` — local file -- `@@` — hunk header showing line numbers -- `-` (red) — line removed in local (upstream has it, you removed it) -- `+` (green) — line added in local (you added this, upstream doesn't have it) - -### Example: Local customization - -```diff ---- a/files/adblock.sh -+++ b/files/adblock.sh -@@ -10,6 +10,8 @@ - UPSTREAM_VERSION="4.1.5" - - # Initialize DNS backend -+# NethSecurity: Force dnsmasq backend -+DNS_BACKEND="dnsmasq" - init_dns_backend() { -``` - -Here: -- Upstream has no hardcoded `DNS_BACKEND` -- Local (your fork) forces `dnsmasq` -- This is a **local customization** to preserve - -### Example: Pure upstream change - -```diff ---- a/files/adblock.sh -+++ b/files/adblock.sh -@@ -234,6 +234,8 @@ - log "Error: Invalid source URL" - return 1 - fi -+ # Upstream fix: sanitize URL before processing -+ sanitize_url "$source_url" - process_source "$source_url" -``` - -Here: -- Upstream added a sanitization step -- Neither the old nor new version is in your local file -- This is a **pure upstream improvement** to adopt - -## Git Workflow for Patches - -If your package has patches in `patches/feeds/packages/`: - -### Understand the current patch -```bash -cd assets/packages/ -git log --oneline -5 # See recent changes -git show HEAD # See the latest change -cat /path/to/your/patches/feeds/packages/*-.patch # Your patch -``` - -### Check if patch still applies -```bash -cd assets/packages/ -git apply --check /path/to/your/patch.patch -``` - -### Regenerate patch if needed -```bash -cd assets/packages/ -# Make your changes to files -git diff > /tmp/regenerated.patch -# Copy to repo -cp /tmp/regenerated.patch /path/to/patches/feeds/packages/ -``` - -## Decision Tree for Categorizing Diffs - -``` -For each hunk in the diff: - -1. Is this hunk in a file you modified locally? - NO → Category A (pure upstream, safe to apply) - YES → go to 2 - -2. Does this hunk match a specific customization documented in README.md or comments? - YES → Category B (intentional customization, preserve) - NO → go to 3 - -3. Does this hunk conflict with your customization (same lines touched)? - YES → Category C (conflict, needs manual merge) - NO → Category B (you customized different parts, keep both) -``` - -## Example: Updating adblock 4.1.5 → 4.5.5 - -**Preparation:** -```bash -scripts/fetch-upstream.sh HEAD adblock ../../../packages/adblock -``` - -**Review output:** -- Makefile version bumped from 4.1.5 to 4.5.5 (upstream) -- New categories in `adblock.categories` (upstream) -- New sources in `adblock.sources` (upstream) -- Custom DNS backend forcing in `adblock.sh` (local) -- Custom email notification logic in `adblock.sh` (local) - -**Categorization:** -- Makefile version: Category A (pure upstream) → adopt -- New categories: Category A → adopt -- New sources: Category A → adopt -- Custom DNS backend: Category B → preserve -- Custom email: Category B → preserve - -**Actions:** -1. Copy upstream Makefile, update locally to preserve DNS backend forcing -2. Copy upstream categories file -3. Copy upstream sources file -4. Merge adblock.sh: take upstream as base, reapply the two local customizations -5. Update Makefile PKG_VERSION=4.5.5, PKG_RELEASE=1 -6. Re-run script to verify remaining diffs are just your customizations - -## Troubleshooting - -### Script fails to clone the feed -**Cause:** Network connectivity or git configuration issue -**Solution:** Verify git works: `git clone https://github.com/openwrt/packages.git /tmp/test` - -### Script can't find the package -**Cause:** Package name doesn't match directory name, or doesn't exist in that release -**Solution:** Manually browse the assets folder: `ls assets/packages/net/ | grep ` - -### Diff is huge (thousands of lines) -**Cause:** You're comparing very different versions -**Solution:** -- Try an intermediate version first -- Use `diff -r --exclude-from=.diffignore` to skip binary files -- Split review into categories (Makefile, scripts, configs) - -### Cloned repo is getting too large -**Cause:** Multiple runs accumulate git history -**Solution:** Delete the assets folder: `rm -rf assets/packages/`, script will re-clone on next run diff --git a/.agents/skills/openwrt-package-update/references/WORKFLOW.md b/.agents/skills/openwrt-package-update/references/WORKFLOW.md new file mode 100644 index 000000000..ec4af820c --- /dev/null +++ b/.agents/skills/openwrt-package-update/references/WORKFLOW.md @@ -0,0 +1,88 @@ +# Detailed Workflow: Updating Upstream Packages + +## 1. Identify the package + +Confirm the package exists and is in scope. Read `packages//Makefile` to understand the current version: +- Note PKG_VERSION and PKG_RELEASE +- Check if `packages//patches/` exists (indicates local patches) +- Scan `files/` directory to understand what configuration files and scripts are included + +## 2. Setup snapshots + +Run the setup script: +```bash +bash scripts/setup-package.sh +``` + +This script will: +- Auto-discover the upstream path in openwrt/packages (e.g., `net/adblock`) +- Use git pickaxe to find the commit that introduced the current PKG_VERSION +- Then find the commit that set the current PKG_RELEASE +- Extract both versions into `assets/_old/` and `assets/_new/` +- Output summary with commit hashes and paths + +## 3. Compare and classify + +Examine both snapshots using your file reading tools. Run: +```bash +bash scripts/diff-package.sh +``` + +Classify each change: +- **Makefile**: version bumps, new dependencies, build flags +- **files/*.sh** / **files/*.init**: behavioral changes to shell scripts +- **files/*.conf**: UCI configuration schema changes (new/removed/renamed options) +- **files/*.sources**, **files/*.categories**, data files: content-only updates +- **New files** added / **files removed** + +## 4. Apply upstream changes + +For each changed file: +- If the local copy is identical to old version or has no NethSecurity-specific changes → take upstream as-is +- If the local copy has customizations → merge carefully: + - Use the diff tool to understand what changed upstream + - Manually integrate the upstream changes into the local file + - Preserve any NethSecurity-specific logic or configuration + +Always: +- Update PKG_VERSION and PKG_RELEASE in `packages//Makefile` +- If `packages//patches/` exists, re-verify each patch still applies cleanly with the new version +- Test build: `make package/feeds/nethsecurity//compile V=sc` inside the build container + +## 5. Cross-package impact detection + +For each changed UCI option name, function name, or config key in the diff: +- Grep the workspace for references: + ```bash + grep -r "" packages/ files/ docs/ --include="*.py" --include="*.sh" --include="*.json" --include="*.conf" --include="*.md" + ``` +- Grep `packages/ns-migration/` for migration scripts that reference old option names +- Grep `packages/ns-api/` for API scripts that read/write this package's configuration +- Present findings to user before making changes + +## 6. Apply cross-package fixes (after user confirmation) + +Update affected files: +- **ns-api scripts**: if UCI schema changed, update handlers in `packages/ns-api/files/ns.` +- **Migration scripts**: if old config options are removed, add migration logic to `packages/ns-migration/files/` +- **files/ overlay defaults**: if default values changed, update `files/etc/uci-defaults/` or `files/etc/config/` +- **Documentation**: update `docs/` if user-facing behavior changed + +## Edge cases + +**mwan3**: This is a deep fork with significant local modifications. Apply upstream changes one by one, preferring manual review for any behavioral change. Verify the result against firewall rules and failover behavior. + +**snort3, openvpn-easy-rsa**: These packages have local patches (see `packages//patches/`). After updating upstream files, re-run `quilt refresh` or regenerate patches. If patches no longer apply cleanly, update them to match the new upstream. + +**Empty diff**: If upstream and local versions are the same, only update PKG_VERSION/PKG_RELEASE in the Makefile if tracking a newer release. Otherwise, the package is up-to-date. + +**Version not found**: If the script cannot find the current PKG_VERSION in upstream history, it means the local version is very old or from a different source. Check git history in the bare repo (`assets/.openwrt-packages-repo/`) to understand the version trajectory. + +## Helpful commands + +Inside the skill directory: +- `bash scripts/get-target-commit.sh` — print the target packages feed commit hash from feeds.conf.default +- `bash scripts/setup-package.sh ` — extract old/new snapshots +- `bash scripts/diff-package.sh ` — show recursive diff +- `find assets/_old/ -type f` — list all files in old snapshot +- `cat assets/_old/Makefile | grep PKG_` — check old version details diff --git a/.agents/skills/openwrt-package-update/scripts/diff-package.sh b/.agents/skills/openwrt-package-update/scripts/diff-package.sh new file mode 100755 index 000000000..a32cc4ee8 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/diff-package.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Diff old and new package snapshots +# +# Usage: diff-package.sh +# +# Outputs a recursive diff between assets/_old/ and assets/_new/ +# + +set -euo pipefail + +PACKAGE_NAME="${1:-}" + +if [[ -z "$PACKAGE_NAME" ]]; then + echo "Usage: diff-package.sh " >&2 + exit 1 +fi + +# Find skill root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILL_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ASSETS_DIR="$SKILL_ROOT/assets" + +OLD_DIR="$ASSETS_DIR/${PACKAGE_NAME}_old" +NEW_DIR="$ASSETS_DIR/${PACKAGE_NAME}_new" + +if [[ ! -d "$OLD_DIR" ]]; then + echo "Error: $OLD_DIR not found. Run setup-package.sh first." >&2 + exit 1 +fi + +if [[ ! -d "$NEW_DIR" ]]; then + echo "Error: $NEW_DIR not found. Run setup-package.sh first." >&2 + exit 1 +fi + +# Run recursive diff, showing added/removed/modified files +diff -rN "$OLD_DIR" "$NEW_DIR" || true diff --git a/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh b/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh deleted file mode 100755 index ab7cb2761..000000000 --- a/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2026 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# -# Fetch upstream OpenWrt package and compare with local fork. -# -# Usage: ./fetch-upstream.sh -# -# Arguments: -# openwrt-tag - OpenWrt release tag (e.g., v24.10.5) or literal 'HEAD' for latest -# package-name - Name of the package (e.g., 'adblock', 'mwan3') -# local-package-dir - Path to the local forked package (relative or absolute) -# -# Examples: -# ./fetch-upstream.sh v24.10.5 adblock ../../../packages/adblock -# ./fetch-upstream.sh HEAD mwan3 /absolute/path/to/packages/mwan3 - -set -o pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ASSETS_DIR="$SCRIPT_DIR/assets/packages" - -OPENWRT_TAG="$1" -PACKAGE_NAME="$2" -LOCAL_PKG_DIR="$3" - -# Validate arguments -if [[ -z "$OPENWRT_TAG" || -z "$PACKAGE_NAME" || -z "$LOCAL_PKG_DIR" ]]; then - echo "ERROR: Missing arguments" - echo "Usage: $0 " - exit 1 -fi - -# Resolve local package directory to absolute path -if [[ ! "$LOCAL_PKG_DIR" = /* ]]; then - LOCAL_PKG_DIR="$(cd "$LOCAL_PKG_DIR" 2>/dev/null && pwd)" || { - echo "ERROR: Local package directory does not exist: $LOCAL_PKG_DIR" - exit 1 - } -else - if [[ ! -d "$LOCAL_PKG_DIR" ]]; then - echo "ERROR: Local package directory does not exist: $LOCAL_PKG_DIR" - exit 1 - fi -fi - -FEEDS_URL="https://raw.githubusercontent.com/openwrt/openwrt" -PACKAGES_REPO="https://github.com/openwrt/packages.git" -TARGET_HASH="" - -# Step 1: Determine the packages feed hash -echo "=== Step 1: Fetching feeds.conf.default from OpenWrt $OPENWRT_TAG ===" - -if [[ "$OPENWRT_TAG" == "HEAD" ]]; then - echo "Using HEAD of packages feed (no pinned commit hash)" - TARGET_HASH="HEAD" -else - FEEDS_CONF_URL="$FEEDS_URL/$OPENWRT_TAG/feeds.conf.default" - echo "Fetching: $FEEDS_CONF_URL" - - FEEDS_CONF=$(curl -sS "$FEEDS_CONF_URL" 2>&1) || { - echo "ERROR: Failed to fetch feeds.conf.default from $FEEDS_CONF_URL" - echo "$FEEDS_CONF" - exit 1 - } - - # Extract the packages feed line and parse the hash after '^' - PACKAGES_LINE=$(echo "$FEEDS_CONF" | grep "^src-git packages") - if [[ -z "$PACKAGES_LINE" ]]; then - echo "ERROR: Could not find 'src-git packages' line in feeds.conf.default" - exit 1 - fi - - # Hash is after the '^' character if present - TARGET_HASH=$(echo "$PACKAGES_LINE" | sed -n 's/.*\^\([a-f0-9]*\).*/\1/p') - - if [[ -z "$TARGET_HASH" ]]; then - echo "WARNING: No pinned commit hash found in feeds.conf.default" - echo " Line: $PACKAGES_LINE" - echo " Using HEAD instead" - TARGET_HASH="HEAD" - else - echo "Found packages feed hash: $TARGET_HASH" - fi -fi - -# Step 2: Clone or update the packages feed -echo "" -echo "=== Step 2: Cloning/updating packages feed to $ASSETS_DIR ===" - -if [[ -d "$ASSETS_DIR/.git" ]]; then - echo "Repository already exists, fetching latest..." - cd "$ASSETS_DIR" - git fetch origin "$TARGET_HASH" 2>&1 | grep -v "^From https" | grep -v "^ " || true -else - echo "Cloning packages feed..." - mkdir -p "$ASSETS_DIR" - git clone --depth 1 "$PACKAGES_REPO" "$ASSETS_DIR" 2>&1 | grep -E "(Cloning|fatal)" || true - cd "$ASSETS_DIR" -fi - -# Checkout the target hash/branch -echo "Checking out $TARGET_HASH..." -git checkout "$TARGET_HASH" 2>&1 | grep -v "^Already on" || true - -# Step 3: Find the package directory in the upstream feed -echo "" -echo "=== Step 3: Locating package '$PACKAGE_NAME' in upstream feed ===" - -UPSTREAM_PKG_DIR=$(find "$ASSETS_DIR" -maxdepth 3 -type d -name "$PACKAGE_NAME" 2>/dev/null | head -1) - -if [[ -z "$UPSTREAM_PKG_DIR" ]]; then - echo "ERROR: Package '$PACKAGE_NAME' not found in upstream feed" - echo "Searched in: $ASSETS_DIR" - exit 1 -fi - -echo "Found upstream package at: $UPSTREAM_PKG_DIR" - -# Step 4: Compare the directories -echo "" -echo "=== Step 4: Comparing upstream vs local package ===" -echo "" -echo "Upstream: $UPSTREAM_PKG_DIR" -echo "Local: $LOCAL_PKG_DIR" -echo "" -echo "--- Diff (upstream vs local) ---" -echo "" - -diff -ru "$UPSTREAM_PKG_DIR" "$LOCAL_PKG_DIR" || true - -echo "" -echo "=== Diff Summary ===" -echo "If no changes are shown above, the local package matches upstream." -echo "" -echo "To update the local package:" -echo " 1. Review the diff above carefully" -echo " 2. Copy relevant files from: $UPSTREAM_PKG_DIR" -echo " 3. Preserve intentional local customizations" -echo " 4. Update PKG_VERSION and PKG_RELEASE in Makefile" -echo " 5. Re-run this script to verify changes" diff --git a/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh b/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh new file mode 100755 index 000000000..687a5ee67 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Get the target OpenWrt packages feed commit hash from feeds.conf.default +# at the version specified in build.conf.defaults +# +# Outputs the commit hash to stdout, or exits with error if fetch/parse fails. +# + +set -euo pipefail + +# Find workspace root +WORKSPACE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && git rev-parse --show-toplevel 2>/dev/null || echo ".") + +# Read OWRT_VERSION from build.conf.defaults +if [[ ! -f "$WORKSPACE_ROOT/build.conf.defaults" ]]; then + echo "Error: build.conf.defaults not found at $WORKSPACE_ROOT" >&2 + exit 1 +fi + +OWRT_VERSION=$(grep '^OWRT_VERSION=' "$WORKSPACE_ROOT/build.conf.defaults" | cut -d'=' -f2 | tr -d ' ') + +if [[ -z "$OWRT_VERSION" ]]; then + echo "Error: OWRT_VERSION not found or empty in build.conf.defaults" >&2 + exit 1 +fi + +# Fetch feeds.conf.default from OpenWrt at the specified tag +FEEDS_URL="https://raw.githubusercontent.com/openwrt/openwrt/$OWRT_VERSION/feeds.conf.default" + +FEEDS_CONTENT=$(curl -fsSL "$FEEDS_URL" 2>/dev/null || { + echo "Error: Failed to fetch $FEEDS_URL" >&2 + exit 1 +}) + +# Parse the packages line to extract the commit hash (after the ^) +# Expected format: src-git packages https://git.openwrt.org/feed/packages.git^ +PACKAGES_COMMIT=$(echo "$FEEDS_CONTENT" | grep '^src-git packages' | sed 's/.*\^//g' | head -1) + +if [[ -z "$PACKAGES_COMMIT" ]]; then + echo "Error: Could not parse packages commit hash from feeds.conf.default" >&2 + exit 1 +fi + +echo "$PACKAGES_COMMIT" diff --git a/.agents/skills/openwrt-package-update/scripts/setup-package.sh b/.agents/skills/openwrt-package-update/scripts/setup-package.sh new file mode 100755 index 000000000..0ce5bf135 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/setup-package.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# +# Setup old and new package snapshots for comparison +# +# Usage: setup-package.sh +# +# Outputs: +# - Creates assets/_old/ with the current pinned upstream version +# - Creates assets/_new/ with the target upstream version (from feeds.conf.default) +# - Prints to stdout: package name, CURRENT_COMMIT, TARGET_COMMIT, upstream path +# + +set -euo pipefail + +PACKAGE_NAME="${1:-}" + +if [[ -z "$PACKAGE_NAME" ]]; then + echo "Usage: setup-package.sh " >&2 + exit 1 +fi + +# Find workspace root (convert to absolute path) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE_ROOT=$(cd "$SCRIPT_DIR/../../../.." && git rev-parse --show-toplevel 2>/dev/null || pwd) +SKILL_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ASSETS_DIR="$SKILL_ROOT/assets" + +# Ensure assets dir exists +mkdir -p "$ASSETS_DIR" + +# Read current package version from local Makefile +LOCAL_MAKEFILE="$WORKSPACE_ROOT/packages/$PACKAGE_NAME/Makefile" +if [[ ! -f "$LOCAL_MAKEFILE" ]]; then + echo "Error: $LOCAL_MAKEFILE not found" >&2 + exit 1 +fi + +PKG_VERSION=$(grep '^PKG_VERSION:=' "$LOCAL_MAKEFILE" | cut -d'=' -f2 | tr -d ' ') +PKG_RELEASE=$(grep '^PKG_RELEASE:=' "$LOCAL_MAKEFILE" | cut -d'=' -f2 | tr -d ' ') + +if [[ -z "$PKG_VERSION" ]]; then + echo "Error: PKG_VERSION not found in $LOCAL_MAKEFILE" >&2 + exit 1 +fi + +echo "Package: $PACKAGE_NAME (version $PKG_VERSION-$PKG_RELEASE)" >&2 + +# Get target commit from feeds.conf.default +TARGET_COMMIT=$("$SKILL_ROOT/scripts/get-target-commit.sh") +echo "Target commit: $TARGET_COMMIT" >&2 + +# Clone/fetch openwrt/packages bare repo +BARE_REPO="$ASSETS_DIR/.openwrt-packages-repo" + +if [[ ! -d "$BARE_REPO" ]]; then + echo "Cloning openwrt/packages (bare repo)..." >&2 + git clone --bare --filter=blob:none https://github.com/openwrt/packages.git "$BARE_REPO" 2>/dev/null || { + echo "Error: Failed to clone openwrt/packages" >&2 + exit 1 + } +else + echo "Fetching openwrt/packages..." >&2 + cd "$BARE_REPO" + git fetch origin 2>/dev/null || { + echo "Error: Failed to fetch openwrt/packages" >&2 + exit 1 + } + cd - > /dev/null +fi + +# Auto-discover the upstream path by searching for /Makefile +# Look in the target commit first to find the exact path +echo "Discovering upstream package path..." >&2 +UPSTREAM_PATH=$(cd "$BARE_REPO" && git ls-tree -r --name-only "$TARGET_COMMIT" | grep -E "^[^/]+/$PACKAGE_NAME/Makefile$" | sed 's|/Makefile$||' | head -1) + +if [[ -z "$UPSTREAM_PATH" ]]; then + echo "Error: Could not find $PACKAGE_NAME in upstream openwrt/packages at $TARGET_COMMIT" >&2 + exit 1 +fi + +echo "Upstream path: $UPSTREAM_PATH" >&2 + +# Find the commit that introduced the current version +# Step 1: Find when PKG_VERSION was introduced using pickaxe +# Step 2: Find when PKG_RELEASE was set using pickaxe +echo "Finding commit that matches local version ($PKG_VERSION-$PKG_RELEASE)..." >&2 + +CURRENT_COMMIT="" + +# Step 1: Find commits that introduced PKG_VERSION +echo " Step 1: Finding commits with PKG_VERSION:=$PKG_VERSION..." >&2 +version_commits=$(cd "$BARE_REPO" && git log -S "PKG_VERSION:=$PKG_VERSION" --all --format="%H" -- "$UPSTREAM_PATH/Makefile" 2>/dev/null) + +if [[ -z "$version_commits" ]]; then + echo "Error: Could not find PKG_VERSION:=$PKG_VERSION in git history" >&2 + exit 1 +fi + +# Get the oldest commit where PKG_VERSION was introduced (last in the list when searching backwards) +version_commit=$(echo "$version_commits" | tail -1) +echo " PKG_VERSION introduced at: $version_commit" >&2 + +# Step 2: From that commit forward, find when PKG_RELEASE was set +# We search commits from version_commit onwards for PKG_RELEASE change +echo " Step 2: Finding commits with PKG_RELEASE:=$PKG_RELEASE after version introduction..." >&2 +release_commits=$(cd "$BARE_REPO" && git log -S "PKG_RELEASE:=$PKG_RELEASE" --all --format="%H" -- "$UPSTREAM_PATH/Makefile" 2>/dev/null) + +if [[ -n "$release_commits" ]]; then + # Find the first commit in release_commits that is an ancestor of or after version_commit + # Also verify both PKG_VERSION and PKG_RELEASE are present + while IFS= read -r commit; do + makefile_content=$(cd "$BARE_REPO" && git show "$commit:$UPSTREAM_PATH/Makefile" 2>/dev/null || echo "") + + if echo "$makefile_content" | grep -q "^PKG_VERSION:=$PKG_VERSION" && \ + echo "$makefile_content" | grep -q "^PKG_RELEASE:=$PKG_RELEASE"; then + CURRENT_COMMIT="$commit" + break + fi + done <<< "$release_commits" +fi + +if [[ -z "$CURRENT_COMMIT" ]]; then + echo "Error: Could not find a commit matching $PKG_VERSION-$PKG_RELEASE in upstream history" >&2 + echo "Available version range (last 20 commits):" >&2 + cd "$BARE_REPO" && git log --all --oneline -20 -- "$UPSTREAM_PATH/Makefile" >&2 + exit 1 +fi + +echo "Current commit: $CURRENT_COMMIT" >&2 + +# Extract the old version (current commit) +OLD_DIR="$ASSETS_DIR/${PACKAGE_NAME}_old" +rm -rf "$OLD_DIR" +mkdir -p "$OLD_DIR" + +echo "Extracting old version to $OLD_DIR..." >&2 +strip_level=$(echo "$UPSTREAM_PATH" | awk -F/ '{print NF}') +cd "$BARE_REPO" && git archive "$CURRENT_COMMIT" "$UPSTREAM_PATH/" | tar -x --strip-components=$strip_level -C "$OLD_DIR" 2>/dev/null || { + echo "Error: Failed to extract old version" >&2 + exit 1 +} + +# Extract the new version (target commit) +NEW_DIR="$ASSETS_DIR/${PACKAGE_NAME}_new" +rm -rf "$NEW_DIR" +mkdir -p "$NEW_DIR" + +echo "Extracting new version to $NEW_DIR..." >&2 +cd "$BARE_REPO" && git archive "$TARGET_COMMIT" "$UPSTREAM_PATH/" | tar -x --strip-components=$strip_level -C "$NEW_DIR" 2>/dev/null || { + echo "Error: Failed to extract new version" >&2 + exit 1 +} + +# Output summary to stdout +echo "" +echo "===== SETUP COMPLETE =====" +echo "Package: $PACKAGE_NAME" +echo "Current upstream commit: $CURRENT_COMMIT" +echo "Target upstream commit: $TARGET_COMMIT" +echo "Upstream path: $UPSTREAM_PATH" +echo "Old version snapshot: $OLD_DIR" +echo "New version snapshot: $NEW_DIR" diff --git a/AGENTS.md b/AGENTS.md index 6f3be5b70..2b4437365 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ This project has domain-specific skills available. You MUST activate the relevan - `ns-api` — **ACTIVATE** when writing, modifying, or reviewing Python RPCD API scripts. Triggers: creating or updating `ns.*` RPCD API endpoints, handling UCI configuration changes, managing pre/post-commit hooks, defining ACL permissions, documenting methods in OpenAPI 3.1.0, or when user mentions ns-api, API endpoints, hooks, or references `/usr/libexec/rpcd/ns.` files. Covers stdin/stdout JSON protocol, error handling, naming conventions, code style, and spec file updates. - `openwrt-package` — **ACTIVATE** when creating or modifying OpenWrt `ns-*` packages. Triggers: building new packages for NethSecurity, managing package dependencies, patching upstream feeds, modifying Makefiles, or when user mentions Makefile, package structure, config fragments, or upstream patches. Covers naming conventions, required Makefile fields, architecture selection, external version management, and patch workflows. -- `openwrt-package-update` — **ACTIVATE** when updating forked OpenWrt upstream packages to newer versions. Triggers: updating packages like adblock, mwan3, or other non-ns- prefixed upstream packages, comparing local forks against the canonical OpenWrt packages feed, or when user mentions upstream package updates and merging improvements. Covers version updates, preserving local customizations, and identifying upstream improvements. +- `openwrt-package-update` — **ACTIVATE** when updating forked OpenWrt packages from the upstream feed (adblock, mwan3, banip, etc.). Triggers: updating non-ns- packages, comparing local forks against openwrt/packages, merging upstream improvements, or when user mentions upstream package updates. Auto-discovers packages; extracts old/new snapshots for side-by-side comparison; guides cross-package impact detection. - `python-nethsecurity` — **ACTIVATE** when writing or modifying Python scripts for NethSecurity packages. Triggers: creating new Python scripts, configuring package build systems, writing utilities, or when user mentions Python code in packages/ns-* or references Python scripts in the NethSecurity package tree. Covers shebang, license headers, extension handling, ruff compliance, available modules, and UCI commit conventions. --- From 1352b74f8a07a7b1a74a86b4bf8dc1e79da2caff Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Mon, 11 May 2026 15:40:20 +0200 Subject: [PATCH 44/48] chore: updated adblock to 4.5.5-3 Sync the local adblock fork to upstream 4.5.5-3 while keeping the NethSecurity-specific ts-dns hooks, bypass migration, and nft bypass rules intact. Assisted-by: Copilot:gpt-5.4 --- packages/adblock/Makefile | 32 +- .../adblock/files/95-adblock-housekeeping | 114 + packages/adblock/files/README.md | 519 ++- .../{adblock.blacklist => adblock.allowlist} | 0 .../{adblock.whitelist => adblock.blocklist} | 0 packages/adblock/files/adblock.categories | 114 +- packages/adblock/files/adblock.conf | 12 +- packages/adblock/files/adblock.custom.feeds | 0 packages/adblock/files/adblock.feeds | 194 + packages/adblock/files/adblock.init | 266 +- packages/adblock/files/adblock.mail | 84 +- packages/adblock/files/adblock.sh | 3140 +++++++++++------ packages/adblock/files/adblock.sources | 352 -- packages/ns-api/Makefile | 2 + packages/ns-api/files/ns.threatshield | 86 +- packages/ns-threat_shield/Makefile | 3 +- packages/ns-threat_shield/README.md | 9 +- packages/ns-threat_shield/files/ts-dns | 40 +- 18 files changed, 2993 insertions(+), 1974 deletions(-) create mode 100755 packages/adblock/files/95-adblock-housekeeping rename packages/adblock/files/{adblock.blacklist => adblock.allowlist} (100%) rename packages/adblock/files/{adblock.whitelist => adblock.blocklist} (100%) create mode 100644 packages/adblock/files/adblock.custom.feeds create mode 100644 packages/adblock/files/adblock.feeds delete mode 100644 packages/adblock/files/adblock.sources diff --git a/packages/adblock/Makefile b/packages/adblock/Makefile index 9cfeef42c..e7afb7727 100644 --- a/packages/adblock/Makefile +++ b/packages/adblock/Makefile @@ -1,13 +1,13 @@ -# -# Copyright (c) 2015-2023 Dirk Brenken (dev@brenken.org) +# dns based ad/abuse domain blocking +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=4.1.5 -PKG_RELEASE:=9 +PKG_VERSION:=4.5.5 +PKG_RELEASE:=3 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken @@ -16,22 +16,23 @@ include $(INCLUDE_DIR)/package.mk define Package/adblock SECTION:=net CATEGORY:=Network - TITLE:=Powerful adblock script to block ad/abuse domains by using DNS - DEPENDS:=+jshn +jsonfilter +coreutils +coreutils-sort +ca-bundle +opkg + TITLE:=adblock blocks ad/abuse domains by using DNS + DEPENDS:=+jshn +jsonfilter +firewall4 +coreutils +coreutils-sort +gawk +ca-bundle +rpcd +rpcd-mod-rpcsys PKGARCH:=all endef define Package/adblock/description -Powerful adblock solution to block ad/abuse domains via dnsmasq, unbound, named or kresd. -The script supports many domain blacklist sites plus manual black- and whitelist overrides. +adblock blocks ad/abuse domains via dnsmasq, unbound, named, smartdns or kresd. +adblock consumes a minimum of memory, is very fast and supports many domain blocklist sites plus local block- and allowlist overrides. Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information. endef define Package/adblock/conffiles /etc/config/adblock -/etc/adblock/adblock.whitelist -/etc/adblock/adblock.blacklist +/etc/adblock/adblock.allowlist +/etc/adblock/adblock.blocklist +/etc/adblock/adblock.custom.feeds endef define Build/Prepare @@ -55,11 +56,14 @@ define Package/adblock/install $(INSTALL_DIR) $(1)/etc/adblock $(INSTALL_BIN) ./files/adblock.mail $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.allowlist $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.blocklist $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.categories $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.sources $(1)/etc/adblock - gzip -9n $(1)/etc/adblock/adblock.sources + $(INSTALL_CONF) ./files/adblock.feeds $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.custom.feeds $(1)/etc/adblock + + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/95-adblock-housekeeping $(1)/etc/uci-defaults endef $(eval $(call BuildPackage,adblock)) diff --git a/packages/adblock/files/95-adblock-housekeeping b/packages/adblock/files/95-adblock-housekeeping new file mode 100755 index 000000000..3cef57d79 --- /dev/null +++ b/packages/adblock/files/95-adblock-housekeeping @@ -0,0 +1,114 @@ +#!/bin/sh +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) +# This is free software, licensed under the GNU General Public License v3. + +# (s)hellcheck exceptions +# shellcheck disable=all + +export LC_ALL=C +export PATH="/usr/sbin:/usr/bin:/sbin:/bin" + +config="adblock" +old_options="adb_sources adb_forcedns adb_fetchutil adb_hag_sources adb_hst_sources adb_stb_sources adb_utc_sources \ + adb_maxqueue adb_backup adb_dnsfilereset adb_tmpbase adb_mailcnt adb_safesearchmod adb_srcfile adb_srcarc adb_nice \ + adb_hag_feed adb_jaildir adb_dnsdenyip adb_dnsallowip adb_zonelist adb_portlist adb_dnsforce adb_replisten" + +for option in ${old_options}; do + inplace="0" + if uci -q get ${config}.global.${option} >/dev/null 2>&1; then + old_values="$(uci -q get ${config}.global.${option})" + for value in ${old_values}; do + case "${option}" in + "adb_sources") + if ! uci -q get ${config}.global.adb_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_feed="${value}" + fi + ;; + "adb_hag_sources") + if ! uci -q get ${config}.global.adb_hag_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_hag_feed="${value}" + fi + ;; + "adb_hst_sources") + if ! uci -q get ${config}.global.adb_hst_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_hst_feed="${value}" + fi + ;; + "adb_stb_sources") + if ! uci -q get ${config}.global.adb_stb_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_stb_feed="${value}" + fi + ;; + "adb_utc_sources") + if ! uci -q get ${config}.global.adb_utc_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_utc_feed="${value}" + fi + ;; + "adb_fetchutil") + uci -q set ${config}.global.adb_fetchcmd="${value}" + ;; + "adb_tmpbase") + uci -q set ${config}.global.adb_basedir="${value}" + ;; + "adb_nice") + uci -q set ${config}.global.adb_nicelimit="${value}" + ;; + "adb_hag_feed") + inplace="1" + if ! printf '%s' "${value}" | grep -qE "^(wildcard/|domains/)"; then + uci -q del_list ${config}.global.adb_hag_feed="${value}" + uci -q add_list ${config}.global.adb_hag_feed="wildcard/${value}" + fi + ;; + "adb_forcedns" | "adb_dnsforce") + uci -q set ${config}.global.adb_nftforce="${value}" + ;; + "adb_zonelist") + if ! uci -q get ${config}.global.adb_nftdevforce | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_nftdevforce="${value}" + fi + ;; + "adb_portlist") + if ! uci -q get ${config}.global.adb_nftportforce | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_nftportforce="${value}" + fi + ;; + "adb_replisten") + if ! uci -q get ${config}.global.adb_repport | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_repport="${value}" + fi + ;; + esac + done + [ "${inplace}" = "0" ] && uci -q delete ${config}.global.${option} + fi +done + +# NethSecurity: migrate adb_bypass to ns_tsdns_bypass +# +if uci -q get ${config}.global.adb_bypass >/dev/null 2>&1; then + old_bypass="$(uci -q get ${config}.global.adb_bypass)" + for value in ${old_bypass}; do + if ! uci -q get ${config}.global.ns_tsdns_bypass | grep -q "${value}"; then + uci -q add_list ${config}.global.ns_tsdns_bypass="${value}" + fi + done + uci -q delete ${config}.global.adb_bypass +fi + +[ -n "$(uci -q changes ${config})" ] && uci -q commit ${config} + +# remove former adblock-related firewall sections (adblock_* redirects and tsdns_bypass ipset) +# +fwcfg="$(uci -qNX show "firewall" | awk 'BEGIN{FS="[.=]"};/adblock_/{if(zone==$2){next}else{ORS=" ";zone=$2;print zone}}')" +for section in ${fwcfg}; do + uci -q delete firewall."${section}" +done +if uci -q get firewall.tsdns_bypass >/dev/null 2>&1; then + uci -q delete firewall.tsdns_bypass +fi +if [ -n "$(uci -q changes firewall)" ]; then + uci -q commit firewall + /etc/init.d/firewall reload +fi +exit 0 diff --git a/packages/adblock/files/README.md b/packages/adblock/files/README.md index 688bdb7cc..725aade52 100644 --- a/packages/adblock/files/README.md +++ b/packages/adblock/files/README.md @@ -2,54 +2,41 @@ # DNS based ad/abuse domain blocking + ## Description -A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other (wlan) gadget!? Getting rid of annoying ads, trackers and other abuse sites (like facebook) is simple: block them with your router. When the DNS server on your router receives DNS requests, you will sort out queries that ask for the resource records of ad servers and return a simple 'NXDOMAIN'. This is nothing but **N**on-e**X**istent Internet or Intranet domain name, if domain name is unable to resolved using the DNS server, a condition called the 'NXDOMAIN' occurred. +A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other (wlan) gadget!? Getting rid of annoying ads, trackers and other abuse sites (like facebook) is simple: block them with your router. +When the DNS server on your router receives DNS requests, you will sort out queries that ask for the resource records of ad servers and return a simple 'NXDOMAIN'. This is nothing but **N**on-e**X**istent Internet or Intranet domain name, if a domain name cannot be resolved using the DNS server, a condition called the 'NXDOMAIN' occurred. + + ## Main Features -* Support of the following fully pre-configured domain blocklist sources (free for private usage, for commercial use please check their individual licenses) +* Support of the following fully pre-configured domain blocklist feeds (free for private usage, for commercial use please check their individual licenses) -| Source | Enabled | Size | Focus | Information | +| Feed | Enabled | Size | Focus | Information | | :------------------ | :-----: | :--- | :--------------- | :-------------------------------------------------------------------------------- | -| adaway | x | S | mobile | [Link](https://github.com/AdAway/adaway.github.io) | -| adguard | x | L | general | [Link](https://adguard.com) | -| adguard_tracking | | S | tracking | [Link](https://github.com/AdguardTeam/cname-trackers) | +| 1Hosts | | VAR | compilation | [Link](https://github.com/badmojr/1Hosts) | +| adguard | x | L | general | [Link](https://adguard.com) | +| adguard_tracking | x | L | tracking | [Link](https://github.com/AdguardTeam/cname-trackers) | | android_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | andryou | | L | compilation | [Link](https://gitlab.com/andryou/block/-/blob/master/readme.md) | | anti_ad | | L | compilation | [Link](https://github.com/privacy-protection-tools/anti-AD/blob/master/README.md) | -| antipopads | | L | compilation | [Link](https://github.com/AdroitAdorKhan/antipopads-re) | | anudeep | | M | compilation | [Link](https://github.com/anudeepND/blacklist) | | bitcoin | | S | mining | [Link](https://github.com/hoshsadiq/adblock-nocoin-list) | +| certpl | x | L | phishing | [Link](https://cert.pl/en/warning-list/) | | cpbl | | XL | compilation | [Link](https://github.com/bongochong/CombinedPrivacyBlockLists) | -| disconnect | x | S | general | [Link](https://disconnect.me) | +| disconnect | | S | general | [Link](https://disconnect.me) | +| divested | | XXL | compilation | [Link](https://divested.dev/pages/dnsbl) | | doh_blocklist | | S | doh_server | [Link](https://github.com/dibdot/DoH-IP-blocklists) | -| easylist | | M | compilation | [Link](https://easylist.to) | -| easyprivacy | | M | tracking | [Link](https://easylist.to) | | firetv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | games_tracking | | S | tracking | [Link](https://www.gameindustry.eu) | +| hagezi | | VAR | compilation | [Link](https://github.com/hagezi/dns-blocklists) | | hblock | | XL | compilation | [Link](https://hblock.molinero.dev) | -| lightswitch05 | | XL | compilation | [Link](https://github.com/lightswitch05/hosts) | -| notracking | | XL | tracking | [Link](https://github.com/notracking/hosts-blocklists) | +| ipfire_dbl | | VAR | compilation | [Link](https://www.ipfire.org/dbl) | | oisd_big | | XXL | general | [Link](https://oisd.nl) | | oisd_nsfw | | XXL | porn | [Link](https://oisd.nl) | +| oisd_nsfw_small | | M | porn | [Link](https://oisd.nl) | | oisd_small | | L | general | [Link](https://oisd.nl) | -| openphish | | S | phishing | [Link](https://openphish.com) | | phishing_army | | S | phishing | [Link](https://phishing.army) | -| reg_cn | | S | reg_china | [Link](https://easylist.to) | -| reg_cz | | S | reg_czech+slovak | [Link](https://easylist.to) | -| reg_de | | S | reg_germany | [Link](https://easylist.to) | -| reg_es | | S | reg_espania | [Link](https://easylist.to) | -| reg_fi | | S | reg_finland | [Link](https://github.com/finnish-easylist-addition) | -| reg_fr | | M | reg_france | [Link](https://forums.lanik.us/viewforum.php?f=91) | -| reg_id | | S | reg_indonesia | [Link](https://easylist.to) | -| reg_it | | S | reg_italy | [Link](https://easylist.to) | -| reg_jp | | S | reg_japan | [Link](https://github.com/k2jp/abp-japanese-filters) | -| reg_kr | | S | reg_korea | [Link](https://github.com/List-KR/List-KR) | -| reg_nl | | S | reg_netherlands | [Link](https://easylist.to) | -| reg_pl | | M | reg_poland | [Link](https://kadantiscam.netlify.com) | -| reg_ro | | S | reg_romania | [Link](https://easylist.to) | -| reg_ru | | S | reg_russia | [Link](https://easylist.to) | -| reg_se | | S | reg_sweden | [Link](https://github.com/lassekongo83/Frellwits-filter-lists) | -| reg_vn | | S | reg_vietnam | [Link](https://bigdargon.github.io/hostsVN) | | smarttv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | spam404 | | S | general | [Link](https://github.com/Dawsey21) | | stevenblack | | VAR | compilation | [Link](https://github.com/StevenBlack/hosts) | @@ -57,68 +44,76 @@ A lot of people already use adblocker plugins within their desktop browsers, but | utcapitole | | VAR | general | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | | wally3k | | S | compilation | [Link](https://firebog.net/about) | | whocares | | M | general | [Link](https://someonewhocares.org) | -| winhelp | | S | general | [Link](https://winhelp2002.mvps.org) | | winspy | | S | win_telemetry | [Link](https://github.com/crazy-max/WindowsSpyBlocker) | -| yoyo | x | S | general | [Link](https://pgl.yoyo.org/adservers) | - -* List of supported and fully pre-configured adblock sources, already active sources are pre-selected. - To avoid OOM errors, please do not select too many lists! - List size information with the respective domain ranges as follows: - • S (-10k), M (10k-30k) and L (30k-80k) should work for 128 MByte devices, - • XL (80k-200k) should work for 256-512 MByte devices, - • XXL (200k-) needs more RAM and Multicore support, e.g. x86 or raspberry devices. - • VAR (50k-500k) variable size depending on the selection. +| yoyo | | S | general | [Link](https://pgl.yoyo.org/adservers) | + +* List of supported and fully pre-configured adblock sources, already active sources are pre-selected. + To avoid OOM errors, please do not select too many lists! + List size information with the respective domain ranges as follows: + • S (-10k), M (10k-30k) and L (30k-80k) should work for 128 MByte devices + • XL (80k-200k) should work for 256-512 MByte devices + • XXL (200k-) needs more RAM and Multicore support, e.g. x86 or raspberry devices + • VAR (50k-900k) variable size depending on the selection * Zero-conf like automatic installation & setup, usually no manual changes needed * Simple but yet powerful adblock engine: adblock does not use error prone external iptables rulesets, http pixel server instances and things like that -* Supports five different DNS backend formats: dnsmasq, unbound, named (bind), kresd or raw (e.g. used by dnscrypt-proxy) -* Supports four different SSL-enabled download utilities: uclient-fetch, wget, curl or aria2c -* Supports SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay -* Supports RPZ-trigger 'RPZ-CLIENT-IP' to always allow/deny certain DNS clients based on their IP address (currently only supported by bind dns backend) +* Supports six different DNS backend formats: dnsmasq, unbound, named (bind), kresd, smartdns or raw (e.g. used by dnscrypt-proxy) +* Supports three different SSL-enabled download utilities: uclient-fetch, full wget or curl +* Supports SafeSearch for google, bing, brave, duckduckgo, yandex, youtube and pixabay * Fast downloads & list processing as they are handled in parallel running background jobs with multicore support +* The download engine supports ETAG headers to download only updated feeds * Supports a wide range of router modes, even AP modes are supported * Full IPv4 and IPv6 support * Provides top level domain compression ('tld compression'), this feature removes thousands of needless host entries from the blocklist and lowers the memory footprint for the DNS backend -* Provides a 'DNS File Reset', where the generated DNS blocklist file will be purged after DNS backend loading to save storage space -* Source parsing by fast & flexible regex rulesets, all rules and source information are placed in an external/compredd JSON file ('/etc/adblock/adblock.sources.gz') +* Provides a 'DNS Blocklist Shift', where the generated final DNS blocklist is moved to the backup directory and only a soft link to this file is set in memory. As long as your backup directory is located on an external drive, you should activate this option to save valuable RAM. +* Feed parsing by a very fast & secure domain validator, all domain rules and feed information are placed in an external JSON file ('/etc/adblock/adblock.feeds') * Overall duplicate removal in generated blocklist file 'adb_list.overall' -* Additional local blacklist for manual overrides, located in '/etc/adblock/adblock.blacklist' -* Additional local whitelist for manual overrides, located in '/etc/adblock/adblock.whitelist' -* Quality checks during blocklist update to ensure a reliable DNS backend service +* Additional local allowlist for manual overrides, located in '/etc/adblock/adblock.allowlist' (only exact matches). +* Additional local blocklist for manual overrides, located in '/etc/adblock/adblock.blocklist' +* Implements firewall‑based DNS Control to force DNS interfaces/ports and to redirect to external unfiltered/filtered DNS server +* Includes firewall‑based Remote DNS Allow, a CGI-Interface to allow certain MACs temporary bypass the local adblock DNS +* Supports firewall‑based temporary DNS Bridging, to ensure a Zero‑Downtime during adblock-related DNS Restarts +* Connection checks during blocklist update to ensure a reliable DNS backend service * Minimal status & error logging to syslog, enable debug logging to receive more output -* Procd based init system support ('start', 'stop', 'restart', 'reload', 'enable', 'disable', 'running', 'status', 'suspend', 'resume', 'query', 'report', 'list', 'timer') +* Procd based init system support ('start', 'stop', 'restart', 'reload', 'enable', 'disable', 'running', 'status', 'suspend', 'resume', 'search', 'report') * Auto-Startup via procd network interface trigger or via classic time based startup -* Suspend & Resume adblock temporarily without blocklist reloading +* Suspend & Resume adblock temporarily without blocklist re-processing * Provides comprehensive runtime information -* Provides a detailed DNS Query Report with DNS related information about client requests, top (blocked) domains and more -* Provides a powerful query function to quickly find blocked (sub-)domains, e.g. for whitelisting -* Provides an easily configurable blocklist update scheduler called 'Refresh Timer' -* Includes an option to generate an additional, restrictive 'adb_list.jail' to block access to all domains except those listed in the whitelist file. You can use this restrictive blocklist manually e.g. for guest wifi or kidsafe configurations -* Includes an option to force DNS requests to the local resolver +* Provides a detailed DNS Report with DNS related information about client requests, top (blocked) domains and more +* Provides a powerful search function to quickly find blocked (sub-)domains, e.g. to allow certain domains +* Implements a jail mode - only domains on the allowlist are permitted, all other DNS requests are rejected * Automatic blocklist backup & restore, these backups will be used in case of download errors and during startup -* Send notification E-Mails in case of a processing error or if the overall domain count is ≤ 0 -* Add new adblock sources on your own, see example below +* Send notification E-Mails, see example configuration below +* Add new adblock feeds on your own with the 'Custom Feed Editor' in LuCI or via CLI, see example below * Strong LuCI support, all relevant options are exposed to the web frontend + ## Prerequisites -* [OpenWrt](https://openwrt.org), tested with the stable release series and with the latest rolling snapshot releases. - Please note: Devices with less than 128 MByte RAM are _not_ supported! -* A usual setup with an enabled DNS backend at minimum - dumb AP modes without a working DNS backend are _not_ supported -* A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required +* **[OpenWrt](https://openwrt.org)**, latest stable release or a development snapshot +* A usual setup with a working DNS backend +* A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries or 'curl' is required * A certificate store such as 'ca-bundle' or 'ca-certificates', as adblock checks the validity of the SSL certificates of all download sites by default -* Optional E-Mail notification support: for E-Mail notifications you need to install the additional 'msmtp' package -* Optional DNS Query Report support: for DNS reporting you need to install the additional package 'tcpdump-mini' or 'tcpdump' -* Optional support for gnu awk as alternative to the busybox default, install the additional package 'gawk' +* For E-Mail notifications you need to install and setup the additional 'msmtp' package +* For DNS reporting you need to install the additional package 'tcpdump-mini' or 'tcpdump' +**Please note:** +* Devices with less than 128MB of RAM are **_not_** supported +* For performance reasons, adblock depends on gnu sort and gawk +* Before update from former adblock releases please make a backup of your local allow- and blocklists. In the latest adblock these lists have been renamed to '/etc/adblock/adblock.allowlist' and '/etc/adblock/adblock.blocklist'. There is no automatic content transition to the new files. +* The uci configuration of adblock is automatically migrated during package installation via the uci-defaults mechanism using a housekeeping script + + ## Installation & Usage -* Update your local opkg repository (_opkg update_) -* Install 'adblock' (_opkg install adblock_). The adblock service is enabled by default -* Install the LuCI companion package 'luci-app-adblock' (_opkg install luci-app-adblock_) +* Make a backup and update your local opkg/apk repository +* Install the LuCI companion package 'luci-app-adblock' which also installs the main 'adblock' package as a dependency +* Enable the adblock system service (System -> Startup) and enable adblock itself (adblock -> General Settings) * It's strongly recommended to use the LuCI frontend to easily configure all aspects of adblock, the application is located in LuCI under the 'Services' menu -* Update from a former adblock version is easy. During the update a backup is made of the old configuration '/etc/config/adblock-backup' and replaced by the new config - that's all +* It's also strongly recommended to configure a ‘Startup Trigger Interface’ to ensure automatic adblock startup on WAN-ifup events during boot or reboot of your router -## Adblock CLI Options -* All important adblock functions are accessible via CLI as well. -

+
+## Adblock CLI interface
+* The most important adblock functions are accessible via CLI as well.
+
+```
 ~# /etc/init.d/adblock
 Syntax: /etc/init.d/adblock [command]
 
@@ -132,73 +127,91 @@ Available commands:
 	enabled         Check if service is started on boot
 	suspend         Suspend adblock processing
 	resume          Resume adblock processing
-	query           <domain> Query active blocklists and backups for a specific domain
-	report          [<search>] Print DNS statistics with an optional search parameter
-	list            [<add>|<add_utc>|<add_eng>|<add_stb>|<remove>|<remove_utc>|<remove_eng>|<remove_stb>] <source(s)> List/Edit available sources
-	timer           [<add> <tasks> <hour> [<minute>] [<weekday>]]|[<remove> <line no.>] List/Edit cron update intervals
-	version         Print version information
+	search           Search active blocklists and backups for a specific domain
+	report          [|||] Print DNS statistics
 	running         Check if service is running
 	status          Service status
 	trace           Start with syscall trace
-
+ info Dump procd service info +``` + ## Adblock Config Options * Usually the auto pre-configured adblock setup works quite well and no manual overrides are needed -| Option | Default | Description/Valid Values | -| :----------------- | :--------------------------------- | :--------------------------------------------------------------------------------------------- | -| adb_enabled | 1, enabled | set to 0 to disable the adblock service | -| adb_srcarc | -, /etc/adblock/adblock.sources.gz | full path to the used adblock source archive | -| adb_srcfile | -, /tmp/adb_sources.json | full path to the used adblock source file, which has a higher precedence than the archive file | -| adb_dns | -, auto-detected | 'dnsmasq', 'unbound', 'named', 'kresd' or 'raw' | -| adb_fetchutil | -, auto-detected | 'uclient-fetch', 'wget', 'curl' or 'aria2c' | -| adb_fetchparm | -, auto-detected | manually override the config options for the selected download utility | -| adb_fetchinsecure | 0, disabled | don't check SSL server certificates during download | -| adb_trigger | -, not set | trigger network interface or 'not set' to use a time-based startup | -| adb_triggerdelay | 2 | additional trigger delay in seconds before adblock processing begins | -| adb_debug | 0, disabled | set to 1 to enable the debug output | -| adb_nice | 0, standard prio. | valid nice level range 0-19 of the adblock processes | -| adb_forcedns | 0, disabled | set to 1 to force DNS requests to the local resolver | -| adb_bypass | -, not set | list of IP addresses excluded from `adb_forcedns` option | -| adb_dnsdir | -, auto-detected | path for the generated blocklist file 'adb_list.overall' | -| adb_dnstimeout | 10 | timeout in seconds to wait for a successful DNS backend restart | -| adb_dnsinstance | 0, first instance | set to the relevant dns backend instance used by adblock (dnsmasq only) | -| adb_dnsflush | 0, disabled | set to 1 to flush the DNS Cache before & after adblock processing | -| adb_dnsallow | -, not set | set to 1 to disable selective DNS whitelisting (RPZ-PASSTHRU) | -| adb_lookupdomain | example.com | external domain to check for a successful DNS backend restart or 'false' to disable this check | -| adb_portlist | 53 853 5353 | space separated list of firewall ports which should be redirected locally | -| adb_report | 0, disabled | set to 1 to enable the background tcpdump gathering process for reporting | -| adb_reportdir | /tmp | path for DNS related report files | -| adb_repiface | -, auto-detected | name of the reporting interface or 'any' used by tcpdump | -| adb_replisten | 53 | space separated list of reporting port(s) used by tcpdump | -| adb_repchunkcnt | 5 | report chunk count used by tcpdump | -| adb_repchunksize | 1 | report chunk size used by tcpdump in MB | -| adb_represolve | 0, disabled | resolve reporting IP addresses using reverse DNS (PTR) lookups | -| adb_backup | 1, enabled | set to 0 to disable the backup function | -| adb_backupdir | /tmp | path for adblock backups | -| adb_tmpbase | /tmp | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc. | -| adb_safesearch | 0, disabled | set to 1 to enforce SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay | -| adb_safesearchlist | -, not set | Limit SafeSearch to certain provider (see above) | -| adb_safesearchmod | 0, disabled | set to 1 to enable moderate SafeSearch filters for youtube | -| adb_mail | 0, disabled | set to 1 to enable notification E-Mails in case of a processing errors | -| adb_mailreceiver | -, not set | receiver address for adblock notification E-Mails | -| adb_mailsender | no-reply@adblock | sender address for adblock notification E-Mails | -| adb_mailtopic | adblock notification | topic for adblock notification E-Mails | -| adb_mailprofile | adb_notify | mail profile used in 'msmtp' for adblock notification E-Mails | -| adb_mailcnt | 0 | minimum domain count to trigger E-Mail notifications | -| adb_jail | 0 | set to 1 to enable the additional, restrictive 'adb_list.jail' creation | -| adb_jaildir | /tmp | path for the generated jail list | +| Option | Default | Description/Valid Values | +| :------------------- | :--------------------------------- | :------------------------------------------------------------------------------------------------- | +| adb_enabled | 1, enabled | set to 0 to disable the adblock service | +| adb_feedfile | /etc/adblock/adblock.feeds | full path to the used adblock feed file | +| adb_dns | -, auto-detected | 'dnsmasq', 'unbound', 'named', 'kresd', 'smartdns' or 'raw' | +| adb_cores | -, auto-detected | limit the cpu cores used by adblock to save RAM | +| adb_fetchcmd | -, auto-detected | 'uclient-fetch', 'wget' or 'curl' | +| adb_fetchparm | -, auto-detected | manually override the config options for the selected download utility | +| adb_fetchinsecure | 0, disabled | don't check SSL server certificates during download | +| adb_trigger | -, not set | trigger network interface or 'not set' to use a time-based startup | +| adb_triggerdelay | 5 | additional trigger delay in seconds before adblock processing begins | +| adb_debug | 0, disabled | set to 1 to enable the debug output | +| adb_nicelimit | 0, standard prio. | valid nice level range 0-19 of the adblock processes | +| adb_dnsshift | 0, disabled | shift the blocklist to the backup directory and only set a soft link to this file in memory | +| adb_dnsdir | -, auto-detected | path for the generated blocklist file 'adb_list.overall' | +| adb_dnstimeout | 20 | timeout in seconds to wait for a successful DNS backend restart | +| adb_dnsinstance | 0, first instance | set the relevant dnsmasq backend instance used by adblock | +| adb_dnsflush | 0, disabled | set to 1 to flush the DNS Cache before & after adblock processing | +| adb_lookupdomain | localhost | domain to check for a successful DNS backend restart | +| adb_report | 0, disabled | set to 1 to enable the background tcpdump gathering process for reporting | +| adb_map | 0, disabled | enable a GeoIP Map with blocked domains | +| adb_reportdir | /tmp/adblock-report | path for DNS related report files | +| adb_repiface | -, auto-detected | name of the reporting interface or 'any' used by tcpdump | +| adb_repport | 53 | list of reporting port(s) used by tcpdump | +| adb_repchunkcnt | 5 | report chunk count used by tcpdump | +| adb_repchunksize | 1 | report chunk size used by tcpdump in MB | +| adb_represolve | 0, disabled | resolve reporting IP addresses using reverse DNS (PTR) lookups | +| adb_tld | 1, enabled | set to 0 to disable the top level domain compression (tld) function | +| adb_basedir | /tmp | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc. | +| adb_backupdir | /tmp/adblock-backup | path for adblock backups | +| adb_safesearch | 0, disabled | enforce SafeSearch for google, bing, brave, duckduckgo, yandex, youtube and pixabay | +| adb_safesearchlist | -, not set | limit SafeSearch to certain provider (see above) | +| adb_mail | 0, disabled | set to 1 to enable notification E-Mails in case of a processing errors | +| adb_mailreceiver | -, not set | receiver address for adblock notification E-Mails | +| adb_mailsender | no-reply@adblock | sender address for adblock notification E-Mails | +| adb_mailtopic | adblock notification | topic for adblock notification E-Mails | +| adb_mailprofile | adb_notify | mail profile used in 'msmtp' for adblock notification E-Mails | +| adb_jail | 0 | jail mode - only domains on the allowlist are permitted, all other DNS requests are rejected | +| adb_nftforce | 0, disabled | redirect all local DNS queries from specified LAN zones to the local DNS resolver | +| adb_nftdevforce | -, not set | firewall LAN Devices/VLANs that should be forced locally | +| adb_nftportforce | -, not set | firewall ports that should be forced locally | +| adb_nftallow | 0, disabled | routes MACs or interfaces to an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftmacallow | -, not set | listed MAC addresses will always use the configured unfiltered DNS server | +| adb_nftdevallow | -, not set | entire interfaces or VLANs will be routed to the unfiltered DNS server | +| adb_allowdnsv4 | -, not set | external IPv4 DNS resolver applied to MACs and interfaces using the unfiltered DNS policy | +| adb_allowdnsv6 | -, not set | external IPv6 DNS resolver applied to MACs and interfaces using the unfiltered DNS policy | +| adb_nftremote | 0, disabled | routes MACs to an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftmacremote | -, not set | Allows listed MACs to remotely access an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftremotetimeout | 15 | Time limit in minutes for remote DNS access of the listed MAC addresses | +| adb_remotednsv4 | -, not set | external IPv4 DNS resolver applied to MACs using the unfiltered remote DNS policy | +| adb_remotednsv6 | -, not set | external IPv6 DNS resolver applied to MACs using the unfiltered remote DNS policy | +| adb_nftblock | 0, disabled | routes MACs or interfaces to a filtered external DNS resolver, bypassing local adblock | +| adb_nftmacblock | -, not set | listed MAC addresses will always use the configured filtered DNS server | +| adb_nftdevblock | -, not set | entire interfaces or VLANs will be routed to the filtered DNS server | +| adb_blockdnsv4 | -, not set | external IPv4 DNS resolver applied to MACs and interfaces using the filtered DNS policy | +| adb_blockdnsv6 | -, not set | external IPv6 DNS resolver applied to MACs and interfaces using the filtered DNS policy | +| adb_nftbridge | -, not set | enables a temporary DNS bridge to an external DNS resolver during local DNS restarts | +| adb_bridgednsv4 | -, not set | external IPv4 DNS resolver used during bridging | +| adb_bridgednsv6 | -, not set | external IPv6 DNS resolver used during bridging | + ## Examples -**Change the DNS backend to 'unbound':** -No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' by default. + +**Change the DNS backend to 'unbound':** +No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' by default. To preserve the DNS cache after adblock processing please install the additional package 'unbound-control'. -**Change the DNS backend to 'bind':** -Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind' by default. -To preserve the DNS cache after adblock processing please install the additional package 'bind-rdnc'. +**Change the DNS backend to 'bind':** +Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind' by default. +To preserve the DNS cache after adblock processing please install the additional package 'bind-rndc'. To use the blocklist please modify '/etc/bind/named.conf': -

+
+```
 in the 'options' namespace add:
   response-policy { zone "rpz"; };
 
@@ -209,26 +222,108 @@ and at the end of the file add:
     allow-query { none; };
     allow-transfer { none; };
   };
-
+``` + +**Change the DNS backend to 'kresd':** +Adblock deposits the final blocklist 'adb_list.overall' in '/tmp/kresd', no further configuration needed. + +**Change the DNS backend to 'smartdns':** +No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/tmp/smartdns' by default. + +**Service status output:** +In LuCI you'll see the realtime status in the 'Runtime' section on the overview page. +To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/init.d/adblock status\_service_: + +```sh +root@blackhole:~# /etc/init.d/adblock status +::: adblock runtime information + + adblock_status : enabled + + frontend_ver : 4.5.2-r4 + + backend_ver : 4.5.2-r4 + + blocked_domains : 888 135 + + active_feeds : 1hosts, adguard, adguard_tracking, bitcoin, certpl, doh_blocklist, hagezi, ipfire_dbl, phishing_army, smarttv_tracking, stevenblack, winspy + + dns_backend : unbound (1.24.2-r1), /mnt/data/adblock/backup, 346.57 MB + + run_ifaces : trigger: wan, report: br-lan + + run_information : base: /mnt/data/adblock, dns: /var/lib/unbound, backup: /mnt/data/adblock/backup, report: /mnt/data/adblock/report, error: /dev/null + + run_flags : shift: ✔, custom feed: ✘, ext. DNS (std/prot/remote/bridge): ✘/✔/✔/✔, force: ✔, flush: ✘, tld: ✔, search: ✘, report: ✔, mail: ✔, jail: ✘, debug: ✘ + + last_run : mode: reload, 2026-03-12T19:08:41+01:00, duration: 0m 57s, 1337.01 MB available + + system_info : cores: 4, fetch: curl, Bananapi BPI-R3, mediatek/filogic, OpenWrt SNAPSHOT (r33360-ab0872a734) +``` + + +## Best practice and tweaks + +**Recommendation for low memory systems** +adblock keeps all working data in RAM to avoid unnecessary flash wear. On devices with only 128–256 MB RAM, you can reduce memory pressure with the following optimizations: +* Use external storage: Set adb_basedir, adb_backupdir and adb_reportdir to a USB drive or SSD to offload temporary and persistent data +* Limit CPU parallelism: Set adb_cores=1 to reduce peak memory usage during feed processing +* Enable blocklist shifting: Activate adb_dnsshift to store the generated blocklist on external storage and keep only a symlink in RAM +* Use firewall‑based DNS redirection: Route DNS queries via nftables to external filtered DNS resolvers and keep only a minimal local blocklist active + +**Sensible choice of blocklists** +The following feeds are just my personal recommendation as an initial setup: +* 'adguard', 'adguard_tracking' and 'certpl' + +In total, this feed selection blocks about 280K domains. It may also be useful to include compilations like hagezi, stevenblack or oisd. +Please note: don't just blindly activate too many feeds at once, sooner or later this will lead to OOM conditions. + +**DNS reporting, enable the GeoIP Map** +adblock includes a powerful reporting tool on the DNS Report tab which shows the latest DNS statistics generated by tcpdump. To get the latest statistics always press the "Refresh" button. +In addition to a tabular overview adblock reporting includes a GeoIP map in a modal popup window/iframe that shows the geolocation of your own uplink addresses (in green) and the locations of blocked domains in red. To enable the GeoIP Map set the following option in "Advanced Report Settings" config tab: set 'adb_map' to '1' to include the external components listed below and activate the GeoIP map. + +To make this work, adblock uses the following external components: +* [Leaflet](https://leafletjs.com/) is a lightweight open-source JavaScript library for interactive maps +* [OpenStreetMap](https://www.openstreetmap.org/) provides the map data under an open-source license +* [CARTO basemap styles](https://github.com/CartoDB/basemap-styles) based on [OpenMapTiles](https://openmaptiles.org/schema) +* The free and quite fast [IP Geolocation API](https://ip-api.com/) to resolve the required IP/geolocation information (max. 45 blocked Domains per request) + +**External adblock test** +In addition to the built‑in DNS reporting and GeoIP map, adblock users can verify the effectiveness of their configuration with an external test page. The [Adblock Test](https://adblock.turtlecute.org/) provides a simple way to check whether your current adblock setup is working as expected. It loads a series of test elements (ads, trackers, and other resources) and reports whether they are successfully blocked by your configuration. + +The test runs entirely in the browser and does not require additional configuration. For best results, open the page in the same environment where adblock is active and review the results displayed. + +**Firewall‑Based DNS Control** +adblock provides several advanced firewall‑integrated features that allow you to enforce DNS policies directly at the network layer. These mechanisms operate independently of the local DNS resolver and ensure that DNS traffic follows your filtering rules, even when clients attempt to bypass them. +* Unfiltered external DNS Routing: routes DNS queries from selected devices or interfaces to an external unfiltered DNS resolver +* Filtered external DNS Routing: routes DNS queries from selected devices or interfaces to an external filtered DNS resolver +* Force DNS: blocks or redirects all external DNS traffic to ensure that clients use the local resolver + +The DNS routing allows you to apply external DNS (unfiltered and/or filtered) to specific devices or entire network segments. DNS queries from these targets are transparently redirected to a chosen external resolver (IPv4 and/or IPv6): +* MAC‑based targeting for individual devices +* Interface/VLAN targeting for entire segments +* Separate IPv4/IPv6 resolver selection +* Transparent DNS redirection without client‑side configuration +This mode is ideal for guest networks, IoT devices, or environments where certain clients require stricter/lesser DNS filtering. + +force DNS ensures that all DNS traffic on your network by specific devices or entire network segments is processed by the local resolver. Any attempt to use external DNS servers is blocked or redirected. +* Blocks external DNS on port 53 and redirects DNS queries to the local resolver when appropriate +* Also prevents DNS bypassing by clients with hardcoded DNS settings on other ports, e.g. on port 853 +This mode guarantees that adblock’s filtering pipeline is always applied. + +adblock's firewall rules are based on nftables in a separate isolated nftables table (inet adblock) and chains (prerouting), with MAC addresses stored in a nftables set. The configuration is carried out centrally in LuCI on the ‘Firewall Settings’ tab in adblock. + +**Remote DNS Allow (Temporary MAC‑Based Bypass)** +This additional firewall feature lets selected client devices temporarily bypass local DNS blocking and use an external, unfiltered DNS resolver. It is designed for situations where a device needs short‑term access to content normally blocked by the adblock rules. -**Change the DNS backend to 'kresd':** -Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd', no further configuration needed. -Please note: The knot-resolver (kresd) is only available on Turris devices and does not support the SafeSearch functionality yet. +A lightweight CGI endpoint handles the workflow: +* The client opens the URL, e.g. https://\cgi-bin/adblock (preferably transferred via QR code shown in LuCI) +* The script automatically detects the device’s MAC address +* If the MAC is authorized, the script displays the current status: + * Not in the nftables set → option to request a temporary allow (“Renew”) + * Already active → shows remaining timeout +* When renewing, the CGI adds the MAC to an nftables Set with a per‑entry timeout -**Use restrictive jail modes:** -You can enable a restrictive 'adb_list.jail' to block access to all domains except those listed in the whitelist file. Usually this list will be generated as an additional list for guest or kidsafe configurations (for a separate dns server instance). If the jail directory points to your primary dns directory, adblock enables the restrictive jail mode automatically (jail mode only). +The CGI interface is mobile‑friendly and includes a LuCI‑style loading spinner during the renew process, giving immediate visual feedback while the nftables entry is created. All operations are atomic and safe even when multiple devices renew access in parallel. -**Manually override the download options:** -By default adblock uses the following pre-configured download options: -* aria2c: --timeout=20 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o -* curl: --connect-timeout 20 --silent --show-error --location -o -* uclient-fetch: --timeout=20 -O -* wget: --no-cache --no-cookies --max-redirect=0 --timeout=20 -O +**Temporary DNS Bridging (Zero‑Downtime during DNS Restarts)** +Adblock can optionally enable a temporary DNS bridging mode to avoid DNS downtime during DNS backend restarts. +When this feature is enabled, all DNS queries from LAN clients are briefly redirected to an external fallback resolver until the local DNS backend becomes available again. This ensures that DNS resolution continues to work seamlessly for all clients, even while adblock reloads blocklists or restarts the DNS service. Just set the options 'adb_nftbridge', 'adb_bridgednsv4' and 'adb_bridgednsv6' accordingly. -To override the default set 'adb_fetchparm' manually to your needs. +**Jail mode (allowlist-only):** +Enforces a strict allowlist‑only DNS policy in which only domains listed in the allowlist file are resolved, while every other query is rejected. This mode is intended for highly restrictive environments and depends on a carefully maintained allowlist, typically managed manually. -**Enable E-Mail notification via 'msmtp':** -To use the email notification you have to install & configure the package 'msmtp'. +**Enable E-Mail notification via 'msmtp':** +To use the email notification you have to install & configure the package 'msmtp'. Modify the file '/etc/msmtprc':

 [...]
@@ -246,70 +341,102 @@ from            dev.adblock@gmail.com
 user            dev.adblock
 password        xxx
 
-Finally enable E-Mail support and add a valid E-Mail receiver address in LuCI. +Finally enable E-Mail support, add a valid E-Mail receiver address in LuCI and setup an appropriate cron job. -**Service status output:** -In LuCI you'll see the realtime status in the 'Runtime' section on the overview page. -To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/init.d/adblock status\_service_: -

-~#@blackhole:~# /etc/init.d/adblock status
-::: adblock runtime information
-  + adblock_status  : enabled
-  + adblock_version : 4.1.4
-  + blocked_domains : 268355
-  + active_sources  : adaway, adguard, adguard_tracking, android_tracking, bitcoin, disconnect, firetv_tracking, games_t
-                      racking, hblock, oisd_basic, phishing_army, smarttv_tracking, stopforumspam, wally3k, winspy, yoyo
-  + dns_backend     : unbound (unbound-control), /var/lib/unbound
-  + run_utils       : download: /usr/bin/curl, sort: /usr/libexec/sort-coreutils, awk: /bin/busybox
-  + run_ifaces      : trigger: wan, report: br-lan
-  + run_directories : base: /tmp, backup: /mnt/data/adblock-Backup, report: /mnt/data/adblock-Report, jail: /tmp
-  + run_flags       : backup: ✔, flush: ✘, force: ✔, search: ✘, report: ✔, mail: ✔, jail: ✘
-  + last_run        : restart, 3m 17s, 249/73/68, 2022-09-10T13:43:07+02:00
-  + system          : ASUS RT-AX53U, OpenWrt SNAPSHOT r20535-2ca5602864
-
-The 'last\_run' line includes the used start type, the run duration, the memory footprint after DNS backend loading (total/free/available) and the date/time of the last run. +**Automatic adblock feed updates and E-Mail reports** +For a regular, automatic update of the used feeds or other regular adblock tasks set up a cron job. In LuCI you find the cron settings under 'System' => 'Scheduled Tasks'. On the command line the cron file is located at '/etc/crontabs/root': -**Edit, add new adblock sources:** -The adblock blocklist sources are stored in an external, compressed JSON file '/etc/adblock/adblock.sources.gz'. -This file is directly parsed in LuCI and accessible via CLI, just call _/etc/init.d/adblock list_: -

-/etc/init.d/adblock list
-::: Available adblock sources
-:::
-    Name                 Enabled   Size   Focus               Info URL
-    ------------------------------------------------------------------
-  + adaway               x         S      mobile              https://adaway.org
-  + adguard              x         L      general             https://adguard.com
-  + andryou              x         L      compilation         https://gitlab.com/andryou/block/-/blob/master/readme.md
-  + bitcoin              x         S      mining              https://github.com/hoshsadiq/adblock-nocoin-list
-  + disconnect           x         S      general             https://disconnect.me
-  + dshield                        XL     general             https://www.dshield.org
-[...]
-  + winhelp                        S      general             http://winhelp2002.mvps.org
-  + winspy               x         S      win_telemetry       https://github.com/crazy-max/WindowsSpyBlocker
-  + yoyo                 x         S      general             https://pgl.yoyo.org
-
+Example 1 +```sh +# update the adblock feeds every morning at 4 o'clock +00 04 * * * /etc/init.d/adblock reload +``` -To add new or edit existing sources extract the compressed JSON file _gunzip /etc/adblock/adblock.sources.gz_. -A valid JSON source object contains the following required information, e.g.: -

+Example 2
+```sh
+# update the adblock feeds every hour
+0 */1 * * * /etc/init.d/adblock reload
+```
+
+Example 3
+```sh
+# send an adblock E-Mail report every morning at 3 o'clock
+00 03 * * * /etc/init.d/adblock report mail
+```
+
+**Change/add adblock feeds**
+The adblock blocklist feeds are stored in an external JSON file '/etc/adblock/adblock.feeds'. All custom changes should be stored in an external JSON file '/etc/adblock/adblock.custom.feeds' (empty by default). It's recommended to use the LuCI based Custom Feed Editor to make changes to this file.
+A valid JSON source object contains the following information, e.g.:
+
+```json
 	[...]
-	"adaway": {
-		"url": "https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt",
-		"rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}",
-		"size": "S",
-		"focus": "mobile",
-		"descurl": "https://github.com/AdAway/adaway.github.io"
+	"stevenblack": {
+		"url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/",
+		"rule": "feed 0.0.0.0 2",
+		"size": "VAR",
+		"descr": "compilation"
 	},
 	[...]
-
-Add an unique object name, make the required changes to 'url', 'rule', 'size' and 'descurl' and finally compress the changed JSON file _gzip /etc/adblock/adblock.sources_ to use the new source object in adblock. -Please note: if you're going to add new sources on your own, please make a copy of the default file and work with that copy further on, cause the default will be overwritten with every adblock update. To reference your copy set the option 'adb\_srcarc' which points by default to '/etc/adblock/adblock.sources.gz' -Please note: when adblock starts, it looks for the uncompressed 'adb\_srcfile', only if this file is not found the archive 'adb\_srcarc' is unpacked once and then the uncompressed file is used +``` + +Add a unique feed name (no spaces, no special chars) and make the required changes: adapt at least the URL, check/change the rule, the size and the description for a new feed. +The rule consist of max. 4 individual, space separated parameters: +1. type: always 'feed' (required) +2. prefix: an optional search term (a string literal, no regex) to identify valid domain list entries, e.g. '0.0.0.0' +3. column: the domain column within the feed file, e.g. '2' (required) +4. separator: an optional field separator, default is the character class '[[:space:]]' + +**Enable debug mode** +Adblock provides an optional debug mode that writes diagnostic information to the system log and captures internal error output in a dedicated error logfile - by default located in the adblock base directory as '/tmp/adb_error.log'. The log file is automatically cleared at the beginning of each run. Under normal conditions, all error messages are discarded to keep regular runs clean and silent. To enable debug mode, set the option 'adb_debug' to '1'. When enabled, the script produces significantly more log output to assist with troubleshooting. ## Support Please join the adblock discussion in this [forum thread](https://forum.openwrt.org/t/adblock-support-thread/507) or contact me by mail -Have fun! +## Removal +Stop all adblock related services with _/etc/init.d/adblock stop_ and remove the adblock package if necessary. + +## Donations +You like this project - is there a way to donate? Generally speaking "No" - I have a well-paying full-time job and my OpenWrt projects are just a hobby of mine in my spare time. + +If you still insist to donate some bucks ... +* I would be happy if you put your money in kind into other, social projects in your area, e.g. a children's hospice +* Let's meet and invite me for a coffee if you are in my area, the “Markgräfler Land” in southern Germany or in Switzerland (Basel) +* Send your money to my [PayPal account](https://www.paypal.me/DirkBrenken) and I will collect your donations over the year to support various social projects in my area + +No matter what you decide - thank you very much for your support! + +Have fun! Dirk +--- + +## NethSecurity Integration + +NethSecurity ships adblock as the DNS-blocking engine for the Threat Shield DNS feature. + +### Threat Shield DNS + +The Threat Shield DNS integration is controlled by the `ts_enabled` option in `adblock.global`. When enabled, `ts-dns` populates `/etc/adblock/adblock.custom.feeds` with Nethesis enterprise feeds (if a subscription is active) and community free feeds, which adblock reads automatically. + +Relevant UCI options set by NethSecurity (in addition to standard adblock options): + +| Option | Default | Description | +| :--- | :--- | :--- | +| `ts_enabled` | `0` | Set to `1` by NethSecurity to activate Threat Shield DNS mode | +| `ns_tsdns_bypass` | -, not set | List of IP addresses or subnets excluded from `adb_nftforce` DNS redirection | + +### IP-based DNS bypass + +The standard adblock `adb_nftforce` feature forces DNS queries from specified LAN devices/VLANs through the local resolver. NethSecurity extends this with an **IP-based bypass** list (`ns_tsdns_bypass`): any source IP or subnet in that list is exempt from DNS redirection, even when DNS enforcement is active. + +To add a bypass address or subnet: +```sh +uci add_list adblock.global.ns_tsdns_bypass=192.168.100.2 +uci commit adblock +/etc/init.d/adblock restart +``` + +The bypass rules are injected into the `inet adblock pre-routing` chain (adblock's own nftables table) as `return` rules that take effect before the DNS redirect rules. No fw4/firewall rules are involved. + +### Disabling the CGI remote allow page +The upstream `adblock.cgi` CGI endpoint is **not installed** in NethSecurity. All adblock management is handled through the NethSecurity API (`ns.threatshield`). diff --git a/packages/adblock/files/adblock.blacklist b/packages/adblock/files/adblock.allowlist similarity index 100% rename from packages/adblock/files/adblock.blacklist rename to packages/adblock/files/adblock.allowlist diff --git a/packages/adblock/files/adblock.whitelist b/packages/adblock/files/adblock.blocklist similarity index 100% rename from packages/adblock/files/adblock.whitelist rename to packages/adblock/files/adblock.blocklist diff --git a/packages/adblock/files/adblock.categories b/packages/adblock/files/adblock.categories index 1d1118837..fad1e4abc 100644 --- a/packages/adblock/files/adblock.categories +++ b/packages/adblock/files/adblock.categories @@ -1,21 +1,100 @@ -stb;fakenews;alternates/fakenews/hosts -stb;fakenews-gambling;alternates/fakenews-gambling/hosts -stb;fakenews-gambling-porn;alternates/fakenews-gambling-porn/hosts -stb;fakenews-gambling-porn-social;alternates/fakenews-porn-social/hosts -stb;fakenews-gambling-social;alternates/fakenews-gambling-social/hosts -stb;fakenews-porn;alternates/fakenews-porn/hosts -stb;fakenews-porn-social;alternates/fakenews-porn-social/hosts -stb;fakenews-social;alternates/fakenews-social/hosts -stb;gambling;alternates/gambling/hosts -stb;gambling-porn;alternates/gambling-porn/hosts -stb;gambling-porn-social;alternates/gambling-porn-social/hosts -stb;gambling-social;alternates/gambling-social/hosts -stb;porn;alternates/porn/hosts -stb;porn-social;alternates/porn-social/hosts -stb;social;alternates/social/hosts +hag;multi-light;wildcard/light-onlydomains.txt +hag;multi-normal;wildcard/multi-onlydomains.txt +hag;multi-pro;wildcard/pro-onlydomains.txt +hag;multi-pro.mini;wildcard/pro.mini-onlydomains.txt +hag;multi-pro.plus;wildcard/pro.plus-onlydomains.txt +hag;multi-pro.plus.mini;wildcard/pro.plus.mini-onlydomains.txt +hag;multi-ultimate;wildcard/ultimate-onlydomains.txt +hag;multi-ultimate.mini;wildcard/ultimate.mini-onlydomains.txt +hag;threat-intelligence;wildcard/tif-onlydomains.txt +hag;threat-intelligence.medium;wildcard/tif.medium-onlydomains.txt +hag;threat-intelligence.mini;wildcard/tif.mini-onlydomains.txt +hag;anti.piracy;wildcard/anti.piracy-onlydomains.txt +hag;blocklist-referral;wildcard/blocklist-referral-onlydomains.txt +hag;doh;wildcard/doh-onlydomains.txt +hag;doh-vpn-proxy-bypass;wildcard/doh-vpn-proxy-bypass-onlydomains.txt +hag;dyndns;wildcard/dyndns-onlydomains.txt +hag;fake;wildcard/fake-onlydomains.txt +hag;gambling;wildcard/gambling-onlydomains.txt +hag;gambling.medium;wildcard/gambling.medium-onlydomains.txt +hag;gambling.mini;wildcard/gambling.mini-onlydomains.txt +hag;hoster;wildcard/hoster-onlydomains.txt +hag;nsfw;wildcard/nsfw-onlydomains.txt +hag;tracker.amazon;wildcard/native.amazon-onlydomains.txt +hag;tracker.apple;wildcard/native.apple-onlydomains.txt +hag;tracker.huawei;wildcard/native.huawei-onlydomains.txt +hag;tracker.lgwebos;wildcard/native.lgwebos-onlydomains.txt +hag;tracker.oppo-realme;wildcard/native.oppo-realme-onlydomains.txt +hag;tracker.roku;wildcard/native.roku-onlydomains.txt +hag;tracker.samsung;wildcard/native.samsung-onlydomains.txt +hag;tracker.tiktok;wildcard/native.tiktok-onlydomains.txt +hag;tracker.tiktok.extended;wildcard/native.tiktok.extended-onlydomains.txt +hag;tracker.vivo;wildcard/native.vivo-onlydomains.txt +hag;tracker.winoffice;wildcard/native.winoffice-onlydomains.txt +hag;tracker.xiaomi;wildcard/native.xiaomi-onlydomains.txt +hag;nosafesearch;wildcard/nosafesearch-onlydomains.txt +hag;popupads;wildcard/popupads-onlydomains.txt +hag;urlshortener;wildcard/urlshortener-onlydomains.txt +hag;abusetlds;wildcard/spam-tlds-onlydomains.txt +hag;social;wildcard/social-onlydomains.txt +hag;dga-7days;domains/dga7.txt +hag;dga-14days;domains/dga14.txt +hag;dga-30days;domains/dga30.txt +hag;nrd-7days;domains/nrd7.txt +hag;nrd-14days;domains/nrd14-8.txt +hag;nrd-21days;domains/nrd21-15.txt +hag;nrd-28days;domains/nrd28-22.txt +hag;nrd-35days;domains/nrd35-29.txt +hst;lite;Lite/domains.wildcards +hst;xtra;Xtra/domains.wildcards +ipf;ads;ads/domains.txt +ipf;dating;dating/domains.txt +ipf;doh;doh/domains.txt +ipf;gambling;gambling/domains.txt +ipf;games;games/domains.txt +ipf;malware;malware/domains.txt +ipf;phishing;phishing/domains.txt +ipf;piracy;piracy/domains.txt +ipf;porn;porn/domains.txt +ipf;shopping;shopping/domains.txt +ipf;smart-tv;smart-tv/domains.txt +ipf;social;social/domains.txt +ipf;streaming;streaming/domains.txt +ipf;violence;violence/domains.txt stb;standard;hosts +stb;standard-fakenews;alternates/fakenews/hosts +stb;standard-fakenews-gambling;alternates/fakenews-gambling/hosts +stb;standard-fakenews-gambling-porn;alternates/fakenews-gambling-porn/hosts +stb;standard-fakenews-gambling-porn-social;alternates/fakenews-gambling-porn-social/hosts +stb;standard-fakenews-gambling-social;alternates/fakenews-gambling-social/hosts +stb;standard-fakenews-porn;alternates/fakenews-porn/hosts +stb;standard-fakenews-porn-social;alternates/fakenews-porn-social/hosts +stb;standard-fakenews-social;alternates/fakenews-social/hosts +stb;standard-gambling;alternates/gambling/hosts +stb;standard-gambling-porn;alternates/gambling-porn/hosts +stb;standard-gambling-porn-social;alternates/gambling-porn-social/hosts +stb;standard-gambling-social;alternates/gambling-social/hosts +stb;standard-porn;alternates/porn/hosts +stb;standard-porn-social;alternates/porn-social/hosts +stb;standard-social;alternates/social/hosts +stb;fakenews;alternates/fakenews-only/hosts +stb;fakenews-gambling;alternates/fakenews-gambling-only/hosts +stb;fakenews-gambling-porn;alternates/fakenews-gambling-porn-only/hosts +stb;fakenews-gambling-porn-social;alternates/fakenews-gambling-porn-social-only/hosts +stb;fakenews-gambling-social;alternates/fakenews-gambling-social-only/hosts +stb;fakenews-porn;alternates/fakenews-porn-only/hosts +stb;fakenews-porn-social;alternates/fakenews-porn-social-only/hosts +stb;fakenews-social;alternates/fakenews-social-only/hosts +stb;gambling;alternates/gambling-only/hosts +stb;gambling-porn;alternates/gambling-porn-only/hosts +stb;gambling-porn-social;alternates/gambling-porn-social-only/hosts +stb;gambling-social;alternates/gambling-social-only/hosts +stb;porn;alternates/porn-only/hosts +stb;porn-social;alternates/porn-social-only/hosts +stb;social;alternates/social-only/hosts utc;adult utc;agressif +utc;ai utc;arjel utc;associations_religieuses utc;astrology @@ -36,9 +115,11 @@ utc;dialer utc;doh utc;download utc;drogue +utc;dynamic-dns utc;educational_games utc;examen_pix utc;exceptions_liste_bu +utc;fakenews utc;filehosting utc;financial utc;forums @@ -61,6 +142,7 @@ utc;radio utc;reaffected utc;redirector utc;remote-control +utc;residential-proxies utc;sect utc;sexual_education utc;shopping @@ -73,7 +155,9 @@ utc;strict_redirector utc;strong_redirector utc;translation utc;tricheur +utc;tricheur_pix utc;update utc;vpn utc;warez +utc;webhosting utc;webmail diff --git a/packages/adblock/files/adblock.conf b/packages/adblock/files/adblock.conf index 8b50b4211..37fd67aa6 100644 --- a/packages/adblock/files/adblock.conf +++ b/packages/adblock/files/adblock.conf @@ -2,13 +2,11 @@ config adblock 'global' option adb_enabled '0' option adb_debug '0' - option adb_forcedns '0' + option adb_dnsforce '0' + option adb_dnsshift '0' option adb_safesearch '0' - option adb_dnsfilereset '0' option adb_mail '0' option adb_report '0' - option adb_backup '1' - list adb_sources 'adaway' - list adb_sources 'adguard' - list adb_sources 'disconnect' - list adb_sources 'yoyo' + list adb_feed 'adguard' + list adb_feed 'adguard_tracking' + list adb_feed 'certpl' diff --git a/packages/adblock/files/adblock.custom.feeds b/packages/adblock/files/adblock.custom.feeds new file mode 100644 index 000000000..e69de29bb diff --git a/packages/adblock/files/adblock.feeds b/packages/adblock/files/adblock.feeds new file mode 100644 index 000000000..34a437261 --- /dev/null +++ b/packages/adblock/files/adblock.feeds @@ -0,0 +1,194 @@ +{ + "1hosts": { + "url": "https://raw.githubusercontent.com/badmojr/1Hosts/master/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "adguard": { + "url": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", + "rule": "feed || 3 [|^]", + "size": "L", + "descr": "general" + }, + "adguard_tracking": { + "url": "https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_trackers_justdomains.txt", + "rule": "feed 1", + "size": "L", + "descr": "tracking" + }, + "android_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "andryou": { + "url": "https://gitlab.com/andryou/block/raw/master/kouhai-compressed-domains", + "rule": "feed 1", + "size": "L", + "descr": "compilation" + }, + "anti_ad": { + "url": "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-domains.txt", + "rule": "feed 1", + "size": "L", + "descr": "compilation" + }, + "anudeep": { + "url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", + "rule": "feed 0.0.0.0 2", + "size": "M", + "descr": "compilation" + }, + "bitcoin": { + "url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "mining" + }, + "certpl": { + "url": "https://hole.cert.pl/domains/v2/domains.txt", + "rule": "feed 1", + "size": "L", + "descr": "phishing" + }, + "cpbl": { + "url": "https://raw.githubusercontent.com/bongochong/CombinedPrivacyBlockLists/master/NoFormatting/cpbl-ctld.txt", + "rule": "feed 1", + "size": "XL", + "descr": "compilation" + }, + "disconnect": { + "url": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", + "rule": "feed 1", + "size": "S", + "descr": "general" + }, + "divested": { + "url": "https://divested.dev/hosts-domains-wildcards", + "rule": "feed 1", + "size": "XXL", + "descr": "compilation" + }, + "doh_blocklist": { + "url": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains_overall.txt", + "rule": "feed 1", + "size": "S", + "descr": "doh_server" + }, + "firetv_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "games_tracking": { + "url": "https://raw.githubusercontent.com/KodoPengin/GameIndustry-hosts-Template/master/Main-Template/hosts", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "tracking" + }, + "hagezi": { + "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "hblock": { + "url": "https://hblock.molinero.dev/hosts_domains.txt", + "rule": "feed 1", + "size": "XL", + "descr": "compilation" + }, + "ipfire_dbl": { + "url": "https://dbl.ipfire.org/lists/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "oisd_big": { + "url": "https://big.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "XXL", + "descr": "general" + }, + "oisd_nsfw": { + "url": "https://nsfw.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "XXL", + "descr": "porn" + }, + "oisd_nsfw_small": { + "url": "https://nsfw-small.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "M", + "descr": "porn" + }, + "oisd_small": { + "url": "https://small.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "L", + "descr": "general" + }, + "phishing_army": { + "url": "https://phishing.army/download/phishing_army_blocklist_extended.txt", + "rule": "feed 1", + "size": "S", + "descr": "phishing" + }, + "smarttv_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "spam404": { + "url": "https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt", + "rule": "feed 1", + "size": "S", + "descr": "general" + }, + "stevenblack": { + "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/", + "rule": "feed 0.0.0.0 2", + "size": "VAR", + "descr": "compilation" + }, + "stopforumspam": { + "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt", + "rule": "feed 1", + "size": "S", + "descr": "spam" + }, + "utcapitole": { + "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", + "rule": "feed 1", + "size": "VAR", + "descr": "general" + }, + "wally3k": { + "url": "https://v.firebog.net/hosts/static/w3kbl.txt", + "rule": "feed 1", + "size": "S", + "descr": "compilation" + }, + "whocares": { + "url": "https://someonewhocares.org/hosts/hosts", + "rule": "feed 127.0.0.1 2", + "size": "M", + "descr": "general" + }, + "winspy": { + "url": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "win_telemetry" + }, + "yoyo": { + "url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext", + "rule": "feed 1", + "size": "S", + "descr": "general" + } +} diff --git a/packages/adblock/files/adblock.init b/packages/adblock/files/adblock.init index 44e6228f8..b0bd5570b 100755 --- a/packages/adblock/files/adblock.init +++ b/packages/adblock/files/adblock.init @@ -1,8 +1,8 @@ #!/bin/sh /etc/rc.common -# Copyright (c) 2015-2022 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. -# disable (s)hellcheck in release +# (s)hellcheck exceptions # shellcheck disable=all START=30 @@ -10,25 +10,47 @@ USE_PROCD=1 extra_command "suspend" "Suspend adblock processing" extra_command "resume" "Resume adblock processing" -extra_command "query" " Query active blocklists and backups for a specific domain" -extra_command "report" "[[|||] [] [] []] Print DNS statistics with an optional search parameter" -extra_command "list" "[|||||||] List/Edit available sources" -extra_command "timer" "[ [] []]|[ ] List/Edit cron update intervals" +extra_command "search" " Search active blocklists and backups for a specific domain" +extra_command "report" "[|||] Print DNS statistics" adb_init="/etc/init.d/adblock" adb_script="/usr/bin/adblock.sh" -adb_pidfile="/var/run/adblock.pid" - -if [ -s "${adb_pidfile}" ] && { [ "${action}" = "start" ] || [ "${action}" = "stop" ] || - [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "report" ] || - [ "${action}" = "suspend" ] || [ "${action}" = "resume" ] || [ "${action}" = "query" ] || - { [ "${action}" = "list" ] && [ -n "${1}" ]; }; }; then - return 0 +adb_rundir="/var/run/adblock" +adb_pidfile="/var/run/adblock/adblock.pid" + +if [ -z "${IPKG_INSTROOT}" ]; then + + # ensure runtime directory exists + # + [ ! -d "${adb_rundir}" ] && mkdir -p "${adb_rundir}" + + # check for running instance and handle boot trigger + # + case "${action}" in + "boot") + "${adb_init}" running && exit 0 + ;; + esac + + # reset pidfile if no/stale process is found, + # otherwise exit with error to prevent multiple instances + # + if [ -s "${adb_pidfile}" ]; then + pid="$(cat "${adb_pidfile}" 2>/dev/null)" + if [ -n "${pid}" ] && kill -0 "${pid}" 2>/dev/null; then + case "${action}" in + "start" | "stop" | "restart" | "reload" | "report" | "suspend" | "resume" | "search") + exit 1 + ;; + esac + else + : >"${adb_pidfile}" + fi + fi fi boot() { - [ -s "${adb_pidfile}" ] && : >"${adb_pidfile}" - rc_procd start_service + rc_procd start_service boot } start_service() { @@ -38,29 +60,28 @@ start_service() { [ -n "$(uci_get adblock global adb_trigger)" ] && return 0 fi procd_open_instance "adblock" - procd_set_param command "${adb_script}" "${@}" + procd_set_param command "${adb_script}" "${@:-"${action}"}" procd_set_param pidfile "${adb_pidfile}" - procd_set_param nice "$(uci_get adblock global adb_nice "0")" - procd_set_param stdout 1 + procd_set_param nice "$(uci_get adblock global adb_nicelimit "0")" + procd_set_param stdout 0 procd_set_param stderr 1 procd_close_instance fi } +restart() { + /usr/sbin/ts-dns # configure threat shield dns, if needed + stop_service "restart" + rc_procd start_service restart +} + reload_service() { /usr/sbin/ts-dns # configure threat shield dns, if needed rc_procd start_service reload - /etc/init.d/firewall restart } stop_service() { - rc_procd "${adb_script}" stop -} - -restart() { - /usr/sbin/ts-dns # configure threat shield dns, if needed - rc_procd start_service restart - /etc/init.d/firewall restart + [ -z "${1}" ] && rc_procd "${adb_script}" stop } suspend() { @@ -71,204 +92,49 @@ resume() { rc_procd start_service resume } -query() { - rc_procd "${adb_script}" query "${1}" +search() { + rc_procd "${adb_script}" search "${1}" } report() { rc_procd "${adb_script}" report "${1:-"cli"}" "${2}" "${3}" "${4}" } -list() { - local src_archive src_file src_enabled enabled name utc_list size focus descurl action="${1}" - - if [ "${action%_*}" = "add" ] || [ "${action%_*}" = "remove" ]; then - shift - for name in "${@}"; do - case "${action}" in - "add") - if ! uci_get adblock global adb_sources | grep -q "${name}"; then - uci_add_list adblock global adb_sources "${name}" - printf "%s\n" "::: adblock source '${name}' added to config" - fi - ;; - "remove") - if uci_get adblock global adb_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_sources "${name}" - printf "%s\n" "::: adblock source '${name}' removed from config" - fi - ;; - "add_utc") - if ! uci_get adblock global adb_utc_sources | grep -q "${name}"; then - uci_add_list adblock global adb_utc_sources "${name}" - printf "%s\n" "::: adblock utcapitole '${name}' added to config" - fi - ;; - "remove_utc") - if uci_get adblock global adb_utc_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_utc_sources "${name}" - printf "%s\n" "::: adblock utcapitole '${name}' removed from config" - fi - ;; - "add_eng") - if ! uci_get adblock global adb_eng_sources | grep -q "${name}"; then - uci_add_list adblock global adb_eng_sources "${name}" - printf "%s\n" "::: adblock energized '${name}' added to config" - fi - ;; - "remove_eng") - if uci_get adblock global adb_eng_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_eng_sources "${name}" - printf "%s\n" "::: adblock energized '${name}' removed from config" - fi - ;; - "add_stb") - if ! uci_get adblock global adb_stb_sources | grep -q "${name}"; then - uci_add_list adblock global adb_stb_sources "${name}" - printf "%s\n" "::: adblock stevenblack '${name}' added to config" - fi - ;; - "remove_stb") - if uci_get adblock global adb_stb_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_stb_sources "${name}" - printf "%s\n" "::: adblock stevenblack '${name}' removed from config" - fi - ;; - esac - done - [ -n "$(uci -q changes adblock)" ] && { uci_commit adblock; "${adb_init}" start; } - else - src_archive="$(uci_get adblock global adb_srcarc "/etc/adblock/adblock.sources.gz")" - src_file="$(uci_get adblock global adb_srcfile "/tmp/adb_sources.json")" - src_enabled="$(uci -q show adblock.global.adb_sources)" - [ -r "${src_archive}" ] && zcat "${src_archive}" >"${src_file}" || printf "%s\n" "::: adblock source archive '${src_archive}' not found" - - if [ -r "${src_file}" ]; then - src_enabled="${src_enabled#*=}" - src_enabled="${src_enabled//\'}" - printf "%s\n" "::: Available adblock sources" - printf "%s\n" ":::" - printf "%-25s%-10s%-7s%-21s%s\n" " Name" "Enabled" "Size" "Focus" "Info URL" - printf "%s\n" " -------------------------------------------------------------------" - json_load_file "${src_file}" - json_get_keys keylist - for key in ${keylist}; do - json_select "${key}" - json_get_var size "size" - json_get_var focus "focus" - json_get_var descurl "descurl" - json_get_var url "url" - json_get_var rule "rule" - if [ -n "${url}" ] && [ -n "${rule}" ]; then - if printf "%s" "${src_enabled}" | grep -q "${key}"; then - enabled="x" - else - enabled=" " - fi - src_enabled="${src_enabled/${key}}" - printf " + %-21s%-10s%-7s%-21s%s\n" "${key:0:20}" "${enabled}" "${size:0:3}" "${focus:0:20}" "${descurl:0:50}" - else - src_enabled="${src_enabled} ${key}" - fi - json_select .. - done - utc_list="$(uci_get adblock global adb_utc_sources "-")" - eng_list="$(uci_get adblock global adb_eng_sources "-")" - stb_list="$(uci_get adblock global adb_stb_sources "-")" - printf "%s\n" " ---------------------------------------------------------------------------" - printf " * %s\n" "Configured utcapitole categories: ${utc_list// /, }" - printf " * %s\n" "Configured energized variants: ${eng_list// /, }" - printf " * %s\n" "Configured stevenblack variants: ${stb_list// /, }" - - if [ -n "${src_enabled// }" ]; then - printf "%s\n" " ---------------------------------------------------------------------------" - printf "%s\n" " Sources with invalid configuration" - printf "%s\n" " ---------------------------------------------------------------------------" - for key in ${src_enabled}; do - printf " - %s\n" "${key:0:20}" - done - fi - else - printf "%s\n" "::: adblock source file '${src_file}' not found" - fi - fi -} - status() { status_service } status_service() { - local key keylist value idxval values type rtfile - - rtfile="$(uci_get adblock global adb_rtfile "/tmp/adb_runtime.json")" + local key keylist type value values - json_load_file "${rtfile}" >/dev/null 2>&1 + json_init + json_load_file "/var/run/adblock/adblock.runtime.json" >/dev/null 2>&1 json_get_keys keylist if [ -n "${keylist}" ]; then - printf "%s\n" "::: adblock runtime information" + printf '%s\n' "::: adblock runtime information" for key in ${keylist}; do - json_get_var value "${key}" >/dev/null 2>&1 - if [ "${key%_*}" = "active" ]; then - printf " + %-15s : " "${key}" - json_select "${key}" >/dev/null 2>&1 - values="" - index="1" - while json_get_type type "${index}" && [ "${type}" = "object" ]; do - json_get_values idxval "${index}" >/dev/null 2>&1 - if [ "${index}" = "1" ]; then - values="${idxval}" - else - values="${values}, ${idxval}" - fi - index="$((index + 1))" - done - values="$(printf "%s" "${values}" | awk '{NR=1;max=98;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{printf"%-22s%s\n","",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" - printf "%s\n" "${values:-"-"}" - json_select ".." + json_get_type type "${key}" >/dev/null 2>&1 + if [ "${type}" = "array" ]; then + json_get_values values "${key}" >/dev/null 2>&1 + value="${values}" else - printf " + %-15s : %s\n" "${key}" "${value:-"-"}" + json_get_var value "${key}" >/dev/null 2>&1 fi + printf ' + %-15s : %s\n' "${key}" "${value:-"-"}" done else - printf "%s\n" "::: no adblock runtime information available" - fi -} - -timer() { - local cron_file cron_content cron_lineno action="${1:-"list"}" cron_tasks="${2}" hour="${3}" minute="${4:-0}" weekday="${5:-"*"}" - - cron_file="/etc/crontabs/root" - - if [ -s "${cron_file}" ] && [ "${action}" = "list" ]; then - awk '{print NR "> " $0}' "${cron_file}" - elif [ -x "/etc/init.d/cron" ] && [ "${action}" = "add" ]; then - hour="${hour//[[:alpha:]]/}" - minute="${minute//[[:alpha:]]/}" - if [ -n "${cron_tasks}" ] && [ -n "${hour}" ] && [ -n "${minute}" ] && [ -n "${weekday}" ] && - [ "${hour}" -ge 0 ] && [ "${hour}" -le 23 ] && - [ "${minute}" -ge 0 ] && [ "${minute}" -le 59 ]; then - printf "%02d %02d %s\n" "${minute}" "${hour}" "* * ${weekday} ${adb_init} ${cron_tasks}" >>"${cron_file}" - /etc/init.d/cron restart - fi - elif [ -x "/etc/init.d/cron" ] && [ -s "${cron_file}" ] && [ "${action}" = "remove" ]; then - cron_tasks="${cron_tasks//[[:alpha:]]/}" - cron_lineno="$(awk 'END{print NR}' "${cron_file}")" - cron_content="$(awk '{print $0}' "${cron_file}")" - if [ "${cron_tasks:-"0"}" -le "${cron_lineno:-"1"}" ] && [ -n "${cron_content}" ]; then - printf "%s\n" "${cron_content}" | awk "NR!~/^${cron_tasks}$/" >"${cron_file}" - /etc/init.d/cron restart - fi + printf '%s\n' "::: no adblock runtime information available" fi } service_triggers() { - local iface delay + local iface delay trigger - iface="$(uci_get adblock global adb_trigger)" delay="$(uci_get adblock global adb_triggerdelay "5")" - PROCD_RELOAD_DELAY="$((delay * 1000))" + trigger="$(uci_get adblock global adb_trigger)" - [ -n "${iface}" ] && procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" "start" - procd_add_reload_trigger "adblock" + PROCD_RELOAD_DELAY="$((delay * 1000))" + for iface in ${trigger}; do + procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" start + done } diff --git a/packages/adblock/files/adblock.mail b/packages/adblock/files/adblock.mail index 67fc011aa..98f4bb1cc 100755 --- a/packages/adblock/files/adblock.mail +++ b/packages/adblock/files/adblock.mail @@ -1,6 +1,6 @@ #!/bin/sh # send mail script for adblock notifications -# Copyright (c) 2015-2022 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # Please note: you have to manually install and configure the package 'msmtp' before using this script @@ -8,70 +8,66 @@ # set (s)hellcheck exceptions # shellcheck disable=all -LC_ALL=C -PATH="/usr/sbin:/usr/bin:/sbin:/bin" - -[ -r "/lib/functions.sh" ] && . "/lib/functions.sh" +[ -r "/usr/bin/adblock.sh" ] && . "/usr/bin/adblock.sh" "mail" adb_debug="$(uci_get adblock global adb_debug "0")" adb_mailsender="$(uci_get adblock global adb_mailsender "no-reply@adblock")" adb_mailreceiver="$(uci_get adblock global adb_mailreceiver)" adb_mailtopic="$(uci_get adblock global adb_mailtopic "adblock notification")" adb_mailprofile="$(uci_get adblock global adb_mailprofile "adb_notify")" -adb_ver="${1}" -adb_mail="$(command -v msmtp)" -adb_logger="$(command -v logger)" -adb_logread="$(command -v logread)" -adb_rc="1" - -f_log() { - local class="${1}" log_msg="${2}" - - if [ -x "${adb_logger}" ]; then - "${adb_logger}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" - else - printf "%s %s %s\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" - fi -} -if [ -z "${adb_mailreceiver}" ]; then - f_log "err" "please set the mail receiver with the 'adb_mailreceiver' option" - exit ${adb_rc} -fi +[ -z "${adb_mailreceiver}" ] && f_log "info" "please set the mail receiver with the 'adb_mailreceiver' option" [ "${adb_debug}" = "1" ] && debug="--debug" -adb_mailhead="From: ${adb_mailsender}\nTo: ${adb_mailreceiver}\nSubject: ${adb_mailtopic}\nReply-to: ${adb_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n" - # info preparation # sys_info="$( strings /etc/banner 2>/dev/null - ubus call system board | sed -e 's/\"release\": {//' | sed -e 's/^[ \t]*//' | sed -e 's/[{}\",]//g' | sed -e 's/[ ]/ \t/' | sed '/^$/d' 2>/dev/null + "${adb_ubuscmd}" call system board | "${adb_awkcmd}" 'BEGIN{FS="[{}\"]"}{if($2=="kernel"||$2=="hostname"||$2=="system"||$2=="model"||$2=="description")printf " + %-12s: %s\n",$2,$4}' 2>/dev/null )" adb_info="$(/etc/init.d/adblock status 2>/dev/null)" -rep_info="${2}" -if [ -x "${adb_logread}" ]; then - log_info="$("${adb_logread}" -l 100 -e "adblock-" | awk '{NR=1;max=120;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{print substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" +rep_info="${1}" +if [ -x "${adb_logreadcmd}" ]; then + log_info="$("${adb_logreadcmd}" -l 100 -e "adblock-" 2>/dev/null)" fi # mail body # -adb_mailtext="
"
-adb_mailtext="${adb_mailtext}\n++\n++ System Information ++\n++\n${sys_info}"
-adb_mailtext="${adb_mailtext}\n\n++\n++ Adblock Information ++\n++\n${adb_info}"
-if [ -n "${rep_info}" ]; then
-	adb_mailtext="${adb_mailtext}\n\n++\n++ Report Information ++\n++\n${rep_info}"
-fi
-adb_mailtext="${adb_mailtext}\n\n++\n++ Logfile Information ++\n++\n${log_info}"
-adb_mailtext="${adb_mailtext}
" +adb_mailtext="$( + printf '%s\n' "
"
+	printf '\n%s\n' "++
+++ System Information ++
+++"
+	printf '%s\n' "${sys_info:-"-"}"
+	printf '\n%s\n' "++
+++ Adblock Information ++
+++"
+	printf '%s\n' "${adb_info:-"-"}"
+	if [ -n "${rep_info}" ]; then
+		printf '\n%s\n' "++
+++ Report Information ++
+++"
+		printf '%s\n' "${rep_info}"
+	fi
+	printf '\n%s\n' "++
+++ Logfile Information ++
+++"
+	printf '%s\n' "${log_info:-"-"}"
+	printf '%s\n' "
" +)" # send mail # -if [ -x "${adb_mail}" ]; then - printf "%b" "${adb_mailhead}${adb_mailtext}" 2>/dev/null | "${adb_mail}" ${debug} -a "${adb_mailprofile}" "${adb_mailreceiver}" >/dev/null 2>&1 - adb_rc=${?} - f_log "info" "mail sent to '${adb_mailreceiver}' with rc '${adb_rc}'" +if [ -x "${adb_mailcmd}" ]; then + adb_mailhead="From: ${adb_mailsender}\nTo: ${adb_mailreceiver}\nSubject: ${adb_mailtopic}\nReply-to: ${adb_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n" + printf '%b' "${adb_mailhead}${adb_mailtext}" 2>/dev/null | "${adb_mailcmd}" ${debug} -a "${adb_mailprofile}" "${adb_mailreceiver}" 2>>"${adb_errorlog}" + mail_rc="${?}" + if [ "${mail_rc}" = "0" ]; then + f_log "info" "mail successfully sent to '${adb_mailreceiver}'" + else + f_log "info" "failed to send mail to '${adb_mailreceiver}' with rc '${mail_rc}'" + fi else - f_log "err" "msmtp mail daemon not found" + f_log "info" "msmtp mail daemon not found" fi -exit ${adb_rc} +exit 0 diff --git a/packages/adblock/files/adblock.sh b/packages/adblock/files/adblock.sh index fe6fc10fe..6c0d738ef 100755 --- a/packages/adblock/files/adblock.sh +++ b/packages/adblock/files/adblock.sh @@ -1,9 +1,9 @@ #!/bin/sh # dns based ad/abuse domain blocking -# Copyright (c) 2015-2023 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. -# disable (s)hellcheck in release +# (s)hellcheck exceptions # shellcheck disable=all # set initial defaults @@ -11,125 +11,179 @@ export LC_ALL=C export PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="4.1.5" adb_enabled="0" adb_debug="0" -adb_forcedns="0" +adb_nftforce="0" +adb_nftdevforce="" +adb_nftportforce="" +ns_tsdns_bypass="" +adb_nftallow="0" +adb_nftmacallow="" +adb_nftdevallow="" +adb_nftblock="0" +adb_nftmacblock="" +adb_nftdevblock="" +adb_nftremote="0" +adb_nftremotetimeout="15" +adb_nftmacremote="" +adb_nftbridge="0" +adb_allowdnsv4="" +adb_allowdnsv6="" +adb_remotednsv4="" +adb_remotednsv6="" +adb_blockdnsv4="" +adb_blockdnsv6="" +adb_bridgednsv4="" +adb_bridgednsv6="" +adb_dnsshift="0" adb_dnsflush="0" -adb_dnstimeout="20" +adb_dnstimeout="30" adb_safesearch="0" -adb_safesearchlist="" -adb_safesearchmod="0" adb_report="0" adb_trigger="" -adb_triggerdelay="0" -adb_backup="1" +adb_triggerdelay="5" adb_mail="0" -adb_mailcnt="0" adb_jail="0" +adb_map="0" +adb_tld="1" adb_dns="" -adb_dnsprefix="adb_list" -adb_locallist="blacklist whitelist iplist" -adb_tmpbase="/tmp" -adb_backupdir="${adb_tmpbase}/adblock-Backup" -adb_reportdir="${adb_tmpbase}/adblock-Report" -adb_jaildir="/tmp" -adb_pidfile="/var/run/adblock.pid" -adb_blacklist="/etc/adblock/adblock.blacklist" -adb_whitelist="/etc/adblock/adblock.whitelist" +adb_dnspid="" +adb_locallist="allowlist blocklist" +adb_basedir="/tmp" +adb_finaldir="" +adb_backupdir="/tmp/adblock-backup" +adb_reportdir="/tmp/adblock-report" +adb_rundir="/var/run/adblock" +adb_pidfile="${adb_rundir}/adblock.pid" +adb_rtfile="${adb_rundir}/adblock.runtime.json" +adb_etaglock="${adb_rundir}/adblock.etag.lock" +adb_allowlist="/etc/adblock/adblock.allowlist" +adb_blocklist="/etc/adblock/adblock.blocklist" adb_mailservice="/etc/adblock/adblock.mail" -adb_dnsfile="${adb_dnsprefix}.overall" -adb_dnsjail="${adb_dnsprefix}.jail" -adb_srcarc="/etc/adblock/adblock.sources.gz" -adb_srcfile="${adb_tmpbase}/adb_sources.json" -adb_rtfile="${adb_tmpbase}/adb_runtime.json" -adb_loggercmd="$(command -v logger)" -adb_dumpcmd="$(command -v tcpdump)" -adb_lookupcmd="$(command -v nslookup)" -adb_fetchutil="" +adb_dnsfile="adb_list.overall" +adb_feedfile="/etc/adblock/adblock.feeds" +adb_customfeedfile="/etc/adblock/adblock.custom.feeds" +adb_errorlog="/dev/null" +adb_fetchcmd="" adb_fetchinsecure="" -adb_zonelist="" -adb_portlist="" +adb_fetchretry="5" +adb_fetchparm="" +adb_etagparm="" +adb_geoparm="" +adb_geourl="http://ip-api.com/json" adb_repiface="" -adb_replisten="53" +adb_repport="53" adb_repchunkcnt="5" adb_repchunksize="1" adb_represolve="0" -adb_lookupdomain="example.com" -adb_action="${1:-"start"}" +adb_lookupdomain="localhost" +adb_action="${1}" adb_packages="" -adb_sources="" adb_cnt="" -# load & check adblock environment +# helper function to find a command in the system and check if it's executable, +# if not try alternative command or log an error if not found # -f_load() { - local bg_pid iface port ports cpu core +f_cmd() { + local cmd pri_cmd="${1}" sec_cmd="${2}" + + cmd="$(command -v "${pri_cmd}" 2>/dev/null)" + if [ -z "${cmd}" ]; then + if [ -n "${sec_cmd}" ]; then + [ "${sec_cmd}" = "optional" ] && return + cmd="$(command -v "${sec_cmd}" 2>/dev/null)" + fi + if [ -n "${cmd}" ]; then + printf '%s' "${cmd}" + else + f_log "emerg" "command '${pri_cmd:-"-"}'/'${sec_cmd:-"-"}' not found" + fi + else + printf '%s' "${cmd}" + fi +} - adb_sysver="$(ubus -S call system board 2>/dev/null | jsonfilter -q -e '@.model' -e '@.release.description' | - "${adb_awk}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s",$1,$2}')" - adb_memory="$("${adb_awk}" '/^MemTotal|^MemFree|^MemAvailable/{ORS="/"; print int($2/1000)}' "/proc/meminfo" 2>/dev/null | - "${adb_awk}" '{print substr($0,1,length($0)-1)}')" +# load adblock environment +# +f_load() { + local cnt bg_pid port filter tcpdump_filter cpu + # load adblock config and set debug log file + # f_conf + if [ "${adb_debug}" = "1" ] && [ -d "${adb_basedir}" ]; then + adb_errorlog="${adb_basedir}/adb_error.log" + else + adb_errorlog="/dev/null" + fi - cpu="$(grep -c '^processor' /proc/cpuinfo 2>/dev/null)" - core="$(grep -cm1 '^core id' /proc/cpuinfo 2>/dev/null)" - [ "${cpu}" = "0" ] && cpu="1" - [ "${core}" = "0" ] && core="1" - adb_cores="$((cpu * core))" + # fetch installed packages and system information + # + adb_packages="$("${adb_ubuscmd}" -S call rpc-sys packagelist '{ "all": true }' 2>>"${adb_errorlog}")" + adb_bver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e '@.packages.adblock')" + adb_fver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e '@.packages["luci-app-adblock"]')" + adb_sysver="$("${adb_ubuscmd}" -S call system board 2>>"${adb_errorlog}" | + "${adb_jsoncmd}" -ql1 -e '@.model' -e '@.release.target' -e '@.release.distribution' -e '@.release.version' -e '@.release.revision' | + "${adb_awkcmd}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s, %s %s (%s)",$1,$2,$3,$4,$5}')" + + # detect cpu cores for parallel processing + # + if [ -z "${adb_cores}" ]; then + cpu="$("${adb_grepcmd}" -cm16 '^processor' /proc/cpuinfo 2>>"${adb_errorlog}")" + [ "${cpu}" = "0" ] && cpu="1" + adb_cores="${cpu}" + fi - if [ "${adb_action}" != "report" ]; then + # load dns backend and fetch utility + # + if [ "${adb_action}" != "report" ] && [ "${adb_action}" != "mail" ]; then f_dns f_fetch fi - if [ "${adb_enabled}" = "0" ]; then - f_extconf - f_temp - f_rmdns - f_jsnup "disabled" - f_log "info" "adblock is currently disabled, please set the config option 'adb_enabled' to '1' to use this service" - exit 0 - fi - + # check if reporting is enabled and tcpdump is available + # if [ "${adb_report}" = "1" ] && [ ! -x "${adb_dumpcmd}" ]; then - f_log "info" "Please install the package 'tcpdump' or 'tcpdump-mini' to use the reporting feature" - elif [ "${adb_report}" = "0" ] && [ "${adb_action}" = "report" ]; then - f_log "info" "Please enable the 'DNS Report' option to use the reporting feature" - exit 0 - fi - - bg_pid="$(pgrep -f "^${adb_dumpcmd}.*adb_report\\.pcap$" | "${adb_awk}" '{ORS=" "; print $1}')" - if [ -x "${adb_dumpcmd}" ] && { [ "${adb_report}" = "0" ] || { [ -n "${bg_pid}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; }; }; }; then - if [ -n "${bg_pid}" ]; then - kill -HUP "${bg_pid}" 2>/dev/null - while kill -0 "${bg_pid}" 2>/dev/null; do - sleep 1 - done - unset bg_pid + f_log "info" "please install the package 'tcpdump' or 'tcpdump-mini' to use the reporting feature" + elif [ -x "${adb_dumpcmd}" ]; then + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + if [ -n "${bg_pid}" ] && { [ "${adb_report}" = "0" ] || [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; }; then + if kill -HUP "${bg_pid}" 2>>"${adb_errorlog}"; then + for cnt in 1 2 3; do + kill -0 "${bg_pid}" >/dev/null 2>&1 || break + sleep 1 + done + fi + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + "${adb_rmcmd}" -f "${adb_reportdir}"/adb_report.pcap* fi - fi - if [ -x "${adb_dumpcmd}" ] && [ "${adb_report}" = "1" ] && [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ] && [ "${adb_action}" != "stop" ]; then - for port in ${adb_replisten}; do - [ -z "${ports}" ] && ports="port ${port}" || ports="${ports} or port ${port}" - done - if [ -z "${adb_repiface}" ]; then - network_get_device iface "lan" - [ -z "${iface}" ] && network_get_physdev iface "lan" - [ -n "${iface}" ] && adb_repiface="${iface}" - [ -n "${adb_repiface}" ] && { uci_set adblock global adb_repiface "${adb_repiface}"; f_uci "adblock"; } - fi - if [ -n "${adb_reportdir}" ] && [ ! -d "${adb_reportdir}" ]; then - mkdir -p "${adb_reportdir}" - f_log "info" "report directory '${adb_reportdir}' created" - fi - if [ -n "${adb_repiface}" ] && [ -d "${adb_reportdir}" ]; then - ("${adb_dumpcmd}" -nn -p -s0 -l -i ${adb_repiface} ${ports} -C${adb_repchunksize} -W${adb_repchunkcnt} -w "${adb_reportdir}/adb_report.pcap" >/dev/null 2>&1 &) - bg_pid="$(pgrep -f "^${adb_dumpcmd}.*adb_report\\.pcap$" | "${adb_awk}" '{ORS=" "; print $1}')" - else - f_log "info" "Please set the name of the reporting network device 'adb_repiface' manually" + if [ "${adb_report}" = "1" ] && [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ] && [ "${adb_action}" != "stop" ]; then + [ ! -d "${adb_reportdir}" ] && mkdir -p "${adb_reportdir}" + if [ -z "${adb_repiface}" ]; then + network_get_device adb_repiface "lan" + [ -z "${adb_repiface}" ] && network_get_physdev adb_repiface "lan" + uci_set adblock global adb_repiface "${adb_repiface:-"any"}" + f_uci "adblock" + fi + for port in ${adb_repport}; do + [ -n "${filter}" ] && filter="${filter} or " + filter="${filter}(udp port ${port}) or (tcp port ${port})" + done + tcpdump_filter="(${filter}) and greater 28" + if [ -n "${adb_repiface}" ] && [ -d "${adb_reportdir}" ]; then + ( + "${adb_dumpcmd}" --immediate-mode -nn -p -s0 -i "${adb_repiface}" \ + "${tcpdump_filter}" \ + -C "${adb_repchunksize}" -W "${adb_repchunkcnt}" \ + -w "${adb_reportdir}/adb_report.pcap" >/dev/null 2>&1 & + ) + sleep 1 + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + f_log "info" "tcpdump background process started for interface: ${adb_repiface}, port: ${adb_repport}, dir: ${adb_reportdir}, pid: ${bg_pid}" + else + f_log "info" "please set the reporting interface 'adb_repiface' and reporting directory 'adb_reportdir' manually" + fi fi fi } @@ -137,514 +191,591 @@ f_load() { # check & set environment # f_env() { - adb_starttime="$(date "+%s")" - f_log "info" "adblock instance started ::: action: ${adb_action}, priority: ${adb_nice:-"0"}, pid: ${$}" - f_jsnup "running" + : >"${adb_errorlog}" + read -r adb_starttime _ <"/proc/uptime" + adb_starttime="${adb_starttime%.*}" + f_log "info" "adblock instance started ::: action: ${adb_action}, priority: ${adb_nicelimit:-"0"}, pid: ${$}" + f_jsnup "processing" f_extconf f_temp - - if [ "${adb_dnsflush}" = "1" ] || [ "${adb_memory##*/}" -lt "64" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_dnsup - fi - - if [ ! -r "${adb_srcfile}" ]; then - if [ -r "${adb_srcarc}" ]; then - zcat "${adb_srcarc}" >"${adb_srcfile}" + f_nftadd + json_init + if [ -s "${adb_customfeedfile}" ]; then + if json_load_file "${adb_customfeedfile}" >/dev/null 2>&1; then + return else - f_log "err" "adblock source archive not found" + f_log "info" "can't load adblock custom feed file" fi fi - if [ -r "${adb_srcfile}" ] && [ "${adb_action}" != "report" ]; then - json_init - json_load_file "${adb_srcfile}" + if [ -s "${adb_feedfile}" ] && json_load_file "${adb_feedfile}" >/dev/null 2>&1; then + return else - f_log "err" "adblock source file not found" + f_log "err" "can't load adblock feed file" fi } # load adblock config # f_conf() { - local cnt="0" cnt_max="10" - - [ ! -r "/etc/config/adblock" ] && f_log "err" "no valid adblock config found, please re-install the package via opkg with the '--force-reinstall --force-maintainer' options" - config_cb() { option_cb() { - local option="${1}" - local value="${2}" - eval "${option}=\"${value}\"" + local option="${1}" value="${2//\"/\\\"}" + + case "${option}" in + *[!a-zA-Z0-9_]*) ;; + + *) + eval "${option}=\"\${value}\"" + ;; + esac } list_cb() { - local option="${1}" - local value="${2}" - if [ "${option}" = "adb_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_sources}") ${value}\"" - elif [ "${option}" = "adb_eng_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_eng_sources}") ${value}\"" - elif [ "${option}" = "adb_stb_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_stb_sources}") ${value}\"" - elif [ "${option}" = "adb_utc_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_utc_sources}") ${value}\"" - elif [ "${option}" = "adb_denyip" ]; then - eval "${option}=\"$(printf "%s" "${adb_denyip}") ${value}\"" - elif [ "${option}" = "adb_allowip" ]; then - eval "${option}=\"$(printf "%s" "${adb_allowip}") ${value}\"" - elif [ "${option}" = "adb_safesearchlist" ]; then - eval "${option}=\"$(printf "%s" "${adb_safesearchlist}") ${value}\"" - elif [ "${option}" = "adb_zonelist" ]; then - eval "${option}=\"$(printf "%s" "${adb_zonelist}") ${value}\"" - elif [ "${option}" = "adb_portlist" ]; then - eval "${option}=\"$(printf "%s" "${adb_portlist}") ${value}\"" - elif [ "${option}" = "adb_bypass" ]; then - eval "${option}=\"$(printf "%s" "${adb_bypass}") ${value}\"" - fi + local append option="${1}" value="${2//\"/\\\"}" + + case "${option}" in + *[!a-zA-Z0-9_]*) ;; + + *) + eval "append=\"\${${option}}\"" + if [ -n "${append}" ]; then + eval "${option}=\"\${append} \${value}\"" + else + eval "${option}=\"\${value}\"" + fi + ;; + esac } } config_load adblock - - if [ -z "${adb_fetchutil}" ] || [ -z "${adb_dns}" ]; then - while [ -z "${adb_packages}" ] && [ "${cnt}" -le "${cnt_max}" ]; do - adb_packages="$(opkg list-installed 2>/dev/null)" - cnt="$((cnt + 1))" - sleep 1 - done - [ -z "${adb_packages}" ] && f_log "err" "local opkg package repository is not available, please set 'adb_fetchutil' and 'adb_dns' manually" - fi } -# status helper function +# domain validation # -f_char() { - local result input="${1}" +f_chkdom() { + local type prefix column separator + + case "${1}" in + "feed" | "local") + type="${1}" + case "${2}" in + [0-9]) + prefix="" + column="${2}" + separator="${3:-[[:space:]]+}" + ;; + *) + prefix="${2}" + column="${3}" + separator="${4:-[[:space:]]+}" + ;; + esac + ;; + "google") + type="${1}" + prefix="" + column="${2}" + separator="${3:-[[:space:]]+}" + ;; + esac - if [ "${input}" = "1" ]; then - result="✔" - else - result="✘" - fi - printf "%s" "${result}" + "${adb_awkcmd}" -v type="${type}" -v pre="${prefix}" -v col="${column}" -v chk="${adb_lookupdomain}" -F "${separator}" ' + { + domain = $col + # remove carriage returns and trim the input + gsub(/\r|^[[:space:]]+|[[:space:]]+$/, "", domain) + # add www. for google safe search + if (type=="google" && domain ~ /^\.+/) { sub(/^\.+/, "", domain); domain="www."domain } + # check optional search prefix + if (pre != "" && index($0, pre) != 1) next + # skip empty lines, comments and special domains + if (domain == "" || domain ~ /^(#|localhost|loopback)/ || index(domain, chk) == 1) next + # no domain with trailing dot + if (substr(domain, length(domain), 1) == ".") next + # check total length (253 characters) + if (length(domain) > 253) next + n = split(domain, L, ".") + valid = 1 + for (i = 1; i <= n; i++) { + l = L[i] + len = length(l) + # label length 1–63 + if (len < 1 || len > 63) { valid = 0; break } + # no leading/trailing hyphen + if (l ~ /^-/ || l ~ /-$/) { valid = 0; break } + # ASCII + hyphen + if (l !~ /^[A-Za-z0-9-]+$/) { valid = 0; break } + } + # TLD must start with a letter or "xn--" + if (valid && L[n] !~ /^[A-Za-z]/ && L[n] !~ /^xn--/) valid = 0 + if (valid) print tolower(domain) + }' + + f_log "debug" "f_chkdom ::: name: ${src_name}, type: ${type}, prefix: ${prefix:-"-"}, column: ${column:-"-"}, separator: ${separator:-"-"}" } # load dns backend config # f_dns() { - local util utils dns_up cnt="0" + local dns dns_list dns_section dns_info free_mem dir + + free_mem="$("${adb_awkcmd}" '/^MemAvailable/{printf "%s",int($2/1000)}' "/proc/meminfo" 2>>"${adb_errorlog}")" + if [ "${adb_action}" = "boot" ] && [ -z "${adb_trigger}" ]; then + sleep ${adb_triggerdelay:-"5"} + fi if [ -z "${adb_dns}" ]; then - utils="knot-resolver bind unbound dnsmasq raw" - for util in ${utils}; do - if [ "${util}" = "raw" ] || printf "%s" "${adb_packages}" | grep -q "^${util}"; then - if [ "${util}" = "knot-resolver" ]; then - util="kresd" - elif [ "${util}" = "bind" ]; then - util="named" - fi - if [ "${util}" = "raw" ] || [ -x "$(command -v "${util}")" ]; then - adb_dns="${util}" - uci_set adblock global adb_dns "${util}" + dns_list="knot-resolver bind-server unbound-daemon smartdns dnsmasq-full dnsmasq-dhcpv6 dnsmasq" + for dns in ${dns_list}; do + if printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e "@.packages[\"${dns}\"]" >/dev/null 2>&1; then + case "${dns}" in + "knot-resolver") + dns="kresd" + ;; + "bind-server") + dns="named" + ;; + "unbound-daemon") + dns="unbound" + ;; + "dnsmasq-full" | "dnsmasq-dhcpv6") + dns="dnsmasq" + ;; + esac + + if command -v "${dns}" >/dev/null 2>&1; then + adb_dns="${dns}" + uci_set adblock global adb_dns "${dns}" f_uci "adblock" break fi fi done - elif [ "${adb_dns}" != "raw" ] && [ ! -x "$(command -v "${adb_dns}")" ]; then - unset adb_dns fi - if [ -n "${adb_dns}" ]; then - case "${adb_dns}" in - "dnsmasq") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"dnsmasq"}" - adb_dnsdir="${adb_dnsdir:-"/tmp/dnsmasq.d"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"local=/\"\$0\"/\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"local=/\"\$0\"/#\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{print \"address=/\"\$0\"/\"item\"\"}'"}" - adb_dnsstop="${adb_dnsstop:-"address=/#/"}" - ;; - "unbound") - adb_dnscachecmd="$(command -v unbound-control || printf "%s" "-")" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"unbound"}" - adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"local-zone: \\042\"\$0\"\\042 always_nxdomain\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"local-zone: \\042\"\$0\"\\042 always_transparent\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{type=\"AAAA\";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type=\"A\"}}{print \"local-data: \\042\"\$0\" \"type\" \"item\"\\042\"}'"}" - adb_dnsstop="${adb_dnsstop:-"local-zone: \".\" always_nxdomain"}" - ;; - "named") - adb_dnscachecmd="$(command -v rndc || printf "%s" "-")" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"bind"}" - adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" - adb_dnsheader="${adb_dnsheader:-"\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n IN NS localhost.\n"}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"\"\$0\" CNAME .\\n*.\"\$0\" CNAME .\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"\"\$0\" CNAME rpz-passthru.\\n*.\"\$0\" CNAME rpz-passthru.\"}'"}" - adb_dnsdenyip="${adb_dnsdenyip:-"${adb_awk} '{print \"\"\$0\".rpz-client-ip CNAME .\"}'"}" - adb_dnsallowip="${adb_dnsallowip:-"${adb_awk} '{print \"\"\$0\".rpz-client-ip CNAME rpz-passthru.\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{print \"\"\$0\" CNAME \"item\".\\n*.\"\$0\" CNAME \"item\".\"}'"}" - adb_dnsstop="${adb_dnsstop:-"* CNAME ."}" - ;; - "kresd") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"root"}" - adb_dnsdir="${adb_dnsdir:-"/etc/kresd"}" - adb_dnsheader="${adb_dnsheader:-"\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n"}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"\"\$0\" CNAME .\\n*.\"\$0\" CNAME .\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"\"\$0\" CNAME rpz-passthru.\\n*.\"\$0\" CNAME rpz-passthru.\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{type=\"AAAA\";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type=\"A\"}}{print \"\"\$0\" \"type\" \"item\"\"}'"}" - adb_dnsstop="${adb_dnsstop:-"* CNAME ."}" - ;; - "raw") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"root"}" - adb_dnsdir="${adb_dnsdir:-"/tmp"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"0"}" - adb_dnsallow="${adb_dnsallow:-"1"}" - adb_dnssafesearch="${adb_dnssafesearch:-"0"}" - adb_dnsstop="${adb_dnsstop:-"0"}" - ;; - esac - fi - - if [ "${adb_dns}" != "raw" ] && { [ -z "${adb_dns}" ] || [ ! -x "$(command -v "${adb_dns}")" ]; }; then + if [ "${adb_dns}" != "raw" ] && ! command -v "${adb_dns}" >/dev/null 2>&1; then f_log "err" "dns backend not found, please set 'adb_dns' manually" fi - if [ "${adb_dns}" != "raw" ] && { [ "${adb_dnsdir}" = "${adb_tmpbase}" ] || [ "${adb_dnsdir}" = "${adb_backupdir}" ] || [ "${adb_dnsdir}" = "${adb_reportdir}" ]; }; then - f_log "err" "dns directory '${adb_dnsdir}' has been misconfigured, it must not point to the 'adb_tmpbase', 'adb_backupdir', 'adb_reportdir'" - fi - - if [ "${adb_action}" = "start" ] && [ -z "${adb_trigger}" ]; then - sleep ${adb_triggerdelay} - fi - - if [ "${adb_dns}" != "raw" ] && [ "${adb_action}" != "stop" ]; then - while [ "${cnt}" -le 30 ]; do - dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" 2>/dev/null | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running" 2>/dev/null)" - if [ "${dns_up}" = "true" ]; then - break - fi - sleep 1 - cnt="$((cnt + 1))" - done - fi - - if [ "${adb_action}" != "stop" ]; then - if [ -n "${adb_dnsdir}" ] && [ ! -d "${adb_dnsdir}" ]; then - if mkdir -p "${adb_dnsdir}"; then - f_log "info" "dns backend directory '${adb_dnsdir}' created" + case "${adb_dns}" in + "dnsmasq") + adb_dnscachecmd="" + adb_dnsinstance="${adb_dnsinstance:-"0"}" + adb_dnsuser="dnsmasq" + adb_dnsdir="${adb_dnsdir:-""}" + if [ -z "${adb_dnsdir}" ]; then + dns_section="$("${adb_ubuscmd}" -S call uci get "{\"config\":\"dhcp\", \"section\":\"@dnsmasq[${adb_dnsinstance}]\", \"type\":\"dnsmasq\"}" 2>>"${adb_errorlog}")" + dns_info="$(printf '%s' "${dns_section}" | "${adb_jsoncmd}" -l1 -e '@.values["confdir"]')" + if [ -n "${dns_info}" ]; then + adb_dnsdir="${dns_info}" else - f_log "err" "dns backend directory '${adb_dnsdir}' could not be created" + dns_info="$(printf '%s' "${dns_section}" | "${adb_jsoncmd}" -l1 -e '@.values[".name"]')" + [ -n "${dns_info}" ] && adb_dnsdir="/tmp/dnsmasq.${dns_info}.d" fi fi - [ ! -f "${adb_dnsdir}/${adb_dnsfile}" ] && printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="address=/#/\nlocal=/#/" + f_dnsdeny() { + "${adb_awkcmd}" '{print "local=/"$0"/"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "local=/"$0"/#"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ "${dns_up}" != "true" ]; then - if ! f_dnsup 4; then - f_log "err" "dns backend '${adb_dns}' not running or executable" - fi - fi + shift + "${adb_awkcmd}" -v item="${item}" '{print "address=/"$0"/"item"";print "local=/"$0"/"}' "${@}" + } + ;; + "unbound") + adb_dnscachecmd="$(f_cmd unbound-control optional)" + adb_dnsinstance="" + adb_dnsuser="unbound" + adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="local-zone: \".\" always_nxdomain" + f_dnsdeny() { + "${adb_awkcmd}" '{print "local-zone: \042"$0"\042 always_nxdomain"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "local-zone: \042"$0"\042 always_transparent"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ "${adb_backup}" = "1" ] && [ -n "${adb_backupdir}" ] && [ ! -d "${adb_backupdir}" ]; then - if mkdir -p "${adb_backupdir}"; then - f_log "info" "backup directory '${adb_backupdir}' created" - else - f_log "err" "backup directory '${adb_backupdir}' could not be created" - fi - fi + shift + "${adb_awkcmd}" -v item="${item}" \ + '{type="AAAA";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type="A"}}{print "local-data: \042"$0" "type" "item"\042"}' "${@}" + } + ;; + "named") + adb_dnscachecmd="$(f_cmd rndc optional)" + adb_dnsinstance="" + adb_dnsuser="bind" + adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" + adb_dnsheader="\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n IN NS localhost.\n" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="* CNAME ." + f_dnsdeny() { + "${adb_awkcmd}" '{print ""$0" CNAME .\n*."$0" CNAME ."}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print ""$0" CNAME rpz-passthru.\n*."$0" CNAME rpz-passthru."}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ -n "${adb_jaildir}" ] && [ ! -d "${adb_jaildir}" ]; then - if mkdir -p "${adb_jaildir}"; then - f_log "info" "jail directory '${adb_jaildir}' created" - else - f_log "err" "jail directory '${adb_jaildir}' could not be created" - fi + shift + "${adb_awkcmd}" -v item="${item}" '{print ""$0" CNAME "item".\n*."$0" CNAME "item"."}' "${@}" + } + ;; + "kresd") + adb_dnscachecmd="" + adb_dnsinstance="" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp/kresd"}" + adb_dnsheader="\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="* CNAME ." + f_dnsdeny() { + "${adb_awkcmd}" '{print ""$0" CNAME .\n*."$0" CNAME ."}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print ""$0" CNAME rpz-passthru.\n*."$0" CNAME rpz-passthru."}' "${@}" + } + f_dnssafesearch() { + local item="${1}" + + shift + "${adb_awkcmd}" -v item="${item}" '{print ""$0" CNAME "item".\n*."$0" CNAME "item"."}' "${@}" + } + ;; + "smartdns") + adb_dnscachecmd="" + adb_dnsinstance="${adb_dnsinstance:-"0"}" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp/smartdns"}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="address #" + f_dnsdeny() { + "${adb_awkcmd}" '{print "address /"$0"/#"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "address /"$0"/-"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" + + shift + "${adb_awkcmd}" -v item="${item}" '{print "cname /"$0"/"item""}' "${@}" + } + ;; + "raw") + adb_dnscachecmd="" + adb_dnsinstance="" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp"}" + adb_dnsheader="" + adb_dnsdeny="" + adb_dnsallow="" + adb_dnssafesearch="" + adb_dnsstop="" + ;; + esac + + # determine final dns file directory based on dns shifting + # + if [ "${adb_dnsshift}" = "0" ]; then + adb_finaldir="${adb_dnsdir}" + [ -L "${adb_dnsdir}/${adb_dnsfile}" ] && "${adb_rmcmd}" -f "${adb_dnsdir}/${adb_dnsfile}" + else + adb_finaldir="${adb_backupdir}" + fi + + # create dns file with header if it doesn't exist or dns flushing is enabled, also create backup and final directories if they don't exist + # + if [ "${adb_action}" != "stop" ]; then + for dir in "${adb_dnsdir:-"/tmp"}" "${adb_backupdir:-"/tmp"}"; do + [ ! -d "${dir}" ] && mkdir -p "${dir}" + done + if [ "${adb_dnsflush}" = "1" ] || [ "${free_mem:-"0"}" -lt "64" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_dnsup + elif [ ! -f "${adb_finaldir}/${adb_dnsfile}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" fi fi - f_log "debug" "f_dns ::: dns: ${adb_dns}, dns_dir: ${adb_dnsdir}, dns_file: ${adb_dnsfile}, dns_user: ${adb_dnsuser}, dns_instance: ${adb_dnsinstance}, backup: ${adb_backup}, backup_dir: ${adb_backupdir}, jail_dir: ${adb_jaildir}" + + # check if adblock is enabled + # + if [ "${adb_enabled}" = "0" ]; then + f_extconf + f_temp + f_nftremove + f_rmdns + f_jsnup "disabled" + f_log "info" "adblock is currently disabled, please set the config option 'adb_enabled' to '1' to use this service" + exit 0 + fi + + f_log "debug" "f_dns ::: dns: ${adb_dns}, dns_instance: ${adb_dnsinstance:-"-"}, dns_user: ${adb_dnsuser}, dns_dir: ${adb_dnsdir}, backup_dir: ${adb_backupdir}, final_dir: ${adb_finaldir}" } # load fetch utility # f_fetch() { - local util utils insecure cnt="0" - - if [ -z "${adb_fetchutil}" ]; then - utils="aria2c curl wget uclient-fetch" - for util in ${utils}; do - if { [ "${util}" = "uclient-fetch" ] && printf "%s" "${adb_packages}" | grep -q "^libustream-"; } || - { [ "${util}" = "wget" ] && printf "%s" "${adb_packages}" | grep -q "^wget -"; } || - [ "${util}" = "curl" ] || [ "${util}" = "aria2c" ]; then - if [ -x "$(command -v "${util}")" ]; then - adb_fetchutil="${util}" - uci_set adblock global adb_fetchutil "${util}" + local fetch fetch_list insecure update="0" + + adb_fetchcmd="$(command -v "${adb_fetchcmd}" 2>/dev/null)" + if [ -z "${adb_fetchcmd}" ]; then + fetch_list="curl wget-ssl libustream-openssl libustream-wolfssl libustream-mbedtls" + for fetch in ${fetch_list}; do + case "${adb_packages}" in *"\"${fetch}\""*) + case "${fetch}" in + "wget-ssl") + fetch="wget" + ;; + "libustream-openssl" | "libustream-wolfssl" | "libustream-mbedtls") + fetch="uclient-fetch" + ;; + esac + adb_fetchcmd="$(command -v "${fetch}" 2>/dev/null)" + if [ -n "${adb_fetchcmd}" ]; then + update="1" + uci_set adblock global adb_fetchcmd "${fetch}" f_uci "adblock" break fi - fi + ;; + esac done - elif [ ! -x "$(command -v "${adb_fetchutil}")" ]; then - unset adb_fetchutil fi - case "${adb_fetchutil}" in - "aria2c") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--check-certificate=false" - adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o"}" - ;; - "curl") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--insecure" - adb_fetchparm="${adb_fetchparm:-"${insecure} --connect-timeout 20 --fail --silent --show-error --location -o"}" - ;; - "uclient-fetch") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 -O"}" - ;; - "wget") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - adb_fetchparm="${adb_fetchparm:-"${insecure} --no-cache --no-cookies --max-redirect=0 --timeout=20 -O"}" - ;; + + [ -z "${adb_fetchcmd}" ] && f_log "err" "download utility with SSL support not found, please set 'adb_fetchcmd' manually" + + case "${adb_fetchcmd##*/}" in + "curl") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--insecure" + adb_fetchparm="${adb_fetchparm:-"${insecure} --connect-timeout 20 --retry-delay 10 --retry $((adb_fetchretry - 1)) --retry-max-time $(((adb_fetchretry - 1) * 20)) --retry-all-errors --fail --silent --show-error --location -o"}" + adb_etagparm="--connect-timeout 5 --silent --location --head" + adb_geoparm="--connect-timeout 5 --silent --location" + ;; + "wget") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + adb_fetchparm="${adb_fetchparm:-"${insecure} --no-cache --no-cookies --timeout=20 --waitretry=10 --tries=${adb_fetchretry} --retry-connrefused -O"}" + adb_etagparm="--timeout=5 --spider --server-response" + adb_geoparm="--timeout=5 --quiet -O-" + ;; + "uclient-fetch") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 -O"}" + adb_geoparm="--timeout=5 --quiet -O-" + ;; esac - if [ -n "${adb_fetchutil}" ] && [ -n "${adb_fetchparm}" ]; then - adb_fetchutil="$(command -v "${adb_fetchutil}")" - else - f_log "err" "download utility with SSL support not found, please install 'uclient-fetch' with a 'libustream-*' variant or another download utility like 'wget', 'curl' or 'aria2'" - fi - f_log "debug" "f_fetch ::: fetch_util: ${adb_fetchutil:-"-"}, fetch_parm: ${adb_fetchparm:-"-"}" + + f_log "debug" "f_fetch ::: update: ${update}, cmd: ${adb_fetchcmd:-"-"}, parm: ${adb_fetchparm:-"-"}, etag_parm: ${adb_etagparm:-"-"}, geo_parm: ${adb_geoparm:-"-"}" } # create temporary files, directories and set dependent options # f_temp() { - if [ -d "${adb_tmpbase}" ]; then - adb_tmpdir="$(mktemp -p "${adb_tmpbase}" -d)" + if [ -d "${adb_basedir}" ]; then + adb_tmpdir="$(mktemp -p "${adb_basedir}" -d)" adb_tmpload="$(mktemp -p "${adb_tmpdir}" -tu)" adb_tmpfile="$(mktemp -p "${adb_tmpdir}" -tu)" adb_srtopts="--temporary-directory=${adb_tmpdir} --compress-program=gzip --parallel=${adb_cores}" else - f_log "err" "the temp base directory '${adb_tmpbase}' does not exist/is not mounted yet, please create the directory or raise the 'adb_triggerdelay' to defer the adblock start" + f_log "err" "the base directory '${adb_basedir}' does not exist/is not mounted yet, please create the directory or raise the 'adb_triggerdelay' to defer the adblock start" fi - [ ! -s "${adb_pidfile}" ] && printf "%s" "${$}" >"${adb_pidfile}" - f_log "debug" "f_temp ::: tmp_base: ${adb_tmpbase:-"-"}, tmp_dir: ${adb_tmpdir:-"-"}, sort_options: ${adb_srtopts}, pid_file: ${adb_pidfile:-"-"}" + [ ! -s "${adb_pidfile}" ] && printf '%s' "${$}" >"${adb_pidfile}" } # remove temporary files and directories # f_rmtemp() { - [ -d "${adb_tmpdir}" ] && rm -rf "${adb_tmpdir}" - rm -f "${adb_srcfile}" + [ -f "${adb_errorlog}" ] && [ ! -s "${adb_errorlog}" ] && "${adb_rmcmd}" -f "${adb_errorlog}" + [ -d "${adb_tmpdir}" ] && "${adb_rmcmd}" -rf "${adb_tmpdir}" : >"${adb_pidfile}" - f_log "debug" "f_rmtemp ::: tmp_dir: ${adb_tmpdir:-"-"}, src_file: ${adb_srcfile:-"-"}, pid_file: ${adb_pidfile:-"-"}" } # remove dns related files # f_rmdns() { - local status - - status="$(ubus -S call service list '{"name":"adblock"}' 2>/dev/null | jsonfilter -l1 -e '@["adblock"].instances.*.running' 2>/dev/null)" - if [ "${adb_dns}" = "raw" ] || { [ -n "${adb_dns}" ] && [ -n "${status}" ]; }; then - : >"${adb_rtfile}" - [ "${adb_backup}" = "1" ] && rm -f "${adb_backupdir}/${adb_dnsprefix}".*.gz - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_dnsup 4 + if [ -n "${adb_finaldir}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_dnsup fi f_rmtemp - f_log "debug" "f_rmdns ::: dns: ${adb_dns}, status: ${status:-"-"}, dns_dir: ${adb_dnsdir}, dns_file: ${adb_dnsfile}, rt_file: ${adb_rtfile}, backup_dir: ${adb_backupdir:-"-"}" + if [ -d "${adb_backupdir}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_enabled}" = "0" ]; }; then + "${adb_findcmd}" "${adb_backupdir}" -maxdepth 1 -type f -name '*.gz' -exec "${adb_rmcmd}" -f {} + + fi } # commit uci changes # f_uci() { - local change config="${1}" - - if [ -n "${config}" ]; then - change="$(uci -q changes "${config}" | "${adb_awk}" '{ORS=" "; print $0}')" - if [ -n "${change}" ]; then - uci_commit "${config}" - case "${config}" in - "firewall") - "/etc/init.d/firewall" reload >/dev/null 2>&1 - ;; - "resolver") - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_count - f_jsnup "running" - "/etc/init.d/${adb_dns}" reload >/dev/null 2>&1 - ;; - esac + local config="${1}" + + if [ -n "$(uci -q changes "${config}")" ]; then + uci_commit "${config}" + if [ "${config}" = "resolver" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + adb_cnt="0" + f_jsnup "processing" + "/etc/init.d/${adb_dns}" reload >/dev/null 2>&1 fi - f_log "debug" "f_uci ::: config: ${config}, change: ${change}" fi } # get list counter # f_count() { - local file mode="${1}" name="${2}" + local files mode="${1}" file="${2}" var="${3}" adb_cnt="0" - case "${mode}" in - "iplist") - [ -s "${adb_tmpdir}/tmp.add.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.add.${name}")" - ;; - "blacklist") - [ -s "${adb_tmpfile}.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpfile}.${name}")" - ;; - "whitelist") - [ -s "${adb_tmpdir}/tmp.raw.${name}" ] && { adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.raw.${name}")"; rm -f "${adb_tmpdir}/tmp.raw.${name}"; } - ;; - "safesearch") - [ -s "${adb_tmpdir}/tmp.safesearch.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.safesearch.${name}")" - ;; - "merge") - [ -s "${adb_tmpdir}/${adb_dnsfile}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/${adb_dnsfile}")" - ;; - "download" | "backup" | "restore") - [ -s "${src_tmpfile}" ] && adb_cnt="$(wc -l 2>/dev/null <"${src_tmpfile}")" - ;; - "final") - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ]; then - adb_cnt="$(wc -l 2>/dev/null <"${adb_dnsdir}/${adb_dnsfile}")" - if [ -s "${adb_tmpdir}/tmp.add.whitelist" ]; then - adb_cnt="$((adb_cnt - $(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.add.whitelist")))" - fi - for file in "${adb_tmpdir}/tmp.safesearch".*; do - if [ -r "${file}" ]; then - adb_cnt="$((adb_cnt - $(wc -l 2>/dev/null <"${file}")))" - fi - done - [ -n "${adb_dnsheader}" ] && adb_cnt="$(((adb_cnt - $(printf "%b" "${adb_dnsheader}" | grep -c "^")) / 2))" - fi - ;; - esac + if [ -s "${file}" ]; then + if [ -n "${var}" ] || [ "${mode}" != "final" ]; then + adb_cnt="$("${adb_awkcmd}" 'END{print NR}' "${file}")" + [ -n "${var}" ] && printf '%s' "${adb_cnt}" + else + # build argument list: main file first, then all subtraction files + # + files="${file}" + [ -s "${adb_tmpdir}/tmp.add.allowlist" ] && files="${files} ${adb_tmpdir}/tmp.add.allowlist" + for file in "${adb_tmpdir}/tmp.safesearch".*; do + [ -s "${file}" ] && files="${files} ${file}" + done + adb_cnt="$("${adb_awkcmd}" -v hdr="${adb_dnsheader}" ' + NR == FNR { main++; next } + { sub_cnt++ } + END { + cnt = main - sub_cnt + if (hdr != "") { + hlines = gsub(/\n/, "", hdr) + cnt = int((cnt - hlines) / 2) + } + if (cnt < 0) cnt = 0 + res = "" + pos = 0 + s = sprintf("%d", cnt) + for (i = length(s); i > 0; i--) { + res = substr(s, i, 1) res + if (++pos == 3 && i > 1) { res = " " res; pos = 0 } + } + print res + } + ' ${files})" + fi + fi } # set external config options # f_extconf() { - local config config_dir config_file section zone port fwcfg + local config case "${adb_dns}" in - "dnsmasq") - config="dhcp" - config_dir="$(uci_get dhcp "@dnsmasq[${adb_dnsinstance}]" confdir | grep -Fo "${adb_dnsdir}")" - if [ "${adb_enabled}" = "1" ] && [ -z "${config_dir}" ]; then - uci_set dhcp "@dnsmasq[${adb_dnsinstance}]" confdir "${adb_dnsdir}" 2>/dev/null - fi - ;; - "kresd") - config="resolver" - config_file="$(uci_get resolver kresd rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" - if [ "${adb_enabled}" = "1" ] && [ -z "${config_file}" ]; then - uci -q add_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - elif [ "${adb_enabled}" = "0" ] && [ -n "${config_file}" ]; then - uci -q del_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - fi - ;; - esac - f_uci "${config}" - - config="firewall" - fwcfg="$(uci -qNX show "${config}" | "${adb_awk}" 'BEGIN{FS="[.=]"};/adblock_/{if(zone==$2){next}else{ORS=" ";zone=$2;print zone}}')" - if [ "${adb_enabled}" = "1" ] && [ "${adb_forcedns}" = "1" ] && - /etc/init.d/firewall enabled; then - for zone in ${adb_zonelist}; do - for port in ${adb_portlist}; do - if ! printf "%s" "${fwcfg}" | grep -q "adblock_${zone}${port}[ |\$]"; then - uci -q batch <<-EOC - set firewall."adblock_${zone}${port}"="redirect" - set firewall."adblock_${zone}${port}".name="Adblock DNS (${zone}, ${port})" - set firewall."adblock_${zone}${port}".src="${zone}" - set firewall."adblock_${zone}${port}".proto="tcp udp" - set firewall."adblock_${zone}${port}".src_dport="${port}" - set firewall."adblock_${zone}${port}".dest_port="${port}" - set firewall."adblock_${zone}${port}".target="DNAT" - set firewall."adblock_${zone}${port}".family="any" - set firewall."adblock_${zone}${port}".ipset="!tsdns_bypass" - add_list firewall."adblock_${zone}${port}".ns_tag="automated" - EOC - fi - fwcfg="${fwcfg/adblock_${zone}${port}[ |\$]/}" - done - done - fwcfg="${fwcfg#"${fwcfg%%[![:space:]]*}"}" - fwcfg="${fwcfg%"${fwcfg##*[![:space:]]}"}" - fi - if [ "${adb_enabled}" = "0" ] || [ "${adb_forcedns}" = "0" ] || [ -n "${fwcfg}" ]; then - for section in ${fwcfg}; do - uci_remove firewall "${section}" - done - fi - - # add adb_bypass - if [ "${adb_enabled}" = "1" ] && [ "${adb_forcedns}" = "1" ] && /etc/init.d/firewall enabled; then - if ! uci -q get firewall.tsdns_bypass >/dev/null; then - uci -q batch <<-EOC - set firewall.tsdns_bypass="ipset" - set firewall.tsdns_bypass.name="tsdns_bypass" - set firewall.tsdns_bypass.match="src_net" - set firewall.tsdns_bypass.enabled="1" - EOC + "dnsmasq") + config="dhcp" + if [ "${adb_dnsshift}" = "1" ] && + ! uci_get ${config} @dnsmasq[${adb_dnsinstance}] addnmount | "${adb_grepcmd}" -q "${adb_backupdir}"; then + uci -q add_list ${config}.@dnsmasq[${adb_dnsinstance}].addnmount="${adb_backupdir}" + elif [ "${adb_dnsshift}" = "0" ] && + uci_get ${config} @dnsmasq[${adb_dnsinstance}] addnmount | "${adb_grepcmd}" -q "${adb_backupdir}"; then + uci -q del_list ${config}.@dnsmasq[${adb_dnsinstance}].addnmount="${adb_backupdir}" fi - # note: adb_bypass var contains an extra space at the beginning - if [ " $(uci -q get firewall.tsdns_bypass.entry)" != "${adb_bypass}" ]; then - # make sure bypass list is always empty - uci -q delete firewall.tsdns_bypass.entry - for src in ${adb_bypass} - do - uci -q add_list firewall.tsdns_bypass.entry="${src}" - done + ;; + "kresd") + config="resolver" + if [ "${adb_enabled}" = "1" ] && + ! uci_get ${config} kresd rpz_file | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q add_list ${config}.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_enabled}" = "0" ] && + uci_get ${config} kresd rpz_file | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q del_list ${config}.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" fi - fi - - # remove adb_bypass - if [ "${adb_enabled}" = "0" ] || [ "${adb_forcedns}" = "0" ]; then - uci -q delete firewall.tsdns_bypass - fi - + ;; + "smartdns") + config="smartdns" + if [ "${adb_enabled}" = "1" ] && + ! uci_get ${config} @${config}[${adb_dnsinstance}] conf_files | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q add_list ${config}.@${config}[${adb_dnsinstance}].conf_files="${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_enabled}" = "0" ] && + uci_get ${config} @${config}[${adb_dnsinstance}] conf_files | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q del_list ${config}.@${config}[${adb_dnsinstance}].conf_files="${adb_dnsdir}/${adb_dnsfile}" + fi + ;; + esac f_uci "${config}" } # restart dns backend # f_dnsup() { - local rset dns_service dns_up dns_pid restart_rc cnt="0" out_rc="4" in_rc="${1:-0}" + local restart_rc nft_rc cnt="0" out_rc="4" - if [ "${adb_dns}" = "raw" ]; then + if [ "${adb_dns}" = "raw" ] || [ -z "${adb_dns}" ]; then out_rc="0" else - if [ "${in_rc}" = "0" ] && [ "${adb_dnsflush}" = "0" ]; then + + # load external dns bridge + # + if { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; } && [ "${adb_nftbridge}" = "1" ]; then + if "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if [ -n "${adb_bridgednsv4}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_bridgednsv4}:53 2>>"${adb_errorlog}" + nft_rc="${?}" + fi + if [ -n "${adb_bridgednsv6}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_bridgednsv6}]:53 2>>"${adb_errorlog}" + nft_rc="$((nft_rc + $?))" + fi + if [ "${nft_rc}" = "0" ]; then + f_log "debug" "external DNS bridge loaded: ${adb_bridgednsv4:-"-"} / ${adb_bridgednsv6:-"-"}" + else + f_log "err" "failed to load external DNS bridge: ${adb_bridgednsv4:-"-"} / ${adb_bridgednsv6:-"-"}" + fi + fi + fi + + # restart dns backend + # + if [ "${adb_dnsflush}" = "0" ]; then case "${adb_dns}" in - "unbound") - if [ -x "${adb_dnscachecmd}" ] && [ -d "${adb_tmpdir}" ] && [ -f "${adb_dnsdir}/unbound.conf" ]; then - "${adb_dnscachecmd}" -c "${adb_dnsdir}/unbound.conf" dump_cache >"${adb_tmpdir}/adb_cache.dump" 2>/dev/null - fi - "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + "unbound") + if [ -x "${adb_dnscachecmd}" ] && [ -d "${adb_tmpdir}" ] && [ -f "${adb_dnsdir}/unbound.conf" ]; then + "${adb_dnscachecmd}" -c "${adb_dnsdir}/unbound.conf" dump_cache >"${adb_tmpdir}/adb_cache.dump" 2>>"${adb_errorlog}" + fi + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + restart_rc="${?}" + ;; + "named") + if [ -x "${adb_dnscachecmd}" ] && [ -f "/etc/bind/rndc.conf" ]; then + "${adb_dnscachecmd}" -c "/etc/bind/rndc.conf" reload >/dev/null 2>&1 restart_rc="${?}" - ;; - "named") - if [ -x "${adb_dnscachecmd}" ] && [ -f "/etc/bind/rndc.conf" ]; then - "${adb_dnscachecmd}" -c "/etc/bind/rndc.conf" reload >/dev/null 2>&1 - restart_rc="${?}" - fi - if [ -z "${restart_rc}" ] || { [ -n "${restart_rc}" ] && [ "${restart_rc}" != "0" ]; }; then - "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 - restart_rc="${?}" - fi - ;; - *) + fi + if [ -z "${restart_rc}" ] || { [ -n "${restart_rc}" ] && [ "${restart_rc}" != "0" ]; }; then "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 restart_rc="${?}" - ;; + fi + ;; + *) + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + restart_rc="${?}" + ;; esac fi if [ -z "${restart_rc}" ]; then @@ -652,24 +783,14 @@ f_dnsup() { restart_rc="${?}" fi fi + + # check if dns backend is responsive, restore dns cache for unbound and get dns backend pid + # if [ "${restart_rc}" = "0" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" while [ "${cnt}" -le "${adb_dnstimeout}" ]; do - dns_service="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}")" - dns_up="$(printf "%s" "${dns_service}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running")" - dns_pid="$(printf "%s" "${dns_service}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.pid")" - if [ "${dns_up}" = "true" ] && [ -n "${dns_pid}" ] && ! ls "/proc/${dns_pid}/fd/${adb_dnsdir}/${adb_dnsfile}" >/dev/null 2>&1; then - if [ -x "${adb_lookupcmd}" ] && [ -n "$(printf "%s" "${adb_lookupdomain}" | "${adb_awk}" "${rset}")" ]; then - if "${adb_lookupcmd}" "${adb_lookupdomain}" >/dev/null 2>&1; then - out_rc="0" - break - fi - else - sleep ${adb_dnstimeout} - cnt=${adb_dnstimeout} - out_rc="0" - break - fi + if "${adb_lookupcmd}" "${adb_lookupdomain}." >/dev/null 2>&1; then + out_rc="0" + break fi cnt="$((cnt + 1))" sleep 1 @@ -681,231 +802,519 @@ f_dnsup() { fi fi fi - f_log "debug" "f_dnsup ::: dns: ${adb_dns}, cache_cmd: ${adb_dnscachecmd:-"-"}, lookup_cmd: ${adb_lookupcmd:-"-"}, lookup_domain: ${adb_lookupdomain:-"-"}, restart_rc: ${restart_rc:-"-"}, dns_flush: ${adb_dnsflush}, dns_timeout: ${adb_dnstimeout}, dns_cnt: ${cnt}, in_rc: ${in_rc}, out_rc: ${out_rc}" + + # remove external dns bridge + # + if [ "${adb_nftbridge}" = "1" ] && "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + "${adb_nftcmd}" flush chain inet adblock dns-bridge 2>>"${adb_errorlog}" + nft_rc="${?}" + if [ "${nft_rc}" = "0" ]; then + f_log "debug" "external DNS bridge removed" + else + f_log "err" "failed to remove external DNS bridge" + fi + fi + + f_log "debug" "f_dnsup ::: dns: ${adb_dns}, cache_cmd: ${adb_dnscachecmd:-"-"}, lookup_domain: ${adb_lookupdomain:-"-"}, restart_rc: ${restart_rc:-"-"}, dns_flush: ${adb_dnsflush}, dns_timeout: ${adb_dnstimeout}, dns_cnt: ${cnt}, nft_rc: ${nft_rc:-"-"}, rc: ${out_rc}" return "${out_rc}" } +# handle etag http header +# +f_etag() { + local http_head http_code etag_id etag_cnt out_rc="4" feed="${1}" feed_url="${2}" feed_suffix="${3}" feed_cnt="${4:-"1"}" + + if [ -n "${adb_etagparm}" ]; then + + # ensure etag file exists + # + [ ! -f "${adb_backupdir}/adblock.etag" ] && : >"${adb_backupdir}/adblock.etag" + + # fetch http headers and extract http code and etag/last-modified header + # + http_head="$("${adb_fetchcmd}" ${adb_etagparm} "${feed_url}${feed_suffix}" 2>&1)" + http_code="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*http\/[0-9.]+ /{printf "%s",$2}')" + etag_id="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*etag: /{gsub("\"","");printf "%s",$2}')" + + # if etag header is not present, try to use last-modified header as fallback for change detection + # + if [ -z "${etag_id}" ]; then + etag_id="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*last-modified: /{gsub(/[Ll]ast-[Mm]odified:|[[:space:]]|,|:/,"");printf "%s\n",$1}')" + fi + + # acquire exclusive lock on etag file to serialize concurrent read-modify-write from parallel feeds + # + exec 9>"${adb_etaglock}" + "${adb_flockcmd}" -x 9 + + # compare http code and etag id with stored values, update etag file and return code accordingly + # + etag_cnt="$("${adb_awkcmd}" -v f="${feed}" '$1 == f { n++ } END { print n+0 }' "${adb_backupdir}/adblock.etag")" + if [ "${http_code}" = "200" ] && [ "${etag_cnt}" = "${feed_cnt}" ] && [ -n "${etag_id}" ] && + "${adb_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" -v e="${etag_id}" ' + BEGIN { rc = 1; p = f " " s } + index($0, p) == 1 { + rest = substr($0, length(p) + 1) + sub(/^[[:space:]]+/, "", rest) + if (rest == e) { rc = 0; exit } + } + END { exit rc }' "${adb_backupdir}/adblock.etag"; then + out_rc="0" + elif [ -n "${etag_id}" ]; then + + # if feed count is less than etag count, it means the feed source has been removed or disabled, so remove all entries for this feed, + # otherwise only remove the entry with the matching feed suffix (feed url) to allow multiple sources for the same feed + # + if [ "${feed_cnt}" -lt "${etag_cnt}" ]; then + "${adb_awkcmd}" -v f="${feed}" '$1 != f' \ + "${adb_backupdir}/adblock.etag" >"${adb_backupdir}/adblock.etag.new" + else + "${adb_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" ' + BEGIN { p = f " " s } + index($0, p) != 1' \ + "${adb_backupdir}/adblock.etag" >"${adb_backupdir}/adblock.etag.new" + fi + "${adb_mvcmd}" -f "${adb_backupdir}/adblock.etag.new" "${adb_backupdir}/adblock.etag" + printf '%s\t%s\n' "${feed} ${feed_suffix}" "${etag_id}" >>"${adb_backupdir}/adblock.etag" + out_rc="2" + fi + + # release lock + # + exec 9>&- + fi + + f_log "debug" "f_etag ::: feed: ${feed}, suffix: ${feed_suffix:-"-"}, http_code: ${http_code:-"-"}, feed/etag: ${feed_cnt}/${etag_cnt:-"0"}, rc: ${out_rc}" + return "${out_rc}" +} + +# add adblock-related nft rules +# +f_nftadd() { + local devices device port file="${adb_tmpdir}/adb_nft.add" + + # only proceed if at least one feature is enabled + # + if [ "${adb_nftallow}" = "0" ] && [ "${adb_nftblock}" = "0" ] && + [ "${adb_nftremote}" = "0" ] && [ "${adb_nftforce}" = "0" ] && + [ "${adb_nftbridge}" = "0" ]; then + return + fi + + # remove existing adblock-related nft rules on restart + # + [ "${adb_action}" = "restart" ] && f_nftremove + + # do not proceed if adblock-related nft rules already exists + # + if "${adb_nftcmd}" -t list table inet adblock >/dev/null 2>&1; then + return + fi + + # do not proceed if action is stop/suspend/resume + # + if [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "suspend" ] || [ "${adb_action}" = "resume" ]; then + return + fi + + # prepare nftables rules for adblock features + # + { + # nft header (tables, sets, base and regular chains) + # + printf '%s\n\n' "#!${adb_nftcmd} -f" + printf '%s\n' "add table inet adblock" + + # allow Set + # + if [ "${adb_nftallow}" = "1" ] && [ -n "${adb_nftmacallow}" ]; then + printf '%s\n' "add set inet adblock mac_allow { type ether_addr; flags interval; auto-merge; elements = { ${adb_nftmacallow// /, } }; }" + fi + + # remote allow Set with timeout, for MACs that should be temporary allowed to bypass dns blocking + # + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ]; then + printf '%s\n' "add set inet adblock mac_remote { type ether_addr; flags timeout; timeout ${adb_nftremotetimeout}m; }" + fi + + # adblock pre-routing chain for allow/block rules + # + if [ "${adb_nftblock}" = "1" ] && [ -n "${adb_nftmacblock}" ]; then + printf '%s\n' "add set inet adblock mac_block { type ether_addr; flags interval; auto-merge; elements = { ${adb_nftmacblock// /, } }; }" + fi + printf '%s\n' "add chain inet adblock pre-routing { type nat hook prerouting priority -150; policy accept; }" + printf '%s\n' "add chain inet adblock _reject" + + # dns-bridge base chain + # + printf '%s\n' "add chain inet adblock dns-bridge { type nat hook prerouting priority -160; policy accept; }" + + # reject chain rules + # + printf '%s\n' "add rule inet adblock _reject meta l4proto tcp counter reject with tcp reset" + printf '%s\n' "add rule inet adblock _reject counter reject with icmpx host-unreachable" + + # external allow rules + # + if [ "${adb_nftallow}" = "1" ]; then + if [ -n "${adb_nftmacallow}" ]; then + [ -n "${adb_allowdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_allow meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_allowdnsv4}:53" + [ -n "${adb_allowdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_allow meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_allowdnsv6}]:53" + fi + for device in ${adb_nftdevallow}; do + [ -n "${adb_allowdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_allowdnsv4}:53" + [ -n "${adb_allowdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_allowdnsv6}]:53" + done + f_log "debug" "adblock-related nft allow rules prepared for external DNS ${adb_allowdnsv4:-"-"} / ${adb_allowdnsv6:-"-"}" + fi + + # external remote allow rules + # + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ]; then + [ -n "${adb_remotednsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_remote meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_remotednsv4}:53" + [ -n "${adb_remotednsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_remote meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_remotednsv6}]:53" + f_log "debug" "adblock-related nft remote allow rules prepared for external DNS ${adb_remotednsv4:-"-"} / ${adb_remotednsv6:-"-"} with timeout of ${adb_nftremotetimeout} minutes" + fi + + # external block rules + # + if [ "${adb_nftblock}" = "1" ]; then + if [ -n "${adb_nftmacblock}" ]; then + [ -n "${adb_blockdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_block meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_blockdnsv4}:53" + [ -n "${adb_blockdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_block meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_blockdnsv6}]:53" + fi + for device in ${adb_nftdevblock}; do + [ -n "${adb_blockdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_blockdnsv4}:53" + [ -n "${adb_blockdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_blockdnsv6}]:53" + done + f_log "debug" "adblock-related nft block rules prepared for external DNS ${adb_blockdnsv4:-"-"} / ${adb_blockdnsv6:-"-"}" + fi + + # local dns enforcement + # + if [ "${adb_nftforce}" = "1" ]; then + + # device/vlan exceptions + # + for device in ${adb_nftdevallow} ${adb_nftdevblock}; do + case " ${devices} " in + *" ${device} "*) ;; + + *) + [ -n "${devices}" ] && devices="${devices} ${device}" || devices="${device}" + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" return" + ;; + esac + done + + # mac exceptions + # + for device in ${adb_nftdevforce}; do + if [ "${adb_nftallow}" = "1" ] && [ -n "${adb_nftmacallow}" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ether saddr @mac_allow return" + fi + if [ "${adb_nftblock}" = "1" ] && [ -n "${adb_nftmacblock}" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ether saddr @mac_block return" + fi + + # NethSecurity: IP-based bypass exceptions + # + for bypass_ip in ${ns_tsdns_bypass}; do + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ip saddr ${bypass_ip} return" + done + + # dns enforce rules + # + for port in ${adb_nftportforce}; do + if [ "${port}" = "53" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto { ipv4, ipv6 } meta l4proto { udp, tcp } th dport ${port} counter redirect to :${port}" + else + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto { ipv4, ipv6 } meta l4proto { udp, tcp } th dport ${port} counter goto _reject" + fi + done + done + f_log "debug" "adblock-related nft local DNS enforcement rules prepared for devices: ${adb_nftdevforce// /, } and ports: ${adb_nftportforce// /, }" + fi + } >"${file}" + if "${adb_nftcmd}" -f "${file}" 2>>"${adb_errorlog}"; then + f_log "info" "adblock-related nft rules loaded" + else + f_log "err" "failed to load adblock-related nft rules" + fi +} + +# remove adblock-related nft rules +# +f_nftremove() { + local file="${adb_tmpdir}/adb_nft.remove" + + if "${adb_nftcmd}" -t list table inet adblock >/dev/null 2>&1; then + { + printf '%s\n' "#!${adb_nftcmd} -f" + printf '%s\n' "delete table inet adblock" + } >"${file}" + + if "${adb_nftcmd}" -f "${file}" 2>>"${adb_errorlog}"; then + f_log "info" "adblock-related nft rules removed" + else + f_log "err" "failed to remove adblock-related nft rules" + fi + fi +} + # backup/restore/remove blocklists # f_list() { - local hold file rset item array safe_url safe_ips safe_cname safe_domains ip out_rc mode="${1}" src_name="${2:-"${src_name}"}" in_rc="${src_rc:-0}" cnt ffiles="-maxdepth 1 -name ${adb_dnsprefix}.*.gz" + local file files item array safe_url safe_ips safe_cname safe_domains ip out_rc file_name name + local mode="${1}" src_name="${2:-"${src_name}"}" in_rc="${src_rc:-0}" use_cname="0" case "${mode}" in - "iplist") - src_name="${mode}" - if [ "${adb_dns}" = "named" ]; then - rset="BEGIN{FS=\"[.:]\";pfx=\"32\"}{if(match(\$0,/:/))pfx=\"128\"}{printf \"%s.\",pfx;for(seg=NF;seg>=1;seg--)if(seg==1)printf \"%s\n\",\$seg;else if(\$seg>=0)printf \"%s.\",\$seg; else printf \"%s.\",\"zz\"}" - if [ -n "${adb_allowip}" ]; then - : >"${adb_tmpdir}/tmp.raw.${src_name}" - for ip in ${adb_allowip}; do - printf "%s" "${ip}" | "${adb_awk}" "${rset}" >>"${adb_tmpdir}/tmp.raw.${src_name}" - done - eval "${adb_dnsallowip}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.add.${src_name}" + "blocklist" | "allowlist") + src_name="${mode}" + case "${src_name}" in + "blocklist") + if [ -f "${adb_blocklist}" ]; then + file_name="${adb_tmpfile}.${src_name}" + f_chkdom local 1 <"${adb_blocklist}" >"${adb_tmpdir}/tmp.raw.${src_name}" + if [ "${adb_tld}" = "1" ]; then + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" ' + NR==FNR { member[$1]; next } + !($1 in member) { + n = split($1, seg, ".") + for (f = n; f > 1; f--) printf "%s.", seg[f] + print seg[1] + } + ' "${adb_tmpdir}/tmp.rem.allowlist" "${adb_tmpdir}/tmp.raw.${src_name}" + else + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${adb_tmpdir}/tmp.raw.${src_name}" + fi | "${adb_sortcmd}" ${adb_srtopts} -u >"${file_name}" out_rc="${?}" - fi - if [ -n "${adb_denyip}" ] && { [ -z "${out_rc}" ] || [ "${out_rc}" = "0" ]; }; then - : >"${adb_tmpdir}/tmp.raw.${src_name}" - for ip in ${adb_denyip}; do - printf "%s" "${ip}" | "${adb_awk}" "${rset}" >>"${adb_tmpdir}/tmp.raw.${src_name}" - done - eval "${adb_dnsdenyip}" "${adb_tmpdir}/tmp.raw.${src_name}" >>"${adb_tmpdir}/tmp.add.${src_name}" + else + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" 'NR==FNR{member[$1];next}!($1 in member)' "${adb_tmpdir}/tmp.rem.allowlist" "${adb_tmpdir}/tmp.raw.${src_name}" | + "${adb_sortcmd}" ${adb_srtopts} -u >"${file_name}" 2>>"${adb_errorlog}" + else + "${adb_sortcmd}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.raw.${src_name}" 2>>"${adb_errorlog}" >"${file_name}" + fi out_rc="${?}" fi - rm -f "${adb_tmpdir}/tmp.raw.${src_name}" fi ;; - "blacklist" | "whitelist") - src_name="${mode}" - if [ "${src_name}" = "blacklist" ] && [ -f "${adb_blacklist}" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" - "${adb_awk}" "${rset}" "${adb_blacklist}" >"${adb_tmpdir}/tmp.raw.${src_name}" - if [ -s "${adb_whitelist}" ]; then - "${adb_awk}" 'NR==FNR{member[$1];next}!($1 in member)' "${adb_whitelist}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.deduplicate.${src_name}" - else - cat "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.deduplicate.${src_name}" - fi - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${adb_tmpdir}/tmp.deduplicate.${src_name}" >"${adb_tmpdir}/tmp.raw.${src_name}" - "${adb_sort}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.raw.${src_name}" 2>/dev/null >"${adb_tmpfile}.${src_name}" - out_rc="${?}" - rm -f "${adb_tmpdir}/tmp.raw.${src_name}" - elif [ "${src_name}" = "whitelist" ] && [ -f "${adb_whitelist}" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" - printf "%s\n" "${adb_lookupdomain}" | "${adb_awk}" "${rset}" >"${adb_tmpdir}/tmp.raw.${src_name}" - "${adb_awk}" "${rset}" "${adb_whitelist}" >>"${adb_tmpdir}/tmp.raw.${src_name}" + "allowlist") + if [ -f "${adb_allowlist}" ] && [ "${adb_dnsallow}" = "1" ]; then + file_name="${adb_tmpdir}/tmp.raw.${src_name}" + [ "${adb_lookupdomain}" != "localhost" ] && { printf '%s\n' "${adb_lookupdomain}" | f_chkdom local 1; } >"${file_name}" + f_chkdom local 1 <"${adb_allowlist}" >>"${file_name}" + "${adb_catcmd}" "${file_name}" >"${adb_tmpdir}/tmp.rem.${src_name}" + f_dnsallow "${file_name}" >"${adb_tmpdir}/tmp.add.${src_name}" out_rc="${?}" - if [ "${out_rc}" = "0" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{gsub(\"\\\\.\",\"\\\\.\",\$1);print tolower(\"^(|.*\\\\.)\"\$1\"$\")}" - "${adb_awk}" "${rset}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.rem.${src_name}" - out_rc="${?}" - if [ "${out_rc}" = "0" ] && [ "${adb_dnsallow}" != "1" ]; then - eval "${adb_dnsallow}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.add.${src_name}" - out_rc="${?}" - if [ "${out_rc}" = "0" ] && [ "${adb_jail}" = "1" ] && [ "${adb_dnsstop}" != "0" ]; then - : >"${adb_jaildir}/${adb_dnsjail}" - [ -n "${adb_dnsheader}" ] && printf "%b" "${adb_dnsheader}" >>"${adb_jaildir}/${adb_dnsjail}" - cat "${adb_tmpdir}/tmp.add.${src_name}" >>"${adb_jaildir}/${adb_dnsjail}" - printf "%s\n" "${adb_dnsstop}" >>"${adb_jaildir}/${adb_dnsjail}" - fi - fi + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_tmpdir}/${adb_dnsfile}" + "${adb_catcmd}" "${adb_tmpdir}/tmp.add.${src_name}" >>"${adb_tmpdir}/${adb_dnsfile}" + printf '%b\n' "${adb_dnsstop}" >>"${adb_tmpdir}/${adb_dnsfile}" fi fi ;; - "safesearch") - case "${src_name}" in - "google") - rset="/^\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{printf \"%s\n%s\n\",tolower(\"www\"\$1),tolower(substr(\$1,2,length(\$1)))}" - safe_url="https://www.google.com/supported_domains" - safe_cname="forcesafesearch.google.com" - safe_domains="${adb_tmpdir}/tmp.load.safesearch.${src_name}" - if [ "${adb_backup}" = "1" ] && [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ]; then - zcat "${adb_backupdir}/safesearch.${src_name}.gz" >"${safe_domains}" - out_rc="${?}" - else - "${adb_fetchutil}" ${adb_fetchparm} "${safe_domains}" "${safe_url}" 2>/dev/null - out_rc="${?}" - if [ "${adb_backup}" = "1" ] && [ "${out_rc}" = "0" ]; then - gzip -cf "${safe_domains}" >"${adb_backupdir}/safesearch.${src_name}.gz" - out_rc="${?}" - fi - fi - if [ "${out_rc}" = "0" ]; then - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && "${adb_awk}" "${rset}" "${safe_domains}" >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - fi - ;; - "bing") - safe_cname="strict.bing.com" - safe_domains="www.bing.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "duckduckgo") - safe_cname="safe.duckduckgo.com" - safe_domains="duckduckgo.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "pixabay") - safe_cname="safesearch.pixabay.com" - safe_domains="pixabay.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "yandex") - safe_cname="familysearch.yandex.ru" - safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az yandex.kz" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "youtube") - if [ "${adb_safesearchmod}" = "0" ]; then - safe_cname="restrict.youtube.com" - else - safe_cname="restrictmoderate.youtube.com" - fi - safe_domains="www.youtube.com m.youtube.com youtubei.googleapis.com youtube.googleapis.com www.youtube-nocookie.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - esac - if [ "${out_rc}" = "0" ] && [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ]; then - : >"${adb_tmpdir}/tmp.safesearch.${src_name}" - [ "${adb_dns}" = "named" ] && array="${safe_cname}" || array="${safe_ips}" - for item in ${array}; do - if ! eval "${adb_dnssafesearch}" "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" >>"${adb_tmpdir}/tmp.safesearch.${src_name}"; then - rm -f "${adb_tmpdir}/tmp.safesearch.${src_name}" - break - fi - done - out_rc="${?}" - rm -f "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + esac + ;; + "safesearch") + file_name="${adb_tmpdir}/tmp.safesearch.${src_name}" + case "${adb_dns}" in + "named" | "kresd" | "smartdns") + use_cname="1" + ;; + esac + case "${src_name}" in + "google") + safe_url="https://www.google.com/supported_domains" + safe_cname="forcesafesearch.google.com" + + # refresh if cache is missing or older than 30 days + # + if [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ] && + [ -z "$("${adb_findcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" -mtime +30 2>>"${adb_errorlog}")" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" >"${adb_tmpdir}/tmp.load.safesearch.${src_name}" + else + "${adb_fetchcmd}" ${adb_fetchparm} "${adb_tmpdir}/tmp.load.safesearch.${src_name}" "${safe_url}" 2>>"${adb_errorlog}" + if [ -s "${adb_tmpdir}/tmp.load.safesearch.${src_name}" ]; then + "${adb_gzipcmd}" -cf "${adb_tmpdir}/tmp.load.safesearch.${src_name}" >"${adb_backupdir}/safesearch.${src_name}.gz" + elif [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" >"${adb_tmpdir}/tmp.load.safesearch.${src_name}" + fi fi ;; - "backup") - ( - gzip -cf "${src_tmpfile}" >"${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" - out_rc="${?}" - ) & + "bing") + safe_cname="strict.bing.com" + safe_domains="www.bing.com" ;; - "restore") - if [ -n "${src_name}" ] && [ -s "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ]; then - zcat "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" >"${src_tmpfile}" - out_rc="${?}" - elif [ -z "${src_name}" ]; then - cnt="1" - for file in "${adb_backupdir}/${adb_dnsprefix}".*.gz; do - if [ -r "${file}" ]; then - name="${file##*/}" - name="${name%.*}" - zcat "${file}" >"${adb_tmpfile}.${name}" & - hold="$((cnt % adb_cores))" - if [ "${hold}" = "0" ]; then - wait - fi - cnt="$((cnt + 1))" - fi - done - wait - out_rc="${?}" + "brave") + safe_cname="forcesafe.search.brave.com" + safe_domains="search.brave.com" + ;; + "duckduckgo") + safe_cname="safe.duckduckgo.com" + safe_domains="duckduckgo.com" + ;; + "pixabay") + safe_cname="safesearch.pixabay.com" + safe_domains="pixabay.com" + ;; + "yandex") + safe_cname="familysearch.yandex.ru" + safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az yandex.kz" + ;; + "youtube") + safe_cname="restrict.youtube.com" + safe_domains="www.youtube.com m.youtube.com youtubei.googleapis.com youtube.googleapis.com www.youtube-nocookie.com" + ;; + esac + if [ -n "${safe_domains}" ] && [ -n "${safe_cname}" ]; then + if [ "${use_cname}" = "0" ]; then + safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>>"${adb_errorlog}" | "${adb_awkcmd}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" + fi + if [ -n "${safe_ips}" ] || [ "${use_cname}" = "1" ]; then + printf '%s\n' ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + [ "${use_cname}" = "1" ] && array="${safe_cname}" || array="${safe_ips}" + fi + fi + if [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ]; then + : >"${file_name}" + for item in ${array}; do + if ! f_dnssafesearch "${item}" "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" >>"${file_name}"; then + : >"${file_name}" + break + fi + done + : >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + out_rc="0" + fi + ;; + "prepare") + file_name="${src_tmpfile}" + if [ -s "${src_tmpload}" ]; then + if [ "${adb_tld}" = "1" ]; then + f_chkdom ${src_rset} <"${src_tmpload}" | + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | + "${adb_sortcmd}" ${adb_srtopts} -u >"${src_tmpfile}" 2>>"${adb_errorlog}" else - out_rc=4 + f_chkdom ${src_rset} <"${src_tmpload}" | + "${adb_sortcmd}" ${adb_srtopts} -u >"${src_tmpfile}" 2>>"${adb_errorlog}" fi - if [ "${adb_action}" != "start" ] && [ "${adb_action}" != "resume" ] && [ -n "${src_name}" ] && [ "${out_rc}" != "0" ]; then - adb_sources="${adb_sources/${src_name}}" + out_rc="${?}" + if [ "${out_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then + f_list backup + elif [ "${adb_action}" != "boot" ] && [ "${adb_action}" != "start" ]; then + f_log "info" "preparation of '${src_name}' failed, rc: ${src_rc}" + f_list restore + out_rc="${?}" + : >"${src_tmpfile}" fi - ;; - "remove") - [ "${adb_backup}" = "1" ] && rm "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 2>/dev/null + else + f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}" + if [ "${adb_action}" != "boot" ] && [ "${adb_action}" != "start" ]; then + f_list restore + out_rc="${?}" + fi + fi + ;; + "backup") + file_name="${src_tmpfile}" + "${adb_gzipcmd}" -cf "${src_tmpfile}" >"${adb_backupdir}/adb_list.${src_name}.gz" + out_rc="${?}" + ;; + "restore") + file_name="${src_tmpfile}" + if [ -n "${src_name}" ] && [ -s "${adb_backupdir}/adb_list.${src_name}.gz" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/adb_list.${src_name}.gz" >"${src_tmpfile}" out_rc="${?}" - adb_sources="${adb_sources/${src_name}}" - ;; - "merge") - if [ "${adb_backup}" = "1" ]; then - for src_name in ${adb_sources}; do - ffiles="${ffiles} -a ! -name ${adb_dnsprefix}.${src_name}.gz" - done - if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ]; then - ffiles="${ffiles} -a ! -name safesearch.google.gz" + elif [ -z "${src_name}" ]; then + for file in "${adb_backupdir}/adb_list."*.gz; do + if [ -r "${file}" ]; then + name="${file##*/}" + name="${name%.*}" + "${adb_zcatcmd}" "${file}" >"${adb_tmpfile}.${name}" + out_rc="${?}" + [ "${out_rc}" != "0" ] && break fi - find "${adb_backupdir}" ${ffiles} -print0 2>/dev/null | xargs -0 rm 2>/dev/null + done + else + out_rc="4" + fi + case "${adb_action}" in + "boot" | "start" | "restart" | "resume") ;; + + *) + if [ -n "${src_name}" ] && [ "${out_rc}" != "0" ]; then + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" fi - unset src_name - "${adb_sort}" ${adb_srtopts} -mu "${adb_tmpfile}".* 2>/dev/null >"${adb_tmpdir}/${adb_dnsfile}" - out_rc="${?}" - rm -f "${adb_tmpfile}".* ;; - "final") - unset src_name - { [ -n "${adb_dnsheader}" ] && printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}"; } || : >"${adb_dnsdir}/${adb_dnsfile}" - [ -s "${adb_tmpdir}/tmp.add.iplist" ] && cat "${adb_tmpdir}/tmp.add.iplist" >>"${adb_dnsdir}/${adb_dnsfile}" - [ -s "${adb_tmpdir}/tmp.add.whitelist" ] && cat "${adb_tmpdir}/tmp.add.whitelist" >>"${adb_dnsdir}/${adb_dnsfile}" - for file in "${adb_tmpdir}/tmp.safesearch".*; do - [ -r "${file}" ] && cat "${file}" >>"${adb_dnsdir}/${adb_dnsfile}" - done - { [ "${adb_dnsdeny}" != "0" ] && eval "${adb_dnsdeny}" "${adb_tmpdir}/${adb_dnsfile}" >>"${adb_dnsdir}/${adb_dnsfile}"; } || mv "${adb_tmpdir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + esac + ;; + "remove") + "${adb_rmcmd}" "${adb_backupdir}/adb_list.${src_name}.gz" 2>>"${adb_errorlog}" + out_rc="${?}" + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" + ;; + "merge") + src_name="" + file_name="${adb_tmpdir}/${adb_dnsfile}" + + # remove stale backup files + # + files="" + for file in ${adb_feed}; do + files="${files} ! -name adb_list.${file}.gz" + done + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" = "1" ]; then + files="${files} ! -name safesearch.google.gz" + fi + "${adb_findcmd}" "${adb_backupdir}" -maxdepth 1 -type f -name '*.gz' ${files} -print0 2>>"${adb_errorlog}" | "${adb_xargscmd}" -0r "${adb_rmcmd}" -f + + # merge files + # + for file in "${adb_tmpfile}".*; do + [ -e "${file}" ] || continue + "${adb_sortcmd}" ${adb_srtopts} -mu "${adb_tmpfile}".* 2>>"${adb_errorlog}" >"${file_name}" out_rc="${?}" - ;; + break + done + [ -z "${out_rc}" ] && { + : >"${file_name}" + out_rc="4" + } + "${adb_rmcmd}" -f "${adb_tmpfile}".* + ;; + "final") + src_name="" + file_name="${adb_finaldir}/${adb_dnsfile}" + { + [ -n "${adb_dnsheader}" ] && printf '%b' "${adb_dnsheader}" + [ -s "${adb_tmpdir}/tmp.add.allowlist" ] && "${adb_sortcmd}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.add.allowlist" + [ "${adb_safesearch}" = "1" ] && "${adb_catcmd}" "${adb_tmpdir}/tmp.safesearch."* 2>>"${adb_errorlog}" + if [ "${adb_dnsdeny}" = "1" ]; then + f_dnsdeny "${adb_tmpdir}/${adb_dnsfile}" + else + "${adb_catcmd}" "${adb_tmpdir}/${adb_dnsfile}" + fi + } >"${file_name}" + if [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${file_name}" "${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_dnsshift}" = "0" ] && [ -s "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_backupdir}/${adb_dnsfile}" + fi + out_rc="0" + ;; esac - f_count "${mode}" "${src_name}" + f_count "${mode}" "${file_name}" out_rc="${out_rc:-"${in_rc}"}" + f_log "debug" "f_list ::: name: ${src_name:-"-"}, mode: ${mode}, cnt: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${out_rc}" return "${out_rc}" } @@ -913,188 +1322,404 @@ f_list() { # top level domain compression # f_tld() { - local cnt cnt_tld source="${1}" temp_tld="${1}.tld" + local cnt_tld cnt_rem source="${1}" temp_tld="${1}.tld" - if "${adb_awk}" '{if(NR==1){tld=$NF};while(getline){if(index($NF,tld".")==0){print tld;tld=$NF}}print tld}' "${source}" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${temp_tld}"; then - mv -f "${temp_tld}" "${source}" - cnt_tld="$(wc -l 2>/dev/null <"${source}")" - else - rm -f "${temp_tld}" + # reverse domain, get unique tlds and unreverse them back to original form + # + if "${adb_awkcmd}" ' + function unreverse(dom, n, seg, out, i) { + n = split(dom, seg, ".") + out = seg[n] + for (i = n-1; i >= 1; i--) out = out "." seg[i] + return out + } + { + if (NR == 1) { parent = $NF } + else if (index($NF, parent ".") == 0) { print unreverse(parent); parent = $NF } + } + END { if (parent) print unreverse(parent) } + ' "${source}" >"${temp_tld}"; then + [ "${adb_debug}" = "1" ] && cnt_tld="$(f_count tld "${temp_tld}" "var")" + + # remove allowlisted (sub-) domains from tld list if allowlist is enabled and not empty + # + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" ' + NR==FNR { + del[$0] + next + } + { + dominated = 0 + n = split($0, seg, ".") + for (i = 1; i <= n; i++) { + parent = seg[i] + for (j = i + 1; j <= n; j++) parent = parent "." seg[j] + if (parent in del) { dominated = 1; break } + } + if (!dominated) print + } + ' "${adb_tmpdir}/tmp.rem.allowlist" "${temp_tld}" >"${source}" + "${adb_rmcmd}" -f "${temp_tld}" + [ "${adb_debug}" = "1" ] && cnt_rem="$(f_count tld "${source}" "var")" + else + "${adb_mvcmd}" -f "${temp_tld}" "${source}" + fi fi - f_log "debug" "f_tld ::: source: ${source}, cnt: ${adb_cnt:-"-"}, cnt_tld: ${cnt_tld:-"-"}" + + f_log "debug" "f_tld ::: name: -, cnt: ${adb_cnt:-"-"}, cnt_tld: ${cnt_tld:-"-"}, cnt_rem: ${cnt_rem:-"-"}" } -# suspend/resume adblock processing -# -f_switch() { - local status entry done="false" mode="${1}" +# suspend/resume adblock processing +# +f_switch() { + local status switch rc="0" mode="${1}" + + json_init + json_load_file "${adb_rtfile}" >/dev/null 2>&1 + json_get_var status "adblock_status" + f_env + + # suspend adblock processing + # + if [ "${status}" = "enabled" ] && [ "${mode}" = "suspend" ]; then + + # suspend via external DNS bridge + # + if { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; } && [ "${adb_nftbridge}" = "1" ]; then + if "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if [ -n "${adb_bridgednsv4}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_bridgednsv4}:53 2>>"${adb_errorlog}" + rc="${?}" + fi + if [ -n "${adb_bridgednsv6}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_bridgednsv6}]:53 2>>"${adb_errorlog}" + rc="$((rc + $?))" + fi + [ "${rc}" = "0" ] && switch="nft" + fi + + # suspend via local DNS + # + else + if [ "${adb_dnsshift}" = "0" ] && [ -f "${adb_finaldir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_finaldir}/${adb_dnsfile}" "${adb_backupdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + switch="dns" + elif [ "${adb_dnsshift}" = "1" ] && [ -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_dnsdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + switch="dns" + fi + fi + + # resume adblock processing + # + elif [ "${status}" = "paused" ] && [ "${mode}" = "resume" ]; then - json_init - json_load_file "${adb_rtfile}" >/dev/null 2>&1 - json_select "data" >/dev/null 2>&1 - json_get_var status "adblock_status" - if [ "${mode}" = "suspend" ] && [ "${status}" = "enabled" ]; then - f_env - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - if [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_jaildir}/${adb_dnsjail}" - elif [ -f "${adb_dnsdir}/${adb_dnsjail}" ]; then - rm -f "${adb_dnsdir}/${adb_dnsjail}" + # resume via external DNS bridge + # + if [ "${adb_nftbridge}" = "1" ] && "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if "${adb_nftcmd}" flush chain inet adblock dns-bridge 2>>"${adb_errorlog}"; then + switch="nft" + fi + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + + # resume via local DNS + # + else + if [ "${adb_dnsshift}" = "0" ] && [ -f "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_backupdir}/${adb_dnsfile}" "${adb_finaldir}/${adb_dnsfile}" + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + switch="dns" + elif [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${adb_finaldir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + switch="dns" + fi fi - f_count - done="true" - elif [ "${mode}" = "resume" ] && [ "${status}" = "paused" ]; then - f_env - f_main - done="true" fi - if [ "${done}" = "true" ]; then - [ "${mode}" = "suspend" ] && f_dnsup + + # update runtime information and log action + # + if [ "${switch}" = "nft" ]; then + f_jsnup "${mode}" + f_log "info" "${mode} adblock service via external DNS bridge" + elif [ "${switch}" = "dns" ]; then + f_dnsup f_jsnup "${mode}" - f_log "info" "${mode} adblock processing" + f_log "info" "${mode} adblock service via local DNS" + else + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + f_jsnup "${status}" fi f_rmtemp } -# query blocklist for certain (sub-)domains +# search blocklist for certain (sub-)domains # -f_query() { - local search result prefix suffix field query_start query_end query_timeout=30 domain="${1}" tld="${1#*.}" +f_search() { + local rc search res result tmp_result prefix suffix field search_start search_end search_timeout=30 domain="${1}" tld="${1#*.}" - if [ -z "${domain}" ] || [ "${domain}" = "${tld}" ]; then - printf "%s\n" "::: invalid input, please submit a single (sub-)domain :::" - else - case "${adb_dns}" in - "dnsmasq") - prefix=".*[\\/\\.]" - suffix="(\\/)" - field="2" - ;; - "unbound") - prefix=".*[\"\\.]" - suffix="(always_nxdomain)" - field="3" - ;; - "named") - prefix="[^\\*].*[\\.]" - suffix="( \\.)" - field="1" - ;; - "kresd") - prefix="[^\\*].*[\\.]" - suffix="( \\.)" - field="1" - ;; - "raw") - prefix=".*[\\.]" - suffix="" - field="1" - ;; - esac - query_start="$(date "+%s")" - while [ "${domain}" != "${tld}" ]; do - search="${domain//[+*~%\$&\"\']/}" - search="${search//./\\.}" - result="$("${adb_awk}" -F '/|\"|\t| ' "/^(${search}|${prefix}+${search}.*${suffix})$/{i++;if(i<=9){printf \" + %s\n\",\$${field}}else if(i==10){printf \" + %s\n\",\"[...]\";exit}}" "${adb_dnsdir}/${adb_dnsfile}")" - printf "%s\n%s\n%s\n" ":::" "::: domain '${domain}' in active blocklist" ":::" - printf "%s\n\n" "${result:-" - no match"}" - domain="${tld}" - tld="${domain#*.}" - done - if [ "${adb_backup}" = "1" ] && [ -d "${adb_backupdir}" ]; then - search="${1//[+*~%\$&\"\']/}" - search="${search//./\\.}" - printf "%s\n%s\n%s\n" ":::" "::: domain '${1}' in backups and black-/whitelist" ":::" - for file in "${adb_backupdir}/${adb_dnsprefix}".*.gz "${adb_blacklist}" "${adb_whitelist}"; do - suffix="${file##*.}" - if [ "${suffix}" = "gz" ]; then - zcat "${file}" 2>/dev/null | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" + # prepare result file + # + tmp_result="${adb_rundir}/adblock.search.tmp" + result="${adb_rundir}/adblock.search" + + # input validation + # + case "${domain}" in + "" | *[!a-zA-Z0-9.-]* | -* | *- | *..* | *.) + printf '%s\n' "::: invalid input, please submit a single (sub-)domain :::" + printf '%s\n' "::: invalid input, please submit a single (sub-)domain :::" >"${result}" + return + ;; + esac + + # length validation for domain part, max. 253 characters according to RFC 1035 + # + case "${#domain}" in + [0-9] | [1-9][0-9] | 1[0-9][0-9] | 2[0-4][0-9] | 25[0-3]) ;; + + *) + printf '%s\n' "::: invalid input, domain exceeds 253 characters :::" + printf '%s\n' "::: invalid input, domain exceeds 253 characters :::" >"${result}" + return + ;; + esac + + # search blocklist + # + case "${adb_dns}" in + "dnsmasq") + prefix='local=.*[\/\.]' + suffix='\/' + field="2" + ;; + "unbound") + prefix='local-zone: .*["\.]' + suffix='" always_nxdomain' + field="3" + ;; + "named") + prefix="" + suffix=' CNAME \.' + field="1" + ;; + "kresd") + prefix="" + suffix=' CNAME \.' + field="1" + ;; + "smartdns") + prefix='address .*.*[\/\.]' + suffix='\/#' + field="3" + ;; + "raw") + prefix="" + suffix="" + field="1" + ;; + esac + + # initialize tmp_result and start search + # + : >"${tmp_result}" + read -r search_start _ <"/proc/uptime" + search_start="${search_start%.*}" + + # search recursively for domain and its parent domains until tld is reached + # + while :; do + search="${domain//./\\.}" + res="$("${adb_awkcmd}" -F '/|\"|\t| ' "/^(${prefix}${search}${suffix})$/{i++;if(i<=9){printf \" + %s\n\",\$${field}}else if(i==10){printf \" + %s\n\",\"[...]\";exit}}" "${adb_finaldir}/${adb_dnsfile}")" + printf '%s\n%s\n%s\n' ":::" "::: domain '${domain}' in active blocklist" ":::" >>"${tmp_result}" + printf '%s\n\n' "${res:-" - no match"}" >>"${tmp_result}" + [ "${domain}" = "${tld}" ] && break + domain="${tld}" + tld="${domain#*.}" + done + + # search exactly for domain in backup files and local block-/allowlist + # + if [ -d "${adb_backupdir}" ]; then + search="${1//./\\.}" + printf '%s\n%s\n%s\n' ":::" "::: domain '${1}' in backups and in local block-/allowlist" ":::" >>"${tmp_result}" + for file in "${adb_backupdir}/adb_list".*.gz "${adb_blocklist}" "${adb_allowlist}"; do + suffix="${file##*.}" + if [ "${suffix}" = "gz" ]; then + if [ "${adb_tld}" = "1" ]; then + "${adb_zcatcmd}" "${file}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" >>"${tmp_result}" else - "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" "${file}" + "${adb_zcatcmd}" "${file}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" >>"${tmp_result}" fi - if [ "${?}" = "0" ]; then - result="true" - query_end="$(date "+%s")" - if [ "$((query_end - query_start))" -gt "${query_timeout}" ]; then - printf "%s\n\n" " - [...]" - break - fi + rc="${?}" + else + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" "${file}" >>"${tmp_result}" + rc="${?}" + fi + if [ "${rc}" = "0" ]; then + res="true" + read -r search_end _ <"/proc/uptime" + search_end="${search_end%.*}" + if [ "$((search_end - search_start))" -gt "${search_timeout}" ]; then + printf '%s\n\n' " - [...]" >>"${tmp_result}" + break fi - done - [ "${result}" != "true" ] && printf "%s\n\n" " - no match" - fi + fi + done + [ "${res}" != "true" ] && printf '%s\n\n' " - no match" >>"${tmp_result}" fi + "${adb_mvcmd}" -f "${tmp_result}" "${result}" + "${adb_catcmd}" "${result}" 2>>"${adb_errorlog}" } # update runtime information # f_jsnup() { - local entry sources runtime utils bg_pid status="${1:-"enabled"}" + local s_shift s_custom s_unfiltered s_filtered s_remote s_bridge s_force s_flush s_tld s_search s_report s_mail + local s_jail s_debug pid pids object feeds end_time runtime dns dns_ver free_mem custom_feed="0" status="${1:-"enabled"}" + local vm_mem dns_mem="0" duration jail="0" nft_unfiltered="0" nft_filtered="0" nft_remote="0" nft_bridge="0" nft_force="0" - adb_memory="$("${adb_awk}" '/^MemTotal|^MemFree|^MemAvailable/{ORS="/"; print int($2/1000)}' "/proc/meminfo" 2>/dev/null | - "${adb_awk}" '{print substr($0,1,length($0)-1)}')" - - case "${status}" in - "enabled" | "error") - adb_endtime="$(date "+%s")" - if [ "$(((adb_endtime - adb_starttime) / 60))" -lt 60 ]; then - runtime="${adb_action}, $(((adb_endtime - adb_starttime) / 60))m $(((adb_endtime - adb_starttime) % 60))s, ${adb_memory:-0}, $(date -Iseconds)" - else - runtime="${adb_action}, n/a, ${adb_memory:-0}, $(date -Iseconds)" - fi - [ "${status}" = "error" ] && adb_cnt="0" + # get DNS memory usage and version + # + if adb_dnspid="$("${adb_ubuscmd}" -S call service list 2>>"${adb_errorlog}" | + "${adb_jsoncmd}" -l1 -e "@[\"${adb_dns}\"].instances.*.pid")" && [ -n "${adb_dnspid}" ]; then + pids="$("${adb_pgrepcmd}" -P "${adb_dnspid}" 2>>"${adb_errorlog}")" + for pid in ${adb_dnspid} ${pids}; do + vm_mem="$("${adb_awkcmd}" '/^VmRSS/{printf "%d", $2}' "/proc/${pid}/status" 2>>"${adb_errorlog}")" + dns_mem="$((dns_mem + ${vm_mem:-0}))" + done + case "${adb_dns}" in + "kresd") + dns="knot-resolver" ;; - "suspend") - status="paused" + "named") + dns="bind-server" ;; - "resume") - status="" + "unbound") + dns="unbound-daemon" + ;; + "dnsmasq") + dns='dnsmasq", "dnsmasq-full", "dnsmasq-dhcpv6' ;; + esac + dns_ver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e "@.packages[\"${dns:-"${adb_dns}"}\"]")" + dns_mem="$("${adb_awkcmd}" -v mem="${dns_mem}" 'BEGIN{printf "%.2f", mem/1024}' 2>>"${adb_errorlog}")" + fi + free_mem="$("${adb_awkcmd}" '/^MemAvailable/{printf "%.2f", $2/1024}' "/proc/meminfo" 2>>"${adb_errorlog}")" + + # check for custom feed and nft rules + # + [ -s "${adb_customfeedfile}" ] && custom_feed="1" + if [ "${adb_nftforce}" = "1" ] && [ -n "${adb_nftdevforce}" ] && [ -n "${adb_nftportforce}" ]; then + nft_force="1" + fi + if [ "${adb_nftallow}" = "1" ] && + { [ -n "${adb_nftmacallow}" ] || [ -n "${adb_nftdevallow}" ]; } && + { [ -n "${adb_allowdnsv4}" ] || [ -n "${adb_allowdnsv6}" ]; }; then + nft_unfiltered="1" + fi + if [ "${adb_nftblock}" = "1" ] && + { [ -n "${adb_nftmacblock}" ] || [ -n "${adb_nftdevblock}" ]; } && + { [ -n "${adb_blockdnsv4}" ] || [ -n "${adb_blockdnsv6}" ]; }; then + nft_filtered="1" + fi + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ] && + { [ -n "${adb_remotednsv4}" ] || [ -n "${adb_remotednsv6}" ]; }; then + nft_remote="1" + fi + if [ "${adb_nftbridge}" = "1" ] && + { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; }; then + nft_bridge="1" + fi + + # update runtime information based on status + # + case "${status}" in + "enabled") + if [ -n "${adb_starttime}" ]; then + read -r end_time _ <"/proc/uptime" + end_time="${end_time%.*}" + duration="$(((end_time - adb_starttime) / 60))m $(((end_time - adb_starttime) % 60))s" + fi + runtime="mode: ${adb_action}, date / time: $(date "+%d/%m/%Y %H:%M:%S"), duration: ${duration:-"-"}, memory: ${free_mem:-0} MB available" + ;; + "resume") + status="enabled" + ;; + "suspend") + adb_cnt="0" + status="paused" + ;; + *) + adb_cnt="0" + ;; esac + + # update runtime file and send mail if enabled + # json_init if json_load_file "${adb_rtfile}" >/dev/null 2>&1; then - utils="download: $(readlink -fn "${adb_fetchutil}"), sort: $(readlink -fn "${adb_sort}"), awk: $(readlink -fn "${adb_awk}")" - [ -z "${adb_cnt}" ] && { json_get_var adb_cnt "blocked_domains"; adb_cnt="${adb_cnt%% *}"; } + [ -z "${adb_cnt}" ] && json_get_var adb_cnt "blocked_domains" [ -z "${runtime}" ] && json_get_var runtime "last_run" - fi - if [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - adb_cnt="0" - sources="restrictive_jail" - else - sources="$(printf "%s\n" ${adb_sources} | "${adb_sort}" | "${adb_awk}" '{ORS=" ";print $0}')" + if [ "${status}" = "enabled" ]; then + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + jail="1" + adb_cnt="0" + feeds="restrictive jail (allowlist-only)" + else + feeds="$(printf '%s\n' ${adb_feed// /, } | "${adb_sortcmd}" | "${adb_xargscmd}")" + fi + fi fi - : >"${adb_rtfile}" + # map flag values to status characters + # + case "${adb_dnsshift}" in "1") s_shift="✔" ;; *) s_shift="✘" ;; esac + case "${custom_feed}" in "1") s_custom="✔" ;; *) s_custom="✘" ;; esac + case "${nft_unfiltered}" in "1") s_unfiltered="✔" ;; *) s_unfiltered="✘" ;; esac + case "${nft_filtered}" in "1") s_filtered="✔" ;; *) s_filtered="✘" ;; esac + case "${nft_remote}" in "1") s_remote="✔" ;; *) s_remote="✘" ;; esac + case "${nft_bridge}" in "1") s_bridge="✔" ;; *) s_bridge="✘" ;; esac + case "${nft_force}" in "1") s_force="✔" ;; *) s_force="✘" ;; esac + case "${adb_dnsflush}" in "1") s_flush="✔" ;; *) s_flush="✘" ;; esac + case "${adb_tld}" in "1") s_tld="✔" ;; *) s_tld="✘" ;; esac + case "${adb_safesearch}" in "1") s_search="✔" ;; *) s_search="✘" ;; esac + case "${adb_report}" in "1") s_report="✔" ;; *) s_report="✘" ;; esac + case "${adb_mail}" in "1") s_mail="✔" ;; *) s_mail="✘" ;; esac + case "${jail}" in "1") s_jail="✔" ;; *) s_jail="✘" ;; esac + case "${adb_debug}" in "1") s_debug="✔" ;; *) s_debug="✘" ;; esac + + # update runtime file + # + printf '%s\n' "{}" >"${adb_rtfile}" json_init json_load_file "${adb_rtfile}" >/dev/null 2>&1 - json_init - json_add_string "adblock_status" "${status:-"enabled"}" - json_add_string "adblock_version" "${adb_ver}" - json_add_string "blocked_domains" "${adb_cnt:-0}" - json_add_array "active_sources" - for entry in ${sources}; do - json_add_object - json_add_string "source" "${entry}" - json_close_object + json_add_string "adblock_status" "${status}" + json_add_string "frontend_ver" "${adb_fver}" + json_add_string "backend_ver" "${adb_bver}" + json_add_string "blocked_domains" "${adb_cnt:-"0"}" + json_add_array "active_feeds" + for object in ${feeds:-"-"}; do + json_add_string "${object}" "${object}" done json_close_array - json_add_string "dns_backend" "${adb_dns:-"-"} (${adb_dnscachecmd##*/}), ${adb_dnsdir:-"-"}" - json_add_string "run_utils" "${utils:-"-"}" + json_add_string "dns_backend" "${adb_dns:-"-"} (${dns_ver:-"-"}), ${adb_finaldir:-"-"}, ${dns_mem:-"0"} MB" json_add_string "run_ifaces" "trigger: ${adb_trigger:-"-"}, report: ${adb_repiface:-"-"}" - json_add_string "run_directories" "base: ${adb_tmpbase}, backup: ${adb_backupdir}, report: ${adb_reportdir}, jail: ${adb_jaildir}" - json_add_string "run_flags" "backup: $(f_char ${adb_backup}), flush: $(f_char ${adb_dnsflush}), force: $(f_char ${adb_forcedns}), search: $(f_char ${adb_safesearch}), report: $(f_char ${adb_report}), mail: $(f_char ${adb_mail}), jail: $(f_char ${adb_jail})" + json_add_string "run_information" "base: ${adb_basedir}, dns: ${adb_dnsdir}, backup: ${adb_backupdir}, report: ${adb_reportdir}, error: ${adb_errorlog}" + json_add_string "run_flags" "shift: ${s_shift}, custom feed: ${s_custom}, ext. DNS (std/prot/remote/bridge): ${s_unfiltered}/${s_filtered}/${s_remote}/${s_bridge}, force: ${s_force}, flush: ${s_flush}, tld: ${s_tld}, search: ${s_search}, report: ${s_report}, mail: ${s_mail}, jail: ${s_jail}, debug: ${s_debug}" json_add_string "last_run" "${runtime:-"-"}" - json_add_string "system" "${adb_sysver}" + json_add_string "system_info" "cores: ${adb_cores}, fetch: ${adb_fetchcmd##*/}, ${adb_sysver}" json_dump >"${adb_rtfile}" - if [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && - { [ "${status}" = "error" ] || { [ "${status}" = "enabled" ] && [ "${adb_cnt}" -le "${adb_mailcnt}" ]; }; }; then - ("${adb_mailservice}" "${adb_ver}" >/dev/null 2>&1) & - bg_pid="${!}" + if [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && [ "${status}" = "enabled" ] && [ "${adb_action}" != "resume" ]; then + "${adb_mailservice}" >/dev/null 2>&1 fi - f_log "debug" "f_jsnup ::: status: ${status:-"-"}, cnt: ${adb_cnt}, mail: ${adb_mail}, mail_service: ${adb_mailservice}, mail_cnt: ${adb_mailcnt}, mail_pid: ${bg_pid:-"-"}" } # write to syslog @@ -1103,10 +1728,13 @@ f_log() { local class="${1}" log_msg="${2}" if [ -n "${log_msg}" ] && { [ "${class}" != "debug" ] || [ "${adb_debug}" = "1" ]; }; then - [ -x "${adb_loggercmd}" ] && "${adb_loggercmd}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" || \ - printf "%s %s %s\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" - if [ "${class}" = "err" ]; then - f_rmdns + if [ -x "${adb_loggercmd}" ]; then + "${adb_loggercmd}" -p "${class}" -t "adblock-${adb_bver}[${$}]" "${log_msg::512}" + else + printf '%s %s %s\n' "${class}" "adblock-${adb_bver}[${$}]" "${log_msg::512}" + fi + if [ "${class}" = "err" ] || [ "${class}" = "emerg" ]; then + [ "${adb_action}" != "mail" ] && f_rmdns f_jsnup "error" exit 1 fi @@ -1116,19 +1744,33 @@ f_log() { # main function for blocklist processing # f_main() { - local src_tmpload src_tmpfile src_name src_rset src_url src_log src_arc src_cat src_item src_list src_entries src_suffix src_rc entry cnt + local src_name src_domain src_rset src_url src_cat src_item src_list src_entries src_suffix src_rc entry cnt + local src_tmpcat src_tmparchive src_tmpload src_tmpfile seen_domains feed_restore map_domain - f_log "debug" "f_main ::: memory: ${adb_memory:-0}, cores: ${adb_cores}, safe_search: ${adb_safesearch}, force_dns: ${adb_forcedns}, awk: ${adb_awk}" - - # white- and blacklist preparation + # allow- and blocklist preparation # for entry in ${adb_locallist}; do - (f_list "${entry}" "${entry}") & + f_list "${entry}" "${entry}" done - if [ "${adb_dns}" != "raw" ] && [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - chown "${adb_dnsuser}" "${adb_jaildir}/${adb_dnsjail}" 2>/dev/null + # jail mode preparation + # + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + if [ -s "${adb_tmpdir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_tmpdir}/${adb_dnsfile}" "${adb_finaldir}/${adb_dnsfile}" + else + f_log "info" "jail mode active without allowlist, blocking all queries" + { + printf '%b' "${adb_dnsheader}" + printf '%b\n' "${adb_dnsstop}" + } >"${adb_finaldir}/${adb_dnsfile}" + fi + chown "${adb_dnsuser}" "${adb_finaldir}/${adb_dnsfile}" 2>>"${adb_errorlog}" + if [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${adb_finaldir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_dnsshift}" = "0" ] && [ -s "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_backupdir}/${adb_dnsfile}" + fi if f_dnsup; then if [ "${adb_action}" != "resume" ]; then f_jsnup "enabled" @@ -1139,160 +1781,233 @@ f_main() { fi f_rmtemp return - elif [ -f "${adb_dnsdir}/${adb_dnsjail}" ]; then - rm -f "${adb_dnsdir}/${adb_dnsjail}" - f_dnsup fi # safe search preparation # - if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ]; then - [ -z "${adb_safesearchlist}" ] && adb_safesearchlist="google bing duckduckgo pixabay yandex youtube" + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" = "1" ]; then + [ -z "${adb_safesearchlist}" ] && adb_safesearchlist="google bing brave duckduckgo pixabay yandex youtube" cnt="1" for entry in ${adb_safesearchlist}; do - (f_list safesearch "${entry}") & - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + ( + f_list safesearch "${entry}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done + wait + fi + + # add map service domain to allowlist if map is enabled + # + if [ "${adb_map}" = "1" ] && [ "${adb_dnsallow}" = "1" ] && [ -n "${adb_geourl}" ]; then + map_domain="${adb_geourl#*://}" + map_domain="${map_domain%%/*}" + if [ -n "${map_domain}" ]; then + printf '%s\n' "${map_domain}" | f_dnsallow >>"${adb_tmpdir}/tmp.add.allowlist" + printf '%s\n' "${map_domain}" >>"${adb_tmpdir}/tmp.rem.allowlist" + fi fi - wait # main loop # cnt="1" - for src_name in ${adb_sources}; do + seen_domains="${map_domain}" + + # determine if feed restore should be attempted based on action + # + case "${adb_action}" in + "boot" | "start" | "restart" | "resume") + feed_restore="1" + ;; + *) + feed_restore="0" + ;; + esac + + # loop through feeds defined in configuration and process them + # + for src_name in ${adb_feed}; do + + # check if feed is defined in configuration, if not remove it from feed list and continue with next one + # if ! json_select "${src_name}" >/dev/null 2>&1; then - adb_sources="${adb_sources/${src_name}/}" + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" continue fi + + # get feed information + # json_get_var src_url "url" >/dev/null 2>&1 json_get_var src_rset "rule" >/dev/null 2>&1 json_select .. src_tmpcat="${adb_tmpload}.${src_name}.cat" src_tmpload="${adb_tmpload}.${src_name}.load" - src_tmpsort="${adb_tmpload}.${src_name}.sort" + src_tmparchive="${adb_tmpload}.${src_name}.archive" src_tmpfile="${adb_tmpfile}.${src_name}" src_rc=4 # basic pre-checks # - if [ -z "${src_url}" ] || [ -z "${src_rset}" ]; then + if [ -z "${src_url}" ] || [ -z "${src_rset}" ] || + [ "${src_rset%% *}" != "feed" ]; then f_list remove continue fi - # backup mode + # add domains of active feed URLs to the allowlist # - if [ "${adb_backup}" = "1" ] && { [ "${adb_action}" = "start" ] || [ "${adb_action}" = "resume" ]; }; then - if f_list restore && [ -s "${src_tmpfile}" ]; then - continue - fi + src_domain="${src_url#*://}" + src_domain="${src_domain%%/*}" + if [ -n "${src_domain}" ] && [ "${adb_dnsallow}" = "1" ]; then + case " ${seen_domains} " in + *" ${src_domain} "*) ;; + + *) + seen_domains="${seen_domains} ${src_domain}" + printf '%s\n' "${src_domain}" | f_dnsallow >>"${adb_tmpdir}/tmp.add.allowlist" + printf '%s\n' "${src_domain}" >>"${adb_tmpdir}/tmp.rem.allowlist" + ;; + esac fi # download queue processing # - unset src_cat src_entries - if [ "${src_name}" = "utcapitole" ] && [ -n "${adb_utc_sources}" ]; then - src_cat="${adb_utc_sources}" - if [ -n "${src_cat}" ]; then - ( - src_arc="${adb_tmpdir}/${src_url##*/}" - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${src_url}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" = "0" ] && [ -s "${src_arc}" ]; then - src_suffix="$(eval printf "%s" \"\$\{adb_src_suffix_${src_name}:-\"domains\"\}\")" - src_list="$(tar -tzf "${src_arc}" 2>/dev/null)" - for src_item in ${src_cat}; do - src_entries="${src_entries} $(printf "%s" "${src_list}" | grep -E "${src_item}/${src_suffix}$")" - done - if [ -n "${src_entries}" ]; then - tar -xOzf "${src_arc}" ${src_entries} 2>/dev/null >"${src_tmpload}" - src_rc="${?}" - fi - : >"${src_arc}" - else - src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" - f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - fi - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ]; then - if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ]; then - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - else - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - fi - : >"${src_tmpload}" - "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null >"${src_tmpfile}" - src_rc="${?}" - : >"${src_tmpsort}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then - f_list download - [ "${adb_backup}" = "1" ] && f_list backup - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "archive preparation of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" - f_list restore - rm -f "${src_tmpfile}" - fi - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "archive extraction of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" - f_list restore - fi - ) & + src_cat="" + src_entries="" + + # category handling for feeds with fixed categories + # + case "${src_name}" in + "1hosts") + src_cat="${adb_hst_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue fi - else - if [ "${src_name}" = "energized" ] && [ -n "${adb_eng_sources}" ]; then - src_cat="${adb_eng_sources}" - elif [ "${src_name}" = "stevenblack" ] && [ -n "${adb_stb_sources}" ]; then - src_cat="${adb_stb_sources}" - elif { [ "${src_name}" = "energized" ] && [ -z "${adb_eng_sources}" ]; } || - { [ "${src_name}" = "stevenblack" ] && [ -z "${adb_stb_sources}" ]; }; then + ;; + "hagezi") + src_cat="${adb_hag_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue + fi + ;; + "ipfire_dbl") + src_cat="${adb_ipf_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue + fi + ;; + "stevenblack") + src_cat="${adb_stb_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" continue fi + ;; + esac + + # category handling for feeds with multiple categories + # + if [ -n "${src_cat}" ]; then ( - for suffix in ${src_cat:-${src_url}}; do - if [ "${src_url}" != "${suffix}" ]; then - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpcat}" "${src_url}${suffix}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpcat}" ]; then - cat "${src_tmpcat}" >>"${src_tmpload}" - : >"${src_tmpcat}" + # restore handling on boot, resume or (re-)start + # + if [ "${feed_restore}" = "1" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + + # etag handling on reload + # + if [ -n "${adb_etagparm}" ] && [ "${adb_action}" = "reload" ]; then + etag_rc="0" + src_cnt="0" + for _ in ${src_cat}; do + src_cnt="$((src_cnt + 1))" + done + for suffix in ${src_cat}; do + if ! f_etag "${src_name}" "${src_url}" "${suffix}" "${src_cnt}"; then + etag_rc="$((etag_rc + 1))" fi - else - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>&1)" - src_rc="${?}" + done + if [ "${etag_rc}" = "0" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + fi + + # category download + # + for suffix in ${src_cat}; do + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmpcat}" "${src_url}${suffix}" 2>>"${adb_errorlog}" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpcat}" ]; then + "${adb_catcmd}" "${src_tmpcat}" >>"${src_tmpload}" + : >"${src_tmpcat}" fi done - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ]; then - if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ]; then - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - else - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" + f_list prepare + ) & + + # normal handling for feeds without categories + # + else + ( + [ "${src_name}" = "utcapitole" ] && src_cat="${adb_utc_feed}" + + # restore handling on boot, resume or (re-)start + # + if [ "${feed_restore}" = "1" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 fi - : >"${src_tmpload}" - "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null >"${src_tmpfile}" - src_rc="${?}" - : >"${src_tmpsort}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then - f_list download - [ "${adb_backup}" = "1" ] && f_list backup - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "preparation of '${src_name}' failed, rc: ${src_rc}" - f_list restore - rm -f "${src_tmpfile}" + fi + + # etag handling on reload + # + if [ -n "${adb_etagparm}" ] && [ "${adb_action}" = "reload" ]; then + if f_etag "${src_name}" "${src_url}"; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + fi + + # download feed and extract categories if necessary + # + if [ "${src_name}" = "utcapitole" ]; then + if [ -n "${src_cat}" ]; then + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmparchive}" "${src_url}" 2>>"${adb_errorlog}" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmparchive}" ]; then + src_suffix="${adb_src_suffix_utcapitole:-"domains"}" + src_entries="$(tar -tzf "${src_tmparchive}" 2>>"${adb_errorlog}" | "${adb_awkcmd}" \ + -v cats="${src_cat}" -v sfx="${src_suffix}" ' + BEGIN { n = split(cats, c, " ") } + { for (i = 1; i <= n; i++) if ($0 ~ "(^|/)" c[i] "/" sfx "$") print }')" + if [ -n "${src_entries}" ]; then + tar -xOzf "${src_tmparchive}" ${src_entries} 2>>"${adb_errorlog}" >"${src_tmpload}" + src_rc="${?}" + fi + : >"${src_tmparchive}" + fi fi else - src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" - f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ] && f_list restore + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>>"${adb_errorlog}" + src_rc="${?}" fi + f_list prepare ) & fi - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done wait @@ -1300,15 +2015,16 @@ f_main() { # tld compression and dns restart # if f_list merge && [ -s "${adb_tmpdir}/${adb_dnsfile}" ]; then - f_tld "${adb_tmpdir}/${adb_dnsfile}" + [ "${adb_tld}" = "1" ] && f_tld "${adb_tmpdir}/${adb_dnsfile}" f_list final else - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_log "info" "no merge input, only header written to ${adb_finaldir}/${adb_dnsfile}" fi - chown "${adb_dnsuser}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null + chown "${adb_dnsuser}" "${adb_finaldir}/${adb_dnsfile}" 2>>"${adb_errorlog}" if f_dnsup; then - [ "${adb_action}" != "resume" ] && f_jsnup "enabled" f_log "info" "blocklist with overall ${adb_cnt} blocked domains loaded successfully (${adb_sysver})" + [ "${adb_action}" != "resume" ] && f_jsnup "enabled" else f_log "err" "dns backend restart with adblock blocklist failed" fi @@ -1318,149 +2034,412 @@ f_main() { # trace dns queries via tcpdump and prepare a report # f_report() { - local report_raw report_txt content status total start end start_date start_time end_date end_time blocked percent top_list top array item index hold ports value key key_list cnt="0" resolve="-nn" action="${1}" top_count="${2:-"10"}" res_count="${3:-"50"}" search="${4:-"+"}" + local report_raw report_txt content status total start end start_date start_time end_date end_time blocked percent top_list top array item index value key key_list + local domain rc map_seen ip request requests iface_v4 iface_v6 ip_v4 ip_v6 map_jsn cnt report_srt report_jsn top_tmpclients top_tmpdomains top_tmpblocked + local file jsn resolve="-nn" action="${1}" top_count="${2:-"10"}" res_count="${3:-"50"}" search="${4:-"+"}" report_raw="${adb_reportdir}/adb_report.raw" report_srt="${adb_reportdir}/adb_report.srt" - report_jsn="${adb_reportdir}/adb_report.json" + report_jsn="${adb_reportdir}/adb_report.jsn" report_txt="${adb_reportdir}/adb_mailreport.txt" + top_tmpclients="${adb_reportdir}/top_clients.tmp" + top_tmpdomains="${adb_reportdir}/top_domains.tmp" + top_tmpblocked="${adb_reportdir}/top_blocked.tmp" + map_jsn="${adb_reportdir}/adb_map.jsn" - # build json file + # build report # if [ "${action}" != "json" ]; then - : >"${report_raw}" - : >"${report_srt}" - : >"${report_txt}" - : >"${report_jsn}" + : >"${report_srt}" >"${report_txt}" >"${report_jsn}" >"${map_jsn}" + : >"${top_tmpclients}" >"${top_tmpdomains}" >"${top_tmpblocked}" [ "${adb_represolve}" = "1" ] && resolve="" + cnt="1" for file in "${adb_reportdir}/adb_report.pcap"*; do + [ -s "${file}" ] || continue ( - "${adb_dumpcmd}" "${resolve}" -tttt -r "${file}" 2>/dev/null | - "${adb_awk}" -v cnt="${cnt}" '!/\.lan\. |PTR\? | SOA\? /&&/ A[\? ]+|NXDomain|0\.0\.0\.0/{a=$1;b=substr($2,0,8);c=$4;sub(/\.[0-9]+$/,"",c);gsub(/[^[:alnum:]\.:-]/,"",c);d=cnt $7;sub(/\*$/,"",d); - e=$(NF-1);sub(/[0-9]\/[0-9]\/[0-9]|0\.0\.0\.0/,"NX",e);sub(/\.$/,"",e);sub(/([0-9]{1,3}\.){3}[0-9]{1,3}/,"OK",e);gsub(/[^[:alnum:]\.-]/,"",e);if(e==""){e="err"};printf "%s\t%s\t%s\t%s\t%s\n",d,e,a,b,c}' >>"${report_raw}" + "${adb_dumpcmd}" ${resolve} --immediate-mode -tttt -T domain -r "${file}" 2>/dev/null | + "${adb_awkcmd}" -v repiface="${adb_repiface}" ' + BEGIN { + pending = 0 + } + # ignore Reverse DNS + /\.in-addr\.arpa/ || /\.ip6\.arpa/ { next } + # domain request parser (with optional EDNS marker support) + /\+[[:space:]]+(\[[0-9a-z]*\][[:space:]]+)?(A\?|AAAA\?)/ { + # drop unresolved previous query + if (pending) + pending = 0 + date = $1 + split($2, t, ":") + time = t[1] ":" t[2] ":" substr(t[3],1,2) + interface = repiface + client = $4 + if (repiface == "any") { + interface = $3 + client = $6 + } + sub(/\.[0-9]+$/, "", client) + domain = $(NF-1) + sub(/[,\.]+$/, "", domain) + if (domain ~ /\.lan$/) next + if (domain !~ /\./) next + if (domain ~ /[\/:]/) next + qtype = $(NF-2) + sub(/\?$/, "", qtype) + last_date = date + last_time = time + last_client = client + last_interface = interface + last_domain = domain + last_qtype = qtype + pending = 1 + next + } + # ok answer + / (A|AAAA|CNAME) / && !/NXDomain/ && !/ServFail/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tOK\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + # nxdomain answer + / NXDomain/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tNX\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + # servfail answer + / ServFail/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tSF\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + ' >"${report_raw}.${cnt}" ) & - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done wait - if [ -s "${report_raw}" ]; then - "${adb_sort}" ${adb_srtopts} -k1 -k3 -k4 -k5 -k1 -ur "${report_raw}" | - "${adb_awk}" '{currA=($1+0);currB=$1;currC=substr($1,length($1),1);if(reqA==currB){reqA=0;printf "%s\t%s\n",d,$2}else if(currC=="+"){reqA=currA;d=$3"\t"$4"\t"$5"\t"$2}}' | - "${adb_sort}" ${adb_srtopts} -k1 -k2 -k3 -k4 -ur >"${report_srt}" - rm -f "${report_raw}" - fi + for file in "${report_raw}".*; do + [ -s "${file}" ] || continue + "${adb_sortcmd}" ${adb_srtopts} -ru "${report_raw}".* >"${report_srt}" + "${adb_rmcmd}" -f "${report_raw}".* + break + done + # build json + # if [ -s "${report_srt}" ]; then - start="$("${adb_awk}" 'END{printf "%s_%s",$1,$2}' "${report_srt}")" - end="$("${adb_awk}" 'NR==1{printf "%s_%s",$1,$2}' "${report_srt}")" - total="$(wc -l <"${report_srt}")" - blocked="$("${adb_awk}" '{if($5=="NX")cnt++}END{printf "%s",cnt}' "${report_srt}")" - percent="$("${adb_awk}" -v t="${total}" -v b="${blocked}" 'BEGIN{printf "%.2f%s",b/t*100,"%"}')" - : >"${report_jsn}" + start="$("${adb_awkcmd}" 'END{printf "%s_%s",$1,$2}' "${report_srt}")" + end="$("${adb_awkcmd}" 'NR==1{printf "%s_%s",$1,$2}' "${report_srt}")" + total="$(f_count tld "${report_srt}" "var")" + blocked="$("${adb_awkcmd}" '{if($7=="NX")cnt++}END{printf "%s",cnt}' "${report_srt}")" + percent="$("${adb_awkcmd}" -v t="${total}" -v b="${blocked}" 'BEGIN{ if(t>0) printf "%.2f%s",b/t*100,"%"; else printf "0.00%%"}')" { - printf "%s\n" "{ " - printf "\t%s\n" "\"start_date\": \"${start%_*}\", " - printf "\t%s\n" "\"start_time\": \"${start#*_}\", " - printf "\t%s\n" "\"end_date\": \"${end%_*}\", " - printf "\t%s\n" "\"end_time\": \"${end#*_}\", " - printf "\t%s\n" "\"total\": \"${total}\", " - printf "\t%s\n" "\"blocked\": \"${blocked}\", " - printf "\t%s\n" "\"percent\": \"${percent}\", " - } >>"${report_jsn}" + printf '%s\n' "{ " + printf '\t%s\n' "\"start_date\": \"${start%_*}\", " + printf '\t%s\n' "\"start_time\": \"${start#*_}\", " + printf '\t%s\n' "\"end_date\": \"${end%_*}\", " + printf '\t%s\n' "\"end_time\": \"${end#*_}\", " + printf '\t%s\n' "\"total\": \"${total}\", " + printf '\t%s\n' "\"blocked\": \"${blocked}\", " + printf '\t%s\n' "\"percent\": \"${percent}\", " + } >"${report_jsn}" + + # build top list counters + # + "${adb_awkcmd}" ' + { + if (NF < 7) { + next + } + client = $3 + domain = $6 + rc = $7 + + if (domain == "" || domain == "-") { + next + } + + sub(/[\.]+$/, "", domain) + domain = tolower(domain) + + clients[client]++ + if (rc == "OK") { + ok_domain[domain]++ + } + else if (rc == "NX") { + nx_domain[domain]++ + } + all_domain[domain]++ + } + END { + for (c in clients) { + printf "%d %s\n", clients[c], c > "'"${top_tmpclients}"'" + } + for (d in all_domain) { + if (d in ok_domain) { + printf "%d %s\n", ok_domain[d], d > "'"${top_tmpdomains}"'" + } + if (d in nx_domain) { + printf "%d %s\n", nx_domain[d], d > "'"${top_tmpblocked}"'" + } + } + } + ' "${report_srt}" + + # build json top lists + # top_list="top_clients top_domains top_blocked" for top in ${top_list}; do - printf "\t%s" "\"${top}\": [ " >>"${report_jsn}" + printf '\t"%s": [ ' "${top}" >>"${report_jsn}" case "${top}" in - "top_clients") - "${adb_awk}" '{print $3}' "${report_srt}" | "${adb_sort}" ${adb_srtopts} | uniq -c | - "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; - "top_domains") - "${adb_awk}" '{if($5!="NX")print $4}' "${report_srt}" | "${adb_sort}" ${adb_srtopts} | uniq -c | - "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; - "top_blocked") - "${adb_awk}" '{if($5=="NX")print $4}' "${report_srt}" | - "${adb_sort}" ${adb_srtopts} | uniq -c | "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; + top_clients) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpclients}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; + top_domains) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpdomains}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; + top_blocked) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpblocked}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; esac - printf "\n\t%s\n" "]," >>"${report_jsn}" + printf '\n\t],\n' >>"${report_jsn}" + done + "${adb_rmcmd}" -f "${top_tmpclients}" "${top_tmpdomains}" "${top_tmpblocked}" + + # build json request list + # + search="${search//[!a-zA-Z0-9._-]/}" + case "${res_count}" in + '' | *[!0-9]*) + res_count="50" + ;; + esac + "${adb_awkcmd}" -v search="${search}" -v res_count="${res_count}" ' + BEGIN { + i = 0 + printf "\t\"requests\": [\n" + } + + # only match if search is empty or non-empty and NF == 7 + ((search == "" || index($0, search)) && NF == 7) { + i++ + if (res_count > 0 && i > res_count) { + next + } + if (i > 1) { + printf ",\n" + } + + printf "\n\t\t{\n" + printf "\t\t\t\"date\": \"%s\",\n", $1 + printf "\t\t\t\"time\": \"%s\",\n", $2 + printf "\t\t\t\"client\": \"%s\",\n", $3 + printf "\t\t\t\"iface\": \"%s\",\n", $4 + printf "\t\t\t\"type\": \"%s\",\n", $5 + printf "\t\t\t\"domain\": \"%s\",\n", $6 + printf "\t\t\t\"rc\": \"%s\"\n", $7 + printf "\t\t}" + } + END { + printf "\n\t]\n}\n" + }' "${report_srt}" >>"${report_jsn}" + "${adb_rmcmd}" -f "${report_srt}" + fi + + # retrieve/prepare map data + # + if [ "${adb_map}" = "1" ] && [ -s "${report_jsn}" ]; then + cnt="1" + network_find_wan iface_v4 && network_get_ipaddr ip_v4 "${iface_v4}" + network_find_wan6 iface_v6 && network_get_ipaddr6 ip_v6 "${iface_v6}" + if [ -n "${ip_v4}" ] || [ -n "${ip_v6}" ]; then + f_fetch + printf '%s' ",[{}" >"${map_jsn}" + fi + for ip in ${ip_v4} ${ip_v6}; do + ( + "${adb_fetchcmd}" ${adb_geoparm} "${adb_geourl}/${ip}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v feed="homeIP" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.${cnt}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + wait + if [ -s "${map_jsn}" ] && [ "${cnt}" -lt "45" ]; then + map_seen="" + json_init + if json_load_file "${report_jsn}" >/dev/null 2>&1; then + json_select "requests" >/dev/null 2>&1 + json_get_keys requests >/dev/null 2>&1 + for request in ${requests}; do + json_select "${request}" >/dev/null 2>&1 + json_get_var rc "rc" >/dev/null 2>&1 + json_get_var domain "domain" >/dev/null 2>&1 + if [ "${rc}" = "NX" ]; then + case " ${map_seen} " in + *" ${domain} "*) ;; + + *) + map_seen="${map_seen} ${domain} " + ( + "${adb_fetchcmd}" ${adb_geoparm} "${adb_geourl}/${domain}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v feed="${domain}" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.${cnt}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n + cnt="$((cnt + 1))" + [ "${cnt}" -ge "45" ] && break + ;; + esac + fi + json_select ".." + done + wait + fi + fi + for file in "${map_jsn}".*; do + [ -s "${file}" ] || continue + "${adb_catcmd}" "${map_jsn}".* >>"${map_jsn}" 2>/dev/null + "${adb_rmcmd}" -f "${map_jsn}".* + break done - search="${search//./\\.}" - search="${search//[+*~%\$&\"\' ]/}" - "${adb_awk}" "BEGIN{i=0;printf \"\t\\\"requests\\\": [\n\"}/(${search})/{i++;if(i==1)printf \"\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5;else if(i<=${res_count})printf \",\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5}END{printf \"\n\t]\n}\n\"}" "${adb_reportdir}/adb_report.srt" >>"${report_jsn}" - rm -f "${report_srt}" fi fi # output preparation # if [ -s "${report_jsn}" ] && { [ "${action}" = "cli" ] || [ "${action}" = "mail" ]; }; then - printf "%s\n%s\n%s\n" ":::" "::: Adblock DNS-Query Report" ":::" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "::: Adblock DNS Report" ":::" >>"${report_txt}" json_init json_load_file "${report_jsn}" json_get_keys key_list for key in ${key_list}; do json_get_var value "${key}" - eval "${key}=\"${value}\"" + case "${key}" in + "start_date") + start_date="${value}" + ;; + "start_time") + start_time="${value}" + ;; + "end_date") + end_date="${value}" + ;; + "end_time") + end_time="${value}" + ;; + "total") + total="${value}" + ;; + "blocked") + blocked="${value}" + ;; + "percent") + percent="${value}" + ;; + esac done - printf " + %s\n + %s\n" "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" >>"${report_txt}" - printf " + %s\n + %s %s\n" "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" >>"${report_txt}" + printf ' + %s\n + %s\n' "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" >>"${report_txt}" + printf ' + %s\n + %s %s\n' "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" >>"${report_txt}" top_list="top_clients top_domains top_blocked requests" for top in ${top_list}; do case "${top}" in - "top_clients") - item="::: Top Clients" - ;; - "top_domains") - item="::: Top Domains" - ;; - "top_blocked") - item="::: Top Blocked Domains" - ;; + "top_clients") + item="::: Top Clients" + ;; + "top_domains") + item="::: Top Domains" + ;; + "top_blocked") + item="::: Top Blocked Domains" + ;; esac if json_get_type status "${top}" && [ "${top}" != "requests" ] && [ "${status}" = "array" ]; then - printf "%s\n%s\n%s\n" ":::" "${item}" ":::" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "${item}" ":::" >>"${report_txt}" json_select "${top}" index="1" item="" while json_get_type status "${index}" && [ "${status}" = "object" ]; do json_get_values item "${index}" - printf " + %-9s::: %s\n" ${item} >>"${report_txt}" + printf ' + %-9s::: %s\n' ${item} >>"${report_txt}" index="$((index + 1))" done elif json_get_type status "${top}" && [ "${top}" = "requests" ] && [ "${status}" = "array" ]; then - printf "%s\n%s\n%s\n" ":::" "::: Latest DNS Queries" ":::" >>"${report_txt}" - printf "%-15s%-15s%-45s%-80s%s\n" "Date" "Time" "Client" "Domain" "Answer" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "::: Latest DNS Queries" ":::" >>"${report_txt}" + printf '%-11s%-9s%-40s%-15s%-5s%-70s%s\n' "Date" "Time" "Client" "Interface" "Type" "Domain" "Answer" >>"${report_txt}" json_select "${top}" index="1" while json_get_type status "${index}" && [ "${status}" = "object" ]; do json_get_values item "${index}" - printf "%-15s%-15s%-45s%-80s%s\n" ${item} >>"${report_txt}" + printf '%-11s%-9s%-40s%-15s%-5s%-70s%s\n' ${item} >>"${report_txt}" index="$((index + 1))" done fi json_select ".." done - content="$(cat "${report_txt}" 2>/dev/null)" - rm -f "${report_txt}" + content="$("${adb_catcmd}" "${report_txt}" 2>>"${adb_errorlog}")" + "${adb_rmcmd}" -f "${report_txt}" fi # report output # - if [ "${action}" = "cli" ]; then - printf "%s\n" "${content}" - elif [ "${action}" = "json" ]; then - cat "${report_jsn}" - elif [ "${action}" = "mail" ] && [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ]; then - ("${adb_mailservice}" "${adb_ver}" "${content}" >/dev/null 2>&1) & - bg_pid="${!}" - fi - f_log "debug" "f_report ::: action: ${action}, top_count: ${top_count}, res_count: ${res_count}, search: ${search}, dump_util: ${adb_dumpcmd}, rep_dir: ${adb_reportdir}, rep_iface: ${adb_repiface:-"-"}, rep_listen: ${adb_replisten}, rep_chunksize: ${adb_repchunksize}, rep_chunkcnt: ${adb_repchunkcnt}, rep_resolve: ${adb_represolve}" + case "${action}" in + "cli") + printf '%s\n' "${content}" + ;; + "json") + if [ "${adb_map}" = "1" ] && [ -s "${map_jsn}" ]; then + jsn="$("${adb_catcmd}" ${report_jsn} ${map_jsn} 2>>"${adb_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]]\n' "${jsn}" + else + jsn="$("${adb_catcmd}" "${report_jsn}" 2>>"${adb_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]\n' "${jsn}" + fi + ;; + "mail") + [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && "${adb_mailservice}" "${content}" >/dev/null 2>&1 + "${adb_rmcmd}" -f "${report_txt}" + ;; + "gen") + printf '%s\n' "1" >"${adb_rundir}/adblock.report" + ;; + esac } # source required system libraries @@ -1473,47 +2452,66 @@ else f_log "err" "system libraries not found" fi -# awk check +# create runtime directory if it doesn't exist # -adb_awk="$(command -v gawk)" -if [ ! -x "${adb_awk}" ]; then - adb_awk="$(command -v awk)" - [ ! -x "${adb_awk}" ] && f_log "err" "awk not found or not executable" -fi +[ ! -d "${adb_rundir}" ] && mkdir -p "${adb_rundir}" -# sort check +# reference required system utilities # -adb_sort="$(command -v sort)" -if [ ! -x "${adb_sort}" ] || ! "${adb_sort}" --version 2>/dev/null | grep -q "coreutils"; then - f_log "err" "coreutils sort not found or not executable" -fi +adb_mvcmd="$(f_cmd mv)" +adb_lncmd="$(f_cmd ln)" +adb_rmcmd="$(f_cmd rm)" +adb_catcmd="$(f_cmd cat)" +adb_zcatcmd="$(f_cmd zcat)" +adb_awkcmd="$(f_cmd gawk awk)" +adb_sortcmd="$(f_cmd sort)" +adb_grepcmd="$(f_cmd grep)" +adb_gzipcmd="$(f_cmd gzip)" +adb_pgrepcmd="$(f_cmd pgrep)" +adb_sedcmd="$(f_cmd sed)" +adb_findcmd="$(f_cmd find)" +adb_jsoncmd="$(f_cmd jsonfilter)" +adb_ubuscmd="$(f_cmd ubus)" +adb_loggercmd="$(f_cmd logger)" +adb_lookupcmd="$(f_cmd nslookup)" +adb_xargscmd="$(f_cmd xargs)" +adb_flockcmd="$(f_cmd flock)" +adb_dumpcmd="$(f_cmd tcpdump optional)" +adb_mailcmd="$(f_cmd msmtp optional)" +adb_logreadcmd="$(f_cmd logread optional)" +adb_nftcmd="$(f_cmd nft)" # handle different adblock actions # f_load case "${adb_action}" in - "stop") - f_rmdns - ;; - "restart") - f_rmdns - f_env - f_main - ;; - "suspend") - [ "${adb_dns}" != "raw" ] && f_switch suspend - ;; - "resume") - [ "${adb_dns}" != "raw" ] && f_switch resume - ;; - "report") - f_report "${2}" "${3}" "${4}" "${5}" - ;; - "query") - f_query "${2}" - ;; - "start" | "reload") - f_env - f_main - ;; +"stop") + f_temp + f_nftremove + f_rmdns + f_jsnup "stopped" + ;; +"suspend") + [ "${adb_dns}" != "raw" ] && f_switch suspend + ;; +"resume") + [ "${adb_dns}" != "raw" ] && f_switch resume + ;; +"report") + f_report "${2}" "${3}" "${4}" "${5}" + ;; +"search") + f_search "${2}" + ;; +"boot" | "start" | "reload") + f_env + f_main + ;; +"restart") + f_temp + f_jsnup "processing" + f_rmdns + f_env + f_main + ;; esac diff --git a/packages/adblock/files/adblock.sources b/packages/adblock/files/adblock.sources deleted file mode 100644 index 85af8602b..000000000 --- a/packages/adblock/files/adblock.sources +++ /dev/null @@ -1,352 +0,0 @@ -{ - "adaway": { - "url": "https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "mobile", - "descurl": "https://github.com/AdAway/adaway.github.io" - }, - "adguard": { - "url": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", - "rule": "BEGIN{FS=\"[\/|^|\\r]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+[\\/\\^\\r]+$/{print tolower($3)}", - "size": "L", - "focus": "general", - "descurl": "https://adguard.com" - }, - "adguard_tracking": { - "url": "https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/combined_disguised_trackers_justdomains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/AdguardTeam/cname-trackers" - }, - "android_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "andryou": { - "url": "https://gitlab.com/andryou/block/raw/master/kouhai-compressed-domains", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://gitlab.com/andryou/block/-/blob/master/readme.md" - }, - "anti_ad": { - "url": "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://github.com/privacy-protection-tools/anti-AD/blob/master/README.md" - }, - "antipopads": { - "url": "https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://github.com/AdroitAdorKhan/antipopads-re" - }, - "anudeep": { - "url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "compilation", - "descurl": "https://github.com/anudeepND/blacklist" - }, - "bitcoin": { - "url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "mining", - "descurl": "https://github.com/hoshsadiq/adblock-nocoin-list" - }, - "cpbl": { - "url": "https://raw.githubusercontent.com/bongochong/CombinedPrivacyBlockLists/master/NoFormatting/cpbl-ctld.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://github.com/bongochong/CombinedPrivacyBlockLists" - }, - "disconnect": { - "url": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://disconnect.me" - }, - "doh_blocklist": { - "url": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains_overall.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "doh_server", - "descurl": "https://github.com/dibdot/DoH-IP-blocklists" - }, - "easylist": { - "url": "https://easylist-downloads.adblockplus.org/easylist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "compilation", - "descurl": "https://easylist.to" - }, - "easyprivacy": { - "url": "https://easylist-downloads.adblockplus.org/easyprivacy.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "tracking", - "descurl": "https://easylist.to" - }, - "firetv_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "games_tracking": { - "url": "https://raw.githubusercontent.com/KodoPengin/GameIndustry-hosts-Template/master/Main-Template/hosts", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "tracking", - "descurl": "https://www.gameindustry.eu" - }, - "hblock": { - "url": "https://hblock.molinero.dev/hosts_domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://hblock.molinero.dev" - }, - "lightswitch05": { - "url": "https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://github.com/lightswitch05/hosts" - }, - "notracking": { - "url": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/dnscrypt-proxy/dnscrypt-proxy.blacklist.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "tracking", - "descurl": "https://github.com/notracking/hosts-blocklists" - }, - "oisd_big": { - "url": "https://big.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XXL", - "focus": "general", - "descurl": "https://oisd.nl" - }, - "oisd_nsfw": { - "url": "https://nsfw.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XXL", - "focus": "porn", - "descurl": "https://oisd.nl" - }, - "oisd_small": { - "url": "https://small.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "L", - "focus": "general", - "descurl": "https://oisd.nl" - }, - "openphish": { - "url": "https://openphish.com/feed.txt", - "rule": "BEGIN{FS=\"\/\"}/^http[s]?:\\/\\/([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+(\\/|$)/{print tolower($3)}", - "size": "S", - "focus": "phishing", - "descurl": "https://openphish.com" - }, - "phishing_army": { - "url": "https://phishing.army/download/phishing_army_blocklist_extended.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "phishing", - "descurl": "https://phishing.army" - }, - "reg_cn": { - "url": "https://easylist-downloads.adblockplus.org/easylistchina.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_china", - "descurl": "https://easylist.to" - }, - "reg_cz": { - "url": "https://easylist-downloads.adblockplus.org/easylistczechslovak.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_czech+slovak", - "descurl": "https://easylist.to" - }, - "reg_de": { - "url": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_germany", - "descurl": "https://easylist.to" - }, - "reg_es": { - "url": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_spain", - "descurl": "https://easylist.to" - }, - "reg_fi": { - "url": "https://raw.githubusercontent.com/finnish-easylist-addition/finnish-easylist-addition/master/Finland_adb.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_finland", - "descurl": "https://github.com/finnish-easylist-addition" - }, - "reg_fr": { - "url": "https://easylist-downloads.adblockplus.org/liste_fr.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "reg_france", - "descurl": "https://forums.lanik.us/viewforum.php?f=91" - }, - "reg_id": { - "url": "https://easylist-downloads.adblockplus.org/abpindo.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_indonesia", - "descurl": "https://easylist.to" - }, - "reg_it": { - "url": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_italy", - "descurl": "https://easylist.to" - }, - "reg_jp": { - "url": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_japan", - "descurl": "https://github.com/k2jp/abp-japanese-filters" - }, - "reg_kr": { - "url": "https://raw.githubusercontent.com/List-KR/List-KR/master/filters-share/adservice.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_korea", - "descurl": "https://github.com/List-KR/List-KR" - }, - "reg_nl": { - "url": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_netherlands", - "descurl": "https://easylist.to" - }, - "reg_pl": { - "url": "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "reg_poland", - "descurl": "https://kadantiscam.netlify.app" - }, - "reg_ro": { - "url": "https://easylist-downloads.adblockplus.org/rolist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_romania", - "descurl": "https://easylist.to" - }, - "reg_ru": { - "url": "https://easylist-downloads.adblockplus.org/ruadlist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_russia", - "descurl": "https://easylist.to" - }, - "reg_se": { - "url": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "reg_sweden", - "descurl": "https://github.com/lassekongo83/Frellwits-filter-lists" - }, - "reg_vn": { - "url": "https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "reg_vietnam", - "descurl": "https://bigdargon.github.io/hostsVN" - }, - "smarttv_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "spam404": { - "url": "https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://github.com/Dawsey21" - }, - "stevenblack": { - "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "VAR", - "focus": "compilation", - "descurl": "https://github.com/StevenBlack/hosts" - }, - "stopforumspam": { - "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "spam", - "descurl": "https://www.stopforumspam.com" - }, - "utcapitole": { - "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "VAR", - "focus": "general", - "descurl": "https://dsi.ut-capitole.fr/blacklists/index_en.php" - }, - "wally3k": { - "url": "https://v.firebog.net/hosts/static/w3kbl.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "compilation", - "descurl": "https://firebog.net/about" - }, - "whocares": { - "url": "https://someonewhocares.org/hosts/hosts", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "general", - "descurl": "https://someonewhocares.org" - }, - "winhelp": { - "url": "https://winhelp2002.mvps.org/hosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "general", - "descurl": "https://winhelp2002.mvps.org" - }, - "winspy": { - "url": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "win_telemetry", - "descurl": "https://github.com/crazy-max/WindowsSpyBlocker" - }, - "yoyo": { - "url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://pgl.yoyo.org/as" - } -} diff --git a/packages/ns-api/Makefile b/packages/ns-api/Makefile index 1ce3a07c5..6ea6d583d 100644 --- a/packages/ns-api/Makefile +++ b/packages/ns-api/Makefile @@ -22,6 +22,7 @@ define Package/ns-api TITLE:=NethSecurity REST API URL:=https://github.com/NethServer/nethsecurity-controller/ DEPENDS:= \ + +adblock \ +coreutils-date \ +coreutils-stty \ +python3-idna \ @@ -32,6 +33,7 @@ define Package/ns-api +python3-urllib \ +sshpass \ +wireguard-tools + EXTRA_DEPENDS:=adblock (>=4.5.3) PKGARCH:=all endef diff --git a/packages/ns-api/files/ns.threatshield b/packages/ns-api/files/ns.threatshield index be8ed3de1..f5bf8f97d 100644 --- a/packages/ns-api/files/ns.threatshield +++ b/packages/ns-api/files/ns.threatshield @@ -120,17 +120,24 @@ def read_gz(file): else: return {} -def list_dns_feeds(enterprise=False): - # Decompress and read the JSON file /etc/adblock/combined.sources.gz - ret = {} - sources = '/etc/adblock/combined.sources.gz' - if not os.path.exists(sources): - ret = read_gz('/usr/share/threat_shield/community-dns.sources.gz') - if enterprise: - ret.update(read_gz('/usr/share/threat_shield/nethesis-dns.sources.gz')) - else: - ret = read_gz(sources) +def read_json(file): + if os.path.exists(file) and os.path.getsize(file) > 0: + with open(file, 'r') as f: + try: + return json.load(f) + except Exception: + return {} + return {} +def list_dns_feeds(enterprise=False): + # Read feeds from adblock.feeds (builtin) and adblock.custom.feeds (NethSecurity/user overrides) + ret = read_json('/etc/adblock/adblock.feeds') + custom = read_json('/etc/adblock/adblock.custom.feeds') + if custom: + ret.update(custom) + elif enterprise: + # fallback: merge nethesis sources gz if no custom feeds file yet + ret.update(read_gz('/usr/share/threat_shield/nethesis-dns.sources.gz')) return ret def get_confidence(f, enterprise=False): @@ -391,7 +398,7 @@ def dns_list_blocklist(e_uci): has_bl = has_bl_entitlement(e_uci) feeds = list_dns_feeds(has_bl) try: - enabled_feeds = list(e_uci.get_all('adblock', 'global', 'adb_sources')) + enabled_feeds = list(e_uci.get_all('adblock', 'global', 'adb_feed')) except: enabled_feeds = [] for f in feeds: @@ -413,7 +420,7 @@ def dns_list_blocklist(e_uci): def dns_edit_blocklist(e_uci, payload): try: - enabled = list(e_uci.get_all('adblock', 'global', 'adb_sources')) + enabled = list(e_uci.get_all('adblock', 'global', 'adb_feed')) except: enabled = [] if payload['enabled'] and payload['blocklist'] not in enabled: @@ -424,7 +431,7 @@ def dns_edit_blocklist(e_uci, payload): has_bl = has_bl_entitlement(e_uci) feeds = list_dns_feeds(has_bl) enabled = [feed for feed in enabled if feed in feeds] - e_uci.set('adblock', 'global', 'adb_sources', enabled) + e_uci.set('adblock', 'global', 'adb_feed', enabled) e_uci.save('adblock') return {'message': 'success'} @@ -438,11 +445,11 @@ def dns_list_zones(e_uci): def dns_list_settings(e_uci): ts_enabled = e_uci.get('adblock', 'global', 'ts_enabled', default='0') try: - zones = list(e_uci.get_all('adblock', 'global', 'adb_zonelist')) + zones = list(e_uci.get_all('adblock', 'global', 'adb_nftdevforce')) except: zones = ['lan'] try: - ports = list(e_uci.get_all('adblock', 'global', 'adb_portlist')) + ports = list(e_uci.get_all('adblock', 'global', 'adb_nftportforce')) except: ports = ['53', '853'] return { 'data': {'enabled': ts_enabled == '1', "zones": zones, "ports": ports} } @@ -453,65 +460,64 @@ def dns_edit_settings(e_uci, payload): raise ValidationError('zones', 'wan_zone_not_allowed', payload['zones']) e_uci.set('adblock', 'global', 'ts_enabled', '1') e_uci.set('adblock', 'global', 'adb_enabled', '1') - e_uci.set('adblock', 'global', 'adb_backup', '1') - e_uci.set('adblock', 'global', 'adb_forcedns', '1') - e_uci.set('adblock', 'global', 'adb_zonelist', payload.get('zones', ['lan'])) - e_uci.set('adblock', 'global', 'adb_portlist', payload.get('ports', ['53', '853'])) + e_uci.set('adblock', 'global', 'adb_nftforce', '1') + e_uci.set('adblock', 'global', 'adb_nftdevforce', payload.get('zones', ['lan'])) + e_uci.set('adblock', 'global', 'adb_nftportforce', payload.get('ports', ['53', '853'])) else: e_uci.set('adblock', 'global', 'ts_enabled', '0') e_uci.set('adblock', 'global', 'adb_enabled', '0') - e_uci.set('adblock', 'global', 'adb_forcedns', '0') + e_uci.set('adblock', 'global', 'adb_nftforce', '0') e_uci.save('adblock') return {'message': 'success'} def dns_list_allowed(): - return { "data": get_allow_list('/etc/adblock/adblock.whitelist') } + return { "data": get_allow_list('/etc/adblock/adblock.allowlist') } def dns_list_blocked(): - return { "data": get_allow_list('/etc/adblock/adblock.blacklist') } + return { "data": get_allow_list('/etc/adblock/adblock.blocklist') } def dns_add_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') + cur = get_allow_list('/etc/adblock/adblock.allowlist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload['description'] }) - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') return {'message': 'success'} def dns_add_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') + cur = get_allow_list('/etc/adblock/adblock.blocklist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload.get('description') }) - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') return {'message': 'success'} def dns_edit_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') + cur = get_allow_list('/etc/adblock/adblock.allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload['description'] break - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') return {'message': 'success'} def dns_edit_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') + cur = get_allow_list('/etc/adblock/adblock.blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload.get('description') break - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') return {'message': 'success'} def dns_delete_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') + cur = get_allow_list('/etc/adblock/adblock.allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -519,11 +525,11 @@ def dns_delete_allowed(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') return {'message': 'success'} def dns_delete_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') + cur = get_allow_list('/etc/adblock/adblock.blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -531,38 +537,38 @@ def dns_delete_blocked(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') return {'message': 'success'} def dns_list_bypass(e_uci): - # adblock.global.adb_bypass + # adblock.global.ns_tsdns_bypass try: - bypass = e_uci.get_all('adblock', 'global', 'adb_bypass') + bypass = e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass') except: bypass = [] return { "data": bypass } def dns_add_bypass(e_uci, payload): try: - bypass = list(e_uci.get_all('adblock', 'global', 'adb_bypass')) + bypass = list(e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass')) except: bypass = [] if payload['address'] in bypass: raise ValidationError('address', 'address_already_present', payload['address']) bypass.append(payload['address']) - e_uci.set('adblock', 'global', 'adb_bypass', bypass) + e_uci.set('adblock', 'global', 'ns_tsdns_bypass', bypass) e_uci.save('adblock') return {'message': 'success'} def dns_delete_bypass(e_uci, payload): try: - bypass = list(e_uci.get_all('adblock', 'global', 'adb_bypass')) + bypass = list(e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass')) except: bypass = [] if payload['address'] not in bypass: raise ValidationError('address', 'address_not_found', payload['address']) bypass.remove(payload['address']) - e_uci.set('adblock', 'global', 'adb_bypass', bypass) + e_uci.set('adblock', 'global', 'ns_tsdns_bypass', bypass) e_uci.save('adblock') return {'message': 'success'} diff --git a/packages/ns-threat_shield/Makefile b/packages/ns-threat_shield/Makefile index 43001ed90..2b66694bf 100644 --- a/packages/ns-threat_shield/Makefile +++ b/packages/ns-threat_shield/Makefile @@ -22,7 +22,8 @@ define Package/ns-threat_shield CATEGORY:=NethSecurity TITLE:=Threat shield block list URL:=https://github.com/NethServer/nethsecurity/ - DEPENDS:=+wget-ssl +adblock +jq + DEPENDS:=+wget-ssl +adblock +jq +ns-api + EXTRA_DEPENDS:=adblock (>=4.5.3), ns-api (>=3.6.2) PKGARCH:=all endef diff --git a/packages/ns-threat_shield/README.md b/packages/ns-threat_shield/README.md index cc8ace162..6943b0ebc 100644 --- a/packages/ns-threat_shield/README.md +++ b/packages/ns-threat_shield/README.md @@ -82,18 +82,13 @@ uci commit adblock /etc/init.d/adblock restart ``` -### Custom categories - -To add custom categories, create a file `/etc/adblock/custom.sources.gz` with the list of categories to block. -If such file is present, the `/usr/share/threat_shield/community-dns.sources.gz` will be ignored. - ### DNS redirect bypass Allow bypass of DNS redirect for a specific source IP: ``` -uci add_list adblock.global.adb_bypass=192.168.100.2 +uci add_list adblock.global.ns_tsdns_bypass=192.168.100.2 uci commit adblock -/etc/init.d/adblock restart +/etc/init.d/adblock reload ``` For more info see [adblock repository](https://github.com/openwrt/packages/tree/master/net/adblock). diff --git a/packages/ns-threat_shield/files/ts-dns b/packages/ns-threat_shield/files/ts-dns index f67484be0..01f6399f6 100755 --- a/packages/ns-threat_shield/files/ts-dns +++ b/packages/ns-threat_shield/files/ts-dns @@ -8,8 +8,8 @@ DEST_DIR=/etc/adblock NETHESIS_SOURCES=/usr/share/threat_shield/nethesis-dns.sources.gz COMMUNITY_SOURCES=/usr/share/threat_shield/community-dns.sources.gz -CUSTOM_SOURCES=/etc/adblock/custom.sources.gz -TMP_FILE=/tmp/combined.sources +CUSTOM_FEEDS="$DEST_DIR/adblock.custom.feeds" +TMP_FILE=/tmp/ts-dns.sources SYSTEM_ID=$(uci -q get ns-plug.config.system_id) SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) @@ -17,42 +17,28 @@ TYPE=$(uci -q get ns-plug.config.type) TS_ENABLED=$(uci -q get adblock.global.ts_enabled) if [ "$TS_ENABLED" = 1 ]; then - # Setup new blacklist source - uci set adblock.global.adb_srcarc="$DEST_DIR"/combined.sources.gz - # Setup dnsmasq as backend uci set adblock.global.adb_dns='dnsmasq' uci set adblock.global.adb_dnsinstance='0' - # Setup wget with compression support - uci set adblock.global.adb_fetchutil='wget' - uci set adblock.global.adb_fetchparm="--compression=gzip --no-cache --no-cookies --max-redirect=0 --timeout=20 -O" - - if [ -f $CUSTOM_SOURCES ]; then - # Add custom sources, if present - gunzip -c $CUSTOM_SOURCES > $TMP_FILE - else - # Use community sources - gunzip -c $COMMUNITY_SOURCES > $TMP_FILE - fi + # Build custom feeds from community sources + gunzip -c "$COMMUNITY_SOURCES" > "$TMP_FILE" - # Setup Nethesis sources if the machine has a subscription - if [ ! -z "$SYSTEM_SECRET" ] && [ ! -z "$SYSTEM_ID" ]; then - # Replaces credentials and compress - gunzip -c $NETHESIS_SOURCES | sed -e "s/__USER__/$SYSTEM_ID/" -e "s/__PASSWORD__/$SYSTEM_SECRET/" -e "s/__TYPE__/$TYPE/" >> $TMP_FILE + # Merge Nethesis sources if the machine has a subscription + if [ -n "$SYSTEM_SECRET" ] && [ -n "$SYSTEM_ID" ]; then + gunzip -c "$NETHESIS_SOURCES" | sed -e "s/__USER__/$SYSTEM_ID/" -e "s/__PASSWORD__/$SYSTEM_SECRET/" -e "s/__TYPE__/$TYPE/" >> "$TMP_FILE" fi - # Merge the source list and compress it to the final file - jq -s 'reduce .[] as $item ({}; . * $item)' $TMP_FILE | gzip -c > "$DEST_DIR"/combined.sources.gz + # Merge all sources into a single JSON object and write to adblock.custom.feeds + jq -s 'reduce .[] as $item ({}; . * $item)' "$TMP_FILE" > "$CUSTOM_FEEDS" # Cleanup - rm -f $TMP_FILE + rm -f "$TMP_FILE" else - # Reset sources to origin file if threat shield is not enabled - uci -q delete adblock.global.adb_srcarc - uci -q delete adblock.global.adb_fetchparam - uci set adblock.global.adb_fetchutil='curl' + # Clear custom feeds when threat shield is disabled + : > "$CUSTOM_FEEDS" fi # Save changes uci commit adblock + From 31b9d8d9db3ac485eef5657f2b0b5ed97a25e073 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 15 May 2026 08:35:31 +0200 Subject: [PATCH 45/48] fix(adblock): reload nft rules Changes: - add a new `nft-reload` action inside adbblock.sh - trigger reload when the configuration has been updated - call nft-reload on reload The above changes will recreated the nft chain when the bypass configuration has been changed. --- packages/adblock/files/adblock.init | 6 ++++++ packages/adblock/files/adblock.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/packages/adblock/files/adblock.init b/packages/adblock/files/adblock.init index b0bd5570b..9a6262f8e 100755 --- a/packages/adblock/files/adblock.init +++ b/packages/adblock/files/adblock.init @@ -76,7 +76,10 @@ restart() { } reload_service() { + # Start NethSecurity patch /usr/sbin/ts-dns # configure threat shield dns, if needed + ${adb_script} nft-reload + # End NethSecurity patch rc_procd start_service reload } @@ -137,4 +140,7 @@ service_triggers() { for iface in ${trigger}; do procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" start done + # Start NethSecurity patch + procd_add_reload_trigger adblock + # End NethSecurity patch } diff --git a/packages/adblock/files/adblock.sh b/packages/adblock/files/adblock.sh index 6c0d738ef..155c6d713 100755 --- a/packages/adblock/files/adblock.sh +++ b/packages/adblock/files/adblock.sh @@ -2507,6 +2507,12 @@ case "${adb_action}" in f_env f_main ;; +# Start NethSecurity patch +"nft-reload") + f_nftremove + f_nftadd + ;; +# End NethSecurity patch "restart") f_temp f_jsnup "processing" From 018237b3c5f1b63805493a1054d3ab8b304dc749 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 15 May 2026 08:52:41 +0200 Subject: [PATCH 46/48] fix(adblock): stage dns list changes Store Threat Shield DNS local allow and block list edits in UCI so rapid API calls no longer rewrite adblock files or restart the service immediately. Write the physical adblock list files during the next reload, add a one-shot migration for existing list files, and document the staged workflow for the affected API methods. Refs #1572 Assisted-by: Copilot:gpt-5.4 --- packages/adblock/Makefile | 5 +- .../adblock/files/99_adblock_migrate_lists.sh | 24 + packages/adblock/files/adblock.allowlist | 0 packages/adblock/files/adblock.blocklist | 0 packages/adblock/files/adblock.init | 38 +- packages/ns-api/README.md | 5 +- packages/ns-api/files/ns.threatshield | 90 +-- packages/ns-api/openapi.yml | 590 ++++++++++++++++-- 8 files changed, 650 insertions(+), 102 deletions(-) create mode 100644 packages/adblock/files/99_adblock_migrate_lists.sh delete mode 100644 packages/adblock/files/adblock.allowlist delete mode 100644 packages/adblock/files/adblock.blocklist diff --git a/packages/adblock/Makefile b/packages/adblock/Makefile index e7afb7727..2a9f153fb 100644 --- a/packages/adblock/Makefile +++ b/packages/adblock/Makefile @@ -30,8 +30,6 @@ endef define Package/adblock/conffiles /etc/config/adblock -/etc/adblock/adblock.allowlist -/etc/adblock/adblock.blocklist /etc/adblock/adblock.custom.feeds endef @@ -56,14 +54,13 @@ define Package/adblock/install $(INSTALL_DIR) $(1)/etc/adblock $(INSTALL_BIN) ./files/adblock.mail $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.allowlist $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.blocklist $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.categories $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.feeds $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.custom.feeds $(1)/etc/adblock $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/95-adblock-housekeeping $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/99_adblock_migrate_lists.sh $(1)/etc/uci-defaults/99_adblock_migrate_lists endef $(eval $(call BuildPackage,adblock)) diff --git a/packages/adblock/files/99_adblock_migrate_lists.sh b/packages/adblock/files/99_adblock_migrate_lists.sh new file mode 100644 index 000000000..ceea7796d --- /dev/null +++ b/packages/adblock/files/99_adblock_migrate_lists.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Migrate local allow/block list files to staged UCI storage. +# Skip once the dedicated section is already present. +uci -q get adblock.ns_lists >/dev/null 2>&1 && exit 0 + +uci set adblock.ns_lists=ns_lists + +for type in allowlist blocklist; do + file="/etc/adblock/adblock.${type}" + [ -f "${file}" ] || continue + + while IFS= read -r line || [ -n "${line}" ]; do + [ -z "${line}" ] && continue + uci add_list "adblock.ns_lists.${type}=${line}" + done < "${file}" +done + +uci commit adblock diff --git a/packages/adblock/files/adblock.allowlist b/packages/adblock/files/adblock.allowlist deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/adblock/files/adblock.blocklist b/packages/adblock/files/adblock.blocklist deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/adblock/files/adblock.init b/packages/adblock/files/adblock.init index 9a6262f8e..14dbc1190 100755 --- a/packages/adblock/files/adblock.init +++ b/packages/adblock/files/adblock.init @@ -53,12 +53,37 @@ boot() { rc_procd start_service boot } +f_append_local_list_entry() { + local entry="${1}" + local file="${2}" + + printf '%s\n' "${entry}" >> "${file}" +} + +f_write_local_lists() { + local allowlist_file='/etc/adblock/adblock.allowlist' + local blocklist_file='/etc/adblock/adblock.blocklist' + + uci -q get adblock.ns_lists >/dev/null 2>&1 || return 0 + + config_load adblock + + : > "${allowlist_file}" + config_list_foreach ns_lists allowlist f_append_local_list_entry "${allowlist_file}" + + : > "${blocklist_file}" + config_list_foreach ns_lists blocklist f_append_local_list_entry "${blocklist_file}" +} + start_service() { if "${adb_init}" enabled; then /usr/sbin/ts-dns # configure threat shield dns, if needed if [ "${action}" = "boot" ]; then [ -n "$(uci_get adblock global adb_trigger)" ] && return 0 fi + # Start NethSecurity patch + f_write_local_lists + # End NethSecurity patch procd_open_instance "adblock" procd_set_param command "${adb_script}" "${@:-"${action}"}" procd_set_param pidfile "${adb_pidfile}" @@ -70,16 +95,19 @@ start_service() { } restart() { + # Start NethSecurity patch /usr/sbin/ts-dns # configure threat shield dns, if needed + # End NethSecurity patch stop_service "restart" rc_procd start_service restart } reload_service() { - # Start NethSecurity patch + # Start NethSecurity patch /usr/sbin/ts-dns # configure threat shield dns, if needed - ${adb_script} nft-reload - # End NethSecurity patch + f_write_local_lists + ${adb_script} nft-reload + # End NethSecurity patch rc_procd start_service reload } @@ -140,7 +168,7 @@ service_triggers() { for iface in ${trigger}; do procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" start done - # Start NethSecurity patch + # Start NethSecurity patch procd_add_reload_trigger adblock - # End NethSecurity patch + # End NethSecurity patch } diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 3f19bdaff..a0265ba4c 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -6259,12 +6259,15 @@ Response example: { "data": [ { - "address": "nethesis.it" + "address": "nethesis.it", + "description": "my allow1" } ] } ``` +The allow and block list methods work on UCI-staged data. Changes are visible immediately through the API and are written to `/etc/adblock/adblock.allowlist` and `/etc/adblock/adblock.blocklist` during the next adblock reload triggered by `ns.commit` or `reload_config`. + ### dns-add-allowed Add a domain which is always allowed: diff --git a/packages/ns-api/files/ns.threatshield b/packages/ns-api/files/ns.threatshield index f5bf8f97d..7ed3e3638 100644 --- a/packages/ns-api/files/ns.threatshield +++ b/packages/ns-api/files/ns.threatshield @@ -101,14 +101,28 @@ def write_allow_list(allow_list, file='/etc/banip/banip.allowlist'): f.write('\n') subprocess.run(["/etc/init.d/banip", "reload"], capture_output=True) -def dns_write_local_list(local_list, file): - with open(file, 'w') as f: - for x in local_list: - f.write(x['address']) - if x['description']: - f.write(' #' + x['description']) - f.write('\n') - subprocess.run(["/etc/init.d/adblock", "restart"], capture_output=True) +def dns_get_local_list(e_uci, list_type): + values = e_uci.get('adblock', 'ns_lists', list_type, list=True, default=[]) + ret = [] + for value in values: + parts = value.split('#', 1) + ret.append({'address': parts[0].strip(), 'description': parts[1].strip() if len(parts) > 1 else ''}) + return ret + +def dns_write_local_list(e_uci, local_list, list_type): + option = 'allowlist' if list_type == 'allowlist' else 'blocklist' + e_uci.set('adblock', 'ns_lists', 'ns_lists') + + values = tuple( + f"{entry['address']} #{entry['description']}" if entry.get('description') else entry['address'] + for entry in local_list + ) + if values: + e_uci.set('adblock', 'ns_lists', option, values) + elif e_uci.get('adblock', 'ns_lists', option, list=True, default=[]): + e_uci.delete('adblock', 'ns_lists', option) + + e_uci.save('adblock') def write_block_list(block_list): write_allow_list(block_list, '/etc/banip/banip.blocklist') @@ -470,54 +484,54 @@ def dns_edit_settings(e_uci, payload): e_uci.save('adblock') return {'message': 'success'} -def dns_list_allowed(): - return { "data": get_allow_list('/etc/adblock/adblock.allowlist') } +def dns_list_allowed(e_uci): + return { "data": dns_get_local_list(e_uci, 'allowlist') } -def dns_list_blocked(): - return { "data": get_allow_list('/etc/adblock/adblock.blocklist') } +def dns_list_blocked(e_uci): + return { "data": dns_get_local_list(e_uci, 'blocklist') } -def dns_add_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.allowlist') +def dns_add_allowed(e_uci, payload): + cur = dns_get_local_list(e_uci, 'allowlist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload['description'] }) - dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') + dns_write_local_list(e_uci, cur, 'allowlist') return {'message': 'success'} -def dns_add_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blocklist') +def dns_add_blocked(e_uci, payload): + cur = dns_get_local_list(e_uci, 'blocklist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload.get('description') }) - dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') + dns_write_local_list(e_uci, cur, 'blocklist') return {'message': 'success'} -def dns_edit_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.allowlist') +def dns_edit_allowed(e_uci, payload): + cur = dns_get_local_list(e_uci, 'allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload['description'] break - dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') + dns_write_local_list(e_uci, cur, 'allowlist') return {'message': 'success'} -def dns_edit_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blocklist') +def dns_edit_blocked(e_uci, payload): + cur = dns_get_local_list(e_uci, 'blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload.get('description') break - dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') + dns_write_local_list(e_uci, cur, 'blocklist') return {'message': 'success'} -def dns_delete_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.allowlist') +def dns_delete_allowed(e_uci, payload): + cur = dns_get_local_list(e_uci, 'allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -525,11 +539,11 @@ def dns_delete_allowed(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') + dns_write_local_list(e_uci, cur, 'allowlist') return {'message': 'success'} -def dns_delete_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blocklist') +def dns_delete_blocked(e_uci, payload): + cur = dns_get_local_list(e_uci, 'blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -537,7 +551,7 @@ def dns_delete_blocked(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') + dns_write_local_list(e_uci, cur, 'blocklist') return {'message': 'success'} def dns_list_bypass(e_uci): @@ -809,15 +823,15 @@ elif cmd == 'call': ret = dns_list_zones(e_uci) elif action == 'dns-add-allowed': payload = json.loads(sys.stdin.read()) - ret = dns_add_allowed(payload) + ret = dns_add_allowed(e_uci, payload) elif action == 'dns-edit-allowed': payload = json.loads(sys.stdin.read()) - ret = dns_edit_allowed(payload) + ret = dns_edit_allowed(e_uci, payload) elif action == 'dns-list-allowed': - ret = dns_list_allowed() + ret = dns_list_allowed(e_uci) elif action == 'dns-delete-allowed': payload = json.loads(sys.stdin.read()) - ret = dns_delete_allowed(payload) + ret = dns_delete_allowed(e_uci, payload) elif action == 'dns-list-bypass': ret = dns_list_bypass(e_uci) elif action == 'dns-add-bypass': @@ -827,16 +841,16 @@ elif cmd == 'call': payload = json.loads(sys.stdin.read()) ret = dns_delete_bypass(e_uci, payload) elif action == 'dns-list-blocked': - ret = dns_list_blocked() + ret = dns_list_blocked(e_uci) elif action == 'dns-add-blocked': payload = json.loads(sys.stdin.read()) - ret = dns_add_blocked(payload) + ret = dns_add_blocked(e_uci, payload) elif action == 'dns-edit-blocked': payload = json.loads(sys.stdin.read()) - ret = dns_edit_blocked(payload) + ret = dns_edit_blocked(e_uci, payload) elif action == 'dns-delete-blocked': payload = json.loads(sys.stdin.read()) - ret = dns_delete_blocked(payload) + ret = dns_delete_blocked(e_uci, payload) print(json.dumps(ret)) except ValidationError as ex: diff --git a/packages/ns-api/openapi.yml b/packages/ns-api/openapi.yml index ece5198e0..cef18dc97 100644 --- a/packages/ns-api/openapi.yml +++ b/packages/ns-api/openapi.yml @@ -79,6 +79,29 @@ components: items: $ref: "#/components/schemas/ValidationErrorDetail" + SuccessResponse: + type: object + required: [message] + properties: + message: + type: string + example: success + + ThreatShieldDnsListEntry: + type: object + required: + - address + - description + properties: + address: + type: string + description: Domain name present in the local Threat Shield DNS list + example: nethesis.it + description: + type: string + description: Optional free-form description associated with the domain + example: my allow1 + securitySchemes: BearerAuth: type: http @@ -188,80 +211,539 @@ paths: example: success - $ref: "#/components/schemas/ValidationError" - $ref: "#/components/schemas/Error" - POST /ubus/ns.telegraf/list-alerts: + POST /ubus/ns.threatshield/dns-list-allowed: + post: + summary: List local Threat Shield DNS allowlist entries + description: Returns the allowlist entries currently staged in UCI. They are written to the adblock file on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-list-allowed + tags: + - threatshield + responses: + "200": + description: Staged allowlist entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/ThreatShieldDnsListEntry" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-allowed: + post: + summary: Add a local Threat Shield DNS allowlist entry + description: Stages the new allowlist entry in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-add-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + - description + properties: + address: + type: string + description: Domain to add to the local allowlist + example: nethesis.it + description: + type: string + description: Free-form description for the domain + example: my allow1 + responses: + "200": + description: Allowlist entry staged or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-allowed: + post: + summary: Edit a local Threat Shield DNS allowlist entry + description: Updates the staged allowlist description in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-edit-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + - description + properties: + address: + type: string + description: Existing domain in the local allowlist + example: nethesis.it + description: + type: string + description: Updated description for the domain + example: my new desc + responses: + "200": + description: Allowlist entry updated or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-allowed: + post: + summary: Delete a local Threat Shield DNS allowlist entry + description: Removes the staged allowlist entry from UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-delete-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local allowlist + example: nethesis.it + responses: + "200": + description: Allowlist entry deleted or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-blocklist: post: - summary: List current monitoring alerts - operationId: ns.telegraf.list-alerts + summary: List available Threat Shield DNS blocklists + operationId: ns.threatshield.dns-list-blocklist tags: - - telegraf + - threatshield responses: "200": - description: Current pending and firing alerts evaluated by vmalert + description: Available blocklists content: application/json: schema: oneOf: - type: object required: - - alerts + - data properties: - alerts: + data: type: array items: type: object required: - - state - name - - labels - - annotations - - activeAt + - type + - enabled + - confidence + - description properties: - state: - type: string - description: Alert state reported by vmalert - example: firing name: type: string - description: Alert name - example: BackupEncryptionDisabled - value: - type: string - description: Evaluated expression value - example: "0" - labels: - type: object - description: Alert labels emitted by vmalert - additionalProperties: - type: string - annotations: - type: object - description: Localized summaries and descriptions emitted by vmalert - additionalProperties: - type: string - activeAt: - type: string - format: date-time - description: Time when the alert became active - example: "2026-05-07T09:18:00Z" - id: - type: string - description: Alert instance identifier - rule_id: + description: Blocklist name + example: adguard + type: type: string - description: Alert rule identifier - group_id: - type: string - description: Alert group identifier - expression: + enum: [enterprise, community] + description: Blocklist category + example: enterprise + enabled: + type: boolean + description: Whether the blocklist is enabled + example: true + confidence: + type: integer + description: Entitlement confidence score + example: 10 + description: + type: [string, "null"] + description: Blocklist description + example: OpenDNS family shield + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-blocked: + post: + summary: List local Threat Shield DNS blocklist entries + description: Returns the blocklist entries currently staged in UCI. They are written to the adblock file on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-list-blocked + tags: + - threatshield + responses: + "200": + description: Staged blocklist entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/ThreatShieldDnsListEntry" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-blocked: + post: + summary: Add a local Threat Shield DNS blocklist entry + description: Stages the new blocklist entry in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-add-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Domain to add to the local blocklist + example: nastydomain.net + description: + type: string + description: Optional free-form description for the domain + example: my block1 + responses: + "200": + description: Blocklist entry staged or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-blocked: + post: + summary: Edit a local Threat Shield DNS blocklist entry + description: Updates the staged blocklist description in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-edit-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local blocklist + example: nastydomain.net + description: + type: string + description: Updated description for the domain + example: My new desc + responses: + "200": + description: Blocklist entry updated or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-blocked: + post: + summary: Delete a local Threat Shield DNS blocklist entry + description: Removes the staged blocklist entry from UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-delete-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local blocklist + example: nastydomain.net + responses: + "200": + description: Blocklist entry deleted or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-settings: + post: + summary: Get Threat Shield DNS enforcement settings + operationId: ns.threatshield.dns-list-settings + tags: + - threatshield + responses: + "200": + description: Current DNS enforcement settings + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: object + required: + - enabled + - zones + - ports + properties: + enabled: + type: boolean + description: Whether Threat Shield DNS is enabled + example: true + zones: + type: array + description: Firewall zones where DNS redirection is enforced + items: type: string - description: Alert rule expression - source: + example: [lan] + ports: + type: array + description: DNS ports enforced locally + items: type: string - description: vmalert source URL for the alert - restored: - type: boolean - description: True when the alert is being restored - stabilizing: - type: boolean - description: True when the alert is stabilizing + example: ["53", "853"] + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-zones: + post: + summary: List firewall zones available for Threat Shield DNS + operationId: ns.threatshield.dns-list-zones + tags: + - threatshield + responses: + "200": + description: Available zones + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + type: string + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-blocklist: + post: + summary: Enable or disable a Threat Shield DNS blocklist + operationId: ns.threatshield.dns-edit-blocklist + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - blocklist + - enabled + properties: + blocklist: + type: string + description: DNS blocklist name + example: adguard + enabled: + type: boolean + description: Whether the blocklist should be enabled + example: true + responses: + "200": + description: Blocklist updated or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-settings: + post: + summary: Update Threat Shield DNS enforcement settings + operationId: ns.threatshield.dns-edit-settings + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - enabled + properties: + enabled: + type: boolean + description: Enable or disable Threat Shield DNS + example: true + zones: + type: array + description: Firewall zones where DNS redirection is enforced + items: + type: string + example: [lan] + ports: + type: array + description: DNS ports enforced locally + items: + type: string + example: ["53", "853"] + responses: + "200": + description: Settings updated or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-bypass: + post: + summary: List Threat Shield DNS bypass entries + operationId: ns.threatshield.dns-list-bypass + tags: + - threatshield + responses: + "200": + description: DNS bypass entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + type: string + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-bypass: + post: + summary: Add a Threat Shield DNS bypass entry + operationId: ns.threatshield.dns-add-bypass + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Source IP address or subnet that should bypass DNS redirection + example: 192.168.1.22 + responses: + "200": + description: Bypass entry added or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-bypass: + post: + summary: Delete a Threat Shield DNS bypass entry + operationId: ns.threatshield.dns-delete-bypass + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Source IP address or subnet to remove from the DNS bypass list + example: 192.168.1.22 + responses: + "200": + description: Bypass entry removed or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" - $ref: "#/components/schemas/Error" From 42324c30a3ab47ff760e8a8f89f781877b2a133d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 15 May 2026 11:57:59 +0200 Subject: [PATCH 47/48] fix(ipsec): add restart dpd_action The init file from upstream replaces the dpd_action option values. Notably it replaces `restart` with `start`, but `start` value is not supported by Strongswan 6. Make sure if `restart` is set, the value is preserved. From the manual: Action to perform for this CHILD_SA on DPD timeout. The default clear closes the CHILD_SA and does not take further action. trap installs a trap policy, which will catch matching traffic and tries to re-negotiate the tunnel on-demand (note that this is redundant if start_action includes trap. restart immediately tries to re-negotiate the CHILD_SA under a fresh IKE_SA. --- .../packages/100-strongswan-dpd-action-restart.patch | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 patches/feeds/packages/100-strongswan-dpd-action-restart.patch diff --git a/patches/feeds/packages/100-strongswan-dpd-action-restart.patch b/patches/feeds/packages/100-strongswan-dpd-action-restart.patch new file mode 100644 index 000000000..8f16df2a3 --- /dev/null +++ b/patches/feeds/packages/100-strongswan-dpd-action-restart.patch @@ -0,0 +1,12 @@ +diff --git a/net/strongswan/files/swanctl.init b/net/strongswan/files/swanctl.init +--- a/net/strongswan/files/swanctl.init ++++ b/net/strongswan/files/swanctl.init +@@ -314,7 +314,7 @@ + hold) + dpdaction="trap" ;; + restart) +- dpdaction="start" ;; ++ dpdaction="restart" ;; + trap|start) + # already using new syntax + ;; From 580493bc7e595c8994e797370d5c872450ee67d4 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 15 May 2026 13:51:15 +0200 Subject: [PATCH 48/48] chore(ui): bump to dev commit --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index ee852564b..db0c137a4 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=6c362f1eb30c8ef6b034d4127885d928e6640553 +PKG_SOURCE_VERSION:=d95a3d9df55b18df2db9afbe01421901e655ac58 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip