From d966d7cf45642af5b95dff91e414a9ff0012c71d Mon Sep 17 00:00:00 2001 From: Harun Baris Bulut Date: Thu, 15 Oct 2026 18:03:11 +0300 Subject: [PATCH 1/4] docs: quickstart, changelog, contributing, issue templates (v0.14.2) Update README status to match v0.14, add a 5-minute quickstart, add CHANGELOG and CONTRIBUTING, issue and PR templates, and commit the usage, API and deployment guides the README links to. Co-Authored-By: Claude Sonnet 5.5 --- .github/ISSUE_TEMPLATE/bug_report.md | 15 + .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.md | 11 + .github/ISSUE_TEMPLATE/plugin_request.md | 11 + .github/pull_request_template.md | 6 + CHANGELOG.md | 55 +++ CONTRIBUTING.md | 36 ++ README.md | 29 +- deploy/nginx/mqtt.plusclouds.com.conf | 9 + docs/api/README.md | 492 ++++++++++++++++++++++ docs/deployment/standalone-vm.md | 328 +++++++++++++++ docs/progress.md | 2 +- docs/usage-guide.md | 408 ++++++++++++++++++ 13 files changed, 1404 insertions(+), 3 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/plugin_request.md create mode 100644 .github/pull_request_template.md create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 deploy/nginx/mqtt.plusclouds.com.conf create mode 100644 docs/api/README.md create mode 100644 docs/deployment/standalone-vm.md create mode 100644 docs/usage-guide.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..7978a42 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,15 @@ +--- +name: Bug report +about: Something does not work as documented +labels: bug +--- + +**Version** (`monitor --version` or image tag): + +**What happened:** + +**What you expected:** + +**Steps to reproduce** (API calls, check config; remove secrets): + +**Logs / output:** diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..d489c2c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Questions and ideas + url: https://github.com/plusclouds/monitoring.server/discussions + about: Ask questions and share setups in Discussions. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..31a49ab --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,11 @@ +--- +name: Feature request +about: Suggest an improvement +labels: enhancement +--- + +**Problem you want to solve:** + +**Proposed solution:** + +**Alternatives considered:** diff --git a/.github/ISSUE_TEMPLATE/plugin_request.md b/.github/ISSUE_TEMPLATE/plugin_request.md new file mode 100644 index 0000000..e566671 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/plugin_request.md @@ -0,0 +1,11 @@ +--- +name: Plugin request +about: Ask for support for a protocol or device type +labels: plugin +--- + +**Device or protocol (vendor, model, MIB/API docs link):** + +**What should be collected:** + +**Can you test it against real hardware?** diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..848772f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ +## What and why + +## Checklist +- [ ] `make check` passes +- [ ] Tests added or updated +- [ ] Docs and CHANGELOG updated if user-visible diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4c95c10 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,55 @@ +# Changelog + +Notable changes per release. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions are git tags. The project is pre-1.0: the API can change between minor versions. Per-milestone detail is in [docs/progress.md](docs/progress.md). + +## [0.14.1] - 2026-10-12 +### Fixed +- Lint failure (unnecessary conversion) in device notifications. + +## [0.14.0] - 2026-10-12 +### Added +- Notification catalog and per-device notifications. + +## [0.13.x] - 2026-10-10 to 2026-10-11 +### Added +- `box.agent` heartbeats (0.13.0), every 5 s in every tenant (0.13.1, 0.13.3). +### Fixed +- Whoopsy relearn behavior (0.13.2). + +## [0.12.x] - 2026-10-07 +### Added +- PlusClouds VM agent telemetry (`vm.agent`) and its weight in usage metering. + +## [0.11.0] - 2026-10-07 +### Added +- Self-monitoring (`monitor.self`) and heartbeat. + +## [0.10.0] - 2026-10-07 +### Added +- Embedded MQTT broker and `push.mqtt` ([ADR-0013](docs/adr/0013-embedded-mqtt-broker.md)). + +## [0.9.0] - 2026-10-07 +### Added +- HTTP push ingest (`push.http`). + +## [0.4.0 to 0.8.4] - 2026-10-04 to 2026-10-07 +### Added +- Alert noise control: grouping, repeat notifications, escalation steps, dependency suppression; webhook completeness (replay, test send, limits, retention). +- Plugins: `snmp.*` (system, get, interfaces, ups, pdu, sensor), `redfish.health`, `xapi.pool` (XCP-ng with discovered devices), `camera.snapshot`, `rtsp.stream`. +- Whoopsy adaptive alerts. +- Grafana provisioning and dashboards (devices, websites, network, servers, power, cameras, XCP-ng). +- `admin rotate-platform-key`, `admin revoke-platform-keys`. + +## [0.3.x] - 2026-10-04 +### Added +- Metrics storage and Grafana data source (F07); usage metering for billing (`GET /v1/usage/tenants`, 0.3.5). + +## [0.2.x] - 2026-10-04 +### Added +- Scheduler/runner, state machine, incidents, signed Standard Webhooks delivery (M2). +- Device `status`, JSON Merge Patch on devices and checks, lookup by external ID (0.2.1). + +## [0.1.x] - 2026-09-29 to 2026-10-04 +First release (`v0.1.0-m1`, 2026-09-29). +### Added +- `monitor` CLI, config, migrations, tenants, users, API keys, hash-chained audit log, plugin SDK, `icmp` and `http` plugins, OpenAPI at `/docs`, Docker image and Compose deployment. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f7518ae --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,36 @@ +# Contributing + +Thanks for helping. Bug reports, plugin ideas, docs fixes and code are all welcome. + +## Ways to contribute + +- **Report a bug or ask for a feature:** open an [issue](https://github.com/plusclouds/monitoring.server/issues/new/choose). Include the version (`monitor --version`), what you expected and what happened. +- **Ask a question or share a setup:** use GitHub Discussions. +- **Add a plugin:** protocols and device types are plugins, never core changes. Start from `pkg/plugin` and an existing plugin such as `plugins/http`, and read the [feature specs](docs/features/README.md). +- **Change the design:** edit the relevant feature spec or add an ADR from the [template](docs/adr/0000-template.md) in the same pull request. +- **Security issues:** do not open a public issue. Email security@plusclouds.com. + +Look for issues labelled `good first issue`. + +## Development + +Requirements: Docker; Go 1.27 to build and test from source. See [Development in the README](README.md#development) to run the stack. + +```sh +make build # bin/monitor +make test # unit and integration tests with -race (needs Docker) +make lint # golangci-lint, including gosec +make check # everything CI runs +``` + +If you change `api/openapi.yaml`, run `make generate` and commit the result. + +## Pull requests + +- Branch from `master`, one topic per pull request. +- Use conventional commit prefixes (`feat:`, `fix:`, `docs:`, `ci:`). +- Add or update tests; `make check` must pass. +- Update the docs and [CHANGELOG.md](CHANGELOG.md) for user-visible changes. +- Dependencies must use a license on the allowlist (see `ALLOWED_LICENSES` in the Makefile). + +By contributing you agree your work is released under the [MIT license](LICENSE). diff --git a/README.md b/README.md index 42fc2af..f73fccc 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,35 @@ An API-first, extensible monitoring engine written in Go. One engine watches network devices, server hardware (BMCs), IP cameras, web endpoints and ports, hypervisors and their VMs, and IoT/facility devices, and turns what it sees into incidents and signed webhook events. -> **Status: milestone M1 (skeleton).** The binary, process config, database schema, tenancy and API keys, audit log, plugin SDK and the `icmp` and `http` plugins exist. Scheduling, state, incidents and notifications (M2) do not yet. The rest of this page describes the intended system; see [Development](#development) for what runs today. +> **Status: v0.14, pre-1.0.** Scheduling, state, incidents, signed webhooks, metrics, Grafana dashboards and plugins for HTTP, ICMP, SNMP, Redfish, XCP-ng, cameras, HTTP/MQTT push and UPS/PDU all run today. Remote probes, VMware/Proxmox and the Phase 2 and 3 items in the [roadmap](#roadmap) do not exist yet. The API can still change before 1.0; see the [changelog](CHANGELOG.md) and [implementation progress](docs/progress.md). Developed by [PlusClouds](https://plusclouds.com) under the MIT license. It runs standalone or embedded in the PlusClouds panel, and it is multi-tenant from day one because customers run it too. +## Quickstart (5 minutes) + +Needs Docker with Compose, `curl` and `jq`. This builds from source and uses plain HTTP: for trying it out, not for production (see [Deploy on a new VM](docs/deployment/standalone-vm.md)). + +```sh +# 1. Start PostgreSQL and the engine +git clone https://github.com/plusclouds/monitoring.server && cd monitoring.server +docker compose -f deploy/docker-compose.yml up -d --build --wait + +# 2. Create your tenant and admin key (shown once) +docker compose -f deploy/docker-compose.yml run --rm monitor admin bootstrap --standalone --tenant "Demo" +export MON_URL=http://127.0.0.1:8443 MON_KEY=mon_... # paste the key +mon() { curl -sS -H "Authorization: Bearer $MON_KEY" -H "Content-Type: application/json" "$@"; } + +# 3. Add a device with an HTTP check (a bad status makes it fail on purpose) +DEVICE=$(mon -X POST $MON_URL/v1/devices -d '{"name":"Demo site","type":"web","address":"https://httpstat.us/500"}' | jq -r .id) +mon -X POST $MON_URL/v1/devices/$DEVICE/checks -d '{"name":"Homepage","plugin":"http","interval_seconds":30,"is_host_check":true}' + +# 4. Point a webhook at a receiver (open https://webhook.site, copy your URL) and route everything to it +WH=$(mon -X POST $MON_URL/v1/webhooks -d '{"name":"demo","url":"https://webhook.site/"}' | jq -r .id) +mon -X POST $MON_URL/v1/alert-routes -d "{\"name\":\"all\",\"endpoint_id\":\"$WH\"}" +``` + +After a few failed runs a signed `monitoring.incident.opened` event arrives at your receiver. Next: the [usage guide](docs/usage-guide.md), and [Grafana dashboards](deploy/grafana/README.md) that read PostgreSQL directly. + --- ## What it does @@ -153,7 +178,7 @@ npx newman run api/postman/monitoring.postman_collection.json \ ## Contributing -The project is in the design phase. Proposals and reviews happen on the feature specs and ADRs: open a pull request that edits the relevant document, or add a new ADR using the [template](docs/adr/0000-template.md). +See [CONTRIBUTING.md](CONTRIBUTING.md). Bugs and ideas go in [issues](https://github.com/plusclouds/monitoring.server/issues); design changes go through the feature specs and ADRs. ## License diff --git a/deploy/nginx/mqtt.plusclouds.com.conf b/deploy/nginx/mqtt.plusclouds.com.conf new file mode 100644 index 0000000..d4e3c3e --- /dev/null +++ b/deploy/nginx/mqtt.plusclouds.com.conf @@ -0,0 +1,9 @@ +# mqtt.plusclouds.com: only the Let's Encrypt challenge for the MQTT broker's +# certificate (webroot renewal, see deploy/compose/mqtt-cert-hook.sh). MQTT +# itself (8883, 1883) goes straight to the container, not through nginx. +server { + listen 80; + server_name mqtt.plusclouds.com; + location /.well-known/acme-challenge/ { root /var/www/letsencrypt; } + location / { return 404; } +} diff --git a/docs/api/README.md b/docs/api/README.md new file mode 100644 index 0000000..bb5400a --- /dev/null +++ b/docs/api/README.md @@ -0,0 +1,492 @@ +# API guide + +The monitoring server is API-first: everything it does is an HTTP call, and there is no other interface. This guide explains the conventions and lists every endpoint. The authoritative, machine-readable description is the OpenAPI 3.1 document [api/openapi.yaml](../../api/openapi.yaml), which the server also publishes: + +| URL | Content | Authentication | +| --- | --- | --- | +| `/docs` | Interactive reference (Redoc) | none | +| `/v1/openapi.json`, `/v1/openapi.yaml` | The spec | none | +| `/healthz` | Liveness, answers `ok` | none | + +If this guide and the spec ever disagree, the spec wins. For task-oriented walkthroughs, see the [usage guide](../usage-guide.md). + +## Contents + +- [Conventions](#conventions) +- [Authentication and roles](#authentication-and-roles) +- [Errors](#errors) +- [Pagination, filtering and updates](#pagination-filtering-and-updates) +- [External IDs](#external-ids) +- [Endpoint reference](#endpoint-reference) +- [Object shapes](#object-shapes) +- [Push ingest API](#push-ingest-api) +- [MQTT ingest](#mqtt-ingest) +- [Webhook events](#webhook-events) +- [Platform key operations](#platform-key-operations) +- [Using the spec: clients, Postman](#using-the-spec-clients-postman) + +## Conventions + +- **Base URL.** `https:///v1`. All paths below are relative to it unless they start with `/ingest`. +- **Format.** JSON in and out (`Content-Type: application/json`). PATCH accepts `application/merge-patch+json` or `application/json`. +- **Identifiers.** UUIDs (v7). +- **Timestamps.** RFC 3339 in UTC, for example `2026-10-10T10:15:00Z`. +- **Limits.** Request bodies up to 4 MiB (`api.max_body`). Requests time out after 90 s. +- **Request IDs.** Send `X-Request-ID` (short string) and the server echoes it in the response and in error bodies. If you do not send one, it makes one. Quote it when you report a problem. +- **Idempotency.** `PUT` and `DELETE` are idempotent. `POST` is not. Use the by-external-ID `PUT` endpoints to create-or-update safely. + +## Authentication and roles + +Send an API key on every request except the unauthenticated URLs above: + +``` +Authorization: Bearer mon__ +``` + +A key is shown once, at creation. The server stores only a hash. Revoked, expired and wrong keys all get `401`. + +| Key kind | Acts as | Created by | +| --- | --- | --- | +| **Tenant key** | One tenant, with a role | `monitor admin bootstrap --standalone` (first admin key) and `POST /v1/api-keys` | +| **Platform key** | The installation: provisions tenants and users, and acts inside a named tenant | `monitor admin bootstrap` without `--standalone`. Meant for a parent system such as the PlusClouds panel. See [Platform key operations](#platform-key-operations). | + +Tenant roles: + +| Role | Can | +| --- | --- | +| `read-only` | Read everything in the tenant (except audit log, keys) | +| `operator` | Everything above, plus acknowledge, comment, resolve incidents, run checks now, and create or change sites, devices, checks, credentials, webhooks and alert routes | +| `admin` | Everything above, plus manage API keys, members, and read the audit log | + +A request outside the role gets `403 forbidden`. + +API key options (`POST /api-keys`): + +```json +{ + "name": "grafana-readonly", + "role": "read-only", + "expires_at": "2027-01-01T00:00:00Z", + "ip_allowlist": ["203.0.113.10/32", "2001:db8::/48"] +} +``` + +`expires_at` and `ip_allowlist` are optional. The client address used for the allowlist, the audit log and rate limits comes from `X-Forwarded-For` only when the request arrives from a `trusted_proxies` address. + +**Rate limits.** Each key has a token bucket: the tenant's `api_rate_per_minute` (default 1200 for tenants, 6000 for platform keys; 0 is unlimited). Beyond it you get `429` with `Retry-After: 1`. + +## Errors + +Errors are RFC 9457 problem documents with `Content-Type: application/problem+json`: + +```json +{ + "type": "https://monitor.plusclouds.com/problems/not-found", + "title": "Not found", + "status": 404, + "instance": "/v1/devices/0192f7c4-...", + "request_id": "b4d1e0c2" +} +``` + +The `type` URI is a stable identifier. It is not a page you can open, and it uses the PlusClouds domain in every installation. Match on the last path segment, or on `status`. `detail` explains the problem in words when there is something to say. + +| Status | `type` suffix | Meaning | +| --- | --- | --- | +| 400 | `invalid-header`, `tenant-required` | Malformed request, bad header, missing tenant header (platform key) | +| 401 | `unauthenticated` | Missing, wrong, revoked or expired key. Carries `WWW-Authenticate: Bearer`. | +| 403 | `forbidden`, `platform-only`, `tenant-suspended`, `not-a-member` | Role too low, platform-only operation, or a suspended tenant (reads work, writes are refused) | +| 404 | `not-found` | No such object in your tenant. Other tenants' objects are indistinguishable from missing ones. | +| 409 | various, for example `limit-reached` | Conflict: duplicate, object in use, tenant limit reached, tenant deleted | +| 422 | `invalid-value` | The body parsed but a value is not allowed: plugin config does not match its schema, interval below the minimum, network not allowed | +| 429 | `rate-limited` | Too many requests | +| 500 | `internal` | Server fault. The cause is in the server log under the `request_id`. | + +## Pagination, filtering and updates + +**Lists** return `{"items": [...], "next_cursor": "..."}`. Pass the cursor back as `?cursor=` for the next page; `next_cursor` is `null` on the last page. `?limit=` is 1 to 500 (default 100). Sort order is documented per endpoint (devices and checks by ID, incidents and audit events newest first). Treat cursors as opaque. + +```sh +cursor="" +while :; do + page=$(curl -sS -H "Authorization: Bearer $MON_KEY" "$MON_URL/v1/devices?limit=200&cursor=$cursor") + echo "$page" | jq -c '.items[]' + cursor=$(echo "$page" | jq -r '.next_cursor // empty') + [ -z "$cursor" ] && break +done +``` + +**Filters** are plain query parameters. The common ones: + +| Endpoint | Filters | +| --- | --- | +| `GET /devices` | `type`, `site_id`, `parent_id`, `availability` (repeatable), `tag=key=value` (repeatable, all must match), `external_source`, `external_type`, `external_id` | +| `GET /checks` | `device_id`, `plugin`, `enabled` | +| `GET /incidents` | `status` (`open`, `acknowledged`, `resolved`, `active`), `severity`, `device_id`, `check_id`, `suppressed`, `object_key` | +| `GET /audit` | `object_type`, `object_id`, `action`, `from`, `to` | +| `GET /webhooks/{id}/deliveries` | `status` (`pending`, `delivered`, `failed`, `cancelled`) | + +**Updating.** + +- `PUT /resource/{id}` replaces the whole writable representation. Omitted optional fields return to their defaults. +- `PATCH /resource/{id}` is a JSON Merge Patch (RFC 7396): fields you leave out stay, `null` clears an optional field, and objects such as `tags` merge key by key. The result is validated like a `PUT`. +- `DELETE` removes the object. Deleting a device deletes its checks and the devices it contains. Deleting a credential, webhook or site that is still in use returns `409`. + +## External IDs + +Sites, devices, credentials, webhooks and alert routes can carry a reference to the object in your own system (`source`, optional `type`, `id`). That lets you manage them idempotently without remembering server UUIDs. + +```json +{ "name": "db-01", "type": "server", "address": "10.0.4.20", + "external": { "source": "cmdb", "type": "host", "id": "srv-0042" } } +``` + +- `PUT /devices/by-external-id/srv-0042?source=cmdb&type=host` creates the device or replaces it. +- `PATCH /devices/by-external-id/{id}?source=...` changes one and never creates (`404` if missing). +- List endpoints filter with `external_source`, `external_type`, `external_id`. +- `source` defaults to the installation's identity source (`MONITOR_IDENTITY_SOURCE`, default `plusclouds`). Standalone users should always pass `source` explicitly, or set the identity source to their own label once, before creating objects. +- Objects created from an external ID report `managed_by: "api"`. + +## Endpoint reference + +Roles: blank = any role; **operator** = operator or admin; **admin** = admin; **platform** = platform key only. "Writes" in the table below follow the rule in [Authentication and roles](#authentication-and-roles): all `POST`, `PUT`, `PATCH`, `DELETE` on monitored objects need `operator`, except the items marked otherwise. The spec is exact; the column marks what it says explicitly. + +### Identity and audit + +| Method | Path | Purpose | Role | +| --- | --- | --- | --- | +| `GET` | `/me` | The resolved caller: key, role, tenant, acting user | | +| `GET` | `/tenant` | Your tenant, with its limits | | +| `GET` | `/api-keys` | List keys (never their secrets) | admin | +| `POST` | `/api-keys` | Create a key; the secret is in the response once | admin | +| `DELETE` | `/api-keys/{key_id}` | Revoke a key | admin | +| `GET` | `/audit` | Query the audit log (hash-chained, tamper evident) | admin | + +### Plugins and credentials + +| Method | Path | Purpose | Role | +| --- | --- | --- | --- | +| `GET` | `/plugins` | Plugins built into this server: config schema, metrics, intervals, credential types | | +| `GET` | `/plugins/{type}` | One plugin's manifest | | +| `GET` | `/credential-types` | Credential types and their field schemas | | +| `GET` | `/credentials` | List credentials, secrets never returned | | +| `POST` | `/credentials` | Store a credential, encrypted at rest | operator | +| `PUT` | `/credentials/by-external-id/{external_id}` | Create or replace | operator | +| `GET`, `PUT`, `DELETE` | `/credentials/{credential_id}` | One credential; delete only when no check uses it | operator for writes | + +### Sites and devices + +| Method | Path | Purpose | Role | +| --- | --- | --- | --- | +| `GET`, `POST` | `/sites` | List, add | operator for POST | +| `PUT`, `PATCH` | `/sites/by-external-id/{external_id}` | Create or replace, change | operator | +| `GET`, `PUT`, `PATCH`, `DELETE` | `/sites/{site_id}` | One site; delete only without devices | operator for writes | +| `GET`, `POST` | `/devices` | List, add | operator for POST | +| `PUT`, `PATCH` | `/devices/by-external-id/{external_id}` | Create or replace, change | operator | +| `GET`, `PUT`, `PATCH`, `DELETE` | `/devices/{device_id}` | One device | operator for writes | +| `GET` | `/devices/{device_id}/children` | Devices this device contains | | +| `GET`, `POST` | `/devices/{device_id}/dependencies` | What the device needs; record a dependency (`{"depends_on_id": "..."}`) | operator for POST | +| `DELETE` | `/devices/{device_id}/dependencies/{depends_on_id}` | Remove a stored dependency | operator | +| `GET` | `/devices/{device_id}/dependents` | Devices that need this one | | +| `GET` | `/devices/{device_id}/impact` | Everything suppressed if this device failed | | +| `POST` | `/devices/{device_id}/test` | Run every enabled check once and return the results | operator | + +Device types: `network`, `server`, `bmc`, `camera`, `web`, `hypervisor_pool`, `hypervisor_host`, `vm`, `iot`, `ups`, `pdu`, `sensor`, `llm_endpoint`, `llm_application`, `other`. + +### Checks, state and incidents + +| Method | Path | Purpose | Role | +| --- | --- | --- | --- | +| `GET`, `POST` | `/devices/{device_id}/checks` | The device's checks; add one | operator for POST | +| `GET` | `/checks` | List checks, filtered | | +| `GET`, `PUT`, `PATCH`, `DELETE` | `/checks/{check_id}` | One check | operator for writes | +| `GET` | `/checks/{check_id}/state` | Current state: status, output, since when | | +| `GET` | `/checks/{check_id}/objects` | Objects of a collector check (interfaces, fans, VMs) and their state | | +| `POST` | `/checks/{check_id}/run-now` | Run the check at once | operator | +| `POST` | `/checks/{check_id}/rotate-token` | New ingest token for a push check; the old one stops at once | operator | +| `GET`, `PUT`, `DELETE` | `/checks/{check_id}/whoopsy` | Adaptive band alerting: read, enable or change, disable | operator for writes | +| `GET` | `/checks/{check_id}/whoopsy/band` | The band of every result, for graphs | | +| `POST` | `/checks/{check_id}/whoopsy/reset` | Forget the learned band | operator | +| `GET` | `/incidents` | List incidents, newest first | | +| `GET` | `/incidents/{incident_id}` | One incident, with comments | | +| `POST` | `/incidents/{incident_id}/ack` | Acknowledge | operator | +| `POST` | `/incidents/{incident_id}/resolve` | Resolve by hand | operator | +| `POST` | `/incidents/{incident_id}/comments` | Comment (`{"body": "..."}`) | operator | + +### Notifications + +| Method | Path | Purpose | Role | +| --- | --- | --- | --- | +| `GET`, `POST` | `/webhooks` | List, add endpoints. The signing secret is returned once. | operator for POST | +| `PUT` | `/webhooks/by-external-id/{external_id}` | Create or replace | operator | +| `GET`, `PUT`, `DELETE` | `/webhooks/{webhook_id}` | One endpoint; delete only when no route uses it | operator for writes | +| `POST` | `/webhooks/{webhook_id}/test` | Send a signed `monitoring.webhook.test` event now, return the receiver's answer | operator | +| `POST` | `/webhooks/{webhook_id}/rotate-secret` | New signing secret; the old one is also valid for 24 h | operator | +| `GET` | `/webhooks/{webhook_id}/deliveries` | The delivery log | | +| `POST` | `/webhooks/{webhook_id}/deliveries/{delivery_id}/replay` | Send one delivery again | operator | +| `POST` | `/webhooks/{webhook_id}/deliveries/replay` | Send failed deliveries again in bulk (`?status=failed&from=&to=`, up to 10,000) | operator | +| `GET`, `POST` | `/alert-routes` | List in evaluation order, add | operator for POST | +| `PUT` | `/alert-routes/by-external-id/{external_id}` | Create or replace | operator | +| `GET`, `PUT`, `DELETE` | `/alert-routes/{route_id}` | One route | operator for writes | +| `POST` | `/alert-routes/test` | Show which routes a sample incident would reach | | +| `GET` | `/notification-catalog` | Every notification a webhook can receive, with an example payload, and the route filters | | +| `GET` | `/devices/{device_id}/notifications` | A device's notification setup: matching routes, webhook health, `would_fire`, and its recent deliveries with payloads (`?status=`, `?event_type=`, `?limit=` 1 to 100, default 20, `?cursor=`) | | + +### Metrics and usage + +| Method | Path | Purpose | Role | +| --- | --- | --- | --- | +| `GET` | `/metrics/series` | Series of a device or check (`device_id` or `check_id` required) | | +| `GET` | `/metrics/query` | Points of the selected series | | +| `GET` | `/metrics/summary` | Statistics over a window: count, min, max, avg, stddev, p50, p95, p99, last | | +| `GET` | `/metrics/retention-classes` | Retention classes and how long each level is kept | | +| `PUT` | `/metrics/retention-classes/{name}` | Create or change a class | platform | +| `GET` | `/usage/hours`, `/usage/current`, `/usage/weights` | Billing-style usage meter of the tenant. Relevant only if you bill your own users. | | +| `GET` | `/usage/tenants` | Usage of every tenant | platform | + +Standalone installs without a platform key set retention with `monitor admin retention` ([deployment guide](../deployment/standalone-vm.md#data-retention)). + +### MQTT ingest management + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET`, `POST` | `/ingest/mqtt-credentials` | List, create (the password is returned once) | +| `GET`, `PATCH`, `DELETE` | `/ingest/mqtt-credentials/{credential_id}` | One credential; rename, enable, disable, change flags | +| `POST` | `/ingest/mqtt-credentials/{credential_id}/rotate` | New password, shown once | +| `GET` | `/ingest/profiles` | Built-in payload profiles (`json`, `fixlean-esp`) | +| `GET` | `/ingest/unregistered` | Device keys whose messages were dropped | +| `GET`, `POST`, `DELETE` | `/devices/{device_id}/mqtt` | Binding and live session; pre-register; unbind | + +## Object shapes + +Only the fields you most often need. The spec lists all of them. + +### Device + +```json +{ + "id": "0192f7c4-...", "name": "db-01", "address": "10.0.4.20", "type": "server", + "tags": { "rack": "a12" }, "notes": null, + "parent_id": null, "site_id": "...", "physical_peer_id": null, + "inventory": {}, "managed_by": "api", "status": { }, + "external": { "source": "cmdb", "type": "host", "id": "srv-0042" }, + "discovered": null, "created_at": "...", "updated_at": "..." +} +``` + +Write fields (`POST`, `PUT`): `name` (required), `type` (required), `address`, `tags` (up to 50, values up to 200 chars), `notes`, `parent_id` (container, for example the hypervisor of a VM), `site_id`, `physical_peer_id` (linked BMC or host), `external`. + +`status` carries the derived availability and the open incident count. `discovered` is set for devices that a collector created (VMs, pool hosts); those follow their collector, do not count toward `max_devices`, and disappear 7 days after the collector stops reporting them. + +### Check + +```json +{ + "id": "...", "device_id": "...", "name": "Homepage", "plugin": "http", + "config": { "keyword": "Welcome" }, + "interval_seconds": 60, "timeout_seconds": null, "enabled": true, + "thresholds": [ { "id": "r1", "metric": "total_ms", + "warning": {"op": ">", "value": 1500}, + "critical": {"op": ">", "value": 5000}, + "for": "5m", "hysteresis": 100 } ], + "failure_count": 3, "recovery_count": 1, + "is_host_check": true, "unknown_is_critical": false, "runbook_url": null, + "credentials": { "auth": "" }, + "whoopsy": null, "push": null, "managed_by": "api" +} +``` + +- `config` is validated against the plugin's `config_schema` (`GET /plugins/{type}`); a mismatch is `422`. +- `interval_seconds` is 1 to 86400. It must meet both the plugin's `min_interval_seconds` and the tenant's `min_check_interval_seconds`. +- `failure_count` consecutive bad results are needed before a state change (default 3; 1 for push checks). `recovery_count` for the way back. +- Threshold `op` is one of `>`, `>=`, `<`, `<=`, `==`, `!=`, `between`, `outside` (the last two with `value_max`). Rules on collector checks can set `object` to `*` or one object key. +- `credentials` maps a role (usually `auth`) to a credential ID. + +### Incident + +`id`, `check_id`, `device_id`, `object_key`, `severity` (`warning` or `critical`), `status` (`open`, `acknowledged`, `resolved`), `summary`, `last_output`, `flapping`, `suppressed`, `root_incident_id`, `root_device_id`, `opened_at`, `acknowledged_at`, `resolved_at`, `resolved_by` (`recovery`, `manual`, `check-deleted`, `check-disabled`, `object-gone`). + +### Metrics query + +``` +GET /v1/metrics/query?check_id=&name=total_ms&name=ttfb_ms + &from=2026-10-10T00:00:00Z&to=2026-10-10T12:00:00Z&step=300&agg=avg +``` + +| Parameter | Meaning | +| --- | --- | +| `device_id` or `check_id` | Required: which series | +| `name` | Metric name; repeat for several | +| `object` | Object key of a collector check | +| `from`, `to` | Window; default the last hour | +| `step` | Bucket seconds; default about 500 points, at least 10 s | +| `agg` | `avg` (default), `min`, `max`, `sum`; `stddev`, `p50`, `p95`, `p99`, `p[.fff]` use raw samples and need `from` inside raw retention | +| `resolution` | `raw`, `5m`, `1h`; default picks by `step` and retention | +| `moving_window` | 1 to 1000: smooth after aggregation | + +At most 10,000 points per series. Buckets start at multiples of `step` counted from 2000-01-01T00:00:00Z, so the first one may begin before `from`. + +## Push ingest API + +The ingest listener is separate from the management API (default port 9443, behind your proxy at `/ingest/`). It authenticates with the check's own push token, not an API key. + +``` +POST /ingest/v1/{check_id} +Authorization: Bearer mpush_<8>_<43> +Content-Type: application/json +``` + +| Item | Detail | +| --- | --- | +| Body | One JSON object, or an array of up to 1,000 objects (oldest first after sorting by timestamp). Up to 64 KiB. | +| Mapping | Defined in the check's `config`: `metrics` (name to JSONPath-subset selector), `units`, `status`, `output`, `timestamp`, `missed_count` (default 3). | +| Timestamps | Accepted from 24 hours in the past to 5 minutes ahead. Outside, or absent, the server time is used. Results older than 2 x the interval update metrics but not state. | +| Rate | 2 requests/s per token, burst 20 (`ingest.http.rate_limit`). | +| Token | Returned once as `push_token` in the response that creates the check (or `rotate-token`). Only its SHA-256 is stored. | + +| Answer | Meaning | +| --- | --- | +| `202 {"accepted": n}` | Taken | +| `400` | Malformed message or mapping failed | +| `401` | Missing or wrong token (the same for an unknown check) | +| `409` | Check disabled | +| `413` | Body too large | +| `429` | Rate exceeded | + +Example: + +```sh +curl -X POST "$MON_URL/ingest/v1/$CHECK_ID" \ + -H "Authorization: Bearer $PUSH_TOKEN" -H "Content-Type: application/json" \ + -d '[{"ts":"2026-10-10T10:00:00Z","sensors":{"temp":21.4}},{"ts":"2026-10-10T10:01:00Z","sensors":{"temp":21.6}}]' +``` + +Silence is the failure: if no message arrives for `interval x missed_count`, the check goes CRITICAL with "no data since ...", and recovers on the next message. + +## MQTT ingest + +With `MONITOR_MQTT_ENABLED=true`, the engine runs an MQTT 3.1.1 and 5 broker on TLS port 8883. + +| Item | Detail | +| --- | --- | +| Auth | Username and password from `/ingest/mqtt-credentials`. The credential decides the tenant. | +| Topic | `//...`. The second segment is the device key (a MAC or serial), compared without case. | +| Direction | Devices publish; nothing may subscribe. Messages are acknowledged (QoS 1) after the pipeline takes them. Clean sessions only. | +| Payload | `json` profile: every numeric field becomes a metric; booleans become 1 and 0; nested objects join with `_` (three levels). | +| Limits | 64 KiB packets; 10 messages/s per client (burst 50); 5 connects/s per source IP (burst 20). | +| Presence | An `mqtt.connection` check follows connect, disconnect and keepalive timeout (1.5 x keepalive), so offline detection does not wait for silence timers. | + +## Webhook events + +The server's only outbound notification is a signed HTTP POST per event, using the CloudEvents 1.0 envelope and the [Standard Webhooks](https://www.standardwebhooks.com/) signature scheme. + +**Request** + +``` +POST +Content-Type: application/json +webhook-id: 0192f7c4-... +webhook-timestamp: 1791626100 +webhook-signature: v1,K5oZfzN95Z9UVu1EsfQmfVNQhnkZ2/3HmN2SFXTqCmU= + +``` + +Verification: HMAC-SHA256 over `"{webhook-id}.{webhook-timestamp}.{raw body}"` with the key `base64decode(secret without "whsec_")`, base64-encoded, compared in constant time with each `v1,` value in the header. During a secret rotation there are two signatures. Reject timestamps older than a few minutes. A Python example is in the [usage guide](../usage-guide.md#receive-and-verify-events). + +**Envelope** + +```json +{ + "specversion": "1.0", + "id": "0192f7c4-...", + "source": "/monitoring//", + "type": "monitoring.incident.opened", + "time": "2026-10-10T10:15:00Z", + "datacontenttype": "application/json", + "subject": "", + "data": { + "account_id": null, + "object_type": "Monitoring\\Incidents", + "object": { }, + "device": { }, + "check": { }, + "route": { "labels": { } }, + "actor": null, + "links": { "incident": "https://..." } + } +} +``` + +`id` equals the `webhook-id` header and is identical on every retry. `data.account_id` is the PlusClouds account UUID and is `null` in standalone installs. `data.object_type` has a PHP-style name for historical reasons and can be ignored. + +**Event types** + +| Type | When | +| --- | --- | +| `monitoring.incident.opened` | An incident opened (or stopped being suppressed) | +| `monitoring.incident.updated` | Severity or output changed | +| `monitoring.incident.acknowledged` | Someone acknowledged it (`data.actor`) | +| `monitoring.incident.resolved` | It recovered or was resolved | +| `monitoring.incident.commented` | A comment was added | +| `monitoring.incident.renotify` | A route's `repeat_interval_seconds` elapsed on an open, unacknowledged incident | +| `monitoring.incident.escalated` | A later route step came due (`data.route.step`) | +| `monitoring.tenant.device_limit_reached`, `monitoring.tenant.check_limit_reached` | A create brought the tenant to its limit | +| `monitoring.webhook.test` | You called `/webhooks/{id}/test` | +| `monitoring.heartbeat` | Dead man's switch to the configured heartbeat URLs (not through routes) | + +**Grouped events.** When a route groups incidents, one event carries `data.objects` (each incident's `object`, `device`, `check`) and `data.group` (`id`, `key`, `count`); `data.object`, `data.device` and `data.check` are null. A group of one is sent as a plain event. + +**Delivery.** At least once. Answer with a 2xx within the endpoint's `timeout_seconds` (default 10, max 30). Anything else is retried after 5 s, 30 s, 2 min, 10 min, 30 min, then hourly, for 24 hours; events of one incident stay in order. Every attempt, with the receiver's status and the first KiB of its answer, is in the delivery log. A `410 Gone` disables the endpoint. Deduplicate on `webhook-id`. + +**Network policy.** Webhook URLs resolving into blocked networks (private ranges by default; see `allowed_target_networks`) are refused with `422` when saved, and again at delivery time. + +## Platform key operations + +Needed only if a parent system (a hosting panel, a billing system) manages many tenants through this server. Standalone single-tenant installs can skip this section. + +The platform key (`monitor admin bootstrap` without `--standalone`) is not tied to a tenant. It can call the `platform` endpoints: + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/tenants` | List tenants | +| `PUT` | `/tenants/by-external-id/{external_id}` | Create or update a tenant by your ID, with name and limits | +| `PATCH` | `/tenants/by-external-id/{external_id}` | Change name, status (suspend) or limits | +| `DELETE` | `/tenants/by-external-id/{external_id}` | Soft-delete; data is purged after `platform.tenant_purge_grace` (30 days) | +| `POST` | `/tenants/by-external-id/{external_id}/restore` | Undelete | +| `GET` | `/tenants/{tenant_id}/members` | Members | +| `PUT`, `DELETE` | `/tenants/{tenant_id}/members/by-external-id/{user_external_id}` | Add or change a member's role, remove one | +| `PATCH` | `/users/by-external-id/{external_id}` | Rename or disable a user everywhere | +| `PUT` | `/metrics/retention-classes/{name}` | Retention classes | +| `GET` | `/usage/tenants` | Usage of every tenant per closed hour | + +Tenant limits (`max_devices`, `max_checks`, `max_webhooks`, `max_alert_routes`, `min_check_interval_seconds`, `api_rate_per_minute`, `allowed_target_networks`, `metric_classes`, `max_api_key_lifetime_days`) are set here. + +To act inside a tenant, add headers to any ordinary request: + +| Header | Meaning | +| --- | --- | +| `X-Tenant-External-ID` or `X-Tenant-ID` | Which tenant. Send one, not both. | +| `X-Actor-External-ID` | Optional: the user on whose behalf the call runs. Their role in that tenant applies and the audit log names them. Without it the call has full rights in the tenant as "platform". | + +Unknown tenants and users named this way are created on the fly (just-in-time provisioning, `platform.jit`). Disable that in the config if you do not want it. + +Example: create a tenant and an admin key for it, from an installation that was bootstrapped with a platform key: + +```sh +curl -X PUT "$MON_URL/v1/tenants/by-external-id/acme" \ + -H "Authorization: Bearer $PLATFORM_KEY" -H "Content-Type: application/json" \ + -d '{"name":"Acme Corp","limits":{"max_devices":200,"allowed_target_networks":["10.0.0.0/8"]}}' + +curl -X POST "$MON_URL/v1/api-keys" \ + -H "Authorization: Bearer $PLATFORM_KEY" -H "X-Tenant-External-ID: acme" \ + -H "Content-Type: application/json" -d '{"name":"acme admin","role":"admin"}' +``` + +(Request body fields for tenants: see `TenantUpsert` and `TenantLimits` in the spec.) + +## Using the spec: clients, Postman + +- **Generate a client** from `/v1/openapi.yaml` with any OpenAPI 3.1 generator, for example `openapi-generator-cli generate -i openapi.yaml -g python -o client/`, or `oapi-codegen` for Go (the server itself is generated from this file with the config in [api/oapi-codegen.yaml](../../api/oapi-codegen.yaml)). Some generators still lack full 3.1 support; if yours fails on nullable types, try a newer version. +- **Postman or Insomnia.** Import [api/postman/monitoring.postman_collection.json](../../api/postman/monitoring.postman_collection.json) and [api/postman/local.postman_environment.json](../../api/postman/local.postman_environment.json). The collection is written for a platform key. For a standalone tenant, put your admin key in the `platformKey` variable (the name is historical) and skip the provisioning folder, whose calls need a platform key. From the command line: `npx newman run ... --env-var platformKey=mon_...`. +- **Browse.** `https:///docs`. +- **Stability.** The spec's `info.version` is `0.1.0`. The API is under `/v1`; this release is a pre-1.0 milestone, so check the spec after each upgrade. diff --git a/docs/deployment/standalone-vm.md b/docs/deployment/standalone-vm.md new file mode 100644 index 0000000..3866f69 --- /dev/null +++ b/docs/deployment/standalone-vm.md @@ -0,0 +1,328 @@ +# Deploying the monitoring server on a new VM (standalone) + +This guide takes a fresh Linux VM to a running monitoring server with no PlusClouds services involved. You end up with a PostgreSQL database, the engine, a TLS endpoint, and an admin API key you can use right away. + +"Standalone" means: + +- You run the server and its database yourself. +- You have your own tenant and your own admin API key. There is no PlusClouds panel, no platform key, no PlusClouds NATS, no PlusClouds container registry. +- You build the container image from source. The published image lives in a private PlusClouds registry. + +When you are done, continue with the [usage guide](../usage-guide.md) and the [API guide](../api/README.md). + +> This procedure was assembled from the repository's compose file, configuration reference and bootstrap code. Run it on a throwaway VM first and report anything that differs. + +## 1. What you need + +| Item | Requirement | +| --- | --- | +| VM | Linux x86-64 or arm64. 2 vCPU, 4 GB RAM and 20 GB disk handle a few hundred devices. Metrics grow with check count and retention, so size the disk for that. | +| Software | Docker Engine with Docker Compose **2.23 or newer** (the compose file uses inline configs), `git`, `openssl`, `curl`. | +| Network, inbound | `443/tcp` for the API (through a TLS proxy). `8883/tcp` only if you use the MQTT broker. | +| Network, outbound | Reach to the things you monitor (ICMP, HTTP, SNMP, Redfish and so on), and to your webhook receivers. | +| DNS | A name for the server, for example `monitor.example.com`, if you want valid TLS. | + +Check the versions: + +```sh +docker --version +docker compose version # must be v2.23 or newer +``` + +## 2. Get the source and build the image + +```sh +git clone https://github.com/plusclouds/monitoring.server.git +cd monitoring.server +docker build \ + --build-arg VERSION="$(git describe --tags --always)" \ + --build-arg COMMIT="$(git rev-parse HEAD)" \ + -t monitoring:local . +``` + +If you received the source as an archive, build from its root the same way (leave out the git arguments and pass `--build-arg VERSION=1.0.0`). + +The build produces a static binary on a distroless image. It needs network access to download Go modules. Confirm it works: + +```sh +docker run --rm monitoring:local version +``` + +## 3. Configure the stack + +All deployment files are in `deploy/compose`. Two files are all it needs: `docker-compose.yml` and `.env`. + +```sh +cd deploy/compose +cp .env.example .env +``` + +Fill in `.env`: + +```sh +# Passwords: letters and digits only (they are placed in connection URLs). Hex is safe. +sed -i "s|^POSTGRES_SUPERUSER_PASSWORD=.*|POSTGRES_SUPERUSER_PASSWORD=$(openssl rand -hex 24)|" .env +sed -i "s|^MONITOR_OWNER_PASSWORD=.*|MONITOR_OWNER_PASSWORD=$(openssl rand -hex 24)|" .env +sed -i "s|^MONITOR_APP_PASSWORD=.*|MONITOR_APP_PASSWORD=$(openssl rand -hex 24)|" .env +sed -i "s|^MONITOR_SYSTEM_PASSWORD=.*|MONITOR_SYSTEM_PASSWORD=$(openssl rand -hex 24)|" .env + +# Key-encryption key for stored device credentials: 32 random bytes, base64. +sed -i "s|^MONITOR_KEK=.*|MONITOR_KEK=$(openssl rand -base64 32)|" .env +``` + +Then edit these values by hand: + +| Variable | Set it to | +| --- | --- | +| `MONITOR_IMAGE` | `monitoring` (the image you built) | +| `MONITOR_VERSION` | `local` | +| `MONITOR_PUBLIC_URL` | The URL clients will use, for example `https://monitor.example.com`. It appears in API links and push-ingest URLs. | +| `MONITOR_IDENTITY_SOURCE` | Optional. Any short label such as `local`. It tags external IDs; standalone installs rarely use it. | +| `MONITOR_TRUSTED_PROXIES` | The address of your TLS proxy as a CIDR, for example `172.17.0.1/32` (see section 6). The default trusts all private ranges. | + +Leave `MONITOR_MQTT_*`, `MONITOR_NATS_*` and `MONITOR_HEARTBEAT_URL` alone for now. NATS is only for PlusClouds VM agents and does not apply to you. + +**Back up `.env` now**, especially `MONITOR_KEK`. Without that key, every stored device credential (SNMP communities, BMC passwords) is unrecoverable. + +### Let the server monitor private networks + +By default a tenant may not check private addresses. HTTP checks, ICMP checks and webhooks to RFC 1918, loopback and link-local ranges are refused. On a typical internal network you want the opposite. Allow your ranges **before** you bootstrap, because the standalone tenant takes its limits from these defaults when it is created. + +Create `deploy/compose/docker-compose.override.yml`: + +```yaml +services: + monitor: + environment: + MONITOR_PLATFORM__TENANT_DEFAULTS__ALLOWED_TARGET_NETWORKS: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" + bootstrap: + environment: + MONITOR_PLATFORM__TENANT_DEFAULTS__ALLOWED_TARGET_NETWORKS: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" +``` + +Narrow the list to the networks you really monitor. The cloud metadata address `169.254.169.254` stays blocked regardless (`outbound.deny_networks`). + +Other tenant defaults, such as `max_devices` (1000), `max_checks` (10000) and `min_check_interval_seconds` (30), can be changed the same way. The environment name is `MONITOR_PLATFORM__TENANT_DEFAULTS__` plus the key in upper case. [deploy/config.example.yaml](../../deploy/config.example.yaml) lists every setting. + +## 4. Start the server and create your admin key + +Do **not** run a plain `docker compose up -d`. The compose file's `bootstrap` service creates a *platform* key for the PlusClouds API. You want a local tenant with an admin key instead, so start the engine alone first, then bootstrap by hand. + +```sh +docker compose up -d monitor # starts PostgreSQL, the role/database setup and the engine +docker compose ps # wait until monitor is "healthy" +docker compose logs monitor | tail # migrations applied, listeners up +``` + +Create the installation, a platform tenant for self-monitoring, and your own tenant with its first admin key: + +```sh +docker compose run --rm bootstrap admin bootstrap --standalone --tenant "My Company" +``` + +The output looks like this: + +``` +installation id: 0192f7c4-... +platform tenant id: 0192f7c4-... +tenant id: 0192f7c4-... + +Admin API key (shown once): +mon_ab12cd34_... +``` + +Store the admin key in a password manager. **It is shown once.** If you lose it: + +```sh +docker compose run --rm bootstrap admin create-admin-key --tenant +``` + +Bootstrap runs only once per database. A second run fails with "already bootstrapped", which is safe. + +After this, `docker compose up -d` is safe to run any time. Its `bootstrap` service sees the installation exists and does nothing. + +### Smoke test (HTTP, on the VM) + +```sh +export MON_URL=http://127.0.0.1:8443 +export MON_KEY=mon_... # the admin key + +curl -s $MON_URL/healthz +curl -s -H "Authorization: Bearer $MON_KEY" $MON_URL/v1/me +curl -s -H "Authorization: Bearer $MON_KEY" $MON_URL/v1/tenant +``` + +`/v1/me` returns your tenant and role. `/v1/tenant` shows the limits, including `allowed_target_networks`. The API reference is served at `http://127.0.0.1:8443/docs`. + +## 5. Make ICMP work (if ping checks fail) + +The container runs as a non-root user. ICMP checks then need unprivileged ICMP sockets. If an ICMP check reports a permission error, allow the container's group to open them by adding to `docker-compose.override.yml`: + +```yaml +services: + monitor: + sysctls: + net.ipv4.ping_group_range: "0 2147483647" +``` + +and run `docker compose up -d monitor`. Alternatively use `cap_add: [NET_RAW]`. HTTP, SNMP, Redfish and other checks need nothing special. + +## 6. Put TLS in front + +The API listens on plain HTTP on `127.0.0.1:8443`, and HTTP push ingest on `127.0.0.1:9443`. Never expose these directly. Terminate TLS with a reverse proxy on the VM, routing `/` to 8443 and `/ingest/` to 9443. + +Caddy (automatic Let's Encrypt certificates), `/etc/caddy/Caddyfile`: + +``` +monitor.example.com { + request_body { + max_size 4MB + } + handle /ingest/* { + reverse_proxy 127.0.0.1:9443 + } + handle { + reverse_proxy 127.0.0.1:8443 { + transport http { + response_header_timeout 120s + } + } + } +} +``` + +nginx: [deploy/nginx/monitoring.plusclouds.com.conf](../../deploy/nginx/monitoring.plusclouds.com.conf) is a complete example. Replace the host name, certificate paths and upstream addresses. + +Then: + +1. Set `MONITOR_PUBLIC_URL=https://monitor.example.com` in `.env`. +2. Set `MONITOR_TRUSTED_PROXIES` to the proxy's address as the engine sees it. With a proxy on the VM host, that is the Docker bridge gateway, usually `172.17.0.1/32` or the gateway of the `monitoring_frontend` network. Check with `docker network inspect monitoring_frontend`. The engine uses `X-Forwarded-For` only from these addresses, for the audit log, API key IP allowlists and rate limits. +3. `docker compose up -d monitor`. +4. Open the firewall for 80 and 443 only. Keep 8443 and 9443 bound to loopback (`MONITOR_API_BIND=127.0.0.1`, the default). + +Test from outside: + +```sh +curl -s https://monitor.example.com/healthz +curl -s -H "Authorization: Bearer $MON_KEY" https://monitor.example.com/v1/me +``` + +## 7. Optional components + +### MQTT broker for sensors + +The engine embeds an MQTT broker for IoT devices. It needs a certificate for a DNS-only name (a CDN or HTTP proxy cannot carry MQTT). + +1. Point `mqtt.example.com` at the VM. Open `8883/tcp`. +2. Put the certificate and key in `deploy/compose/tls/` as `mqtt.crt` and `mqtt.key`, readable by uid 65532. [mqtt-cert-hook.sh](../../deploy/compose/mqtt-cert-hook.sh) copies a Let's Encrypt certificate there on renewal. The broker reloads changed files within a minute. +3. In `.env`: `MONITOR_MQTT_ENABLED=true`. +4. `docker compose up -d monitor`. +5. Create device credentials through the API (see the [usage guide](../usage-guide.md#mqtt-sensors)). + +Plain port 1883 (`MONITOR_MQTT_PLAIN_ENABLED`) exists only to migrate old firmware without TLS. Leave it off. + +### Grafana dashboards + +Grafana reads PostgreSQL through a read-only role with starter dashboards for devices, websites, interfaces, hardware, power, hypervisors and cameras. + +1. Set `MONITOR_GRAFANA_PASSWORD` in `.env` (hex) and run `docker compose up -d`. The setup service lets the `monitor_grafana` role log in. +2. Run Grafana in the same compose project or on a host that can reach the database. The database is on an internal Docker network and is not published. Follow [deploy/grafana/README.md](../../deploy/grafana/README.md) for the provisioning mounts and variables. + +The Grafana views show **every tenant**. Treat that Grafana as an operator tool; give end users access through the API only. + +### Heartbeat (dead man's switch) + +The server can ping an external URL every minute while it is healthy, so you hear about it when the server itself dies. With healthchecks.io or any similar service, put the ping URL in `.env`: + +``` +MONITOR_HEARTBEAT_URL=https://hc-ping.com/ +``` + +### Remote probes + +Probes (the same binary, outbound-only) for remote sites are described in [F09](../features/F09-remote-probes.md). The `probe_service` is disabled in the compose deployment of this release, so a standalone install runs all checks from the server itself. + +## 8. Operations + +### Backups + +Back up two things and test the restore: + +1. **The database.** Online dump: + ```sh + docker compose exec -T postgres pg_dump -U postgres -Fc monitor > monitor-$(date +%F).dump + ``` + Restore into an empty database with `pg_restore`. Or snapshot the `monitoring_pgdata` volume while the stack is stopped. +2. **`.env`** (especially `MONITOR_KEK`) and any files in `deploy/compose/tls`. + +The database holds the configuration, state, incidents, audit log and metrics. The `.env` file holds the key that opens the stored credentials. One is useless without the other. + +### Upgrades + +```sh +cd monitoring.server +git pull +docker build --build-arg VERSION="$(git describe --tags --always)" \ + --build-arg COMMIT="$(git rev-parse HEAD)" -t monitoring:local . +cd deploy/compose +docker compose up -d +``` + +Migrations run when the engine starts (under a database lock). Take a backup before upgrading. Read the release notes in [docs/progress.md](../progress.md) first. + +### Logs and health + +```sh +docker compose logs -f monitor +docker compose exec monitor /usr/local/bin/monitor health # exit 0 when ready +``` + +The node status server listens on loopback inside the container (port 9090): `/healthz`, `/readyz`, `/status` and Prometheus `/metrics`. To scrape it, bind it to a reachable address with a status token and TLS; see `self_monitoring` in the config reference. The engine also stores its own health on a built-in `monitor` device in the platform tenant. + +### Rotating the key-encryption key + +[ADR-0006](../adr/0006-credential-encryption.md) designs rotation as: add a second key, make it active, restart, re-wrap with `monitor admin rewrap-credentials`, then remove the old key. That command does not exist in this release, so do not remove or replace `MONITOR_KEK` on a server that holds credentials. Keep the key safe instead (see Backups). + +### Data retention + +Metrics are kept in classes with a retention per level (raw samples, 5-minute rollups, hourly rollups). Inspect or change them from the command line, since standalone installs have no platform key for the API: + +```sh +docker compose run --rm monitor admin retention +docker compose run --rm monitor admin retention standard --raw-days 14 --rollup-5m-days 90 --rollup-1h-days 730 +``` + +The audit log is kept for one year by default (`platform.audit_retention`). Verify its integrity with `monitor admin verify-audit`. + +### Reset everything + +This deletes all data: + +```sh +docker compose down -v +``` + +## 9. Troubleshooting + +| Symptom | Cause and fix | +| --- | --- | +| `docker compose` complains about `configs` or `content` | Compose is older than 2.23. Upgrade it. | +| `set MONITOR_KEK in .env` (or another variable) | A required variable is empty. Fill it in. | +| `monitor` restarts, logs mention the database | A `MONITOR_*_PASSWORD` has characters other than letters and digits, or `POSTGRES_SUPERUSER_PASSWORD` was changed after the volume was created. Restore the original, or `down -v` on a new install. | +| `already bootstrapped` | Expected on a second bootstrap. Use `create-admin-key` if you lost the key. | +| `docker compose up -d` printed a platform key | You ran `up` for the whole stack before the standalone bootstrap. The installation now has a platform key and no tenant. Either `down -v` and start again as in section 4, or create the tenant through the API with that platform key: `PUT /v1/tenants/by-external-id/{id}`, then create an admin key with `POST /v1/api-keys` while sending `X-Tenant-External-ID` (see the [API guide](../api/README.md#platform-key-operations)). | +| Check output says the address is not allowed | The target is in a blocked range. Allow it in `allowed_target_networks` (section 3), or patch the tenant limits with a platform key. | +| `401 unauthenticated` | Wrong, revoked or expired key, or a missing `Bearer ` prefix. | +| `429 rate-limited` | The key exceeded `api_rate_per_minute` (default 1200 for tenants). Wait for `Retry-After` and slow down. | +| Audit log shows the proxy's IP for every request | `MONITOR_TRUSTED_PROXIES` does not include your proxy. | +| Webhook test fails with a network error | The receiver URL resolves into a blocked range, or the VM cannot reach it. Allow the range (section 3) and check outbound firewalls. | + +## 10. Security checklist + +- TLS on the public endpoint; ports 8443/9443/9090 not reachable from outside. +- `MONITOR_TRUSTED_PROXIES` set to your proxy only. +- `.env` readable by root only (`chmod 600 .env`), backed up with the KEK stored separately from database backups. +- Admin key in a password manager. Create `operator` or `read-only` keys for people and automation, and restrict them with `ip_allowlist` and `expires_at` where you can ([API guide](../api/README.md#api-keys)). +- `allowed_target_networks` limited to what you monitor. +- Database not published. Do not use `deploy/docker-compose.db-port.yml` in production. +- Host patched; Docker images updated regularly. diff --git a/docs/progress.md b/docs/progress.md index 5c852fb..c977abd 100644 --- a/docs/progress.md +++ b/docs/progress.md @@ -1,6 +1,6 @@ # Implementation progress -**Last updated:** 2026-10-07 · **Latest release tag:** `v0.12.0` +**Last updated:** 2026-10-15 · **Latest release tag:** `v0.14.1` This page records which milestones are done and what comes next. The milestone definitions (contents and demo) live in the [feature specs index](features/README.md#suggested-milestones); update this page when a milestone's status changes. diff --git a/docs/usage-guide.md b/docs/usage-guide.md new file mode 100644 index 0000000..f8c340f --- /dev/null +++ b/docs/usage-guide.md @@ -0,0 +1,408 @@ +# Usage guide + +How to use a running monitoring server: add what you want to watch, get alerted, read the data. Everything here uses the REST API with `curl`, because the API is the only interface. The server has no built-in UI. Grafana and your own scripts are the clients. + +To install the server, see [Deploying on a new VM](deployment/standalone-vm.md). For every endpoint and field, see the [API guide](api/README.md). + +The examples assume: + +```sh +export MON_URL=https://monitor.example.com # or http://127.0.0.1:8443 on the VM +export MON_KEY=mon_... # your admin API key +alias mon='curl -sS -H "Authorization: Bearer $MON_KEY" -H "Content-Type: application/json"' +``` + +(Alias lines work in bash and zsh. In fish use a function.) Pipe output through `jq` for readability. + +## The model in five lines + +- A **tenant** is your account: everything below belongs to one. A standalone install has one. +- A **device** is a thing you watch: a server, a switch, a website, a camera, a sensor. +- A **check** runs one **plugin** against a device on an interval (ping, HTTP request, SNMP read, Redfish health). Plugins that receive data instead of polling are *push* checks. +- A check produces a **state** (OK, WARNING, CRITICAL, UNKNOWN) and **metrics**. Failing states open **incidents**. +- Incident changes become signed **webhook** events. **Alert routes** decide which events go to which webhook. + +Noise control is built in: a check must fail several times in a row before it alerts (`failure_count`, default 3), flapping is detected, and if a device's host or parent is down, the incidents it explains are suppressed. + +## 1. See what the server can do + +```sh +mon $MON_URL/v1/me # who you are, your role +mon $MON_URL/v1/tenant # your limits +mon $MON_URL/v1/plugins | jq -r '.items[] | "\(.type)\t\(.kind)\t\(.description)"' +mon $MON_URL/v1/plugins/http | jq .config_schema +``` + +Built-in plugins include: + +| Plugin | Use it for | +| --- | --- | +| `icmp` | Ping: round-trip time and packet loss | +| `http` | Website or API: status code, keyword, timing breakdown | +| `snmp.get`, `snmp.system`, `snmp.interfaces`, `snmp.ups`, `snmp.pdu`, `snmp.sensor` | Network gear, UPS, PDUs, environment sensors | +| `redfish.health` | Server hardware through the BMC: temperatures, fans, PSUs, disks | +| `xapi.pool` | XenServer/XCP-ng pools, their hosts and VMs | +| `camera.snapshot`, `rtsp.stream` | IP cameras | +| `push.http`, `push.mqtt` | Devices that send data to the server | + +`vm.agent` and `box.agent` are PlusClouds-specific ingesters and are of no use outside PlusClouds. `monitor.self` is the server's own health check. + +Each plugin publishes a JSON Schema for its `config`, its metrics, and its default and minimum interval. + +## 2. Monitor a website + +Create a device, then add an HTTP check. + +```sh +DEVICE=$(mon -X POST $MON_URL/v1/devices -d '{ + "name": "Company website", + "type": "web", + "address": "https://www.example.com" +}' | jq -r .id) + +mon -X POST $MON_URL/v1/devices/$DEVICE/checks -d '{ + "name": "Homepage", + "plugin": "http", + "config": { "keyword": "Welcome", "expected_status": [200] }, + "interval_seconds": 60, + "is_host_check": true, + "thresholds": [ + { "metric": "total_ms", + "warning": { "op": ">", "value": 1500 }, + "critical": { "op": ">", "value": 5000 }, + "for": "5m" } + ] +}' +``` + +- `is_host_check` marks the check that says whether the device is up. Other checks of the device are suppressed while it fails. +- Thresholds apply to the plugin's metrics (`GET /v1/plugins/http` lists them). `for` requires the condition to hold that long. `hysteresis` adds a margin before the state clears. +- The check status itself (status code wrong, keyword missing, timeout) is decided by the plugin, independent of thresholds. + +Run it now and read the result: + +```sh +CHECK= +mon -X POST $MON_URL/v1/checks/$CHECK/run-now +mon $MON_URL/v1/checks/$CHECK/state +mon -X POST $MON_URL/v1/devices/$DEVICE/test # runs every enabled check once +``` + +If the check says the address is not allowed, your tenant's `allowed_target_networks` does not include it. See the deployment guide. + +## 3. Monitor a server with ping + +```sh +SRV=$(mon -X POST $MON_URL/v1/devices -d '{ + "name": "db-01", "type": "server", "address": "10.0.4.20", + "tags": { "rack": "a12", "env": "prod" } +}' | jq -r .id) + +mon -X POST $MON_URL/v1/devices/$SRV/checks -d '{ + "name": "Ping", "plugin": "icmp", "interval_seconds": 30, "is_host_check": true, + "thresholds": [ + { "metric": "packet_loss_percent", "warning": {"op": ">", "value": 10}, "critical": {"op": ">=", "value": 100} } + ] +}' +``` + +Tags are free text and are what alert routes and Grafana filter on. + +## 4. Network gear, hardware, hypervisors: credentials + +Plugins that log in (SNMP, Redfish, XAPI, cameras, authenticated HTTP) take a **credential**. Secrets are encrypted at rest and write-only: no endpoint returns them. + +```sh +mon $MON_URL/v1/credential-types | jq # types and their fields + +CRED=$(mon -X POST $MON_URL/v1/credentials -d '{ + "name": "core switches v2c", + "type": "", + "fields": { "...": "..." } # see fields_schema of that type +}' | jq -r .id) + +mon -X POST $MON_URL/v1/devices/$DEVICE/checks -d '{ + "name": "Interfaces", "plugin": "snmp.interfaces", "interval_seconds": 60, + "credentials": { "auth": "'$CRED'" } +}' +``` + +Each type lists its fields in `fields_schema` (secret fields are `writeOnly`), and each plugin lists the `credential_types` it accepts in `GET /v1/plugins/{type}`. A check created without a required credential reports UNKNOWN until one is assigned. One credential can serve many checks; it cannot be deleted while a check uses it. + +Collector plugins (`snmp.interfaces`, `redfish.health`, `xapi.pool`, cameras) create **objects**: one per interface, fan, VM and so on. Incidents and thresholds work per object, and you can list them: + +```sh +mon $MON_URL/v1/checks/$CHECK/objects +``` + +`xapi.pool` also discovers hypervisor hosts and VMs and creates them as child devices. + +## 5. Devices that push data + +For sensors, scripts and agents that report on their own, silence is the failure: no data for `interval × missed_count` makes the check CRITICAL. + +### HTTP push + +```sh +mon -X POST $MON_URL/v1/devices -d '{"name":"Boiler room","type":"sensor"}' # note the id + +mon -X POST $MON_URL/v1/devices/$DEVICE/checks -d '{ + "name": "Boiler telemetry", + "plugin": "push.http", + "interval_seconds": 60, + "config": { + "metrics": { "temperature": "$.sensors.temp", "humidity": "$.sensors.hum" }, + "units": { "temperature": "celsius", "humidity": "percent" }, + "missed_count": 3 + }, + "thresholds": [ { "metric": "temperature", "critical": { "op": ">", "value": 45 } } ] +}' +``` + +The response contains `push_token` (`mpush_...`) **once**. Then the device sends: + +```sh +curl -X POST $MON_URL/ingest/v1/$CHECK \ + -H "Authorization: Bearer mpush_..." -H "Content-Type: application/json" \ + -d '{"sensors": {"temp": 21.4, "hum": 48}}' +# 202 {"accepted":1} +``` + +- Send one object or an array of up to 1,000 with a `timestamp` for each (configure the field with `timestamp`). Timestamps up to 24 hours old or 5 minutes ahead are kept. +- Selectors are a JSONPath subset: `$.a.b`, `$.list[0].v`, `$["name with-dash"]`. +- Set `status` in the config to take the state from a field (`ok`, `warning`, `critical`, `0` to `3`, `up`, `down`, `true`, `false`). +- Rate limit is 2 requests per second per token (burst 20), and bodies are limited to 64 KiB. +- Rotate or revoke a token without touching other devices: `POST /v1/checks/{id}/rotate-token`. + +Answers: `202` accepted, `401` bad token, `400` malformed body, `409` check disabled, `413` too big, `429` too fast. + +### MQTT sensors + +Requires the MQTT broker (deployment guide, section 7). + +```sh +# Credential for one device (the key is usually the MAC or serial) +mon -X POST $MON_URL/v1/ingest/mqtt-credentials -d '{ + "name": "greenhouse-1", "kind": "device", "device_key": "AA:BB:CC:DD:EE:01", "profile": "json" +}' +# -> returns username and password once +``` + +The device connects to `mqtts://mqtt.example.com:8883` with that username and password, and publishes JSON to `//`. The second topic segment is the device key; a device credential may publish only under its own key. + +- With `auto_register` true (default), the first message creates the device with a `push.mqtt` data check and an `mqtt.connection` check. Online/offline follows the MQTT connection and keepalive. +- The `json` profile turns every numeric field into a metric (nested objects joined with `_`). +- Use `kind: "shared"` for fleets that share one login; the device key then comes only from the topic. +- `GET /v1/ingest/unregistered` lists keys whose messages were dropped (auto-registration off or limit reached). `POST /v1/devices/{id}/mqtt` pre-registers a device. + +## 6. Get alerts: webhooks and routes + +The server delivers one thing: signed HTTP POSTs. What happens next (email, Slack, SMS, a ticket) is the receiver's job. n8n, Node-RED, Zapier, a small script or an incident tool all work. + +### Create a webhook + +```sh +WH=$(mon -X POST $MON_URL/v1/webhooks -d '{ + "name": "n8n alerts", "url": "https://n8n.example.com/webhook/monitoring", "timeout_seconds": 10 +}') +echo "$WH" | jq # note id and the signing secret (whsec_...), shown once +``` + +Add static headers (for example an auth token the receiver wants) with `"headers": {"X-Api-Token": "..."}`. They are stored encrypted. Check delivery end to end: + +```sh +mon -X POST $MON_URL/v1/webhooks//test # sends a signed monitoring.webhook.test event +``` + +### Create a route + +No route, no alerts. A route says "incidents matching this go to that webhook". + +```sh +mon -X POST $MON_URL/v1/alert-routes -d '{ + "name": "critical to n8n", + "endpoint_id": "", + "match": { "severity": ["critical"], "device_types": ["server", "network"], "tags": { "env": "prod" } }, + "group_by": ["root_device_id"], + "group_wait_seconds": 30, + "repeat_interval_seconds": 14400, + "labels": { "recipient": "noc" }, + "steps": [ + { "after_seconds": 0 }, + { "after_seconds": 900, "labels": { "recipient": "on-call" }, "only_if_unacknowledged": true } + ] +}' +``` + +- Routes are evaluated in order (`position`). The first match wins unless `continue` is true. An empty `match` takes everything, so a catch-all route is one call. +- `group_by` bundles incidents with the same value into one event, so a switch failure sends one message. +- `repeat_interval_seconds` (300 to 604800) re-sends open, unacknowledged incidents. +- `steps` escalate: each later step is sent that many seconds after the first, unless the incident was resolved (or acknowledged, with `only_if_unacknowledged`). `labels` are passed to the receiver, which decides what they mean. +- Try a route without waiting for a real incident: `POST /v1/alert-routes/test`. + +### Receive and verify events + +Events use the CloudEvents 1.0 envelope, signed with the [Standard Webhooks](https://www.standardwebhooks.com/) scheme. Libraries exist for most languages. Headers: + +| Header | Meaning | +| --- | --- | +| `webhook-id` | Event ID. Identical on retries: deduplicate on it. | +| `webhook-timestamp` | Unix seconds. Reject old ones. | +| `webhook-signature` | `v1,`; several space-separated values while a secret rotates. | + +The signed content is `webhook-id + "." + webhook-timestamp + "." + raw body`; the key is the base64 part of `whsec_`. A minimal Python receiver check: + +```python +import base64, hashlib, hmac, time + +def verify(secret: str, headers: dict, body: bytes, tolerance=300) -> bool: + if abs(time.time() - int(headers["webhook-timestamp"])) > tolerance: + return False + key = base64.b64decode(secret.removeprefix("whsec_")) + signed = f'{headers["webhook-id"]}.{headers["webhook-timestamp"]}.'.encode() + body + expected = base64.b64encode(hmac.new(key, signed, hashlib.sha256).digest()).decode() + return any(hmac.compare_digest(expected, s.split(",", 1)[1]) + for s in headers["webhook-signature"].split() if s.startswith("v1,")) +``` + +Event types: `monitoring.incident.opened`, `.updated`, `.acknowledged`, `.resolved`, `.commented`, `.renotify`, `.escalated`, plus `monitoring.tenant.device_limit_reached`, `monitoring.tenant.check_limit_reached` and `monitoring.webhook.test`. Example: + +```json +{ + "specversion": "1.0", + "id": "0192f7c4-...", + "source": "/monitoring//", + "type": "monitoring.incident.opened", + "time": "2026-10-10T10:15:00Z", + "subject": "", + "data": { + "object": { "severity": "critical", "summary": "...", "...": "..." }, + "device": { "name": "db-01", "address": "10.0.4.20", "tags": { "rack": "a12" } }, + "check": { "name": "Ping", "plugin": "icmp", "thresholds": [ ] }, + "route": { "labels": { "recipient": "noc" } }, + "links": { "incident": "https://..." } + } +} +``` + +(Fields are abbreviated. Capture a real event, from a test delivery or an incident, for the full shape.) Set `notifier.incident_url_template` in the server config to get a `data.links.incident` link into your own tool. + +Delivery is at least once. The server retries failures with growing delays for 24 hours, keeps order per incident, and records every attempt: + +```sh +mon "$MON_URL/v1/webhooks//deliveries" # the delivery log +mon -X POST "$MON_URL/v1/webhooks//deliveries//replay" # send one again +mon -X POST "$MON_URL/v1/webhooks//deliveries/replay?status=failed" # bulk +mon -X POST $MON_URL/v1/webhooks//rotate-secret # old secret stays valid 24 h +``` + +A receiver that answers `410 Gone` has its webhook disabled by the server (`disabled_reason`). + +## 7. Work with incidents + +```sh +mon "$MON_URL/v1/incidents?status=active" +mon "$MON_URL/v1/incidents?status=active&severity=critical&suppressed=false" +mon $MON_URL/v1/incidents/ # with comments +mon -X POST $MON_URL/v1/incidents//ack +mon -X POST $MON_URL/v1/incidents//comments -d '{"body":"Rebooting the node"}' +mon -X POST $MON_URL/v1/incidents//resolve +``` + +Incidents resolve themselves when the check recovers (`recovery_count` good results, default 1). `suppressed: true` means an upstream failure explains the incident, so it was not notified. Tell the server how your devices depend on each other for better suppression: + +```sh +mon -X POST $MON_URL/v1/devices/$SRV/dependencies -d '{"depends_on_id":""}' +mon $MON_URL/v1/devices//impact # everything that goes quiet if the switch fails +``` + +Put a VM or camera inside its host or NVR with `parent_id` when creating the device. Children are suppressed when the container fails. + +## 8. Read metrics + +```sh +# Which series exist for a check +mon "$MON_URL/v1/metrics/series?check_id=$CHECK" + +# Average response time per 5 minutes over the last day +mon "$MON_URL/v1/metrics/query?check_id=$CHECK&name=total_ms&from=$(date -u -d '1 day ago' +%FT%TZ)&step=300&agg=avg" + +# 95th percentile and stats over a window +mon "$MON_URL/v1/metrics/query?check_id=$CHECK&name=total_ms&agg=p95&step=3600&from=2026-10-09T00:00:00Z" +mon "$MON_URL/v1/metrics/summary?check_id=$CHECK&name=total_ms" +``` + +- `name` can repeat; `object` selects an object of a collector check; `device_id` selects every check of a device. +- Without `resolution` the server picks raw data, 5-minute or hourly rollups from the step and retention. A response holds at most 10,000 points per series. +- `avg`, `min`, `max`, `sum` work at every resolution. `stddev` and percentiles (`p50`, `p95`, `p99.9`) use raw samples only, so `from` must be within raw retention. +- `moving_window` smooths the line. + +For dashboards, use Grafana against PostgreSQL ([deploy/grafana](../deploy/grafana/README.md)) or call this endpoint from your own front end. + +## 9. Adaptive alerts: Whoopsy! + +Fixed thresholds do not fit metrics that have a daily rhythm. Whoopsy! learns a band of normal values for a check's metric and alerts when results leave it. + +```sh +mon $MON_URL/v1/checks/$CHECK/whoopsy # current settings and band +mon -X PUT $MON_URL/v1/checks/$CHECK/whoopsy -d '{"window": 12, "deviations": 2, "consecutive": 3, "direction": "above", "severity": "warning"}' +mon "$MON_URL/v1/checks/$CHECK/whoopsy/band?from=...&to=..." # band for graphs +mon -X POST $MON_URL/v1/checks/$CHECK/whoopsy/reset # accept a new normal +mon -X DELETE $MON_URL/v1/checks/$CHECK/whoopsy +``` + +Every field is optional. Defaults: the plugin's default metric (`whoopsy_metric` in `GET /v1/plugins`), a window of 7 results, 1 standard deviation, 3 results in a row, direction `above`, severity `warning`. Whoopsy! is refused on collector checks, and it changes how the check is billed on PlusClouds (irrelevant to you). + +## 10. Organize with sites + +```sh +SITE=$(mon -X POST $MON_URL/v1/sites -d '{"name":"Istanbul DC 1","timezone":"Europe/Istanbul"}' | jq -r .id) +mon -X PATCH $MON_URL/v1/devices/$SRV -d '{"site_id":"'$SITE'"}' +``` + +Routes can match `site_ids`, and lists can filter by site. + +## 11. People and automation: API keys + +Roles: `read-only` reads everything; `operator` also acknowledges, comments, resolves, runs checks and edits what is monitored; `admin` also manages keys and reads the audit log. + +```sh +mon -X POST $MON_URL/v1/api-keys -d '{ + "name": "grafana-readonly", "role": "read-only", + "expires_at": "2027-01-01T00:00:00Z", "ip_allowlist": ["203.0.113.10/32"] +}' # the key is returned once +mon $MON_URL/v1/api-keys +mon -X DELETE $MON_URL/v1/api-keys/ +mon "$MON_URL/v1/audit?limit=50" # who did what, tamper-evident +``` + +Use one key per integration, so you can revoke one without touching the rest. + +## 12. Infrastructure as code + +Every resource that you create with a script can carry an `external` reference (`source`, `type`, `id`) and be addressed by it with an idempotent `PUT .../by-external-id/{id}`. That gives you create-or-update semantics without storing the server's UUIDs: + +```sh +mon -X PUT "$MON_URL/v1/devices/by-external-id/db-01?source=cmdb" -d '{"name":"db-01","type":"server","address":"10.0.4.20"}' +``` + +Devices, sites, credentials, webhooks and alert routes support this. Run the same script after every change in your inventory. + +## 13. Is the monitor itself healthy? + +- Set `MONITOR_HEARTBEAT_URL` to a dead-man's-switch service (healthchecks.io or similar). The engine pings it every minute while its core loop is healthy, so you hear when the server, its database or its notifier stops. +- `docker compose logs monitor`, and `monitor health` for readiness. +- The engine also keeps its own health checks on a built-in `monitor` device in an internal platform tenant. A standalone tenant cannot see or route those; the heartbeat is the supported way to watch the server. + +## Limits to know + +| Limit | Default | Where it is set | +| --- | --- | --- | +| Devices / checks per tenant | 1000 / 10000 | `platform.tenant_defaults`, or the tenant record | +| Webhooks / routes per tenant | 20 / 50 | the same | +| Shortest check interval | 30 s | the same | +| API requests per minute per key | 1200 | the same | +| API body | 4 MiB | `api.max_body` | +| Push body, rate | 64 KiB, 2/s per token | `ingest.http` | + +Beyond a limit, creating returns `409 limit-reached`, and the account also gets a `monitoring.tenant.*_limit_reached` event. From 9bc212781417878a56c76fc3629233e0c65c35df Mon Sep 17 00:00:00 2001 From: Harun Baris Bulut Date: Thu, 15 Oct 2026 21:29:15 +0300 Subject: [PATCH 2/4] docs: state the no-UI choice up front and start Grafana in the quickstart Co-Authored-By: Claude Sonnet 5.5 --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f73fccc..70a959e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ An API-first, extensible monitoring engine written in Go. One engine watches net > **Status: v0.14, pre-1.0.** Scheduling, state, incidents, signed webhooks, metrics, Grafana dashboards and plugins for HTTP, ICMP, SNMP, Redfish, XCP-ng, cameras, HTTP/MQTT push and UPS/PDU all run today. Remote probes, VMware/Proxmox and the Phase 2 and 3 items in the [roadmap](#roadmap) do not exist yet. The API can still change before 1.0; see the [changelog](CHANGELOG.md) and [implementation progress](docs/progress.md). +**No built-in UI, by design.** The engine is an API; dashboards are [Grafana](deploy/grafana/README.md), reading PostgreSQL directly, with seven ready-made dashboards (devices and incidents, websites, network interfaces, server hardware, power, cameras, XCP-ng) provisioned for you. If you want a point-and-click admin UI instead, this is not that tool yet. + + + Developed by [PlusClouds](https://plusclouds.com) under the MIT license. It runs standalone or embedded in the PlusClouds panel, and it is multi-tenant from day one because customers run it too. ## Quickstart (5 minutes) @@ -11,9 +15,9 @@ Developed by [PlusClouds](https://plusclouds.com) under the MIT license. It runs Needs Docker with Compose, `curl` and `jq`. This builds from source and uses plain HTTP: for trying it out, not for production (see [Deploy on a new VM](docs/deployment/standalone-vm.md)). ```sh -# 1. Start PostgreSQL and the engine +# 1. Start PostgreSQL, the engine and Grafana with the dashboards git clone https://github.com/plusclouds/monitoring.server && cd monitoring.server -docker compose -f deploy/docker-compose.yml up -d --build --wait +docker compose -f deploy/docker-compose.yml --profile grafana up -d --build --wait # 2. Create your tenant and admin key (shown once) docker compose -f deploy/docker-compose.yml run --rm monitor admin bootstrap --standalone --tenant "Demo" @@ -29,7 +33,7 @@ WH=$(mon -X POST $MON_URL/v1/webhooks -d '{"name":"demo","url":"https://webhook. mon -X POST $MON_URL/v1/alert-routes -d "{\"name\":\"all\",\"endpoint_id\":\"$WH\"}" ``` -After a few failed runs a signed `monitoring.incident.opened` event arrives at your receiver. Next: the [usage guide](docs/usage-guide.md), and [Grafana dashboards](deploy/grafana/README.md) that read PostgreSQL directly. +After a few failed runs a signed `monitoring.incident.opened` event arrives at your receiver. Open http://127.0.0.1:3000 (login `admin` / `admin`) to see the device, its check and the incident in the **Device overview** dashboard. Next: the [usage guide](docs/usage-guide.md) and the [Grafana notes](deploy/grafana/README.md). --- From 4ffde44fa60581b93a607436a16d023302c29faf Mon Sep 17 00:00:00 2001 From: Harun Baris Bulut Date: Thu, 15 Oct 2026 21:31:03 +0300 Subject: [PATCH 3/4] feat: Grafana image with dashboards baked in Add deploy/grafana/Dockerfile, publish registry.plusclouds.com/monitoring-grafana on version tags, and an optional grafana profile in deploy/compose. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 17 ++++++++++++++++- deploy/compose/.env.example | 7 +++++++ deploy/compose/docker-compose.yml | 28 ++++++++++++++++++++++++++++ deploy/grafana/Dockerfile | 21 +++++++++++++++++++++ deploy/grafana/README.md | 16 ++++++++++++++++ 5 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 deploy/grafana/Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ae4ee1..5bf06dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ env: # Docker stores credentials per host, so login, image tags and push must all use this same name. REGISTRY: registry.plusclouds.com IMAGE: monitoring + IMAGE_GRAFANA: monitoring-grafana jobs: lint: @@ -97,6 +98,7 @@ jobs: version="${GITHUB_REF_NAME#v}" echo "version=$version" >> "$GITHUB_OUTPUT" echo "tags=-t $REGISTRY/$IMAGE:$version -t $REGISTRY/$IMAGE:latest" >> "$GITHUB_OUTPUT" + echo "grafana_tags=-t $REGISTRY/$IMAGE_GRAFANA:$version -t $REGISTRY/$IMAGE_GRAFANA:latest" >> "$GITHUB_OUTPUT" # Building needs no credentials (public base images), so it uses its own # Docker config: ~/.docker/buildx on the shared runner is root-owned @@ -115,11 +117,23 @@ jobs: - name: Check the image runs run: docker run --rm "$REGISTRY/$IMAGE:${{ steps.tags.outputs.version }}" version + # Grafana with the dashboards baked in (deploy/grafana/Dockerfile). + - name: Build Grafana image + env: + DOCKER_CONFIG: ${{ runner.temp }}/docker-build + run: | + docker build --pull \ + --build-arg VERSION=${{ github.ref_name }} \ + --label org.opencontainers.image.revision=${{ github.sha }} \ + ${{ steps.tags.outputs.grafana_tags }} deploy/grafana + # Uses the registry login stored on the self-hosted runner. If a push # fails with "no basic auth credentials", another job logged the runner # out (see plusclouds.api.v4 commit 8105c9dc): log it in again. - name: Push - run: docker image push --all-tags "$REGISTRY/$IMAGE" + run: | + docker image push --all-tags "$REGISTRY/$IMAGE" + docker image push --all-tags "$REGISTRY/$IMAGE_GRAFANA" # Remove only this build's images: a full prune would also delete # other repositories' images and build cache on the shared runner. @@ -127,4 +141,5 @@ jobs: if: always() run: | docker image ls --format '{{.Repository}}:{{.Tag}}' "$REGISTRY/$IMAGE" | xargs -r docker image rm -f + docker image ls --format '{{.Repository}}:{{.Tag}}' "$REGISTRY/$IMAGE_GRAFANA" | xargs -r docker image rm -f docker image prune -f diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 1fe7493..3947902 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -73,3 +73,10 @@ MONITOR_IDENTITY_SOURCE=plusclouds # Optional: password of the read-only monitor_grafana database role (F07). # Leave empty to keep the role unable to log in. See deploy/grafana. MONITOR_GRAFANA_PASSWORD= + +# Optional Grafana image (docker compose --profile grafana up -d). It needs +# MONITOR_GRAFANA_PASSWORD above and an admin password for the Grafana login. +# GRAFANA_ADMIN_PASSWORD= +# GRAFANA_ROOT_URL=http://localhost:3000 +# GRAFANA_BIND=127.0.0.1 +# GRAFANA_PORT=3000 diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index f4098a2..eb9d1e0 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -120,6 +120,33 @@ services: target: /etc/monitor/config.yaml networks: [backend] + # Optional Grafana with the dashboards baked in: docker compose --profile grafana up -d + # Needs MONITOR_GRAFANA_PASSWORD in .env (db-init then lets monitor_grafana + # log in). Published on 127.0.0.1 only: put a TLS proxy in front. + grafana: + image: ${GRAFANA_IMAGE:-registry.plusclouds.com/monitoring-grafana}:${MONITOR_VERSION:-0.1.2-m1} + profiles: [grafana] + restart: unless-stopped + depends_on: + db-init: + condition: service_completed_successfully + environment: + MONITOR_GRAFANA_DB_HOST: postgres:5432 + MONITOR_GRAFANA_DB_SSLMODE: disable + MONITOR_GRAFANA_PASSWORD: ${MONITOR_GRAFANA_PASSWORD:?set MONITOR_GRAFANA_PASSWORD in .env to use Grafana} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env to use Grafana} + GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000} + volumes: + - grafana-data:/var/lib/grafana + networks: [backend, frontend] + ports: + - "${GRAFANA_BIND:-127.0.0.1}:${GRAFANA_PORT:-3000}:3000" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 30 + configs: # Engine settings. Secrets are not here: the environment above supplies the # database DSNs, the key-encryption key and the public URL from .env. @@ -247,3 +274,4 @@ networks: volumes: pgdata: + grafana-data: diff --git a/deploy/grafana/Dockerfile b/deploy/grafana/Dockerfile new file mode 100644 index 0000000..c0afdcc --- /dev/null +++ b/deploy/grafana/Dockerfile @@ -0,0 +1,21 @@ +# Grafana with the starter dashboards and the PostgreSQL data source baked in +# (F07). Build from this directory: +# docker build --build-arg VERSION=$(git describe --tags --always) -t monitoring-grafana deploy/grafana +# Settings at run time (see README.md): MONITOR_GRAFANA_DB_HOST, +# MONITOR_GRAFANA_DB_SSLMODE, MONITOR_GRAFANA_PASSWORD. +ARG GRAFANA_VERSION=13.0.2 +FROM grafana/grafana-oss:${GRAFANA_VERSION} +ARG VERSION=dev +LABEL org.opencontainers.image.title="monitoring.server Grafana" \ + org.opencontainers.image.description="Grafana with the monitoring.server dashboards and data source" \ + org.opencontainers.image.source="https://github.com/plusclouds/monitoring.server" \ + org.opencontainers.image.licenses="MIT" \ + org.opencontainers.image.version="${VERSION}" +ENV MONITOR_GRAFANA_DB_HOST=postgres:5432 \ + MONITOR_GRAFANA_DB_SSLMODE=disable \ + GF_AUTH_ANONYMOUS_ENABLED=false \ + GF_ANALYTICS_REPORTING_ENABLED=false \ + GF_ANALYTICS_CHECK_FOR_UPDATES=false \ + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/monitor/device-overview.json +COPY --chown=472:0 provisioning /etc/grafana/provisioning +COPY --chown=472:0 dashboards /var/lib/grafana/dashboards/monitor diff --git a/deploy/grafana/README.md b/deploy/grafana/README.md index 291473d..add37ef 100644 --- a/deploy/grafana/README.md +++ b/deploy/grafana/README.md @@ -53,6 +53,22 @@ docker compose -f deploy/docker-compose.yml --profile grafana up -d --wait open http://127.0.0.1:3000 # admin / admin ``` +## Grafana image + +`Dockerfile` builds Grafana with the provisioning and dashboards baked in, so nothing needs mounting. CI publishes it with every version tag as `registry.plusclouds.com/monitoring-grafana`, next to the engine image, with the same version. + +```sh +docker build -t monitoring-grafana deploy/grafana +``` + +In `deploy/compose`, set `MONITOR_GRAFANA_PASSWORD` and `GRAFANA_ADMIN_PASSWORD` in `.env`, then: + +```sh +docker compose --profile grafana up -d +``` + +Grafana listens on `127.0.0.1:3000`; put a TLS proxy in front before exposing it. The image reads the same three variables as below. Dashboards are read-only (`allowUiUpdates: false`); change them in `dashboards/` and rebuild. + ## Production 1. Set `MONITOR_GRAFANA_PASSWORD` in `deploy/compose/.env`. On the next `docker compose up`, the `db-init` service lets `monitor_grafana` log in with that password and grants it the views. From f730e166434fcaf771d66794c7be7986dc2fbeaf Mon Sep 17 00:00:00 2001 From: Harun Baris Bulut Date: Mon, 19 Oct 2026 17:04:12 +0300 Subject: [PATCH 4/4] fix: default compose image version to 0.15.0 Co-Authored-By: Claude Sonnet 5.5 --- deploy/compose/.env.example | 2 +- deploy/compose/docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 3947902..131a205 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -65,7 +65,7 @@ MONITOR_TRUSTED_PROXIES=172.16.0.0/12,192.168.0.0/16,10.0.0.0/8 # Image to run. The registry needs `docker login registry.plusclouds.com` # unless the host is already logged in. MONITOR_IMAGE=registry.plusclouds.com/monitoring -MONITOR_VERSION=0.1.2-m1 +MONITOR_VERSION=0.15.0 # external_source of tenants and users provisioned by the platform key. MONITOR_IDENTITY_SOURCE=plusclouds diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index eb9d1e0..5f72fda 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -65,7 +65,7 @@ services: networks: [backend] monitor: - image: ${MONITOR_IMAGE:-registry.plusclouds.com/monitoring}:${MONITOR_VERSION:-0.1.2-m1} + image: ${MONITOR_IMAGE:-registry.plusclouds.com/monitoring}:${MONITOR_VERSION:-0.15.0} restart: unless-stopped depends_on: db-init: @@ -108,7 +108,7 @@ services: # Runs once per `up`: creates the installation ID, the platform tenant and # the platform key on the first start, and does nothing afterwards. bootstrap: - image: ${MONITOR_IMAGE:-registry.plusclouds.com/monitoring}:${MONITOR_VERSION:-0.1.2-m1} + image: ${MONITOR_IMAGE:-registry.plusclouds.com/monitoring}:${MONITOR_VERSION:-0.15.0} command: ["admin", "bootstrap", "--if-needed"] restart: "no" depends_on: @@ -124,7 +124,7 @@ services: # Needs MONITOR_GRAFANA_PASSWORD in .env (db-init then lets monitor_grafana # log in). Published on 127.0.0.1 only: put a TLS proxy in front. grafana: - image: ${GRAFANA_IMAGE:-registry.plusclouds.com/monitoring-grafana}:${MONITOR_VERSION:-0.1.2-m1} + image: ${GRAFANA_IMAGE:-registry.plusclouds.com/monitoring-grafana}:${MONITOR_VERSION:-0.15.0} profiles: [grafana] restart: unless-stopped depends_on: