From 9c594c393310f0af07522fabd4e30d306a0363e0 Mon Sep 17 00:00:00 2001 From: Mauro Ezequiel Moltrasio Date: Mon, 6 Jul 2026 16:27:43 +0200 Subject: [PATCH] docs: add OpenTelemetry output documentation Add docs/otel/ with: - otel.md: how to build with --features otel or make image-otel, configure the OTLP endpoint, and a full Loki + Grafana example - loki-config.yaml: minimal single-instance Loki configuration - grafana-datasource.yaml: auto-provisioned Loki datasource - grafana-dashboard-provider.yaml: dashboard file provider config - grafana-fact-dashboard.json: example dashboard with event rate, events by type/file pie charts, and a log stream panel Assisted-by: claude-opus-4-6 --- docs/otel/grafana-dashboard-provider.yaml | 8 + docs/otel/grafana-datasource.yaml | 9 ++ docs/otel/grafana-fact-dashboard.json | 138 ++++++++++++++++ docs/otel/loki-config.yaml | 26 +++ docs/otel/otel.md | 187 ++++++++++++++++++++++ 5 files changed, 368 insertions(+) create mode 100644 docs/otel/grafana-dashboard-provider.yaml create mode 100644 docs/otel/grafana-datasource.yaml create mode 100644 docs/otel/grafana-fact-dashboard.json create mode 100644 docs/otel/loki-config.yaml create mode 100644 docs/otel/otel.md diff --git a/docs/otel/grafana-dashboard-provider.yaml b/docs/otel/grafana-dashboard-provider.yaml new file mode 100644 index 00000000..75c961c8 --- /dev/null +++ b/docs/otel/grafana-dashboard-provider.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 + +providers: + - name: default + type: file + disableDeletion: true + options: + path: /var/lib/grafana/dashboards diff --git a/docs/otel/grafana-datasource.yaml b/docs/otel/grafana-datasource.yaml new file mode 100644 index 00000000..ed1103d4 --- /dev/null +++ b/docs/otel/grafana-datasource.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + isDefault: true diff --git a/docs/otel/grafana-fact-dashboard.json b/docs/otel/grafana-fact-dashboard.json new file mode 100644 index 00000000..2d87050f --- /dev/null +++ b/docs/otel/grafana-fact-dashboard.json @@ -0,0 +1,138 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "title": "Event rate", + "type": "timeseries", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "fillOpacity": 30, + "stacking": { "mode": "normal" } + }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum by (detected_level) (count_over_time({service_name=\"fact\"} [1m]))", + "legendFormat": "{{detected_level}}", + "refId": "A" + } + ] + }, + { + "title": "Events by type", + "type": "piechart", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "hideFrom": { "legend": false, "tooltip": false, "viz": false } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": ["percent"], + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" }, + "pieType": "donut", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "sort": "desc" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "editorMode": "code", + "expr": "sum by (file_event_type) (count_over_time({service_name=\"fact\"} | file_event_type != `` [$__range]))", + "legendFormat": "{{file_event_type}}", + "queryType": "range", + "direction": "backward", + "refId": "A" + } + ] + }, + { + "title": "Events by file", + "type": "piechart", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "hideFrom": { "legend": false, "tooltip": false, "viz": false } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": ["percent"], + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" }, + "pieType": "donut", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "sort": "desc" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "editorMode": "code", + "expr": "sum by (file_filename) (count_over_time({service_name=\"fact\"} | file_filename != `` [$__range]))", + "legendFormat": "{{file_filename}}", + "queryType": "range", + "direction": "backward", + "refId": "A" + } + ] + }, + { + "title": "Events", + "type": "logs", + "gridPos": { "h": 16, "w": 24, "x": 0, "y": 16 }, + "datasource": { "type": "loki", "uid": "loki" }, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": false, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=\"fact\"}", + "refId": "A" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["fact", "otel"], + "templating": { "list": [] }, + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "fact - File Activity", + "uid": "fact-file-activity", + "version": 1 +} diff --git a/docs/otel/loki-config.yaml b/docs/otel/loki-config.yaml new file mode 100644 index 00000000..25b0b2d4 --- /dev/null +++ b/docs/otel/loki-config.yaml @@ -0,0 +1,26 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h diff --git a/docs/otel/otel.md b/docs/otel/otel.md new file mode 100644 index 00000000..1847271a --- /dev/null +++ b/docs/otel/otel.md @@ -0,0 +1,187 @@ +# `fact` OpenTelemetry output + +`fact` can optionally push file activity events as +[OTLP](https://opentelemetry.io/docs/specs/otlp/) log records over +HTTP (protobuf encoding) to any OTLP-compatible endpoint such as +[Grafana Loki](https://grafana.com/oss/loki/), +[Grafana Alloy](https://grafana.com/docs/alloy/), or a standard +[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/). + +This feature is gated behind the `otel` Cargo feature flag and is not +included in the default build. + +## Building with OTel support + +### Local build + +```sh +cargo build --release --features otel +``` + +### Container image + +```sh +make image-otel +``` + +This invokes the regular `image` target with `CARGO_ARGS=--features otel`. +If you use podman instead of docker, set `DOCKER=podman`: + +```sh +make image-otel DOCKER=podman +``` + +## Configuration + +The OTel output requires a single setting: the OTLP HTTP endpoint URL +where log records will be pushed. It can be configured in three ways +(later entries override earlier ones): + +| Method | Example | +|---|---| +| YAML config file | `otel:` block with `endpoint:` key (see below) | +| Environment variable | `FACT_OTEL_ENDPOINT=http://loki:3100/otlp/v1/logs` | +| CLI flag | `--otel-endpoint http://loki:3100/otlp/v1/logs` | + +YAML example: + +```yaml +otel: + endpoint: http://loki:3100/otlp/v1/logs +``` + +The endpoint value is the full OTLP HTTP logs URL exposed by your +backend. Common values: + +| Backend | Endpoint | +|---|---| +| Grafana Loki | `http://:3100/otlp/v1/logs` | +| OTel Collector | `http://:4318/v1/logs` | + +When no endpoint is configured the OTel client is idle and consumes no +resources. + +## Technical details + +- Events are serialized as structured OTLP `LogRecord` attributes + using the native OpenTelemetry `AnyValue` map format. +- Transport is HTTP with binary protobuf encoding. +- Records are batched automatically by the OpenTelemetry SDK before + export. Batch parameters can be tuned via standard + [OTLP environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#batch-log-record-processor) + such as `OTEL_BLRP_SCHEDULE_DELAY` and + `OTEL_BLRP_MAX_EXPORT_BATCH_SIZE`. +- The OTLP resource `service.name` is set to `fact`. +- All records are emitted with severity `Info`. +- The endpoint supports hot-reload: sending `SIGHUP` to reload the + configuration will reconnect the OTel client to a new endpoint + without restarting the process. + +## Example: pushing events to Loki with Grafana + +This walkthrough sets up a minimal Loki instance with Grafana for +visualization, then runs `fact` with the OTel output pointing at +Loki's OTLP ingestion endpoint. + +All commands below use `docker` but work identically with `podman`. + +### 1. Create a network + +```sh +docker network create fact-loki +``` + +### 2. Start Loki + +This directory contains a minimal Loki configuration in +[loki-config.yaml](loki-config.yaml). Mount it into the container: + +```sh +docker run -d --name loki \ + --network fact-loki \ + -p 3100:3100 \ + -v ./docs/otel/loki-config.yaml:/etc/loki/config.yaml:ro,z \ + docker.io/grafana/loki:3.7.3 +``` + +Loki exposes its OTLP ingestion endpoint at `/otlp/v1/logs` on port +3100. + +### 3. Start Grafana + +Grafana is pre-configured with a Loki datasource +([grafana-datasource.yaml](grafana-datasource.yaml)) and a small +dashboard ([grafana-fact-dashboard.json](grafana-fact-dashboard.json)) +via provisioning files: + +```sh +docker run -d --name grafana \ + --network fact-loki \ + -p 3000:3000 \ + -e GF_AUTH_ANONYMOUS_ENABLED=true \ + -e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin \ + -v ./docs/otel/grafana-datasource.yaml:/etc/grafana/provisioning/datasources/loki.yaml:ro,z \ + -v ./docs/otel/grafana-dashboard-provider.yaml:/etc/grafana/provisioning/dashboards/default.yaml:ro,z \ + -v ./docs/otel/grafana-fact-dashboard.json:/var/lib/grafana/dashboards/fact.json:ro,z \ + docker.io/grafana/grafana:13.1.0 +``` + +### 4. Run `fact` + +#### Option A: container image + +Build the OTel image first (if you haven't already): + +```sh +make image-otel +``` + +Then run it on the same network so it can reach Loki by name: + +```sh +docker run --rm -it \ + --privileged \ + --network fact-loki \ + -e FACT_OTEL_ENDPOINT=http://loki:3100/otlp/v1/logs \ + -e FACT_PATHS='/etc:/etc/**/*' \ + -e FACT_HOST_MOUNT=/host \ + -v /:/host:ro \ + "$(make image-name)" +``` + +#### Option B: local binary + +Since the binary runs on the host, reach Loki via the published port: + +```sh +cargo build --release --features otel +sudo -E FACT_OTEL_ENDPOINT=http://localhost:3100/otlp/v1/logs \ + ./target/release/fact -p /etc +``` + +### 5. View events in Grafana + +Open and select the **fact - File +Activity** dashboard. It contains four panels: + +- **Event rate** — a time series showing the number of events per + minute. +- **Events by type** — a donut chart breaking down events by type + (open, creation, chmod, etc.). +- **Events by file** — a donut chart breaking down events by filename. +- **Events** — a log stream of all file activity events with full + attribute details. + +You can also query events directly in **Explore** +() using LogQL, for example: + +```logql +{service_name="fact"} +``` + +### Cleanup + +```sh +docker rm -f loki grafana +docker network rm fact-loki +```