Add one-time dashboard import and Akvorado datasource provisioning (#436)

* Add one-time dashboard import and Akvorado datasource provisioning

- Stage dashboards and library panels from puppet:///svn staging dir
- Import via Grafana API once (flag file prevents re-import), library panels first
- Add Akvorado datasource provisioning

* Add Akvorado server discovery and clean up import script

- Populate akvorado_servers in grafana.py node classifier
- Fix import script: move base64 import to top, remove unused auth variable, clean up spacing

Author: SoundGoof

modules/grafana.py

@@ -14,10 +14,19 @@ def generate(host, *args):
             domain = 'unknown'
         prometheus_servers[server] = domain
 
+    akvorado_servers = {}
+    for server in sorted(lib.get_nodes_with_package('akvorado').keys()):
+        try:
+            domain = lib.get_domain(server).lower()
+        except lib.NoDomainError:
+            domain = 'unknown'
+        akvorado_servers[server] = domain
+
     info = {
     'grafana': {
         'current_event': lib.get_current_event(),
         'prometheus_servers': prometheus_servers,
+        'akvorado_servers': akvorado_servers,
         }
     }
 

modules/grafana/manifests/init.pp

@@ -11,7 +11,7 @@
 # for package details such as default paths etc.
 #
 
-class grafana($current_event, $prometheus_servers = []) {
+class grafana($current_event, $prometheus_servers = [], $akvorado_servers = []) {
 
   # Adding the apt repository
   package { 'apt-transport-https':
@@ -119,6 +119,43 @@
     notify  => Service['grafana-server'],
   }
 
+  # Akvorado datasource provisioning
+  file { 'grafana-akvorado-datasources':
+    path    => '/etc/grafana/provisioning/datasources/akvorado.yaml',
+    content => template('grafana/akvorado-datasource.yaml.erb'),
+    mode    => '0644',
+    require => Package['grafana'],
+    notify  => Service['grafana-server'],
+  }
+
+  # Stage dashboards and library panels from SVN for one-time import
+  file { '/var/lib/grafana/dashboards-staging':
+    ensure  => directory,
+    source  => "puppet:///svn/${current_event}/services/grafana/staging",
+    recurse => true,
+    owner   => 'grafana',
+    group   => 'grafana',
+    require => Package['grafana'],
+  }
+
+  file { 'grafana-import-script':
+    path    => '/usr/local/bin/grafana-import-dashboards',
+    content => template('grafana/grafana-import-dashboards.py.erb'),
+    mode    => '0755',
+    require => Package['grafana'],
+  }
+
+  # One-time import via API — flag file prevents re-import on subsequent Puppet runs
+  exec { 'import-grafana-dashboards':
+    command => '/usr/local/bin/grafana-import-dashboards',
+    creates => '/var/lib/grafana/.dashboards-imported',
+    require => [
+      Service['grafana-server'],
+      File['/var/lib/grafana/dashboards-staging'],
+      File['grafana-import-script'],
+    ],
+  }
+
   # Setting up the Apache proxy
   apache::proxy { 'grafana-backend':
     url     => '/',

modules/grafana/templates/akvorado-datasource.yaml.erb

@@ -0,0 +1,13 @@
+# Managed by Puppet - do not edit manually
+apiVersion: 1
+
+datasources:
+<% @akvorado_servers.each do |server, domain| -%>
+  - name: akvorado_<%= domain %>
+    uid: akvorado_<%= domain %>
+    type: ovhcloud-akvorado-datasource
+    url: http://<%= server %>
+    access: proxy
+    isDefault: false
+    editable: true
+<% end -%>

modules/grafana/templates/grafana-import-dashboards.py.erb

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# Managed by Puppet - do not edit manually
+# Imports dashboards from staging into Grafana via API.
+# Runs once; flag file prevents re-import on subsequent Puppet runs.
+
+import base64
+import json
+import subprocess
+import sys
+import urllib.request
+import urllib.error
+from pathlib import Path
+
+STAGING = Path('/var/lib/grafana/dashboards-staging')
+FLAG_FILE = Path('/var/lib/grafana/.dashboards-imported')
+GRAFANA_URL = 'http://localhost:3001'
+
+lib_files = sorted((STAGING / 'Library_Panels').glob('*.json')) if (STAGING / 'Library_Panels').is_dir() else []
+dash_files = sorted((STAGING / 'General').glob('*.json')) if (STAGING / 'General').is_dir() else []
+
+if not lib_files and not dash_files:
+    print('No dashboards found in staging, skipping import.')
+    sys.exit(0)
+
+password = subprocess.check_output([
+    '/usr/local/bin/dh-create-service-account',
+    '--type', 'grafana',
+    '--product', 'login',
+    '--format', '{password}',
+], text=True).strip()
+
+def post(path, payload):
+    data = json.dumps(payload).encode()
+    req = urllib.request.Request(
+        GRAFANA_URL + path,
+        data=data,
+        headers={'Content-Type': 'application/json'},
+        method='POST',
+    )
+    req.add_header('Authorization', 'Basic ' + base64.b64encode(b'admin:' + password.encode()).decode())
+    try:
+        with urllib.request.urlopen(req) as resp:
+            print(f'  OK ({resp.status}): {path}')
+    except urllib.error.HTTPError as e:
+        print(f'  ERROR {e.code} importing {path}: {e.read().decode()}', file=sys.stderr)
+        sys.exit(1)
+
+# Import library panels first — dashboards may reference them by UID
+for f in lib_files:
+    print(f'Importing library panel: {f.name}')
+    raw = json.loads(f.read_text())
+    result = raw['result']
+    payload = {
+        'kind': result['kind'],
+        'name': result['name'],
+        'model': result['model'],
+        'uid': result['uid'],
+        'folderId': result['folderId'],
+        'folderUid': result['folderUid'],
+    }
+    post('/api/library-elements', payload)
+
+# Import dashboards after library panels exist
+for f in dash_files:
+    print(f'Importing dashboard: {f.name}')
+    dashboard = json.loads(f.read_text())
+    dashboard['id'] = None  # Let Grafana assign a new ID; uid is preserved for references
+    post('/api/dashboards/db', {'dashboard': dashboard, 'overwrite': False, 'folderId': 0})
+
+FLAG_FILE.touch()
+print('Import complete.')