nixos: add grafana-agent module

Easily ship logs and metrics to Grafana Cloud and other similar targets.

nixos/modules/module-list.nix

@@ -660,6 +660,7 @@
   ./services/monitoring/do-agent.nix
   ./services/monitoring/fusion-inventory.nix
   ./services/monitoring/grafana.nix
+  ./services/monitoring/grafana-agent.nix
   ./services/monitoring/grafana-image-renderer.nix
   ./services/monitoring/grafana-reporter.nix
   ./services/monitoring/graphite.nix

nixos/modules/services/monitoring/grafana-agent.nix

@@ -0,0 +1,152 @@
+{ lib, pkgs, config, generators, ... }:
+with lib;
+let
+  cfg = config.services.grafana-agent;
+  settingsFormat = pkgs.formats.yaml { };
+  configFile = settingsFormat.generate "grafana-agent.yaml" cfg.settings;
+in
+{
+  meta = {
+    maintainers = with maintainers; [ zimbatm ];
+  };
+
+  options.services.grafana-agent = {
+    enable = mkEnableOption "grafana-agent";
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.grafana-agent;
+      defaultText = "pkgs.grafana-agent";
+      description = "The grafana-agent package to use.";
+    };
+
+    credentials = mkOption {
+      description = ''
+        Credentials to load at service startup. Keys that are UPPER_SNAKE will be loaded as env vars. Values are absolute paths to the credentials.
+      '';
+      type = types.attrsOf types.str;
+      default = { };
+
+      example = {
+        logs_remote_write_password = "/run/keys/grafana_agent_logs_remote_write_password";
+        LOGS_REMOTE_WRITE_URL = "/run/keys/grafana_agent_logs_remote_write_url";
+        LOGS_REMOTE_WRITE_USERNAME = "/run/keys/grafana_agent_logs_remote_write_username";
+        metrics_remote_write_password = "/run/keys/grafana_agent_metrics_remote_write_password";
+        METRICS_REMOTE_WRITE_URL = "/run/keys/grafana_agent_metrics_remote_write_url";
+        METRICS_REMOTE_WRITE_USERNAME = "/run/keys/grafana_agent_metrics_remote_write_username";
+      };
+    };
+
+    settings = mkOption {
+      description = ''
+        Configuration for <package>grafana-agent</package>.
+
+        See https://grafana.com/docs/agent/latest/configuration/
+      '';
+
+      type = types.submodule {
+        freeformType = settingsFormat.type;
+      };
+
+      default = {
+        server = {
+          # Don't bind on 0.0.0.0
+          grpc_listen_address = "127.0.0.1";
+          http_listen_address = "127.0.0.1";
+          # Don't bind on the default port 80
+          http_listen_port = 9090;
+        };
+        prometheus = {
+          wal_directory = "\${STATE_DIRECTORY}";
+          global.scrape_interval = "5s";
+        };
+        integrations = {
+          agent.enabled = true;
+          agent.scrape_integration = true;
+          node_exporter.enabled = true;
+          replace_instance_label = true;
+        };
+      };
+
+      example = {
+        loki.configs = [{
+          name = "default";
+          scrape_configs = [
+            {
+              job_name = "journal";
+              journal = {
+                max_age = "12h";
+                labels.job = "systemd-journal";
+              };
+              relabel_configs = [
+                {
+                  source_labels = [ "__journal__systemd_unit" ];
+                  target_label = "systemd_unit";
+                }
+                {
+                  source_labels = [ "__journal__hostname" ];
+                  target_label = "nodename";
+                }
+                {
+                  source_labels = [ "__journal_syslog_identifier" ];
+                  target_label = "syslog_identifier";
+                }
+              ];
+            }
+          ];
+          positions.filename = "\${STATE_DIRECTORY}/loki_positions.yaml";
+          clients = [{
+            url = "\${LOGS_REMOTE_WRITE_URL}";
+            basic_auth.username = "\${LOGS_REMOTE_WRITE_USERNAME}";
+            basic_auth.password_file = "\${CREDENTIALS_DIRECTORY}/logs_remote_write_password";
+          }];
+        }];
+        integrations = {
+          prometheus_remote_write = [{
+            url = "\${METRICS_REMOTE_WRITE_URL}";
+            basic_auth.username = "\${METRICS_REMOTE_WRITE_USERNAME}";
+            basic_auth.password_file = "\${CREDENTIALS_DIRECTORY}/metrics_remote_write_password";
+          }];
+        };
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.grafana-agent = {
+      wantedBy = [ "multi-user.target" ];
+      script = ''
+        set -euo pipefail
+        shopt -u nullglob
+
+        # Load all credentials into env if they are in UPPER_SNAKE form.
+        if [[ -n "''${CREDENTIALS_DIRECTORY:-}" ]]; then
+          for file in "$CREDENTIALS_DIRECTORY"/*; do
+            key=$(basename "$file")
+            if [[ $key =~ ^[A-Z0-9_]+$ ]]; then
+              echo "Environ $key"
+              export "$key=$(< "$file")"
+            fi
+          done
+        fi
+
+        # We can't use Environment=HOSTNAME=%H, as it doesn't include the domain part.
+        export HOSTNAME=$(< /proc/sys/kernel/hostname)
+
+        exec ${cfg.package}/bin/agent -config.expand-env -config.file ${configFile}
+      '';
+      serviceConfig = {
+        Restart = "always";
+        DynamicUser = true;
+        RestartSec = 2;
+        SupplementaryGroups = [
+          # allow to read the systemd journal for loki log forwarding
+          "systemd-journal"
+        ];
+        StateDirectory = "grafana-agent";
+        LoadCredential = lib.mapAttrsToList (key: value: "${key}:${value}") cfg.credentials;
+        Type = "simple";
+      };
+    };
+  };
+}

nixos/tests/all-tests.nix

@@ -189,6 +189,7 @@ in {
   google-oslogin = handleTest ./google-oslogin {};
   gotify-server = handleTest ./gotify-server.nix {};
   grafana = handleTest ./grafana.nix {};
+  grafana-agent = handleTest ./grafana-agent.nix {};
   graphite = handleTest ./graphite.nix {};
   graylog = handleTest ./graylog.nix {};
   grocy = handleTest ./grocy.nix {};

nixos/tests/grafana-agent.nix

@@ -0,0 +1,32 @@
+import ./make-test-python.nix ({ lib, pkgs, ... }:
+
+  let
+    nodes = {
+      machine = {
+        services.grafana-agent = {
+          enable = true;
+        };
+      };
+    };
+  in
+  {
+    name = "grafana-agent";
+
+    meta = with lib.maintainers; {
+      maintainers = [ zimbatm ];
+    };
+
+    inherit nodes;
+
+    testScript = ''
+      start_all()
+
+      with subtest("Grafana-agent is running"):
+          machine.wait_for_unit("grafana-agent.service")
+          machine.wait_for_open_port(9090)
+          machine.succeed(
+              "curl -sSfN http://127.0.0.1:9090/-/healthy"
+          )
+          machine.shutdown()
+    '';
+  })