Merge pull request #288790 from superherointj/nixos-tests-k3s-etcd

nixos/tests/k3s: add etcd backend

nixos/tests/k3s/default.nix

@@ -6,6 +6,11 @@ let
   allK3s = lib.filterAttrs (n: _: lib.strings.hasPrefix "k3s_" n) pkgs;
 in
 {
+  # Testing K3s with Etcd backend
+  etcd = lib.mapAttrs (_: k3s: import ./etcd.nix {
+    inherit system pkgs k3s;
+    inherit (pkgs) etcd;
+  }) allK3s;
   # Run a single node k3s cluster and verify a pod can run
   single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
   # Run a multi-node k3s cluster and verify pod networking works across nodes

nixos/tests/k3s/etcd.nix

@@ -0,0 +1,100 @@
+import ../make-test-python.nix ({ pkgs, lib, k3s, etcd, ... }:
+
+{
+  name = "${k3s.name}-etcd";
+
+  nodes = {
+
+    etcd = { ... }: {
+      services.etcd = {
+        enable = true;
+        openFirewall = true;
+        listenClientUrls = [ "http://192.168.1.1:2379" "http://127.0.0.1:2379" ];
+        listenPeerUrls = [ "http://192.168.1.1:2380" ];
+        initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ];
+        initialCluster = [ "etcd=http://192.168.1.1:2380" ];
+      };
+      networking = {
+        useDHCP = false;
+        defaultGateway = "192.168.1.1";
+        interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
+          { address = "192.168.1.1"; prefixLength = 24; }
+        ];
+      };
+    };
+
+    k3s = { pkgs, ... }: {
+      environment.systemPackages = with pkgs; [ jq ];
+      # k3s uses enough resources the default vm fails.
+      virtualisation.memorySize = 1536;
+      virtualisation.diskSize = 4096;
+
+      services.k3s = {
+        enable = true;
+        role = "server";
+        extraFlags = builtins.toString [
+          "--datastore-endpoint=\"http://192.168.1.1:2379\""
+          "--disable" "coredns"
+          "--disable" "local-storage"
+          "--disable" "metrics-server"
+          "--disable" "servicelb"
+          "--disable" "traefik"
+          "--node-ip" "192.168.1.2"
+        ];
+      };
+
+      networking = {
+        firewall = {
+          allowedTCPPorts = [ 2379 2380 6443 ];
+          allowedUDPPorts = [ 8472 ];
+        };
+        useDHCP = false;
+        defaultGateway = "192.168.1.2";
+        interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
+          { address = "192.168.1.2"; prefixLength = 24; }
+        ];
+      };
+    };
+
+  };
+
+  testScript = ''
+    with subtest("should start etcd"):
+        etcd.start()
+        etcd.wait_for_unit("etcd.service")
+
+    with subtest("should wait for etcdctl endpoint status to succeed"):
+        etcd.wait_until_succeeds("etcdctl endpoint status")
+
+    with subtest("should start k3s"):
+        k3s.start()
+        k3s.wait_for_unit("k3s")
+
+    with subtest("should test if kubectl works"):
+        k3s.wait_until_succeeds("k3s kubectl get node")
+
+    with subtest("should wait for service account to show up; takes a sec"):
+        k3s.wait_until_succeeds("k3s kubectl get serviceaccount default")
+
+    with subtest("should create a sample secret object"):
+        k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
+
+    with subtest("should check if secret is correct"):
+        k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
+
+    with subtest("should have a secret in database"):
+        etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
+
+    with subtest("should delete the secret"):
+        k3s.succeed("k3s kubectl delete secret nixossecret")
+
+    with subtest("should not have a secret in database"):
+        etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
+
+    with subtest("should shutdown k3s and etcd"):
+        k3s.shutdown()
+        etcd.shutdown()
+  '';
+
+  meta.maintainers = etcd.meta.maintainers ++ k3s.meta.maintainers;
+})

pkgs/applications/networking/cluster/k3s/builder.nix

@@ -355,6 +355,7 @@ buildGoModule rec {
   passthru.mkTests = version:
     let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version);
     in {
+      etcd = nixosTests.k3s.etcd.${k3s_version};
       single-node = nixosTests.k3s.single-node.${k3s_version};
       multi-node = nixosTests.k3s.multi-node.${k3s_version};
     };

pkgs/servers/etcd/3.5.nix

@@ -1,4 +1,4 @@
-{ lib, buildGoModule, fetchFromGitHub, symlinkJoin, nixosTests }:
+{ lib, buildGoModule, fetchFromGitHub, symlinkJoin, nixosTests, k3s }:
 
 let
   version = "3.5.12";
@@ -67,7 +67,10 @@ symlinkJoin {
 
   passthru = {
     inherit etcdserver etcdutl etcdctl;
-    tests = { inherit (nixosTests) etcd etcd-cluster; };
+    tests = {
+      inherit (nixosTests) etcd etcd-cluster;
+      k3s = k3s.passthru.tests.etcd;
+    };
   };
 
   paths = [