From 26036ecf319c57700ce06bac8f44a956d8038c42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christina=20S=C3=B8rensen?= <christina@cafkafk.com>
Date: Tue, 20 Feb 2024 06:59:08 +0100
Subject: [PATCH] kubelet: Set Kubelet Parameters Via A intermediate
 Configuration File
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christina Sørensen <christina@cafkafk.com>
---
 .../services/cluster/kubernetes/kubelet.nix   | 59 +++++++++++--------
 1 file changed, 36 insertions(+), 23 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index fd2dce7ee6a2..313dbe234018 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -33,6 +33,41 @@ let
 
   kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
 
+  # Flag based settings are deprecated, use the `--config` flag with a
+  # `KubeletConfiguration` struct.
+  # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
+  #
+  # NOTE: registerWithTaints requires a []core/v1.Taint, therefore requires
+  # additional work to be put in config format.
+  #
+  kubeletConfig = pkgs.writeText "kubelet-config" (builtins.toJSON ({
+    apiVersion = "kubelet.config.k8s.io/v1beta1";
+    kind = "KubeletConfiguration";
+    address = cfg.address;
+    port = cfg.port;
+    authentication = {
+      x509 = lib.optionalAttrs (cfg.clientCaFile != null) { clientCAFile = cfg.clientCaFile; };
+      webhook = {
+        enabled = true;
+        cacheTTL = "10s";
+      };
+    };
+    authorization = {
+      mode = "Webhook";
+    };
+    cgroupDriver = "systemd";
+    hairpinMode = "hairpin-veth";
+    registerNode = cfg.registerNode;
+    containerRuntimeEndpoint = cfg.containerRuntimeEndpoint;
+    healthzPort = cfg.healthz.port;
+    healthzBindAddress = cfg.healthz.bind;
+  } // lib.optionalAttrs (cfg.tlsCertFile != null)  { tlsCertFile = cfg.tlsCertFile; }
+    // lib.optionalAttrs (cfg.tlsKeyFile != null)   { tlsPrivateKeyFile = cfg.tlsKeyFile; }
+    // lib.optionalAttrs (cfg.clusterDomain != "")  { clusterDomain = cfg.clusterDomain; }
+    // lib.optionalAttrs (cfg.clusterDns != "")     { clusterDNS = [ cfg.clusterDns ] ; }
+    // lib.optionalAttrs (cfg.featureGates != [])   { featureGates = cfg.featureGates; }
+  ));
+
   manifestPath = "kubernetes/manifests";
 
   taintOptions = with lib.types; { name, ... }: {
@@ -294,21 +329,7 @@ in
           Restart = "on-failure";
           RestartSec = "1000ms";
           ExecStart = ''${top.package}/bin/kubelet \
-            --address=${cfg.address} \
-            --authentication-token-webhook \
-            --authentication-token-webhook-cache-ttl="10s" \
-            --authorization-mode=Webhook \
-            ${optionalString (cfg.clientCaFile != null)
-              "--client-ca-file=${cfg.clientCaFile}"} \
-            ${optionalString (cfg.clusterDns != "")
-              "--cluster-dns=${cfg.clusterDns}"} \
-            ${optionalString (cfg.clusterDomain != "")
-              "--cluster-domain=${cfg.clusterDomain}"} \
-            ${optionalString (cfg.featureGates != [])
-              "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-            --hairpin-mode=hairpin-veth \
-            --healthz-bind-address=${cfg.healthz.bind} \
-            --healthz-port=${toString cfg.healthz.port} \
+            --config=${kubeletConfig} \
             --hostname-override=${cfg.hostname} \
             --kubeconfig=${kubeconfig} \
             ${optionalString (cfg.nodeIp != null)
@@ -316,18 +337,10 @@ in
             --pod-infra-container-image=pause \
             ${optionalString (cfg.manifests != {})
               "--pod-manifest-path=/etc/${manifestPath}"} \
-            --port=${toString cfg.port} \
-            --register-node=${boolToString cfg.registerNode} \
             ${optionalString (taints != "")
               "--register-with-taints=${taints}"} \
             --root-dir=${top.dataDir} \
-            ${optionalString (cfg.tlsCertFile != null)
-              "--tls-cert-file=${cfg.tlsCertFile}"} \
-            ${optionalString (cfg.tlsKeyFile != null)
-              "--tls-private-key-file=${cfg.tlsKeyFile}"} \
             ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
-            --container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \
-            --cgroup-driver=systemd \
             ${cfg.extraOpts}
           '';
           WorkingDirectory = top.dataDir;