From e7383a32e365371b9d62c30f16089687db7cb5d4 Mon Sep 17 00:00:00 2001 From: talyz Date: Mon, 9 Jan 2023 18:08:25 +0100 Subject: [PATCH 1/3] gitlab-pages: Maintain together with the rest of GitLab + add to... ...update.py --- .../version-management/gitlab}/gitlab-pages/default.nix | 0 pkgs/applications/version-management/gitlab/update.py | 9 +++++++++ pkgs/top-level/all-packages.nix | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) rename pkgs/{servers/http => applications/version-management/gitlab}/gitlab-pages/default.nix (100%) diff --git a/pkgs/servers/http/gitlab-pages/default.nix b/pkgs/applications/version-management/gitlab/gitlab-pages/default.nix similarity index 100% rename from pkgs/servers/http/gitlab-pages/default.nix rename to pkgs/applications/version-management/gitlab/gitlab-pages/default.nix diff --git a/pkgs/applications/version-management/gitlab/update.py b/pkgs/applications/version-management/gitlab/update.py index 0f322a5d375e..76e45cd4469e 100755 --- a/pkgs/applications/version-management/gitlab/update.py +++ b/pkgs/applications/version-management/gitlab/update.py @@ -199,6 +199,14 @@ def update_gitaly(): _call_nix_update('gitaly', gitaly_server_version) +@cli.command('update-gitlab-pages') +def update_gitlab_pages(): + """Update gitlab-shell""" + data = _get_data_json() + gitlab_pages_version = data['passthru']['GITLAB_PAGES_VERSION'] + _call_nix_update('gitlab-pages', gitlab_pages_version) + + @cli.command('update-gitlab-shell') def update_gitlab_shell(): """Update gitlab-shell""" @@ -223,6 +231,7 @@ def update_all(ctx, rev: str): ctx.invoke(update_data, rev=rev) ctx.invoke(update_rubyenv) ctx.invoke(update_gitaly) + ctx.invoke(update_gitlab_pages) ctx.invoke(update_gitlab_shell) ctx.invoke(update_gitlab_workhorse) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 1ffde8e34ba6..b52fc43d6dbd 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7473,6 +7473,8 @@ with pkgs; gitlab-clippy = callPackage ../development/tools/rust/gitlab-clippy { }; + gitlab-pages = callPackage ../applications/version-management/gitlab/gitlab-pages { }; + gitlab-runner = callPackage ../development/tools/continuous-integration/gitlab-runner { }; gitlab-shell = callPackage ../applications/version-management/gitlab/gitlab-shell { }; @@ -24014,8 +24016,6 @@ with pkgs; gatling = callPackage ../servers/http/gatling { }; - gitlab-pages = callPackage ../servers/http/gitlab-pages { }; - glabels = callPackage ../applications/graphics/glabels { }; nats-server = callPackage ../servers/nats-server { }; From dbd563b9b8c20071075bcdd5912b99486b3fcfa3 Mon Sep 17 00:00:00 2001 From: talyz Date: Mon, 9 Jan 2023 18:08:31 +0100 Subject: [PATCH 2/3] nixos/gitlab: Improve support for GitLab Pages - provide options and set defaults for important settings - generate the shared secret - reenable gitlab-pages in test --- nixos/modules/services/misc/gitlab.nix | 169 ++++++++++++++++++++----- nixos/tests/gitlab.nix | 12 +- 2 files changed, 145 insertions(+), 36 deletions(-) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index e7c707228f1b..4efe16748bc7 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -88,11 +88,6 @@ let }; }; - pagesArgs = [ - "-pages-domain" gitlabConfig.production.pages.host - "-pages-root" "${gitlabConfig.production.shared.path}/pages" - ] ++ cfg.pagesExtraArgs; - gitlabConfig = { # These are the default settings from config/gitlab.example.yml production = flip recursiveUpdate cfg.extraConfig { @@ -160,6 +155,12 @@ let }; extra = {}; uploads.storage_path = cfg.statePath; + pages = { + enabled = cfg.pages.enable; + port = 8090; + host = cfg.pages.settings.pages-domain; + secret_file = cfg.pages.settings.api-secret-key; + }; }; }; @@ -245,6 +246,7 @@ in { (mkRenamedOptionModule [ "services" "gitlab" "backupPath" ] [ "services" "gitlab" "backup" "path" ]) (mkRemovedOptionModule [ "services" "gitlab" "satelliteDir" ] "") (mkRemovedOptionModule [ "services" "gitlab" "logrotate" "extraConfig" ] "Modify services.logrotate.settings.gitlab directly instead") + (mkRemovedOptionModule [ "services" "gitlab" "pagesExtraArgs" ] "Use services.gitlab.pages.settings instead") ]; options = { @@ -666,10 +668,107 @@ in { }; }; - pagesExtraArgs = mkOption { - type = types.listOf types.str; - default = [ "-listen-proxy" "127.0.0.1:8090" ]; - description = lib.mdDoc "Arguments to pass to the gitlab-pages daemon"; + pages.enable = mkEnableOption (lib.mdDoc "the GitLab Pages service"); + + pages.settings = mkOption { + description = lib.mdDoc '' + Configuration options to set in the GitLab Pages config + file. + ''; + + type = types.submodule { + freeformType = with types; attrsOf (nullOr (oneOf [ str int bool ])); + + options = { + listen-http = mkOption { + type = with types; listOf str; + apply = x: if x == [] then null else lib.concatStringsSep "," x; + default = []; + description = lib.mdDoc '' + The address(es) to listen on for HTTP requests. + ''; + }; + + listen-https = mkOption { + type = with types; listOf str; + apply = x: if x == [] then null else lib.concatStringsSep "," x; + default = []; + description = lib.mdDoc '' + The address(es) to listen on for HTTPS requests. + ''; + }; + + listen-proxy = mkOption { + type = with types; listOf str; + apply = x: if x == [] then null else lib.concatStringsSep "," x; + default = [ "127.0.0.1:8090" ]; + description = lib.mdDoc '' + The address(es) to listen on for proxy requests. + ''; + }; + + artifacts-server = mkOption { + type = with types; nullOr str; + default = "http${optionalString cfg.https "s"}://${cfg.host}/api/v4"; + defaultText = "http(s):///api/v4"; + example = "https://gitlab.example.com/api/v4"; + description = lib.mdDoc '' + API URL to proxy artifact requests to. + ''; + }; + + gitlab-server = mkOption { + type = with types; nullOr str; + default = "http${optionalString cfg.https "s"}://${cfg.host}"; + defaultText = "http(s)://"; + example = "https://gitlab.example.com"; + description = lib.mdDoc '' + Public GitLab server URL. + ''; + }; + + internal-gitlab-server = mkOption { + type = with types; nullOr str; + default = null; + defaultText = "http(s)://"; + example = "https://gitlab.example.internal"; + description = lib.mdDoc '' + Internal GitLab server used for API requests, useful + if you want to send that traffic over an internal load + balancer. By default, the value of + `services.gitlab.pages.settings.gitlab-server` is + used. + ''; + }; + + api-secret-key = mkOption { + type = with types; nullOr str; + default = "${cfg.statePath}/gitlab_pages_secret"; + internal = true; + description = lib.mdDoc '' + File with secret key used to authenticate with the + GitLab API. + ''; + }; + + pages-domain = mkOption { + type = with types; nullOr str; + example = "example.com"; + description = lib.mdDoc '' + The domain to serve static pages on. + ''; + }; + + pages-root = mkOption { + type = types.str; + default = "${gitlabConfig.production.shared.path}/pages"; + defaultText = literalExpression ''config.${opt.extraConfig}.production.shared.path + "/pages"''; + description = lib.mdDoc '' + The directory where pages are stored. + ''; + }; + }; + }; }; secrets.secretFile = mkOption { @@ -1209,6 +1308,9 @@ in { umask u=rwx,g=,o= openssl rand -hex 32 > ${cfg.statePath}/gitlab_shell_secret + ${optionalString cfg.pages.enable '' + openssl rand -base64 32 > ${cfg.pages.settings.api-secret-key} + ''} rm -f '${cfg.statePath}/config/database.yml' @@ -1357,28 +1459,37 @@ in { }; }; - systemd.services.gitlab-pages = mkIf (gitlabConfig.production.pages.enabled or false) { - description = "GitLab static pages daemon"; - after = [ "network.target" "gitlab-config.service" ]; - bindsTo = [ "gitlab-config.service" ]; - wantedBy = [ "gitlab.target" ]; - partOf = [ "gitlab.target" ]; - - path = [ pkgs.unzip ]; - - serviceConfig = { - Type = "simple"; - TimeoutSec = "infinity"; - Restart = "on-failure"; - - User = cfg.user; - Group = cfg.group; - - ExecStart = "${cfg.packages.pages}/bin/gitlab-pages ${escapeShellArgs pagesArgs}"; - WorkingDirectory = gitlabEnv.HOME; - }; + services.gitlab.pages.settings = { + api-secret-key = "${cfg.statePath}/gitlab_pages_secret"; }; + systemd.services.gitlab-pages = + let + filteredConfig = filterAttrs (_: v: v != null) cfg.pages.settings; + configFile = pkgs.writeText "gitlab-pages.conf" (lib.generators.toKeyValue {} filteredConfig); + in + mkIf cfg.pages.enable { + description = "GitLab static pages daemon"; + after = [ "network.target" "gitlab-config.service" "gitlab.service" ]; + bindsTo = [ "gitlab-config.service" "gitlab.service" ]; + wantedBy = [ "gitlab.target" ]; + partOf = [ "gitlab.target" ]; + + path = [ pkgs.unzip ]; + + serviceConfig = { + Type = "simple"; + TimeoutSec = "infinity"; + Restart = "on-failure"; + + User = cfg.user; + Group = cfg.group; + + ExecStart = "${cfg.packages.pages}/bin/gitlab-pages -config=${configFile}"; + WorkingDirectory = gitlabEnv.HOME; + }; + }; + systemd.services.gitlab-workhorse = { after = [ "network.target" ]; wantedBy = [ "gitlab.target" ]; diff --git a/nixos/tests/gitlab.nix b/nixos/tests/gitlab.nix index d9d75d1cbd89..0b3d321fbbdf 100644 --- a/nixos/tests/gitlab.nix +++ b/nixos/tests/gitlab.nix @@ -69,6 +69,10 @@ in { databasePasswordFile = pkgs.writeText "dbPassword" "xo0daiF4"; initialRootPasswordFile = pkgs.writeText "rootPassword" initialRootPassword; smtp.enable = true; + pages = { + enable = true; + settings.pages-domain = "localhost"; + }; extraConfig = { incoming_email = { enabled = true; @@ -79,11 +83,6 @@ in { host = "localhost"; port = 143; }; - # https://github.com/NixOS/nixpkgs/issues/132295 - # pages = { - # enabled = true; - # host = "localhost"; - # }; }; secrets = { secretFile = pkgs.writeText "secret" "Aig5zaic"; @@ -171,10 +170,9 @@ in { waitForServices = '' gitlab.wait_for_unit("gitaly.service") gitlab.wait_for_unit("gitlab-workhorse.service") - # https://github.com/NixOS/nixpkgs/issues/132295 - # gitlab.wait_for_unit("gitlab-pages.service") gitlab.wait_for_unit("gitlab-mailroom.service") gitlab.wait_for_unit("gitlab.service") + gitlab.wait_for_unit("gitlab-pages.service") gitlab.wait_for_unit("gitlab-sidekiq.service") gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/tmp/sockets/gitlab.socket") gitlab.wait_until_succeeds("curl -sSf http://gitlab/users/sign_in") From 2d4f4e9bdfdcc69ea19299150db5c26c7aa4e44a Mon Sep 17 00:00:00 2001 From: talyz Date: Tue, 17 Jan 2023 17:34:47 +0100 Subject: [PATCH 3/3] nixos/gitlab: Handle secrets in GitLab Pages config --- nixos/modules/services/misc/gitlab.nix | 57 ++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index 4efe16748bc7..13e1a70547ac 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -671,13 +671,33 @@ in { pages.enable = mkEnableOption (lib.mdDoc "the GitLab Pages service"); pages.settings = mkOption { + example = literalExpression '' + { + pages-domain = "example.com"; + auth-client-id = "generated-id-xxxxxxx"; + auth-client-secret = { _secret = "/var/keys/auth-client-secret"; }; + auth-redirect-uri = "https://projects.example.com/auth"; + auth-secret = { _secret = "/var/keys/auth-secret"; }; + auth-server = "https://gitlab.example.com"; + } + ''; + description = lib.mdDoc '' Configuration options to set in the GitLab Pages config file. + + Options containing secret data should be set to an attribute + set containing the attribute `_secret` - a string pointing + to a file containing the value the option should be set + to. See the example to get a better picture of this: in the + resulting configuration file, the `auth-client-secret` and + `auth-secret` keys will be set to the contents of the + {file}`/var/keys/auth-client-secret` and + {file}`/var/keys/auth-secret` files respectively. ''; type = types.submodule { - freeformType = with types; attrsOf (nullOr (oneOf [ str int bool ])); + freeformType = with types; attrsOf (nullOr (oneOf [ str int bool attrs ])); options = { listen-http = mkOption { @@ -1466,7 +1486,24 @@ in { systemd.services.gitlab-pages = let filteredConfig = filterAttrs (_: v: v != null) cfg.pages.settings; - configFile = pkgs.writeText "gitlab-pages.conf" (lib.generators.toKeyValue {} filteredConfig); + isSecret = v: isAttrs v && v ? _secret && isString v._secret; + mkPagesKeyValue = lib.generators.toKeyValue { + mkKeyValue = lib.flip lib.generators.mkKeyValueDefault "=" rec { + mkValueString = v: + if isInt v then toString v + else if isString v then v + else if true == v then "true" + else if false == v then "false" + else if isSecret v then builtins.hashString "sha256" v._secret + else throw "unsupported type ${builtins.typeOf v}: ${(lib.generators.toPretty {}) v}"; + }; + }; + secretPaths = lib.catAttrs "_secret" (lib.collect isSecret filteredConfig); + mkSecretReplacement = file: '' + replace-secret ${lib.escapeShellArgs [ (builtins.hashString "sha256" file) file "/run/gitlab-pages/gitlab-pages.conf" ]} + ''; + secretReplacements = lib.concatMapStrings mkSecretReplacement secretPaths; + configFile = pkgs.writeText "gitlab-pages.conf" (mkPagesKeyValue filteredConfig); in mkIf cfg.pages.enable { description = "GitLab static pages daemon"; @@ -1475,7 +1512,10 @@ in { wantedBy = [ "gitlab.target" ]; partOf = [ "gitlab.target" ]; - path = [ pkgs.unzip ]; + path = with pkgs; [ + unzip + replace-secret + ]; serviceConfig = { Type = "simple"; @@ -1485,8 +1525,17 @@ in { User = cfg.user; Group = cfg.group; - ExecStart = "${cfg.packages.pages}/bin/gitlab-pages -config=${configFile}"; + ExecStartPre = pkgs.writeShellScript "gitlab-pages-pre-start" '' + set -o errexit -o pipefail -o nounset + shopt -s dotglob nullglob inherit_errexit + + install -m u=rw ${configFile} /run/gitlab-pages/gitlab-pages.conf + ${secretReplacements} + ''; + ExecStart = "${cfg.packages.pages}/bin/gitlab-pages -config=/run/gitlab-pages/gitlab-pages.conf"; WorkingDirectory = gitlabEnv.HOME; + RuntimeDirectory = "gitlab-pages"; + RuntimeDirectoryMode = "0700"; }; };