Merge pull request #292835 from sinavir/castopod2

nixos/castopod: updated nixos test and module
2024-04-01 20:27:49 +02:00 · 2024-04-01 20:27:49 +02:00 · a5fcefa1ad
commit a5fcefa1ad
parent 98f2a766ef 301a66e8ea
6 changed files with 272 additions and 89 deletions
--- a/nixos/doc/manual/release-notes/rl-2405.section.md
+++ b/nixos/doc/manual/release-notes/rl-2405.section.md
@ -470,6 +470,8 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m
 - `services.zfs.zed.enableMail` now uses the global `sendmail` wrapper defined by an email module
  (such as msmtp or Postfix). It no longer requires using a special ZFS build with email support.

+- `castopod` has some migration actions to be taken in case of a S3 setup. Some new features may also need some manual migration actions. See [https://code.castopod.org/adaures/castopod/-/releases](https://code.castopod.org/adaures/castopod/-/releases) for more informations.
+
 - `nextcloud-setup.service` no longer changes the group of each file & directory inside `/var/lib/nextcloud/{config,data,store-apps}` if one of these directories has the wrong owner group. This was part of transitioning the group used for `/var/lib/nextcloud`, but isn't necessary anymore.

 - `services.kavita` now uses the freeform option `services.kavita.settings` for the application settings file.
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@ -342,7 +342,6 @@
  ./services/amqp/rabbitmq.nix
  ./services/audio/alsa.nix
  ./services/audio/botamusique.nix
-  ./services/audio/castopod.nix
  ./services/audio/gmediarender.nix
  ./services/audio/gonic.nix
  ./services/audio/goxlr-utility.nix
@ -1303,6 +1302,7 @@
  ./services/web-apps/bookstack.nix
  ./services/web-apps/c2fmzq-server.nix
  ./services/web-apps/calibre-web.nix
+  ./services/web-apps/castopod.nix
  ./services/web-apps/coder.nix
  ./services/web-apps/changedetection-io.nix
  ./services/web-apps/chatgpt-retrieval-plugin.nix
--- a/nixos/modules/services/web-apps/castopod.md
+++ b/nixos/modules/services/web-apps/castopod.md
@ -4,6 +4,7 @@ Castopod is an open-source hosting platform made for podcasters who want to enga

 ## Quickstart {#module-services-castopod-quickstart}

+Configure ACME (https://nixos.org/manual/nixos/unstable/#module-security-acme).
 Use the following configuration to start a public instance of Castopod on `castopod.example.com` domain:

 ```nix
--- a/nixos/modules/services/web-apps/castopod.nix
+++ b/nixos/modules/services/web-apps/castopod.nix
@ -4,7 +4,6 @@ let
  fpm = config.services.phpfpm.pools.castopod;

  user = "castopod";
-  stateDirectory = "/var/lib/castopod";

  # https://docs.castopod.org/getting-started/install.html#requirements
  phpPackage = pkgs.php.withExtensions ({ enabled, all }: with all; [
@ -29,6 +28,15 @@ in
        defaultText = lib.literalMD "pkgs.castopod";
        description = lib.mdDoc "Which Castopod package to use.";
      };
+      dataDir = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/castopod";
+        description = lib.mdDoc ''
+          The path where castopod stores all data. This path must be in sync
+          with the castopod package (where it is hardcoded during the build in
+          accordance with its own `dataDir` argument).
+        '';
+      };
      database = {
        createLocally = lib.mkOption {
          type = lib.types.bool;
@ -59,6 +67,8 @@ in
          description = lib.mdDoc ''
            A file containing the password corresponding to
            [](#opt-services.castopod.database.user).
+
+            This file is loaded using systemd LoadCredentials.
          '';
        };
      };
@ -85,6 +95,8 @@ in
          Environment file to inject e.g. secrets into the configuration.
          See [](https://code.castopod.org/adaures/castopod/-/blob/main/.env.example)
          for available environment variables.
+
+          This file is loaded using systemd LoadCredentials.
        '';
      };
      configureNginx = lib.mkOption {
@ -111,6 +123,19 @@ in
          Options for Castopod's PHP pool. See the documentation on `php-fpm.conf` for details on configuration directives.
        '';
      };
+      maxUploadSize = lib.mkOption {
+        type = lib.types.str;
+        default = "512M";
+        description = lib.mdDoc ''
+          Maximum supported size for a file upload in. Maximum HTTP body
+          size is set to this value for nginx and PHP (because castopod doesn't
+          support chunked uploads yet:
+          https://code.castopod.org/adaures/castopod/-/issues/330).
+
+          Note, that practical upload size limit is smaller. For example, with
+          512 MiB setting - around 500 MiB is possible.
+        '';
+      };
    };
  };

@ -120,13 +145,13 @@ in
        sslEnabled = with config.services.nginx.virtualHosts.${cfg.localDomain}; addSSL || forceSSL || onlySSL || enableACME || useACMEHost != null;
        baseURL = "http${lib.optionalString sslEnabled "s"}://${cfg.localDomain}";
      in
-      lib.mapAttrs (name: lib.mkDefault) {
+      lib.mapAttrs (_: lib.mkDefault) {
        "app.forceGlobalSecureRequests" = sslEnabled;
        "app.baseURL" = baseURL;

-        "media.baseURL" = "/";
+        "media.baseURL" = baseURL;
        "media.root" = "media";
-        "media.storage" = stateDirectory;
+        "media.storage" = cfg.dataDir;

        "admin.gateway" = "admin";
        "auth.gateway" = "auth";
@ -142,13 +167,13 @@ in
    services.phpfpm.pools.castopod = {
      inherit user;
      group = config.services.nginx.group;
-      phpPackage = phpPackage;
+      inherit phpPackage;
      phpOptions = ''
-        # https://code.castopod.org/adaures/castopod/-/blob/main/docker/production/app/uploads.ini
+        # https://code.castopod.org/adaures/castopod/-/blob/develop/docker/production/common/uploads.template.ini
        file_uploads = On
        memory_limit = 512M
-        upload_max_filesize = 500M
-        post_max_size = 512M
+        upload_max_filesize = ${cfg.maxUploadSize}
+        post_max_size = ${cfg.maxUploadSize}
        max_execution_time = 300
        max_input_time = 300
      '';
@ -165,45 +190,50 @@ in
      path = [ pkgs.openssl phpPackage ];
      script =
        let
-          envFile = "${stateDirectory}/.env";
+          envFile = "${cfg.dataDir}/.env";
          media = "${cfg.settings."media.storage"}/${cfg.settings."media.root"}";
        in
        ''
-          mkdir -p ${stateDirectory}/writable/{cache,logs,session,temp,uploads}
+          mkdir -p ${cfg.dataDir}/writable/{cache,logs,session,temp,uploads}

          if [ ! -d ${lib.escapeShellArg media} ]; then
            cp --no-preserve=mode,ownership -r ${cfg.package}/share/castopod/public/media ${lib.escapeShellArg media}
          fi

-          if [ ! -f ${stateDirectory}/salt ]; then
-            openssl rand -base64 33 > ${stateDirectory}/salt
+          if [ ! -f ${cfg.dataDir}/salt ]; then
+            openssl rand -base64 33 > ${cfg.dataDir}/salt
          fi

          cat <<'EOF' > ${envFile}
          ${lib.generators.toKeyValue { } cfg.settings}
          EOF

-          echo "analytics.salt=$(cat ${stateDirectory}/salt)" >> ${envFile}
+          echo "analytics.salt=$(cat ${cfg.dataDir}/salt)" >> ${envFile}

          ${if (cfg.database.passwordFile != null) then ''
-            echo "database.default.password=$(cat ${lib.escapeShellArg cfg.database.passwordFile})" >> ${envFile}
+            echo "database.default.password=$(cat "$CREDENTIALS_DIRECTORY/dbpasswordfile)" >> ${envFile}
          '' else ''
            echo "database.default.password=" >> ${envFile}
          ''}

          ${lib.optionalString (cfg.environmentFile != null) ''
-            cat ${lib.escapeShellArg cfg.environmentFile}) >> ${envFile}
+            cat "$CREDENTIALS_DIRECTORY/envfile" >> ${envFile}
          ''}

-          php spark castopod:database-update
+          php ${cfg.package}/share/castopod/spark castopod:database-update
        '';
      serviceConfig = {
        StateDirectory = "castopod";
+        LoadCredential = lib.optional (cfg.environmentFile != null)
+          "envfile:${cfg.environmentFile}"
+        ++ (lib.optional (cfg.database.passwordFile != null)
+          "dbpasswordfile:${cfg.database.passwordFile}");
        WorkingDirectory = "${cfg.package}/share/castopod";
        Type = "oneshot";
        RemainAfterExit = true;
        User = user;
        Group = config.services.nginx.group;
+        ReadWritePaths = cfg.dataDir;
      };
    };

@ -212,9 +242,7 @@ in
      wantedBy = [ "multi-user.target" ];
      path = [ phpPackage ];
      script = ''
-        php public/index.php scheduled-activities
-        php public/index.php scheduled-websub-publish
-        php public/index.php scheduled-video-clips
+        php ${cfg.package}/share/castopod/spark tasks:run
      '';
      serviceConfig = {
        StateDirectory = "castopod";
@ -222,6 +250,8 @@ in
        Type = "oneshot";
        User = user;
        Group = config.services.nginx.group;
+        ReadWritePaths = cfg.dataDir;
+        LogLevelMax = "notice"; # otherwise periodic tasks flood the journal
      };
    };

@ -251,6 +281,7 @@ in
        extraConfig = ''
          try_files $uri $uri/ /index.php?$args;
          index index.php index.html;
+          client_max_body_size ${cfg.maxUploadSize};
        '';

        locations."^~ /${cfg.settings."media.root"}/" = {
@ -278,7 +309,7 @@ in
      };
    };

-    users.users.${user} = lib.mapAttrs (name: lib.mkDefault) {
+    users.users.${user} = lib.mapAttrs (_: lib.mkDefault) {
      description = "Castopod user";
      isSystemUser = true;
      group = config.services.nginx.group;
--- a/nixos/tests/castopod.nix
+++ b/nixos/tests/castopod.nix
@ -4,74 +4,218 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
  meta = with lib.maintainers; {
    maintainers = [ alexoundos misuzu ];
  };
+
  nodes.castopod = { nodes, ... }: {
+    # otherwise 500 MiB file upload fails!
+    virtualisation.diskSize = 512 + 3 * 512;
+
    networking.firewall.allowedTCPPorts = [ 80 ];
-    networking.extraHosts = ''
-      127.0.0.1 castopod.example.com
-    '';
+    networking.extraHosts =
+      lib.strings.concatStringsSep "\n"
+        (lib.attrsets.mapAttrsToList
+          (name: _: "127.0.0.1 ${name}")
+          nodes.castopod.services.nginx.virtualHosts);
+
    services.castopod = {
      enable = true;
      database.createLocally = true;
      localDomain = "castopod.example.com";
+      maxUploadSize = "512M";
    };
-    environment.systemPackages =
-      let
-        username = "admin";
-        email = "admin@castood.example.com";
-        password = "v82HmEp5";
-        testRunner = pkgs.writers.writePython3Bin "test-runner"
-          {
-            libraries = [ pkgs.python3Packages.selenium ];
-            flakeIgnore = [
-              "E501"
-            ];
-          } ''
-          from selenium.webdriver.common.by import By
-          from selenium.webdriver import Firefox
-          from selenium.webdriver.firefox.options import Options
-          from selenium.webdriver.support.ui import WebDriverWait
-          from selenium.webdriver.support import expected_conditions as EC
-
-          options = Options()
-          options.add_argument('--headless')
-          driver = Firefox(options=options)
-          try:
-              driver.implicitly_wait(20)
-              driver.get('http://castopod.example.com/cp-install')
-
-              wait = WebDriverWait(driver, 10)
-
-              wait.until(EC.title_contains("installer"))
-
-              driver.find_element(By.CSS_SELECTOR, '#username').send_keys(
-                  '${username}'
-              )
-              driver.find_element(By.CSS_SELECTOR, '#email').send_keys(
-                  '${email}'
-              )
-              driver.find_element(By.CSS_SELECTOR, '#password').send_keys(
-                  '${password}'
-              )
-              driver.find_element(By.XPATH, "//button[contains(., 'Finish install')]").click()
-
-              wait.until(EC.title_contains("Auth"))
-
-              driver.find_element(By.CSS_SELECTOR, '#email').send_keys(
-                  '${email}'
-              )
-              driver.find_element(By.CSS_SELECTOR, '#password').send_keys(
-                  '${password}'
-              )
-              driver.find_element(By.XPATH, "//button[contains(., 'Login')]").click()
-
-              wait.until(EC.title_contains("Admin dashboard"))
-          finally:
-              driver.close()
-              driver.quit()
-        '';
-      in
-      [ pkgs.firefox-unwrapped pkgs.geckodriver testRunner ];
  };
+
+  nodes.client = { nodes, pkgs, lib, ... }:
+    let
+      domain = nodes.castopod.services.castopod.localDomain;
+
+      getIP = node:
+        (builtins.head node.networking.interfaces.eth1.ipv4.addresses).address;
+
+      targetPodcastSize = 500 * 1024 * 1024;
+      lameMp3Bitrate = 348300;
+      lameMp3FileAdjust = -800;
+      targetPodcastDuration = toString
+        ((targetPodcastSize + lameMp3FileAdjust) / (lameMp3Bitrate / 8));
+      bannerWidth = 3000;
+      banner = pkgs.runCommand "gen-castopod-cover.jpg" { } ''
+        ${pkgs.imagemagick}/bin/magick `
+        `-background green -bordercolor white -gravity northwest xc:black `
+        `-duplicate 99 `
+        `-seed 1 -resize "%[fx:rand()*72+24]" `
+        `-seed 0 -rotate "%[fx:rand()*360]" -border 6x6 -splice 16x36 `
+        `-seed 0 -rotate "%[fx:floor(rand()*4)*90]" -resize "150x50!" `
+        `+append -crop 10x1@ +repage -roll "+%[fx:(t%2)*72]+0" -append `
+        `-resize ${toString bannerWidth} -quality 1 $out
+      '';
+
+      coverWidth = toString 3000;
+      cover = pkgs.runCommand "gen-castopod-banner.jpg" { } ''
+        ${pkgs.imagemagick}/bin/magick `
+        `-background white -bordercolor white -gravity northwest xc:black `
+        `-duplicate 99 `
+        `-seed 1 -resize "%[fx:rand()*72+24]" `
+        `-seed 0 -rotate "%[fx:rand()*360]" -border 6x6 -splice 36x36 `
+        `-seed 0 -rotate "%[fx:floor(rand()*4)*90]" -resize "144x144!" `
+        `+append -crop 10x1@ +repage -roll "+%[fx:(t%2)*72]+0" -append `
+        `-resize ${coverWidth} -quality 1 $out
+      '';
+    in
+    {
+      networking.extraHosts =
+        lib.strings.concatStringsSep "\n"
+          (lib.attrsets.mapAttrsToList
+            (name: _: "${getIP nodes.castopod} ${name}")
+            nodes.castopod.services.nginx.virtualHosts);
+
+      environment.systemPackages =
+        let
+          username = "admin";
+          email = "admin@${domain}";
+          password = "Abcd1234";
+          podcastTitle = "Some Title";
+          episodeTitle = "Episode Title";
+          browser-test = pkgs.writers.writePython3Bin "browser-test"
+            {
+              libraries = [ pkgs.python3Packages.selenium ];
+              flakeIgnore = [ "E124" "E501" ];
+            } ''
+            from selenium.webdriver.common.by import By
+            from selenium.webdriver import Firefox
+            from selenium.webdriver.firefox.options import Options
+            from selenium.webdriver.firefox.service import Service
+            from selenium.webdriver.support.ui import WebDriverWait
+            from selenium.webdriver.support import expected_conditions as EC
+            from subprocess import STDOUT
+            import logging
+
+            selenium_logger = logging.getLogger("selenium")
+            selenium_logger.setLevel(logging.DEBUG)
+            selenium_logger.addHandler(logging.StreamHandler())
+
+            options = Options()
+            options.add_argument('--headless')
+            service = Service(log_output=STDOUT)
+            driver = Firefox(options=options, service=service)
+            driver = Firefox(options=options)
+            driver.implicitly_wait(30)
+
+            # install ##########################################################
+
+            driver.get('http://${domain}/cp-install')
+
+            wait = WebDriverWait(driver, 20)
+
+            wait.until(EC.title_contains("installer"))
+
+            driver.find_element(By.CSS_SELECTOR, '#username').send_keys(
+                '${username}'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#email').send_keys(
+                '${email}'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#password').send_keys(
+                '${password}'
+            )
+            driver.find_element(By.XPATH,
+                                "//button[contains(., 'Finish install')]"
+            ).click()
+
+            wait.until(EC.title_contains("Auth"))
+
+            driver.find_element(By.CSS_SELECTOR, '#email').send_keys(
+                '${email}'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#password').send_keys(
+                '${password}'
+            )
+            driver.find_element(By.XPATH,
+                                "//button[contains(., 'Login')]"
+            ).click()
+
+            wait.until(EC.title_contains("Admin dashboard"))
+
+            # create podcast ###################################################
+
+            driver.get('http://${domain}/admin/podcasts/new')
+
+            wait.until(EC.title_contains("Create podcast"))
+
+            driver.find_element(By.CSS_SELECTOR, '#cover').send_keys(
+                '${cover}'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#banner').send_keys(
+                '${banner}'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#title').send_keys(
+                '${podcastTitle}'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#handle').send_keys(
+                'some_handle'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#description').send_keys(
+                'Some description'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#owner_name').send_keys(
+                'Owner Name'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#owner_email').send_keys(
+                'owner@email.xyz'
+            )
+            driver.find_element(By.XPATH,
+                                "//button[contains(., 'Create podcast')]"
+            ).click()
+
+            wait.until(EC.title_contains("${podcastTitle}"))
+
+            driver.find_element(By.XPATH,
+                                "//span[contains(., 'Add an episode')]"
+            ).click()
+
+            wait.until(EC.title_contains("Add an episode"))
+
+            # upload podcast ###################################################
+
+            driver.find_element(By.CSS_SELECTOR, '#audio_file').send_keys(
+                '/tmp/podcast.mp3'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#cover').send_keys(
+                '${cover}'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#description').send_keys(
+                'Episode description'
+            )
+            driver.find_element(By.CSS_SELECTOR, '#title').send_keys(
+                '${episodeTitle}'
+            )
+            driver.find_element(By.XPATH,
+                                "//button[contains(., 'Create episode')]"
+            ).click()
+
+            wait.until(EC.title_contains("${episodeTitle}"))
+
+            driver.close()
+            driver.quit()
+          '';
+        in
+        [
+          pkgs.firefox-unwrapped
+          pkgs.geckodriver
+          browser-test
+          (pkgs.writeShellApplication {
+            name = "build-mp3";
+            runtimeInputs = with pkgs; [ sox lame ];
+            text = ''
+              out=/tmp/podcast.mp3
+              sox -n -r 48000 -t wav - synth ${targetPodcastDuration} sine 440 `
+              `| lame --noreplaygain -cbr -q 9 -b 320 - $out
+              FILESIZE="$(stat -c%s $out)"
+              [ "$FILESIZE" -gt 0 ]
+              [ "$FILESIZE" -le "${toString targetPodcastSize}" ]
+            '';
+          })
+        ];
+    };
+
  testScript = ''
    start_all()
    castopod.wait_for_unit("castopod-setup.service")
@ -79,9 +223,11 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
    castopod.wait_for_unit("nginx.service")
    castopod.wait_for_open_port(80)
    castopod.wait_until_succeeds("curl -sS -f http://castopod.example.com")
-    castopod.succeed("curl -s http://localhost/cp-install | grep 'Create your Super Admin account' > /dev/null")

-    with subtest("Create superadmin and log in"):
-        castopod.succeed("PYTHONUNBUFFERED=1 systemd-cat -t test-runner test-runner")
+    client.succeed("build-mp3")
+
+    with subtest("Create superadmin, log in, create and upload a podcast"):
+        client.succeed(\
+          "PYTHONUNBUFFERED=1 systemd-cat -t browser-test browser-test")
  '';
 })
--- a/pkgs/applications/audio/castopod/default.nix
+++ b/pkgs/applications/audio/castopod/default.nix
@ -3,7 +3,7 @@
 , ffmpeg-headless
 , lib
 , nixosTests
-, stateDirectory ? "/var/lib/castopod"
+, dataDir ? "/var/lib/castopod"
 }:
 stdenv.mkDerivation {
  pname = "castopod";
@ -20,13 +20,16 @@ stdenv.mkDerivation {
  postPatch = ''
    # not configurable at runtime unfortunately:
    substituteInPlace app/Config/Paths.php \
-      --replace "__DIR__ . '/../../writable'" "'${stateDirectory}/writable'"
+      --replace "__DIR__ . '/../../writable'" "'${dataDir}/writable'"

-    # configuration file must be writable, place it to ${stateDirectory}
+    substituteInPlace modules/Admin/Controllers/DashboardController.php \
+      --replace "disk_total_space('./')" "disk_total_space('${dataDir}')"
+
+    # configuration file must be writable, place it to ${dataDir}
    substituteInPlace modules/Install/Controllers/InstallController.php \
-      --replace "ROOTPATH" "'${stateDirectory}/'"
+      --replace "ROOTPATH" "'${dataDir}/'"
    substituteInPlace public/index.php spark \
-      --replace "DotEnv(ROOTPATH)" "DotEnv('${stateDirectory}')"
+      --replace "DotEnv(ROOTPATH)" "DotEnv('${dataDir}')"

    # ffmpeg is required for Video Clips feature
    substituteInPlace modules/MediaClipper/VideoClipper.php \