Merge pull request #265368 from chayleaf/maubot

nixos/maubot: init

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -14,7 +14,7 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
 
-- Create the first release note entry in this section!
+- [maubot](https://github.com/maubot/maubot), a plugin-based Matrix bot framework. Available as [services.maubot](#opt-services.maubot.enable).
 
 ## Backward Incompatibilities {#sec-release-24.05-incompatibilities}
 

nixos/modules/module-list.nix

@@ -621,6 +621,7 @@
   ./services/matrix/appservice-irc.nix
   ./services/matrix/conduit.nix
   ./services/matrix/dendrite.nix
+  ./services/matrix/maubot.nix
   ./services/matrix/mautrix-facebook.nix
   ./services/matrix/mautrix-telegram.nix
   ./services/matrix/mautrix-whatsapp.nix

nixos/modules/services/matrix/maubot.md

@@ -0,0 +1,103 @@
+# Maubot {#module-services-maubot}
+
+[Maubot](https://github.com/maubot/maubot) is a plugin-based bot
+framework for Matrix.
+
+## Configuration {#module-services-maubot-configuration}
+
+1. Set [](#opt-services.maubot.enable) to `true`. The service will use
+   SQLite by default.
+2. If you want to use PostgreSQL instead of SQLite, do this:
+
+   ```nix
+   services.maubot.settings.database = "postgresql://maubot@localhost/maubot";
+   ```
+
+   If the PostgreSQL connection requires a password, you will have to
+   add it later on step 8.
+3. If you plan to expose your Maubot interface to the web, do something
+   like this:
+   ```nix
+   services.nginx.virtualHosts."matrix.example.org".locations = {
+     "/_matrix/maubot/" = {
+       proxyPass = "http://127.0.0.1:${toString config.services.maubot.settings.server.port}";
+       proxyWebsockets = true;
+     };
+   };
+   services.maubot.settings.server.public_url = "matrix.example.org";
+   # do the following only if you want to use something other than /_matrix/maubot...
+   services.maubot.settings.server.ui_base_path = "/another/base/path";
+   ```
+4. Optionally, set `services.maubot.pythonPackages` to a list of python3
+   packages to make available for Maubot plugins.
+5. Optionally, set `services.maubot.plugins` to a list of Maubot
+   plugins (full list available at https://plugins.maubot.xyz/):
+   ```nix
+   services.maubot.plugins = with config.services.maubot.package.plugins; [
+     reactbot
+     # This will only change the default config! After you create a
+     # plugin instance, the default config will be copied into that
+     # instance's config in Maubot's database, and further base config
+     # changes won't affect the running plugin.
+     (rss.override {
+       base_config = {
+         update_interval = 60;
+         max_backoff = 7200;
+         spam_sleep = 2;
+         command_prefix = "rss";
+         admins = [ "@chayleaf:pavluk.org" ];
+       };
+     })
+   ];
+   # ...or...
+   services.maubot.plugins = config.services.maubot.package.plugins.allOfficialPlugins;
+   # ...or...
+   services.maubot.plugins = config.services.maubot.package.plugins.allPlugins;
+   # ...or...
+   services.maubot.plugins = with config.services.maubot.package.plugins; [
+     (weather.override {
+       # you can pass base_config as a string
+       base_config = ''
+         default_location: New York
+         default_units: M
+         default_language:
+         show_link: true
+         show_image: false
+       '';
+     })
+   ];
+   ```
+6. Start Maubot at least once before doing the following steps (it's
+   necessary to generate the initial config).
+7. If your PostgreSQL connection requires a password, add
+   `database: postgresql://user:password@localhost/maubot`
+   to `/var/lib/maubot/config.yaml`. This overrides the Nix-provided
+   config. Even then, don't remove the `database` line from Nix config
+   so the module knows you use PostgreSQL!
+8. To create a user account for logging into Maubot web UI and
+   configuring it, generate a password using the shell command
+   `mkpasswd -R 12 -m bcrypt`, and edit `/var/lib/maubot/config.yaml`
+   with the following:
+
+   ```yaml
+   admins:
+       admin_username: $2b$12$g.oIStUeUCvI58ebYoVMtO/vb9QZJo81PsmVOomHiNCFbh0dJpZVa
+   ```
+
+   Where `admin_username` is your username, and `$2b...` is the bcrypted
+   password.
+9. Optional: if you want to be able to register new users with the
+   Maubot CLI (`mbc`), and your homeserver is private, add your
+   homeserver's registration key to `/var/lib/maubot/config.yaml`:
+
+   ```yaml
+   homeservers:
+       matrix.example.org:
+           url: https://matrix.example.org
+           secret: your-very-secret-key
+   ```
+10. Restart Maubot after editing `/var/lib/maubot/config.yaml`,and
+    Maubot will be available at
+    `https://matrix.example.org/_matrix/maubot`. If you want to use the
+    `mbc` CLI, it's available using the `maubot` package (`nix-shell -p
+    maubot`).

nixos/modules/services/matrix/maubot.nix

@@ -0,0 +1,459 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+
+let
+  cfg = config.services.maubot;
+
+  wrapper1 =
+    if cfg.plugins == [ ]
+    then cfg.package
+    else cfg.package.withPlugins (_: cfg.plugins);
+
+  wrapper2 =
+    if cfg.pythonPackages == [ ]
+    then wrapper1
+    else wrapper1.withPythonPackages (_: cfg.pythonPackages);
+
+  settings = lib.recursiveUpdate cfg.settings {
+    plugin_directories.trash =
+      if cfg.settings.plugin_directories.trash == null
+      then "delete"
+      else cfg.settings.plugin_directories.trash;
+    server.unshared_secret = "generate";
+  };
+
+  finalPackage = wrapper2.withBaseConfig settings;
+
+  isPostgresql = db: builtins.isString db && lib.hasPrefix "postgresql://" db;
+  isLocalPostgresDB = db: isPostgresql db && builtins.any (x: lib.hasInfix x db) [
+    "@127.0.0.1/"
+    "@::1/"
+    "@[::1]/"
+    "@localhost/"
+  ];
+  parsePostgresDB = db:
+    let
+      noSchema = lib.removePrefix "postgresql://" db;
+    in {
+      username = builtins.head (lib.splitString "@" noSchema);
+      database = lib.last (lib.splitString "/" noSchema);
+    };
+
+  postgresDBs = [
+    cfg.settings.database
+    cfg.settings.crypto_database
+    cfg.settings.plugin_databases.postgres
+  ];
+
+  localPostgresDBs = builtins.filter isLocalPostgresDB postgresDBs;
+
+  parsedLocalPostgresDBs = map parsePostgresDB localPostgresDBs;
+  parsedPostgresDBs = map parsePostgresDB postgresDBs;
+
+  hasLocalPostgresDB = localPostgresDBs != [ ];
+in
+{
+  options.services.maubot = with lib; {
+    enable = mkEnableOption (mdDoc "maubot");
+
+    package = lib.mkPackageOptionMD pkgs "maubot" { };
+
+    plugins = mkOption {
+      type = types.listOf types.package;
+      default = [ ];
+      example = literalExpression ''
+        with config.services.maubot.package.plugins; [
+          xyz.maubot.reactbot
+          xyz.maubot.rss
+        ];
+      '';
+      description = mdDoc ''
+        List of additional maubot plugins to make available.
+      '';
+    };
+
+    pythonPackages = mkOption {
+      type = types.listOf types.package;
+      default = [ ];
+      example = literalExpression ''
+        with pkgs.python3Packages; [
+          aiohttp
+        ];
+      '';
+      description = mdDoc ''
+        List of additional Python packages to make available for maubot.
+      '';
+    };
+
+    dataDir = mkOption {
+      type = types.str;
+      default = "/var/lib/maubot";
+      description = mdDoc ''
+        The directory where maubot stores its stateful data.
+      '';
+    };
+
+    extraConfigFile = mkOption {
+      type = types.str;
+      default = "./config.yaml";
+      defaultText = literalExpression ''"''${config.services.maubot.dataDir}/config.yaml"'';
+      description = mdDoc ''
+        A file for storing secrets. You can pass homeserver registration keys here.
+        If it already exists, **it must contain `server.unshared_secret`** which is used for signing API keys.
+        If `configMutable` is not set to true, **maubot user must have write access to this file**.
+      '';
+    };
+
+    configMutable = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Whether maubot should write updated config into `extraConfigFile`. **This will make your Nix module settings have no effect besides the initial config, as extraConfigFile takes precedence over NixOS settings!**
+      '';
+    };
+
+    settings = mkOption {
+      default = { };
+      description = mdDoc ''
+        YAML settings for maubot. See the
+        [example configuration](https://github.com/maubot/maubot/blob/master/maubot/example-config.yaml)
+        for more info.
+
+        Secrets should be passed in by using `extraConfigFile`.
+      '';
+      type = with types; submodule {
+        options = {
+          database = mkOption {
+            type = str;
+            default = "sqlite:maubot.db";
+            example = "postgresql://username:password@hostname/dbname";
+            description = mdDoc ''
+              The full URI to the database. SQLite and Postgres are fully supported.
+              Other DBMSes supported by SQLAlchemy may or may not work.
+            '';
+          };
+
+          crypto_database = mkOption {
+            type = str;
+            default = "default";
+            example = "postgresql://username:password@hostname/dbname";
+            description = mdDoc ''
+              Separate database URL for the crypto database. By default, the regular database is also used for crypto.
+            '';
+          };
+
+          database_opts = mkOption {
+            type = types.attrs;
+            default = { };
+            description = mdDoc ''
+              Additional arguments for asyncpg.create_pool() or sqlite3.connect()
+            '';
+          };
+
+          plugin_directories = mkOption {
+            default = { };
+            description = mdDoc "Plugin directory paths";
+            type = submodule {
+              options = {
+                upload = mkOption {
+                  type = types.str;
+                  default = "./plugins";
+                  defaultText = literalExpression ''"''${config.services.maubot.dataDir}/plugins"'';
+                  description = mdDoc ''
+                    The directory where uploaded new plugins should be stored.
+                  '';
+                };
+                load = mkOption {
+                  type = types.listOf types.str;
+                  default = [ "./plugins" ];
+                  defaultText = literalExpression ''[ "''${config.services.maubot.dataDir}/plugins" ]'';
+                  description = mdDoc ''
+                    The directories from which plugins should be loaded. Duplicate plugin IDs will be moved to the trash.
+                  '';
+                };
+                trash = mkOption {
+                  type = with types; nullOr str;
+                  default = "./trash";
+                  defaultText = literalExpression ''"''${config.services.maubot.dataDir}/trash"'';
+                  description = mdDoc ''
+                    The directory where old plugin versions and conflicting plugins should be moved. Set to null to delete files immediately.
+                  '';
+                };
+              };
+            };
+          };
+
+          plugin_databases = mkOption {
+            description = mdDoc "Plugin database settings";
+            default = { };
+            type = submodule {
+              options = {
+                sqlite = mkOption {
+                  type = types.str;
+                  default = "./plugins";
+                  defaultText = literalExpression ''"''${config.services.maubot.dataDir}/plugins"'';
+                  description = mdDoc ''
+                    The directory where SQLite plugin databases should be stored.
+                  '';
+                };
+
+                postgres = mkOption {
+                  type = types.nullOr types.str;
+                  default = if isPostgresql cfg.settings.database then "default" else null;
+                  defaultText = literalExpression ''if isPostgresql config.services.maubot.settings.database then "default" else null'';
+                  description = mdDoc ''
+                    The connection URL for plugin database. See [example config](https://github.com/maubot/maubot/blob/master/maubot/example-config.yaml) for exact format.
+                  '';
+                };
+
+                postgres_max_conns_per_plugin = mkOption {
+                  type = types.nullOr types.int;
+                  default = 3;
+                  description = mdDoc ''
+                    Maximum number of connections per plugin instance.
+                  '';
+                };
+
+                postgres_opts = mkOption {
+                  type = types.attrs;
+                  default = { };
+                  description = mdDoc ''
+                    Overrides for the default database_opts when using a non-default postgres connection URL.
+                  '';
+                };
+              };
+            };
+          };
+
+          server = mkOption {
+            default = { };
+            description = mdDoc "Listener config";
+            type = submodule {
+              options = {
+                hostname = mkOption {
+                  type = types.str;
+                  default = "127.0.0.1";
+                  description = mdDoc ''
+                    The IP to listen on
+                  '';
+                };
+                port = mkOption {
+                  type = types.port;
+                  default = 29316;
+                  description = mdDoc ''
+                    The port to listen on
+                  '';
+                };
+                public_url = mkOption {
+                  type = types.str;
+                  default = "http://${cfg.settings.server.hostname}:${toString cfg.settings.server.port}";
+                  defaultText = literalExpression ''"http://''${config.services.maubot.settings.server.hostname}:''${toString config.services.maubot.settings.server.port}"'';
+                  description = mdDoc ''
+                    Public base URL where the server is visible.
+                  '';
+                };
+                ui_base_path = mkOption {
+                  type = types.str;
+                  default = "/_matrix/maubot";
+                  description = mdDoc ''
+                    The base path for the UI.
+                  '';
+                };
+                plugin_base_path = mkOption {
+                  type = types.str;
+                  default = "${config.services.maubot.settings.server.ui_base_path}/plugin/";
+                  defaultText = literalExpression ''
+                    "''${config.services.maubot.settings.server.ui_base_path}/plugin/"
+                  '';
+                  description = mdDoc ''
+                    The base path for plugin endpoints. The instance ID will be appended directly.
+                  '';
+                };
+                override_resource_path = mkOption {
+                  type = types.nullOr types.str;
+                  default = null;
+                  description = mdDoc ''
+                    Override path from where to load UI resources.
+                  '';
+                };
+              };
+            };
+          };
+
+          homeservers = mkOption {
+            type = types.attrsOf (types.submodule {
+              options = {
+                url = mkOption {
+                  type = types.str;
+                  description = mdDoc ''
+                    Client-server API URL
+                  '';
+                };
+              };
+            });
+            default = {
+              "matrix.org" = {
+                url = "https://matrix-client.matrix.org";
+              };
+            };
+            description = mdDoc ''
+              Known homeservers. This is required for the `mbc auth` command and also allows more convenient access from the management UI.
+              If you want to specify registration secrets, pass this via extraConfigFile instead.
+            '';
+          };
+
+          admins = mkOption {
+            type = types.attrsOf types.str;
+            default = { root = ""; };
+            description = mdDoc ''
+              List of administrator users. Plaintext passwords will be bcrypted on startup. Set empty password
+              to prevent normal login. Root is a special user that can't have a password and will always exist.
+            '';
+          };
+
+          api_features = mkOption {
+            type = types.attrsOf bool;
+            default = {
+              login = true;
+              plugin = true;
+              plugin_upload = true;
+              instance = true;
+              instance_database = true;
+              client = true;
+              client_proxy = true;
+              client_auth = true;
+              dev_open = true;
+              log = true;
+            };
+            description = mdDoc ''
+              API feature switches.
+            '';
+          };
+
+          logging = mkOption {
+            type = types.attrs;
+            description = mdDoc ''
+              Python logging configuration. See [section 16.7.2 of the Python
+              documentation](https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema)
+              for more info.
+            '';
+            default = {
+              version = 1;
+              formatters = {
+                colored = {
+                  "()" = "maubot.lib.color_log.ColorFormatter";
+                  format = "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s";
+                };
+                normal = {
+                  format = "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s";
+                };
+              };
+              handlers = {
+                file = {
+                  class = "logging.handlers.RotatingFileHandler";
+                  formatter = "normal";
+                  filename = "./maubot.log";
+                  maxBytes = 10485760;
+                  backupCount = 10;
+                };
+                console = {
+                  class = "logging.StreamHandler";
+                  formatter = "colored";
+                };
+              };
+              loggers = {
+                maubot = {
+                  level = "DEBUG";
+                };
+                mau = {
+                  level = "DEBUG";
+                };
+                aiohttp = {
+                  level = "INFO";
+                };
+              };
+              root = {
+                level = "DEBUG";
+                handlers = [ "file" "console" ];
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    warnings = lib.optional (builtins.any (x: x.username != x.database) parsedLocalPostgresDBs) ''
+      The Maubot database username doesn't match the database name! This means the user won't be automatically
+      granted ownership of the database. Consider changing either the username or the database name.
+    '';
+    assertions = [
+      {
+        assertion = builtins.all (x: !lib.hasInfix ":" x.username) parsedPostgresDBs;
+        message = ''
+          Putting database passwords in your Nix config makes them world-readable. To securely put passwords
+          in your Maubot config, change /var/lib/maubot/config.yaml after running Maubot at least once as
+          described in the NixOS manual.
+        '';
+      }
+      {
+        assertion = hasLocalPostgresDB -> config.services.postgresql.enable;
+        message = ''
+          Cannot deploy maubot with a configuration for a local postgresql database and a missing postgresql service.
+        '';
+      }
+    ];
+
+    services.postgresql = lib.mkIf hasLocalPostgresDB {
+      enable = true;
+      ensureDatabases = map (x: x.database) parsedLocalPostgresDBs;
+      ensureUsers = lib.flip map parsedLocalPostgresDBs (x: {
+        name = x.username;
+        ensureDBOwnership = lib.mkIf (x.username == x.database) true;
+      });
+    };
+
+    users.users.maubot = {
+      group = "maubot";
+      home = cfg.dataDir;
+      # otherwise StateDirectory is enough
+      createHome = lib.mkIf (cfg.dataDir != "/var/lib/maubot") true;
+      isSystemUser = true;
+    };
+
+    users.groups.maubot = { };
+
+    systemd.services.maubot = rec {
+      description = "maubot - a plugin-based Matrix bot system written in Python";
+      after = [ "network.target" ] ++ wants ++ lib.optional hasLocalPostgresDB "postgresql.service";
+      # all plugins get automatically disabled if maubot starts before synapse
+      wants = lib.optional config.services.matrix-synapse.enable config.services.matrix-synapse.serviceUnit;
+      wantedBy = [ "multi-user.target" ];
+
+      preStart = ''
+        if [ ! -f "${cfg.extraConfigFile}" ]; then
+          echo "server:" > "${cfg.extraConfigFile}"
+          echo "    unshared_secret: $(head -c40 /dev/random | base32 | ${pkgs.gawk}/bin/awk '{print tolower($0)}')" > "${cfg.extraConfigFile}"
+          chmod 640 "${cfg.extraConfigFile}"
+        fi
+      '';
+
+      serviceConfig = {
+        ExecStart = "${finalPackage}/bin/maubot --config ${cfg.extraConfigFile}" + lib.optionalString (!cfg.configMutable) " --no-update";
+        User = "maubot";
+        Group = "maubot";
+        Restart = "on-failure";
+        RestartSec = "10s";
+        StateDirectory = lib.mkIf (cfg.dataDir == "/var/lib/maubot") "maubot";
+        WorkingDirectory = cfg.dataDir;
+      };
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ chayleaf ];
+  meta.doc = ./maubot.md;
+}

pkgs/tools/networking/maubot/allow-building-plugins-from-nix-store.patch

@@ -1,13 +0,0 @@
-diff --git a/maubot/cli/commands/build.py b/maubot/cli/commands/build.py
-index ec3ac26..4de85f2 100644
---- a/maubot/cli/commands/build.py
-+++ b/maubot/cli/commands/build.py
-@@ -84,7 +84,7 @@ def read_output_path(output: str, meta: PluginMeta) -> str | None:
- 
- 
- def write_plugin(meta: PluginMeta, output: str | IO) -> None:
--    with zipfile.ZipFile(output, "w") as zip:
-+    with zipfile.ZipFile(output, "w", strict_timestamps=False) as zip:
-         meta_dump = BytesIO()
-         yaml.dump(meta.serialize(), meta_dump)
-         zip.writestr("maubot.yaml", meta_dump.getvalue())

pkgs/tools/networking/maubot/default.nix

@@ -1,6 +1,7 @@
 { lib
 , fetchPypi
 , fetchpatch
+, callPackage
 , runCommand
 , python3
 , encryptionSupport ? true
@@ -55,8 +56,6 @@ let
         url = "https://github.com/maubot/maubot/commit/283f0a3ed5dfae13062b6f0fd153fbdc477f4381.patch";
         sha256 = "0yn5357z346qzy5v5g124mgiah1xsi9yyfq42zg028c8paiw8s8x";
       })
-      # allow running "mbc build" in a nix derivation
-      ./allow-building-plugins-from-nix-store.patch
     ];
 
     propagatedBuildInputs = with python.pkgs; [
@@ -88,15 +87,6 @@ let
       rm $out/example-config.yaml
     '';
 
-    passthru = {
-      inherit python;
-      tests = {
-        simple = runCommand "${pname}-tests" { } ''
-          ${maubot}/bin/mbc --help > $out
-        '';
-      };
-    };
-
     # Setuptools is trying to do python -m maubot test
     dontUseSetuptoolsCheck = true;
 
@@ -104,6 +94,35 @@ let
       "maubot"
     ];
 
+    passthru = let
+      wrapper = callPackage ./wrapper.nix {
+        unwrapped = maubot;
+        python3 = python;
+      };
+    in
+    {
+      tests = {
+        simple = runCommand "${pname}-tests" { } ''
+          ${maubot}/bin/mbc --help > $out
+        '';
+      };
+
+      inherit python;
+
+      plugins = callPackage ./plugins {
+        maubot = maubot;
+        python3 = python;
+      };
+
+      withPythonPackages = pythonPackages: wrapper { inherit pythonPackages; };
+
+      # This adds the plugins to lib/maubot-plugins
+      withPlugins = plugins: wrapper { inherit plugins; };
+
+      # This changes example-config.yaml in module directory
+      withBaseConfig = baseConfig: wrapper { inherit baseConfig; };
+    };
+
     meta = with lib; {
       description = "A plugin-based Matrix bot system written in Python";
       homepage = "https://maubot.xyz/";

pkgs/tools/networking/maubot/plugins/default.nix

@@ -0,0 +1,68 @@
+{ lib
+, fetchgit
+, fetchFromGitHub
+, fetchFromGitLab
+, fetchFromGitea
+, stdenvNoCC
+, callPackage
+, ensureNewerSourcesForZipFilesHook
+, maubot
+, python3
+, poetry
+, formats
+}:
+
+let
+  # pname: plugin id (example: xyz.maubot.echo)
+  # version: plugin version
+  # other attributes are passed directly to stdenv.mkDerivation (you at least need src)
+  buildMaubotPlugin = attrs@{ version, pname, base_config ? null, ... }:
+    stdenvNoCC.mkDerivation (builtins.removeAttrs attrs [ "base_config" ] // {
+      pluginName = "${pname}-v${version}.mbp";
+      nativeBuildInputs = (attrs.nativeBuildInputs or [ ]) ++ [
+        ensureNewerSourcesForZipFilesHook
+        maubot
+      ];
+      buildPhase = ''
+        runHook preBuild
+
+        mbc build
+
+        runHook postBuild
+      '';
+
+      postPatch = lib.optionalString (base_config != null) ''
+        [ -e base-config.yaml ] || (echo "base-config.yaml doesn't exist, can't override it" && exit 1)
+        cp "${if builtins.isPath base_config || lib.isDerivation base_config then base_config
+          else if builtins.isString base_config then builtins.toFile "base-config.yaml" base_config
+          else (formats.yaml { }).generate "base-config.yaml" base_config}" base-config.yaml
+      '' + attrs.postPatch or "";
+
+      installPhase = ''
+        runHook preInstall
+
+        mkdir -p $out/lib/maubot-plugins
+        install -m 444 $pluginName $out/lib/maubot-plugins
+
+        runHook postInstall
+      '';
+    });
+
+  generated = import ./generated.nix {
+    inherit lib fetchgit fetchFromGitHub fetchFromGitLab
+      fetchFromGitea python3 poetry buildMaubotPlugin;
+  };
+in
+generated // {
+  inherit buildMaubotPlugin;
+
+  allOfficialPlugins =
+    builtins.filter
+      (x: x.isOfficial && !x.meta.broken)
+      (builtins.attrValues generated);
+
+  allPlugins =
+    builtins.filter
+      (x: !x.meta.broken)
+      (builtins.attrValues generated);
+}

pkgs/tools/networking/maubot/plugins/generated.nix

@@ -0,0 +1,74 @@
+{ lib
+, fetchgit
+, fetchFromGitHub
+, fetchFromGitLab
+, fetchFromGitea
+, python3
+, poetry
+, buildMaubotPlugin
+}:
+
+let
+  json = builtins.fromJSON (builtins.readFile ./generated.json);
+in
+
+lib.flip builtins.mapAttrs json (name: entry:
+let
+  inherit (entry) manifest;
+
+  resolveDeps = deps: map
+    (name:
+      let
+        packageName = builtins.head (builtins.match "([^~=<>]*).*" name);
+        lower = lib.toLower packageName;
+        dash = builtins.replaceStrings ["_"] ["-"] packageName;
+        lowerDash = builtins.replaceStrings ["_"] ["-"] lower;
+      in
+        python3.pkgs.${packageName}
+        or python3.pkgs.${lower}
+        or python3.pkgs.${dash}
+        or python3.pkgs.${lowerDash}
+        or null)
+    (builtins.filter (x: x != "maubot" && x != null) deps);
+
+  reqDeps = resolveDeps (lib.toList (manifest.dependencies or null));
+  optDeps = resolveDeps (lib.toList (manifest.soft_dependencies or null));
+in
+
+lib.makeOverridable buildMaubotPlugin (entry.attrs // {
+  pname = manifest.id;
+  inherit (manifest) version;
+
+  src =
+    if entry?github then fetchFromGitHub entry.github
+    else if entry?git then fetchgit entry.git
+    else if entry?gitlab then fetchFromGitLab entry.gitlab
+    else if entry?gitea then fetchFromGitea entry.gitea
+    else throw "Invalid generated entry for ${manifest.id}: missing source";
+
+  propagatedBuildInputs = builtins.filter (x: x != null) (reqDeps ++ optDeps);
+
+  passthru.isOfficial = entry.isOfficial or false;
+
+  meta = entry.attrs.meta // {
+    license =
+      let
+        spdx = entry.attrs.meta.license or manifest.license or "unfree";
+        spdxLicenses = builtins.listToAttrs
+          (map (x: lib.nameValuePair x.spdxId x) (builtins.filter (x: x?spdxId) (builtins.attrValues lib.licenses)));
+      in
+      spdxLicenses.${spdx};
+    broken = builtins.any (x: x == null) reqDeps;
+  };
+} // lib.optionalAttrs (entry.isPoetry or false) {
+  nativeBuildInputs = [
+    poetry
+    (python3.withPackages (p: with p; [ toml ruamel-yaml isort ]))
+  ];
+
+  preBuild = lib.optionalString (entry?attrs.preBuild) (entry.attrs.preBuild + "\n") + ''
+    export HOME=$(mktemp -d)
+    [[ ! -d scripts ]] || patchShebangs --build scripts
+    make maubot.yaml
+  '';
+}))

pkgs/tools/networking/maubot/plugins/update.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p git nurl "(python3.withPackages (ps: with ps; [ toml gitpython requests ruamel-yaml ]))"
+
+import git
+import json
+import os
+import subprocess
+import ruamel.yaml
+import sys
+import toml
+import zipfile
+
+from typing import Dict, List
+
+HOSTNAMES = {
+    'git.skeg1.se': 'gitlab',
+    'edugit.org': 'gitlab',
+    'codeberg.org': 'gitea',
+}
+PLUGINS: Dict[str, dict] = {}
+
+yaml = ruamel.yaml.YAML(typ='safe')
+
+TMP = os.environ.get('TEMPDIR', '/tmp')
+
+def process_repo(path: str, official: bool):
+    global PLUGINS
+    with open(path, 'rt') as f:
+        data = yaml.load(f)
+    name, repourl, license, desc = data['name'], data['repo'], data['license'], data['description']
+    origurl = repourl
+    if '/' in name or ' ' in name:
+        name = os.path.split(path)[-1].removesuffix('.yaml')
+    name = name.replace('_', '-')
+    if name in PLUGINS.keys():
+        raise ValueError(f'Duplicate plugin {name}, refusing to continue')
+    repodir = os.path.join(TMP, 'maubot-plugins', name)
+    plugindir = repodir
+    if '/tree/' in repourl:
+        repourl, rev_path = repourl.split('/tree/')
+        rev, subdir = rev_path.strip('/').split('/')
+        plugindir = os.path.join(plugindir, subdir)
+    else:
+        rev = None
+        subdir = None
+
+    if repourl.startswith('http:'):
+        repourl = 'https' + repourl[4:]
+    repourl = repourl.rstrip('/')
+    if not os.path.exists(repodir):
+        print('Fetching', name)
+        repo = git.Repo.clone_from(repourl + '.git', repodir)
+    else:
+        repo = git.Repo(repodir)
+    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+    tags = list(filter(lambda x: 'rc' not in str(x), tags))
+    if tags:
+        repo.git.checkout(tags[-1])
+        rev = str(tags[-1])
+    else:
+        rev = str(repo.commit('HEAD'))
+    ret: dict = {'attrs':{}}
+    if subdir:
+        ret['attrs']['postPatch'] = f'cd {subdir}'
+    domain, query = repourl.removeprefix('https://').split('/', 1)
+    hash = subprocess.run([
+        'nurl',
+        '--hash',
+        f'file://{repodir}',
+        rev
+    ], capture_output=True, check=True).stdout.decode('utf-8')
+    ret['attrs']['meta'] = {
+        'description': desc,
+        'homepage': origurl,
+    }
+    if domain.endswith('github.com'):
+        owner, repo = query.split('/')
+        ret['github'] = {
+            'owner': owner,
+            'repo': repo,
+            'rev': rev,
+            'hash': hash,
+        }
+        ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'
+        ret['attrs']['meta']['changelog'] = f'{repourl}/releases'
+        repobase = f'{repourl}/blob/{rev}'
+    elif HOSTNAMES.get(domain, 'gitea' if 'gitea.' in domain or 'forgejo.' in domain else None) == 'gitea':
+        owner, repo = query.split('/')
+        ret['gitea'] = {
+            'domain': domain,
+            'owner': owner,
+            'repo': repo,
+            'rev': rev,
+            'hash': hash,
+        }
+        repobase = f'{repourl}/src/commit/{rev}'
+        ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'
+        ret['attrs']['meta']['changelog'] = f'{repourl}/releases'
+    elif HOSTNAMES.get(domain, 'gitlab' if 'gitlab.' in domain else None) == 'gitlab':
+        owner, repo = query.split('/')
+        ret['gitlab'] = {
+            'owner': owner,
+            'repo': repo,
+            'rev': rev,
+            'hash': hash,
+        }
+        if domain != 'gitlab.com':
+            ret['gitlab']['domain'] = domain
+        repobase = f'{repourl}/-/blob/{rev}'
+    else:
+        raise ValueError(f'Is {domain} Gitea or Gitlab, or something else? Please specify in the Python script!')
+    if os.path.exists(os.path.join(plugindir, 'CHANGELOG.md')):
+        ret['attrs']['meta']['changelog'] = f'{repobase}/CHANGELOG.md'
+    if os.path.exists(os.path.join(plugindir, 'maubot.yaml')):
+        with open(os.path.join(plugindir, 'maubot.yaml'), 'rt') as f:
+            ret['manifest'] = yaml.load(f)
+    elif os.path.exists(os.path.join(plugindir, 'pyproject.toml')):
+        ret['isPoetry'] = True
+        with open(os.path.join(plugindir, 'pyproject.toml'), 'rt') as f:
+            data = toml.load(f)
+        deps = []
+        for key, val in data['tool']['poetry'].get('dependencies', {}).items():
+            if key in ['maubot', 'mautrix', 'python']:
+                continue
+            reqs = []
+            for req in val.split(','):
+                reqs.extend(poetry_to_pep(req))
+            deps.append(key + ', '.join(reqs))
+        ret['manifest'] = data['tool']['maubot']
+        ret['manifest']['id'] = data['tool']['poetry']['name']
+        ret['manifest']['version'] = data['tool']['poetry']['version']
+        ret['manifest']['license'] = data['tool']['poetry']['license']
+        if deps:
+            ret['manifest']['dependencies'] = deps
+    else:
+        raise ValueError(f'No maubot.yaml or pyproject.toml found in {repodir}')
+    # normalize non-spdx-conformant licenses this way
+    # (and fill out missing license info)
+    if 'license' not in ret['manifest'] or ret['manifest']['license'] in ['GPLv3', 'AGPL 3.0']:
+        ret['attrs']['meta']['license'] = license
+    elif ret['manifest']['license'] != license:
+        print(f"Warning: licenses for {repourl} don't match! {ret['manifest']['license']} != {license}")
+    if official:
+        ret['isOfficial'] = official
+    PLUGINS[name] = ret
+
+def next_incomp(ver_s: str) -> str:
+    ver = ver_s.split('.')
+    zero = False
+    for i in range(len(ver)):
+        try:
+            seg = int(ver[i])
+        except ValueError:
+            if zero:
+                ver = ver[:i]
+                break
+            continue
+        if zero:
+            ver[i] = '0'
+        elif seg:
+            ver[i] = str(seg + 1)
+            zero = True
+    return '.'.join(ver)
+
+def poetry_to_pep(ver_req: str) -> List[str]:
+    if '*' in ver_req:
+        raise NotImplementedError('Wildcard poetry versions not implemented!')
+    if ver_req.startswith('^'):
+        return ['>=' + ver_req[1:], '<' + next_incomp(ver_req[1:])]
+    if ver_req.startswith('~'):
+        return ['~=' + ver_req[1:]]
+    return [ver_req]
+
+def main():
+    cache_path = os.path.join(TMP, 'maubot-plugins')
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
+        git.Repo.clone_from('https://github.com/maubot/plugins.maubot.xyz', os.path.join(cache_path, '_repo'))
+    else:
+        pass
+
+    repodir = os.path.join(cache_path, '_repo')
+
+    for suffix, official in (('official', True), ('thirdparty', False)):
+        directory = os.path.join(repodir, 'data', 'plugins', suffix)
+        for plugin_name in os.listdir(directory):
+            process_repo(os.path.join(directory, plugin_name), official)
+
+    if os.path.isdir('pkgs/tools/networking/maubot/plugins'):
+        generated = 'pkgs/tools/networking/maubot/plugins/generated.json'
+    else:
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        generated = os.path.join(script_dir, 'generated.json')
+
+    with open(generated, 'wt') as file:
+        json.dump(PLUGINS, file, indent='  ', separators=(',', ': '), sort_keys=True)
+        file.write('\n')
+
+if __name__ == '__main__':
+    main()

pkgs/tools/networking/maubot/wrapper.nix

@@ -0,0 +1,70 @@
+{ lib
+, symlinkJoin
+, runCommand
+, unwrapped
+, python3
+, formats
+}:
+
+let wrapper = { pythonPackages ? (_: [ ]), plugins ? (_: [ ]), baseConfig ? null }:
+  let
+    plugins' = plugins unwrapped.plugins;
+    extraPythonPackages = builtins.concatLists (map (p: p.propagatedBuildInputs or [ ]) plugins');
+  in
+  symlinkJoin {
+    name = "${unwrapped.pname}-with-plugins-${unwrapped.version}";
+
+    inherit unwrapped;
+    paths = lib.optional (baseConfig != null) unwrapped ++ plugins';
+    pythonPath = lib.optional (baseConfig == null) unwrapped ++ pythonPackages python3.pkgs ++ extraPythonPackages;
+
+    nativeBuildInputs = [ python3.pkgs.wrapPython ];
+
+    postBuild = ''
+      rm -f $out/nix-support/propagated-build-inputs
+      rmdir $out/nix-support || true
+      ${lib.optionalString (baseConfig != null) ''
+        rm $out/${python3.sitePackages}/maubot/example-config.yaml
+        substituteAll ${(formats.yaml { }).generate "example-config.yaml" (lib.recursiveUpdate baseConfig {
+          plugin_directories = lib.optionalAttrs (plugins' != []) {
+            load = [ "@out@/lib/maubot-plugins" ] ++ (baseConfig.plugin_directories.load or []);
+          };
+          # Normally it should be set to false by default to take it from package
+          # root, but aiohttp doesn't follow symlinks when serving static files
+          # unless follow_symlinks=True is passed. Instead of patching maubot, use
+          # this non-invasive approach
+          # XXX: would patching maubot be better? See:
+          # https://github.com/maubot/maubot/blob/75879cfb9370aade6fa0e84e1dde47222625139a/maubot/server.py#L106
+          server.override_resource_path =
+            if builtins.isNull (baseConfig.server.override_resource_path or null)
+            then "${unwrapped}/${python3.sitePackages}/maubot/management/frontend/build"
+            else baseConfig.server.override_resource_path;
+        })})} $out/${python3.sitePackages}/maubot/example-config.yaml
+        rm -rf $out/bin
+      ''}
+      mkdir -p $out/bin
+      cp $unwrapped/bin/.mbc-wrapped $out/bin/mbc
+      cp $unwrapped/bin/.maubot-wrapped $out/bin/maubot
+      wrapPythonProgramsIn "$out/bin" "${lib.optionalString (baseConfig != null) "$out "}$pythonPath"
+    '';
+
+    passthru = {
+      inherit unwrapped;
+      python = python3;
+      withPythonPackages = filter: wrapper {
+        pythonPackages = pkgs: pythonPackages pkgs ++ filter pkgs;
+        inherit plugins baseConfig;
+      };
+      withPlugins = filter: wrapper {
+        plugins = pkgs: plugins pkgs ++ filter pkgs;
+        inherit pythonPackages baseConfig;
+      };
+      withBaseConfig = baseConfig: wrapper {
+        inherit baseConfig pythonPackages plugins;
+      };
+    };
+
+    meta.priority = (unwrapped.meta.priority or 0) - 1;
+  };
+in
+wrapper