Merge pull request #282699 from happysalada/update_materialize

materialize: 0.17.0 -> 0.84.0
2024-01-28 20:00:15 +00:00 · 2024-01-28 20:00:15 +00:00 · 48f6e0d286
commit 48f6e0d286
parent 11d08e3cb9 571eb488bd
3 changed files with 6677 additions and 2167 deletions
--- a/pkgs/servers/sql/materialize/Cargo.lock
+++ b/pkgs/servers/sql/materialize/Cargo.lock
--- a/pkgs/servers/sql/materialize/default.nix
+++ b/pkgs/servers/sql/materialize/default.nix
@ -10,7 +10,11 @@
 , libiconv
 , openssl
 , perl
-, pkg-config}:
+, pkg-config
+, protobuf
+, libclang
+, rdkafka
+}:

 let
  fetchNpmPackage = {name, version, hash, js_prod_file, js_dev_file, ...} @ args:
@ -20,16 +24,19 @@ let
      inherit hash;
    };

-    static = "./src/materialized/src/http/static";
-    cssVendor = "./src/materialized/src/http/static/css/vendor";
-    jsProdVendor = "./src/materialized/src/http/static/js/vendor";
-    jsDevVendor = "./src/materialized/src/http/static-dev/js/vendor";
-
    files = with args; [
-      { src = js_prod_file; dst = "${jsProdVendor}/${name}.js"; }
-      { src = js_dev_file;  dst = "${jsDevVendor}/${name}.js"; }
-    ] ++ lib.optional (args ? css_file) { src = css_file; dst = "${cssVendor}/${name}.css"; }
-      ++ lib.optional (args ? extra_file) { src = extra_file.src; dst = "${static}/${extra_file.dst}"; };
+      { src = js_prod_file; dst = "./src/environmentd/src/http/static/js/vendor/${name}.js"; }
+      { src = js_prod_file; dst = "./src/prof-http/src/http/static/js/vendor/${name}.js"; }
+      { src = js_dev_file;  dst = "./src/environmentd/src/http/static-dev/js/vendor/${name}.js"; }
+      { src = js_dev_file;  dst = "./src/prof-http/src/http/static-dev/js/vendor/${name}.js"; }
+    ] ++ lib.optionals (args ? css_file) [
+     { src = css_file; dst = "./src/environmentd/src/http/static/css/vendor/${name}.css"; }
+     { src = css_file; dst = "./src/prof-http/src/http/static/css/vendor/${name}.css"; }
+    ]
+      ++ lib.optionals (args ? extra_file) [
+      { src = extra_file.src; dst = "./src/environmentd/src/http/static/${extra_file.dst}";}
+      { src = extra_file.src; dst = "./src/prof-http/src/http/static/${extra_file.dst}";}
+    ];
  in
    lib.concatStringsSep "\n" (lib.forEach files ({src, dst}: ''
      mkdir -p "${dirOf dst}"
@ -40,44 +47,73 @@ let
 in
 rustPlatform.buildRustPackage rec {
  pname = "materialize";
-  version = "0.17.0";
+  version = "0.84.2";
  MZ_DEV_BUILD_SHA = "9f8cf75b461d288335cb6a7a73aaa670bab4a466";

  src = fetchFromGitHub {
    owner = "MaterializeInc";
    repo = pname;
    rev = "v${version}";
-    hash = "sha256-wKYU5S77VoOX7UA9/d21Puz9NYs/om08eNM69/m3Orc=";
+    hash = "sha256-+cvTCiTbuaPYPIyDxQlMWdJA5/6cbMoiTcSmjj5KPjs=";
+    fetchSubmodules = true;
  };

+  postPatch = ''
+    ${lib.concatStringsSep "\n" (map fetchNpmPackage npmPackages)}
+    substituteInPlace ./misc/dist/materialized.service \
+      --replace /usr/bin $out/bin \
+      --replace _Materialize root
+    substituteInPlace ./src/catalog/build.rs \
+      --replace '&[ ' '&["."'
+  '';
+
+  # needed for internal protobuf c wrapper library
+  env.PROTOC = "${protobuf}/bin/protoc";
+  env.PROTOC_INCLUDE = "${protobuf}/include";
+  env.LIBCLANG_PATH = "${libclang.lib}/lib";
+  # needed to dynamically link rdkafka
+  env.CARGO_FEATURE_DYNAMIC_LINKING=1;
+
  cargoLock = {
    lockFile = ./Cargo.lock;
    outputHashes = {
-      "criterion-0.3.5" = "sha256-gXhwLw99kC08zxHdY6l5LF5EVzAAFasI4GLnopLwhEU=";
-      "differential-dataflow-0.12.0" = "sha256-sDy4502XqCuXqRtwLWuaHSgfX7v9NNochhfpI6n8DrM=";
-      "headers-0.3.5" = "sha256-ipxMfuPgoeH2uR4Im/XBdnxT00+LGzTgW7Ed2armYOU=";
-      "mzcloud-1.0.0" = "sha256-Nt9YCG+DFCCOMbKaNhOD78WF/z3qB5ymUIp6Wep2A9A=";
-      "parquet-format-async-temp-0.2.0" = "sha256-UUQv/90egmwQK0CZCztinEskvWcZ40rKWbJoWYz2oLQ=";
-      "postgres-0.19.1" = "sha256-zH7PF4p7wJCSYpuydTL3HPcOjPU9SlTy6IJREOe2l7U=";
-      "postgres_array-0.11.0" = "sha256-M1nMsQfxK0ay4JxoPqm2cl4Cp8mVZlVUAfWDhhv9nA4=";
-      "prometheus-0.10.0" = "sha256-NTnKt1RGiZ8AxsU8UzhLhpfEi24Pos5kR9g22Mmt444=";
-      "protobuf-3.0.0-alpha.2" = "sha256-8gBGQqAtKJelcBCxnDESanlblBLQ1Th7khHxUkDw7To=";
-      "pubnub-core-0.1.0" = "sha256-YuGEFaStfrhb0ygjVFm2a2eJla9ABc5ifXKuvJxUvgk=";
-      "rdkafka-0.28.0" = "sha256-u2gBTzu+IvXTDvcZGzPaVpSVOgAKhTth7GLwob4urDs=";
-      "timely-0.12.0" = "sha256-PHaDRNm7MezXJWhILWJHTeiCWO3iLUp94Z0V2dreCMk=";
+      "columnation-0.1.0" = "sha256-VRDQqIVLayEnMHeth4cjsS/FYah3B3mwYEGnv8jpKs8=";
+      "differential-dataflow-0.12.0" = "sha256-cEmtDXOZSy4rDFZ7gCd7lx6wH+m1S9vphjb+wO4MSAM=";
+      "eventsource-client-0.11.0" = "sha256-FeEWV2yy1et5mna0TyAnakXlcIR42Aq97Lfjjlom8T0=";
+      "launchdarkly-server-sdk-1.0.0" = "sha256-fSWiV9mNf5WBkWDNckiUR3URQ8lJ4GZURxbYO/753sU=";
+      "librocksdb-sys-0.11.0+8.3.2" = "sha256-bnAvH2z9n26MYFhTN/+Yz+7lEdNKKmHJOoHkxTdZGvw=";
+      "openssh-0.9.9" = "sha256-2jaQN6PhavUtlWwqCn2VXEg213uj7BQ+FIrhnL3rb8Q=";
+      "postgres-0.19.5" = "sha256-i0mURHTCMrgaW1DD1CihWMdZ3zoNI14dCpq/ja8RW9E=";
+      "postgres_array-0.11.0" = "sha256-ealgPVExRIFUt0QVao8H7Q7u/PTuCbpGrk6Tm5jVwZ0=";
+      "proptest-1.0.0" = "sha256-sJbPQIVeHZZiRXssRpJWRbD9l8QnfwVcpGu6knjAe5o=";
+      "rdkafka-0.29.0" = "sha256-48CMvJ4PoVfKyiNMSpCGBtj36j2CF1E8a/QQ/urfiPc=";
+      "reqwest-middleware-0.2.3" = "sha256-zzlQycH5dmgM8ew1gy8m5r6Q2ib7LXnUeX69M3ih+sY=";
+      "serde-value-0.7.0" = "sha256-ewEYsf1+9MmLuSm5KbO326ngGB79i00lAp2NMHuuxw8=";
+      "timely-0.12.0" = "sha256-wJtHJ9ygPVusN5Io8SjZGo1r7lcrrcauESSC+9038AU=";
+      "tonic-build-0.9.2" = "sha256-cGvHjgmdr3NU1phwUfMvEE6uU12fOlhTlL2LoWeOO4I=";
+      "tracing-opentelemetry-0.22.0" = "sha256-mawDGrue/e3dPYVG0ANs9nZ+xmQyd1YTWH8QmE6VD0U=";
    };
  };

-  nativeBuildInputs = [ cmake perl pkg-config ]
+  nativeBuildInputs = [
+    cmake
+    perl
+    pkg-config
+    rustPlatform.bindgenHook
+  ]
    # Provides the mig command used by the krb5-src build script
    ++ lib.optional stdenv.isDarwin bootstrap_cmds;

  # Needed to get openssl-sys to use pkg-config.
  OPENSSL_NO_VENDOR = 1;

-  buildInputs = [ openssl ]
+  buildInputs = [ openssl rdkafka libclang ]
    ++ lib.optionals stdenv.isDarwin [ libiconv DiskArbitration Foundation ];

+  # the check phase requires linking with rocksdb which can be a problem since
+  # the rust rocksdb crate is not updated very often.
+  doCheck = false;
+
  # Skip tests that use the network
  checkFlags = [
    "--exact"
@ -90,14 +126,7 @@ rustPlatform.buildRustPackage rec {
    "--skip test_tls"
  ];

-  postPatch = ''
-    ${lib.concatStringsSep "\n" (map fetchNpmPackage npmPackages)}
-    substituteInPlace ./misc/dist/materialized.service \
-      --replace /usr/bin $out/bin \
-      --replace _Materialize root
-  '';
-
-  cargoBuildFlags = [ "--bin materialized" ];
+  cargoBuildFlags = [ "--bin environmentd --bin clusterd" ];

  postInstall = ''
    install --mode=444 -D ./misc/dist/materialized.service $out/etc/systemd/system/materialized.service
--- a/pkgs/servers/sql/materialize/npm_deps.nix
+++ b/pkgs/servers/sql/materialize/npm_deps.nix
@ -3,7 +3,7 @@
 # files.
 #
 # The list of modules can be found in this file
-# https://github.com/MaterializeInc/materialize/blob/master/src/materialized/build/npm.rs
+# https://github.com/MaterializeInc/materialize/blob/master/src/npm/lib.rs
 [
  {
    name = "@hpcc-js/wasm";
@ -17,9 +17,9 @@
    };
  }
  {
-    name = "babel-standalone";
-    version = "6.26.0";
-    hash = "sha256-zdeTj4aOEzWvrnh90T80jL+RD8qg8PaeG0Ceua82Dz4=";
+    name = "@babel/standalone";
+    version = "7.23.3";
+    hash = "sha256-yxhB4OVOdV8hYNPqcap+5/JXYeaVrNGOSOG8lKpiG9E=";
    js_prod_file = "babel.min.js";
    js_dev_file = "babel.js";
  }