diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index 495a7094f9b4..6495b024b008 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -31,6 +31,9 @@ rec { */ + inherit (import ./formats/java-properties/default.nix { inherit lib pkgs; }) + javaProperties; + json = {}: { type = with lib.types; let diff --git a/pkgs/pkgs-lib/formats/java-properties/default.nix b/pkgs/pkgs-lib/formats/java-properties/default.nix new file mode 100644 index 000000000000..e4f42a61f5c8 --- /dev/null +++ b/pkgs/pkgs-lib/formats/java-properties/default.nix @@ -0,0 +1,85 @@ +{ lib, pkgs }: +{ + javaProperties = {}: { + type = lib.types.attrsOf lib.types.str; + + generate = name: value: + pkgs.runCommandLocal name + { + # Requirements + # ============ + # + # 1. Strings in Nix carry over to the same + # strings in Java => need proper escapes + # 2. Generate files quickly + # - A JVM would have to match the app's + # JVM to avoid build closure bloat + # - Even then, JVM startup would slow + # down config generation. + # + # + # Implementation + # ============== + # + # Escaping has two steps + # + # 1. jq + # Escape known separators, in order not + # to break up the keys and values. + # This handles typical whitespace correctly, + # but may produce garbage for other control + # characters. + # + # 2. iconv + # Escape >ascii code points to java escapes, + # as .properties files are supposed to be + # encoded in ISO 8859-1. It's an old format. + # UTF-8 behavior may exist in some apps and + # libraries, but we can't rely on this in + # general. + + passAsFile = [ "value" ]; + value = builtins.toJSON value; + nativeBuildInputs = [ + pkgs.jq + pkgs.libiconvReal + ]; + + jqCode = + let + main = '' + to_entries + | .[] + | "\( + .key + | ${commonEscapes} + | gsub(" "; "\\ ") + | gsub("="; "\\=") + ) = \( + .value + | ${commonEscapes} + | gsub("^ "; "\\ ") + | gsub("\\n "; "\n\\ ") + )" + ''; + # Most escapes are equal for both keys and values. + commonEscapes = '' + gsub("\\\\"; "\\\\") + | gsub("\\n"; "\\n\\\n") + | gsub("#"; "\\#") + | gsub("!"; "\\!") + | gsub("\\t"; "\\t") + | gsub("\r"; "\\r") + ''; + in + main; + + inputEncoding = "UTF-8"; + + } '' + jq -r --arg hash '#' "$jqCode" "$valuePath" \ + | iconv --from-code "$inputEncoding" --to-code JAVA \ + > "$out" + ''; + }; +} diff --git a/pkgs/pkgs-lib/formats/java-properties/test/Main.java b/pkgs/pkgs-lib/formats/java-properties/test/Main.java new file mode 100644 index 000000000000..dc83944f24b0 --- /dev/null +++ b/pkgs/pkgs-lib/formats/java-properties/test/Main.java @@ -0,0 +1,27 @@ +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.SortedSet; +import java.util.TreeSet; + +class Main { + public static void main (String args[]) { + try { + InputStream input = new FileInputStream(args[0]); + Properties prop = new Properties(); + prop.load(input); + SortedSet keySet = new TreeSet(prop.keySet()); + for (String key : keySet) { + System.out.println("KEY"); + System.out.println(key); + System.out.println("VALUE"); + System.out.println(prop.get(key)); + System.out.println(""); + } + } catch (Exception e) { + e.printStackTrace(); + System.err.println(e.toString()); + System.exit(1); + } + } +} diff --git a/pkgs/pkgs-lib/formats/java-properties/test/default.nix b/pkgs/pkgs-lib/formats/java-properties/test/default.nix new file mode 100644 index 000000000000..4b3845c10296 --- /dev/null +++ b/pkgs/pkgs-lib/formats/java-properties/test/default.nix @@ -0,0 +1,85 @@ +{ fetchurl +, formats +, glibcLocales +, jdk +, lib +, stdenv +}: +let + inherit (lib) concatStrings attrValues mapAttrs; + + javaProperties = formats.javaProperties { }; + + input = { + foo = "bar"; + "empty value" = ""; + "typical.dot.syntax" = "com.sun.awt"; + "" = "empty key's value"; + "1" = "2 3"; + "#" = "not a comment # still not"; + "!" = "not a comment!"; + "!a" = "still not! a comment"; + "!b" = "still not ! a comment"; + "dos paths" = "C:\\Program Files\\Nix For Windows\\nix.exe"; + "a \t\nb" = " c"; + "angry \t\nkey" = '' + multi + ${"\tline\r"} + space- + indented + trailing-space${" "} + trailing-space${" "} + value + ''; + "this=not" = "bad"; + "nor = this" = "bad"; + "all stuff" = "foo = bar"; + "unicode big brain" = "e = mc□"; + "ütf-8" = "dûh"; + # NB: Some editors (vscode) show this _whole_ line in right-to-left order + "الجبر" = "أكثر من مجرد أرقام"; + }; + +in +stdenv.mkDerivation { + name = "pkgs.formats.javaProperties-test-${jdk.name}"; + nativeBuildInputs = [ + jdk + glibcLocales + ]; + + # technically should go through the type.merge first, but that's tested + # in tests/formats.nix. + properties = javaProperties.generate "example.properties" input; + + # Expected output as printed by Main.java + passAsFile = [ "expected" ]; + expected = concatStrings (attrValues ( + mapAttrs + (key: value: + '' + KEY + ${key} + VALUE + ${value} + + '' + ) + input + )); + + src = lib.sourceByRegex ./. [ + ".*\.java" + ]; + LANG = "C.UTF-8"; + buildPhase = '' + javac Main.java + ''; + doCheck = true; + checkPhase = '' + cat -v $properties + java Main $properties >actual + diff -U3 $expectedPath actual + ''; + installPhase = "touch $out"; +} diff --git a/pkgs/pkgs-lib/tests/default.nix b/pkgs/pkgs-lib/tests/default.nix index f3549ea9b0f2..ae91e15aa9ef 100644 --- a/pkgs/pkgs-lib/tests/default.nix +++ b/pkgs/pkgs-lib/tests/default.nix @@ -1,7 +1,45 @@ # Call nix-build on this file to run all tests in this directory -{ pkgs ? import ../../.. {} }: + +# This produces a link farm derivation with the original attrs +# merged on top of it. +# You can run parts of the "hierarchy" with for example: +# nix-build -A java-properties +# See `structured` below. + +{ pkgs ? import ../../.. { } }: let - formats = import ./formats.nix { inherit pkgs; }; -in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [ - { name = "formats"; path = import ./formats.nix { inherit pkgs; }; } -] + inherit (pkgs.lib) mapAttrs mapAttrsToList isDerivation mergeAttrs foldl' attrValues recurseIntoAttrs; + + structured = { + formats = import ./formats.nix { inherit pkgs; }; + java-properties = recurseIntoAttrs { + jdk8 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk8; }; + jdk11 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk11_headless; }; + jdk17 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk17_headless; }; + }; + }; + + flatten = prefix: as: + foldl' + mergeAttrs + { } + (attrValues + (mapAttrs + (k: v: + if isDerivation v + then { "${prefix}${k}" = v; } + else if v?recurseForDerivations + then flatten "${prefix}${k}-" (removeAttrs v [ "recurseForDerivations" ]) + else builtins.trace v throw "expected derivation or recurseIntoAttrs") + as + ) + ); +in + +# It has to be a link farm for inclusion in the hydra unstable jobset. +pkgs.linkFarm "pkgs-lib-formats-tests" + (mapAttrsToList + (k: v: { name = k; path = v; }) + (flatten "" structured) + ) +// structured diff --git a/pkgs/pkgs-lib/tests/formats.nix b/pkgs/pkgs-lib/tests/formats.nix index 2bc4e407fe75..71b4a3946a34 100644 --- a/pkgs/pkgs-lib/tests/formats.nix +++ b/pkgs/pkgs-lib/tests/formats.nix @@ -168,4 +168,21 @@ in runBuildTests { level4 = "deep" ''; }; + + # See also java-properties/default.nix for more complete tests + testJavaProperties = { + drv = evalFormat formats.javaProperties {} { + foo = "bar"; + "1" = "2"; + "ütf 8" = "dûh"; + # NB: Some editors (vscode) show this _whole_ line in right-to-left order + "الجبر" = "أكثر من مجرد أرقام"; + }; + expected = '' + 1 = 2 + foo = bar + \u00fctf\ 8 = d\u00fbh + \u0627\u0644\u062c\u0628\u0631 = \u0623\u0643\u062b\u0631 \u0645\u0646 \u0645\u062c\u0631\u062f \u0623\u0631\u0642\u0627\u0645 + ''; + }; }