semgrep{,-core}: 1.15.0 -> 1.27.0
Now fetching semgrep-core from the python wheel as r2c don't add binaries to the GH releases anymore.
This commit is contained in:
06kellyjac
parent
c5c01f0b61
commit
f3dde5bf8c
4 changed files with 102 additions and 69 deletions
|
|
@ -1,54 +1,39 @@
|
|||
{ lib, fetchFromGitHub, fetchzip, stdenv }:
|
||||
{ lib }:
|
||||
|
||||
rec {
|
||||
version = "1.15.0";
|
||||
version = "1.27.0";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "returntocorp";
|
||||
repo = "semgrep";
|
||||
rev = "v${version}";
|
||||
sha256 = "sha256-x+AOt6nn2hN4MODFZCvlq0kZ3VLoS7rVcFGGCEssIu0=";
|
||||
};
|
||||
srcHash = "sha256-F6n3LQY4a5sO6c8SMQF9YjjgOS+v2SH+UQPwhg2EX7Q=";
|
||||
|
||||
# submodule dependencies
|
||||
# these are fetched so we:
|
||||
# 1. don't fetch the many submodules we don't need
|
||||
# 2. avoid fetchSubmodules since it's prone to impurities
|
||||
submodules = {
|
||||
"cli/src/semgrep/lang" = fetchFromGitHub {
|
||||
owner = "returntocorp";
|
||||
repo = "semgrep-langs";
|
||||
rev = "08656cdefc9e6818c64e168cf51ee1e76ea8829e";
|
||||
sha256 = "sha256-vYf33JhfvEDmt/VW0hBOmqailIERS0GdUgrPuCxWt9I=";
|
||||
};
|
||||
"cli/src/semgrep/semgrep_interfaces" = fetchFromGitHub {
|
||||
"cli/src/semgrep/semgrep_interfaces" = {
|
||||
owner = "returntocorp";
|
||||
repo = "semgrep-interfaces";
|
||||
rev = "ba9241ca8f13dea72a4ca5c5eae99f45c071c8b4";
|
||||
sha256 = "sha256-2rcMmN42445AivcyYLPeE+HBYOyxJijQME1UUr9HISA=";
|
||||
rev = "213f67abea73546ca6111e1bbf0ef96aa917c940";
|
||||
hash = "sha256-HeNHJkTje9j16+dwsfyMhoqQn/J18q/7XvQPRwgTw/Y=";
|
||||
};
|
||||
};
|
||||
|
||||
# fetch pre-built semgrep-core since the ocaml build is complex and relies on
|
||||
# the opam package manager at some point
|
||||
core = rec {
|
||||
data = {
|
||||
# pulling it out of the python wheel as r2c no longer release a built binary
|
||||
# on github releases
|
||||
core = {
|
||||
x86_64-linux = {
|
||||
suffix = "-ubuntu-16.04.tgz";
|
||||
sha256 = "sha256-vLtV1WAnOD6HhgrWYIP0NfXHKfvXORksdNp5UTG1QWc=";
|
||||
platform = "any";
|
||||
hash = "sha256-cRj81dXpAE6S0EXajsRikOIAPzlUf42FhiDCWjv+wZQ=";
|
||||
};
|
||||
x86_64-darwin = {
|
||||
suffix = "-osx.zip";
|
||||
sha256 = "sha256-6+ENjOOIJ5TSjpnJ5pDudblrWj/FLUe66UGr6V9c0HQ=";
|
||||
platform = "macosx_10_14_x86_64";
|
||||
hash = "sha256-jqfGVZGF/DFgXkr7kQg6QyqEELSr8AKE3Ga8kTftnIY=";
|
||||
};
|
||||
};
|
||||
src = let
|
||||
inherit (stdenv.hostPlatform) system;
|
||||
selectSystemData = data: data.${system} or (throw "Unsupported system: ${system}");
|
||||
inherit (selectSystemData data) suffix sha256;
|
||||
in fetchzip {
|
||||
url = "https://github.com/returntocorp/semgrep/releases/download/v${version}/semgrep-v${version}${suffix}";
|
||||
inherit sha256;
|
||||
aarch64-darwin = {
|
||||
platform = "macosx_11_0_arm64";
|
||||
hash = "sha256-e/uCSRMdbVD0lvc0hukbiUzheqRNIIh1LgMq6Ae7JYI=";
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -66,7 +51,5 @@ rec {
|
|||
'';
|
||||
license = licenses.lgpl21Plus;
|
||||
maintainers = with maintainers; [ jk ambroisie ];
|
||||
# limited by semgrep-core
|
||||
platforms = [ "x86_64-linux" "x86_64-darwin" ];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
{ lib
|
||||
, fetchFromGitHub
|
||||
, callPackage
|
||||
, semgrep-core
|
||||
, buildPythonApplication
|
||||
, pythonPackages
|
||||
|
|
@ -11,12 +10,20 @@
|
|||
}:
|
||||
|
||||
let
|
||||
common = callPackage ./common.nix { };
|
||||
common = import ./common.nix { inherit lib; };
|
||||
in
|
||||
buildPythonApplication rec {
|
||||
pname = "semgrep";
|
||||
inherit (common) src version;
|
||||
inherit (common) version;
|
||||
src = fetchFromGitHub {
|
||||
owner = "returntocorp";
|
||||
repo = "semgrep";
|
||||
rev = "v${version}";
|
||||
hash = common.srcHash;
|
||||
};
|
||||
|
||||
# prepare a subset of the submodules as we only need a handful
|
||||
# and there are many many submodules total
|
||||
postPatch = (lib.concatStringsSep "\n" (lib.mapAttrsToList
|
||||
(
|
||||
path: submodule: ''
|
||||
|
|
@ -27,7 +34,7 @@ buildPythonApplication rec {
|
|||
ln -s ${submodule}/ ${path}
|
||||
''
|
||||
)
|
||||
common.submodules)) + ''
|
||||
passthru.submodulesSubset)) + ''
|
||||
cd cli
|
||||
'';
|
||||
|
||||
|
|
@ -97,10 +104,12 @@ buildPythonApplication rec {
|
|||
|
||||
passthru = {
|
||||
inherit common;
|
||||
submodulesSubset = lib.mapAttrs (k: args: fetchFromGitHub args) common.submodules;
|
||||
updateScript = ./update.sh;
|
||||
};
|
||||
|
||||
meta = common.meta // {
|
||||
description = common.meta.description + " - cli";
|
||||
inherit (semgrep-core.meta) platforms;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,52 @@
|
|||
{ lib, stdenvNoCC, callPackage }:
|
||||
{ lib, stdenvNoCC, fetchPypi, unzip }:
|
||||
|
||||
let
|
||||
common = callPackage ./common.nix { };
|
||||
common = import ./common.nix { inherit lib; };
|
||||
in
|
||||
stdenvNoCC.mkDerivation rec {
|
||||
pname = "semgrep-core";
|
||||
inherit (common) version;
|
||||
inherit (common.core) src;
|
||||
# fetch pre-built semgrep-core since the ocaml build is complex and relies on
|
||||
# the opam package manager at some point
|
||||
# pulling it out of the python wheel as r2c no longer release a built binary
|
||||
# on github releases
|
||||
src =
|
||||
let
|
||||
inherit (stdenvNoCC.hostPlatform) system;
|
||||
data = common.core.${system} or (throw "Unsupported system: ${system}");
|
||||
in
|
||||
fetchPypi rec {
|
||||
pname = "semgrep";
|
||||
inherit version;
|
||||
format = "wheel";
|
||||
dist = python;
|
||||
python = "cp37.cp38.cp39.py37.py38.py39";
|
||||
inherit (data) platform hash;
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ unzip ];
|
||||
|
||||
# _tryUnzip from unzip's setup-hook doesn't recognise .whl
|
||||
# "do not know how to unpack source archive"
|
||||
# perform unpack by hand
|
||||
unpackPhase = ''
|
||||
runHook preUnpack
|
||||
LANG=en_US.UTF-8 unzip -qq "$src"
|
||||
runHook postUnpack
|
||||
'';
|
||||
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
install -Dm 755 -t $out/bin semgrep-core
|
||||
install -Dm 755 -t $out/bin semgrep-${version}.data/purelib/semgrep/bin/semgrep-core
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
meta = common.meta // {
|
||||
description = common.meta.description + " - core binary";
|
||||
sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ];
|
||||
platforms = lib.attrNames common.core;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p curl gnused jq
|
||||
#!nix-shell -i bash -p curl gnused jq nix-prefetch
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
|
|
@ -33,7 +33,7 @@ NEW_VERSION=$(
|
|||
)
|
||||
# trim v prefix
|
||||
NEW_VERSION="${NEW_VERSION:1}"
|
||||
OLD_VERSION="$(instantiateClean semgrep.common.version)"
|
||||
OLD_VERSION="$(instantiateClean semgrep.passthru.common.version)"
|
||||
|
||||
if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then
|
||||
echo "Already up to date"
|
||||
|
|
@ -50,43 +50,54 @@ fetchgithub() {
|
|||
set -eo pipefail
|
||||
}
|
||||
|
||||
fetchzip() {
|
||||
set +eo pipefail
|
||||
nix-build -E "with import $NIXPKGS_ROOT {}; fetchzip {url = \"$1\"; sha256 = lib.fakeSha256; }" 2>&1 >/dev/null | grep "got:" | cut -d':' -f2 | sed 's| ||g'
|
||||
set -eo pipefail
|
||||
fetch_arch() {
|
||||
VERSION=$1
|
||||
PLATFORM=$2
|
||||
nix-prefetch "{ fetchPypi }:
|
||||
fetchPypi rec {
|
||||
pname = \"semgrep\";
|
||||
version = \"$VERSION\";
|
||||
format = \"wheel\";
|
||||
dist = python;
|
||||
python = \"cp37.cp38.cp39.py37.py38.py39\";
|
||||
platform = \"$PLATFORM\";
|
||||
}
|
||||
"
|
||||
}
|
||||
|
||||
replace "$OLD_VERSION" "$NEW_VERSION" "$COMMON_FILE"
|
||||
|
||||
echo "Updating src"
|
||||
|
||||
OLD_HASH="$(instantiateClean semgrep.common.src.outputHash)"
|
||||
OLD_HASH="$(instantiateClean semgrep.passthru.common.srcHash)"
|
||||
echo "Old hash $OLD_HASH"
|
||||
TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
|
||||
replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
|
||||
NEW_HASH="$(fetchgithub semgrep.common.src)"
|
||||
NEW_HASH="$(fetchgithub semgrep.src)"
|
||||
echo "New hash $NEW_HASH"
|
||||
replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"
|
||||
|
||||
echo "Updated src"
|
||||
|
||||
# loop through platforms for core
|
||||
nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.common.core.data" --eval --strict --json \
|
||||
| jq '.[]' -r \
|
||||
| while read -r PLATFORM; do
|
||||
echo "Updating core for $PLATFORM"
|
||||
SUFFIX=$(instantiateClean semgrep.common.core.data."$PLATFORM".suffix)
|
||||
OLD_HASH=$(instantiateClean semgrep.common.core.data."$PLATFORM".sha256)
|
||||
echo "Old hash $OLD_HASH"
|
||||
|
||||
NEW_URL="https://github.com/returntocorp/semgrep/releases/download/v$NEW_VERSION/semgrep-v$NEW_VERSION$SUFFIX"
|
||||
NEW_HASH="$(fetchzip "$NEW_URL")"
|
||||
echo "New hash $NEW_HASH"
|
||||
update_core_platform() {
|
||||
SYSTEM=$1
|
||||
echo "Updating core src $SYSTEM"
|
||||
|
||||
PLATFORM="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.platform")"
|
||||
|
||||
OLD_HASH="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.hash")"
|
||||
echo "Old core hash $OLD_HASH"
|
||||
NEW_HASH="$(fetch_arch "$NEW_VERSION" "$PLATFORM")"
|
||||
echo "New core hash $NEW_HASH"
|
||||
replace "$OLD_HASH" "$NEW_HASH" "$COMMON_FILE"
|
||||
|
||||
echo "Updated core for $PLATFORM"
|
||||
done
|
||||
echo "Updated core src $SYSTEM"
|
||||
}
|
||||
|
||||
update_core_platform "x86_64-linux"
|
||||
update_core_platform "x86_64-darwin"
|
||||
update_core_platform "aarch64-darwin"
|
||||
|
||||
OLD_PWD=$PWD
|
||||
TMPDIR="$(mktemp -d)"
|
||||
|
|
@ -109,7 +120,7 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas
|
|||
echo "Updating $SUBMODULE"
|
||||
OLD_REV=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".rev)
|
||||
echo "Old commit $OLD_REV"
|
||||
OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".outputHash)
|
||||
OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".hash)
|
||||
echo "Old hash $OLD_HASH"
|
||||
|
||||
NEW_REV=$(get_submodule_commit "$SUBMODULE")
|
||||
|
|
@ -120,13 +131,12 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas
|
|||
continue
|
||||
fi
|
||||
|
||||
NEW_URL=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g")
|
||||
NEW_HASH=$(nix --experimental-features nix-command hash to-sri "sha256:$(nix-prefetch-url "$NEW_URL")")
|
||||
NEW_URL=$(instantiateClean semgrep.passthru.submodulesSubset."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g")
|
||||
|
||||
TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
|
||||
replace "$OLD_REV" "$NEW_REV" "$COMMON_FILE"
|
||||
replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
|
||||
NEW_HASH="$(fetchgithub semgrep.passthru.common.submodules."$SUBMODULE")"
|
||||
NEW_HASH="$(fetchgithub semgrep.passthru.submodulesSubset."$SUBMODULE")"
|
||||
echo "New hash $NEW_HASH"
|
||||
replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue