cafkafk/nixpkgs
cafkafk/nixpkgs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Merge pull request #235226 from natsukium/catboost/update

Browse source 
python3Packages.catboost: 1.0.5 -> 1.2.2; build with cmake
This commit is contained in:
OTABI Tomoya 2023-10-16 14:09:28 +09:00 committed by GitHub
commit 5b87022c29
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 199 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
pkgs/development/libraries/catboost/default.nix Normal file
View file

@ -0,0 +1,113 @@
{ lib
, config
, stdenv
, fetchFromGitHub
, cmake
, libiconv
, llvmPackages
, ninja
, openssl
, python3Packages
, ragel
, yasm
, zlib
, cudaSupport ? config.cudaSupport
, cudaPackages ? {}
, pythonSupport ? false
}:
stdenv.mkDerivation (finalAttrs: {
pname = "catboost";
version = "1.2.2";
src = fetchFromGitHub {
owner = "catboost";
repo = "catboost";
rev = "refs/tags/v${finalAttrs.version}";
hash = "sha256-A1zCIqPOW21dHKBQHRtS+/sstZ2o6F8k71lmJFGn0+g=";
};
patches = [
./remove-conan.patch
];
postPatch = ''
substituteInPlace cmake/common.cmake \
--replace "\''${RAGEL_BIN}" "${ragel}/bin/ragel" \
--replace "\''${YASM_BIN}" "${yasm}/bin/yasm"
shopt -s globstar
for cmakelists in **/CMakeLists.*; do
sed -i "s/OpenSSL::OpenSSL/OpenSSL::SSL/g" $cmakelists
${lib.optionalString (lib.versionOlder cudaPackages.cudaVersion "11.8") ''
sed -i 's/-gencode=arch=compute_89,code=sm_89//g' $cmakelists
sed -i 's/-gencode=arch=compute_90,code=sm_90//g' $cmakelists
''}
done
'';
outputs = [ "out" "dev" ];
nativeBuildInputs = [
cmake
llvmPackages.bintools
ninja
(python3Packages.python.withPackages (ps: with ps; [ six ]))
ragel
yasm
] ++ lib.optionals cudaSupport (with cudaPackages; [
cuda_nvcc
]);
buildInputs = [
openssl
zlib
] ++ lib.optionals stdenv.isDarwin [
libiconv
] ++ lib.optionals cudaSupport (with cudaPackages; [
cuda_cudart
cuda_cccl
libcublas
]);
env = {
CUDAHOSTCXX = lib.optionalString cudaSupport "${stdenv.cc}/bin/cc";
NIX_CFLAGS_LINK = lib.optionalString stdenv.isLinux "-fuse-ld=lld";
NIX_LDFLAGS = "-lc -lm";
};
cmakeFlags = [
"-DCMAKE_BINARY_DIR=$out"
"-DCMAKE_POSITION_INDEPENDENT_CODE=on"
"-DCATBOOST_COMPONENTS=app;libs${lib.optionalString pythonSupport ";python-package"}"
] ++ lib.optionals cudaSupport [
"-DHAVE_CUDA=on"
];
installPhase = ''
runHook preInstall
mkdir $dev
cp -r catboost $dev
install -Dm555 catboost/app/catboost -t $out/bin
install -Dm444 catboost/libs/model_interface/static/lib/libmodel_interface-static-lib.a -t $out/lib
install -Dm444 catboost/libs/model_interface/libcatboostmodel${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib
install -Dm444 catboost/libs/train_interface/libcatboost${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib
runHook postInstall
'';
meta = with lib; {
description = "High-performance library for gradient boosting on decision trees";
longDescription = ''
A fast, scalable, high performance Gradient Boosting on Decision Trees
library, used for ranking, classification, regression and other machine
learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU.
'';
license = licenses.asl20;
platforms = platforms.unix;
homepage = "https://catboost.ai";
maintainers = with maintainers; [ PlushBeaver natsukium ];
mainProgram = "catboost";
};
})

34
pkgs/development/libraries/catboost/remove-conan.patch Normal file
View file

@ -0,0 +1,34 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index becd2ad03c..7e3c8c99b1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -27,7 +27,6 @@ cmake_policy(SET CMP0104 OLD)
include(cmake/archive.cmake)
include(cmake/common.cmake)
-include(cmake/conan.cmake)
include(cmake/cuda.cmake)
include(cmake/cython.cmake)
include(cmake/fbs.cmake)
@@ -37,21 +36,6 @@ include(cmake/recursive_library.cmake)
include(cmake/swig.cmake)
include(cmake/global_vars.cmake)
-if (CMAKE_CROSSCOMPILING)
- include(${CMAKE_BINARY_DIR}/conan_paths.cmake)
-else()
- conan_cmake_autodetect(settings)
- conan_cmake_install(
- PATH_OR_REFERENCE ${CMAKE_SOURCE_DIR}
- INSTALL_FOLDER ${CMAKE_BINARY_DIR}
- BUILD missing
- REMOTE conancenter
- SETTINGS ${settings}
- ENV "CONAN_CMAKE_GENERATOR=${CMAKE_GENERATOR}"
- CONF "tools.cmake.cmaketoolchain:generator=${CMAKE_GENERATOR}"
- )
-endif()
-
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
include(CMakeLists.linux-x86_64.txt)
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND HAVE_CUDA)

94
pkgs/development/python-modules/catboost/default.nix
View file

@ -1,64 +1,50 @@
{ buildPythonPackage, fetchFromGitHub, lib, pythonOlder
, clang_12, python
, graphviz, matplotlib, numpy, pandas, plotly, scipy, six
, withCuda ? false, cudatoolkit }:
{ lib
, buildPythonPackage
, catboost
, python
, graphviz
, matplotlib
, numpy
, pandas
, plotly
, scipy
, setuptools
, six
, wheel
}:
buildPythonPackage rec {
pname = "catboost";
# nixpkgs-update: no auto update
version = "1.0.5";
buildPythonPackage {
inherit (catboost) pname version src meta;
format = "pyproject";
disabled = pythonOlder "3.4";
sourceRoot = "source/catboost/python-package";
src = fetchFromGitHub {
owner = "catboost";
repo = "catboost";
rev = "refs/tags/v${version}";
hash = "sha256-ILemeZUBI9jPb9G6F7QX/T1HaVhQ+g6y7YmsT6DFCJk";
};
nativeBuildInputs = [ clang_12 ];
propagatedBuildInputs = [ graphviz matplotlib numpy pandas scipy plotly six ]
++ lib.optionals withCuda [ cudatoolkit ];
patches = [
./nix-support.patch
nativeBuildInputs = [
setuptools
wheel
];
postPatch = ''
# substituteInPlace is too slow for these large files, and the target has lots of numbers in it that change often.
sed -e 's|\$(YMAKE_PYTHON3-.*)/python3|${python.interpreter}|' -i make/*.makefile
propagatedBuildInputs = [
graphviz
matplotlib
numpy
pandas
plotly
scipy
six
];
buildPhase = ''
runHook preBuild
# these arguments must set after bdist_wheel
${python.pythonForBuild.interpreter} setup.py bdist_wheel --no-widget --prebuilt-extensions-build-root-dir=${lib.getDev catboost}
runHook postBuild
'';
preBuild = ''
cd catboost/python-package
'';
setupPyBuildFlags = [ "--with-ymake=no" ];
CUDA_ROOT = lib.optional withCuda cudatoolkit;
enableParallelBuilding = true;
# setup a test is difficult
doCheck = false;
# Tests use custom "ya" tool, not yet supported.
dontUseSetuptoolsCheck = true;
pythonImportsCheck = [ "catboost" ];
passthru = {
# Do not update to catboost 1.1.x because the patch doesn't apply cleanly
skipBulkUpdate = true;
};
meta = with lib; {
description = "High-performance library for gradient boosting on decision trees.";
longDescription = ''
A fast, scalable, high performance Gradient Boosting on Decision Trees
library, used for ranking, classification, regression and other machine
learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU.
'';
license = licenses.asl20;
platforms = [ "x86_64-linux" ];
homepage = "https://catboost.ai";
maintainers = with maintainers; [ PlushBeaver ];
# _catboost.pyx.cpp:226822:19: error: use of undeclared identifier '_PyGen_Send'
broken = withCuda;
};
}

173
pkgs/development/python-modules/catboost/nix-support.patch
View file

@ -1,173 +0,0 @@
diff --git a/catboost/python-package/setup.py b/catboost/python-package/setup.py
index fe9251a21f..86b880c5d0 100644
--- a/catboost/python-package/setup.py
+++ b/catboost/python-package/setup.py
@@ -80,7 +80,7 @@ class Helper(object):
self.with_cuda = os.environ.get('CUDA_PATH') or os.environ.get('CUDA_ROOT') or None
self.os_sdk = 'local'
self.with_ymake = True
- self.parallel = None
+ self.parallel = os.environ.get('NIX_BUILD_CORES') or None
def finalize_options(self):
if os.path.exists(str(self.with_cuda)):
@@ -222,11 +222,12 @@ class build_ext(_build_ext):
def build_with_make(self, topsrc_dir, build_dir, catboost_ext, put_dir, verbose, dry_run):
logging.info('Buildling {} with gnu make'.format(catboost_ext))
- makefile = 'python{}.{}CLANG11-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '')
+ makefile = 'python{}.{}CLANG12-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '')
make_cmd = [
'make', '-f', '../../make/' + makefile,
- 'CC=clang-11',
- 'CXX=clang++-11',
+ 'CC=clang',
+ 'CXX=clang++',
+ 'PYTHON=python{}'.format(python_version()[0]),
'BUILD_ROOT=' + build_dir,
'SOURCE_ROOT=' + topsrc_dir,
]
diff --git a/make/python2.CLANG12-LINUX-X86_64.makefile b/make/python2.CLANG12-LINUX-X86_64.makefile
index b49a36fb3f..33996af995 100644
--- a/make/python2.CLANG12-LINUX-X86_64.makefile
+++ b/make/python2.CLANG12-LINUX-X86_64.makefile
@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
SOURCE_ROOT = $(shell pwd)
PYTHON = $(shell which python)
-ifneq ($(MAKECMDGOALS),help)
-define _CC_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
-$(info _CC_VERSION = '$(_CC_VERSION)')
-
-ifneq '$(_CC_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
-
-ifneq ($(MAKECMDGOALS),help)
-define _CXX_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
-$(info _CXX_VERSION = '$(_CXX_VERSION)')
-
-ifneq '$(_CXX_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
all\
diff --git a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile
index 82935b297e..093cc86532 100644
--- a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile
+++ b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile
@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
SOURCE_ROOT = $(shell pwd)
PYTHON = $(shell which python)
-ifneq ($(MAKECMDGOALS),help)
-define _CC_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
-$(info _CC_VERSION = '$(_CC_VERSION)')
-
-ifneq '$(_CC_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
-
-ifneq ($(MAKECMDGOALS),help)
-define _CXX_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
-$(info _CXX_VERSION = '$(_CXX_VERSION)')
-
-ifneq '$(_CXX_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
all\
diff --git a/make/python3.CLANG12-LINUX-X86_64.makefile b/make/python3.CLANG12-LINUX-X86_64.makefile
index 1c5d646ae4..6c091fbe17 100644
--- a/make/python3.CLANG12-LINUX-X86_64.makefile
+++ b/make/python3.CLANG12-LINUX-X86_64.makefile
@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
SOURCE_ROOT = $(shell pwd)
PYTHON = $(shell which python)
-ifneq ($(MAKECMDGOALS),help)
-define _CC_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
-$(info _CC_VERSION = '$(_CC_VERSION)')
-
-ifneq '$(_CC_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
-
-ifneq ($(MAKECMDGOALS),help)
-define _CXX_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
-$(info _CXX_VERSION = '$(_CXX_VERSION)')
-
-ifneq '$(_CXX_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
all\
diff --git a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile
index fcdb75a719..4e1dbc3cd7 100644
--- a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile
+++ b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile
@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
SOURCE_ROOT = $(shell pwd)
PYTHON = $(shell which python)
-ifneq ($(MAKECMDGOALS),help)
-define _CC_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
-$(info _CC_VERSION = '$(_CC_VERSION)')
-
-ifneq '$(_CC_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
-
-ifneq ($(MAKECMDGOALS),help)
-define _CXX_TEST
-__clang_major__ __clang_minor__
-endef
-
-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
-$(info _CXX_VERSION = '$(_CXX_VERSION)')
-
-ifneq '$(_CXX_VERSION)' '12 0'
- $(error clang 12.0 is required)
-endif
-endif
all\

6
pkgs/top-level/all-packages.nix
View file

@ -20755,6 +20755,12 @@ with pkgs;
captive-browser = callPackage ../applications/networking/browsers/captive-browser { };
catboost = callPackage ../development/libraries/catboost {
# catboost requires clang 12+ for build
# after bumping the default version of llvm, check for compatibility with the cuda backend and pin it.
inherit (llvmPackages_12) stdenv;
};
ndn-cxx = callPackage ../development/libraries/ndn-cxx { };
ndn-tools = callPackage ../tools/networking/ndn-tools { };

7
pkgs/top-level/python-packages.nix
View file

@ -1803,7 +1803,12 @@ self: super: with self; {
catalogue = callPackage ../development/python-modules/catalogue { };
catboost = callPackage ../development/python-modules/catboost { };
catboost = callPackage ../development/python-modules/catboost {
catboost = pkgs.catboost.override {
pythonSupport = true;
python3Packages = self;
};
};
catppuccin = callPackage ../development/python-modules/catppuccin { };