Merge pull request #235226 from natsukium/catboost/update

python3Packages.catboost: 1.0.5 -> 1.2.2; build with cmake

pkgs/development/libraries/catboost/default.nix

@@ -0,0 +1,113 @@
+{ lib
+, config
+, stdenv
+, fetchFromGitHub
+, cmake
+, libiconv
+, llvmPackages
+, ninja
+, openssl
+, python3Packages
+, ragel
+, yasm
+, zlib
+, cudaSupport ? config.cudaSupport
+, cudaPackages ? {}
+, pythonSupport ? false
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "catboost";
+  version = "1.2.2";
+
+  src = fetchFromGitHub {
+    owner = "catboost";
+    repo = "catboost";
+    rev = "refs/tags/v${finalAttrs.version}";
+    hash = "sha256-A1zCIqPOW21dHKBQHRtS+/sstZ2o6F8k71lmJFGn0+g=";
+  };
+
+  patches = [
+    ./remove-conan.patch
+  ];
+
+  postPatch = ''
+    substituteInPlace cmake/common.cmake \
+      --replace  "\''${RAGEL_BIN}" "${ragel}/bin/ragel" \
+      --replace "\''${YASM_BIN}" "${yasm}/bin/yasm"
+
+    shopt -s globstar
+    for cmakelists in **/CMakeLists.*; do
+      sed -i "s/OpenSSL::OpenSSL/OpenSSL::SSL/g" $cmakelists
+      ${lib.optionalString (lib.versionOlder cudaPackages.cudaVersion "11.8") ''
+        sed -i 's/-gencode=arch=compute_89,code=sm_89//g' $cmakelists
+        sed -i 's/-gencode=arch=compute_90,code=sm_90//g' $cmakelists
+      ''}
+    done
+  '';
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    cmake
+    llvmPackages.bintools
+    ninja
+    (python3Packages.python.withPackages (ps: with ps; [ six ]))
+    ragel
+    yasm
+  ] ++ lib.optionals cudaSupport (with cudaPackages; [
+    cuda_nvcc
+  ]);
+
+  buildInputs = [
+    openssl
+    zlib
+  ] ++ lib.optionals stdenv.isDarwin [
+    libiconv
+  ] ++ lib.optionals cudaSupport (with cudaPackages; [
+    cuda_cudart
+    cuda_cccl
+    libcublas
+  ]);
+
+  env = {
+    CUDAHOSTCXX = lib.optionalString cudaSupport "${stdenv.cc}/bin/cc";
+    NIX_CFLAGS_LINK = lib.optionalString stdenv.isLinux "-fuse-ld=lld";
+    NIX_LDFLAGS = "-lc -lm";
+  };
+
+  cmakeFlags = [
+    "-DCMAKE_BINARY_DIR=$out"
+    "-DCMAKE_POSITION_INDEPENDENT_CODE=on"
+    "-DCATBOOST_COMPONENTS=app;libs${lib.optionalString pythonSupport ";python-package"}"
+  ] ++ lib.optionals cudaSupport [
+    "-DHAVE_CUDA=on"
+  ];
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir $dev
+    cp -r catboost $dev
+    install -Dm555 catboost/app/catboost -t $out/bin
+    install -Dm444 catboost/libs/model_interface/static/lib/libmodel_interface-static-lib.a -t $out/lib
+    install -Dm444 catboost/libs/model_interface/libcatboostmodel${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib
+    install -Dm444 catboost/libs/train_interface/libcatboost${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib
+
+    runHook postInstall
+  '';
+
+  meta = with lib; {
+    description = "High-performance library for gradient boosting on decision trees";
+    longDescription = ''
+      A fast, scalable, high performance Gradient Boosting on Decision Trees
+      library, used for ranking, classification, regression and other machine
+      learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU.
+    '';
+    license = licenses.asl20;
+    platforms = platforms.unix;
+    homepage = "https://catboost.ai";
+    maintainers = with maintainers; [ PlushBeaver natsukium ];
+    mainProgram = "catboost";
+  };
+})

pkgs/development/libraries/catboost/remove-conan.patch

@@ -0,0 +1,34 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index becd2ad03c..7e3c8c99b1 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -27,7 +27,6 @@ cmake_policy(SET CMP0104 OLD)
+ 
+ include(cmake/archive.cmake)
+ include(cmake/common.cmake)
+-include(cmake/conan.cmake)
+ include(cmake/cuda.cmake)
+ include(cmake/cython.cmake)
+ include(cmake/fbs.cmake)
+@@ -37,21 +36,6 @@ include(cmake/recursive_library.cmake)
+ include(cmake/swig.cmake)
+ include(cmake/global_vars.cmake)
+ 
+-if (CMAKE_CROSSCOMPILING)
+-  include(${CMAKE_BINARY_DIR}/conan_paths.cmake)
+-else()
+-  conan_cmake_autodetect(settings)
+-  conan_cmake_install(
+-    PATH_OR_REFERENCE ${CMAKE_SOURCE_DIR}
+-    INSTALL_FOLDER ${CMAKE_BINARY_DIR}
+-    BUILD missing
+-    REMOTE conancenter
+-    SETTINGS ${settings}
+-      ENV "CONAN_CMAKE_GENERATOR=${CMAKE_GENERATOR}"
+-      CONF "tools.cmake.cmaketoolchain:generator=${CMAKE_GENERATOR}"
+-  )
+-endif()
+-
+ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+   include(CMakeLists.linux-x86_64.txt)
+ elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND HAVE_CUDA)

pkgs/development/python-modules/catboost/default.nix

@@ -1,64 +1,50 @@
-{ buildPythonPackage, fetchFromGitHub, lib, pythonOlder
+{ lib
-, clang_12, python
+, buildPythonPackage
-, graphviz, matplotlib, numpy, pandas, plotly, scipy, six
+, catboost
-, withCuda ? false, cudatoolkit }:
+, python
+, graphviz
+, matplotlib
+, numpy
+, pandas
+, plotly
+, scipy
+, setuptools
+, six
+, wheel
+}:
 
-buildPythonPackage rec {
+buildPythonPackage {
-  pname = "catboost";
+  inherit (catboost) pname version src meta;
-  # nixpkgs-update: no auto update
+  format = "pyproject";
-  version = "1.0.5";
 
-  disabled = pythonOlder "3.4";
+  sourceRoot = "source/catboost/python-package";
 
-  src = fetchFromGitHub {
+  nativeBuildInputs = [
-    owner = "catboost";
+    setuptools
-    repo = "catboost";
+    wheel
-    rev = "refs/tags/v${version}";
-    hash = "sha256-ILemeZUBI9jPb9G6F7QX/T1HaVhQ+g6y7YmsT6DFCJk";
-  };
-
-  nativeBuildInputs = [ clang_12 ];
-
-  propagatedBuildInputs = [ graphviz matplotlib numpy pandas scipy plotly six ]
-    ++ lib.optionals withCuda [ cudatoolkit ];
-
-  patches = [
-    ./nix-support.patch
   ];
 
-  postPatch = ''
+  propagatedBuildInputs = [
-    # substituteInPlace is too slow for these large files, and the target has lots of numbers in it that change often.
+    graphviz
-    sed -e 's|\$(YMAKE_PYTHON3-.*)/python3|${python.interpreter}|' -i make/*.makefile
+    matplotlib
+    numpy
+    pandas
+    plotly
+    scipy
+    six
+  ];
+
+  buildPhase = ''
+    runHook preBuild
+
+    # these arguments must set after bdist_wheel
+    ${python.pythonForBuild.interpreter} setup.py bdist_wheel --no-widget --prebuilt-extensions-build-root-dir=${lib.getDev catboost}
+
+    runHook postBuild
   '';
 
-  preBuild = ''
+  # setup a test is difficult
-    cd catboost/python-package
+  doCheck = false;
-  '';
-  setupPyBuildFlags = [ "--with-ymake=no" ];
-  CUDA_ROOT = lib.optional withCuda cudatoolkit;
-  enableParallelBuilding = true;
 
-  # Tests use custom "ya" tool, not yet supported.
-  dontUseSetuptoolsCheck = true;
   pythonImportsCheck = [ "catboost" ];
-
-  passthru = {
-    # Do not update to catboost 1.1.x because the patch doesn't apply cleanly
-    skipBulkUpdate = true;
-  };
-
-  meta = with lib; {
-    description = "High-performance library for gradient boosting on decision trees.";
-    longDescription = ''
-      A fast, scalable, high performance Gradient Boosting on Decision Trees
-      library, used for ranking, classification, regression and other machine
-      learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU.
-    '';
-    license = licenses.asl20;
-    platforms = [ "x86_64-linux" ];
-    homepage = "https://catboost.ai";
-    maintainers = with maintainers; [ PlushBeaver ];
-    # _catboost.pyx.cpp:226822:19: error: use of undeclared identifier '_PyGen_Send'
-    broken = withCuda;
-  };
 }

pkgs/development/python-modules/catboost/nix-support.patch

@@ -1,173 +0,0 @@
-diff --git a/catboost/python-package/setup.py b/catboost/python-package/setup.py
-index fe9251a21f..86b880c5d0 100644
---- a/catboost/python-package/setup.py
-+++ b/catboost/python-package/setup.py
-@@ -80,7 +80,7 @@ class Helper(object):
-         self.with_cuda = os.environ.get('CUDA_PATH') or os.environ.get('CUDA_ROOT') or None
-         self.os_sdk = 'local'
-         self.with_ymake = True
--        self.parallel = None
-+        self.parallel = os.environ.get('NIX_BUILD_CORES') or None
- 
-     def finalize_options(self):
-         if os.path.exists(str(self.with_cuda)):
-@@ -222,11 +222,12 @@ class build_ext(_build_ext):
- 
-     def build_with_make(self, topsrc_dir, build_dir, catboost_ext, put_dir, verbose, dry_run):
-         logging.info('Buildling {} with gnu make'.format(catboost_ext))
--        makefile = 'python{}.{}CLANG11-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '')
-+        makefile = 'python{}.{}CLANG12-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '')
-         make_cmd = [
-             'make', '-f', '../../make/' + makefile,
--            'CC=clang-11',
--            'CXX=clang++-11',
-+            'CC=clang',
-+            'CXX=clang++',
-+            'PYTHON=python{}'.format(python_version()[0]),
-             'BUILD_ROOT=' + build_dir,
-             'SOURCE_ROOT=' + topsrc_dir,
-         ]
-diff --git a/make/python2.CLANG12-LINUX-X86_64.makefile b/make/python2.CLANG12-LINUX-X86_64.makefile
-index b49a36fb3f..33996af995 100644
---- a/make/python2.CLANG12-LINUX-X86_64.makefile
-+++ b/make/python2.CLANG12-LINUX-X86_64.makefile
-@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
- SOURCE_ROOT = $(shell pwd)
- PYTHON = $(shell which python)
- 
--ifneq ($(MAKECMDGOALS),help)
--define _CC_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
--$(info _CC_VERSION = '$(_CC_VERSION)')
--
--ifneq '$(_CC_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
--
--ifneq ($(MAKECMDGOALS),help)
--define _CXX_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
--$(info _CXX_VERSION = '$(_CXX_VERSION)')
--
--ifneq '$(_CXX_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
- 
- 
- all\
-diff --git a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile
-index 82935b297e..093cc86532 100644
---- a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile
-+++ b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile
-@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
- SOURCE_ROOT = $(shell pwd)
- PYTHON = $(shell which python)
- 
--ifneq ($(MAKECMDGOALS),help)
--define _CC_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
--$(info _CC_VERSION = '$(_CC_VERSION)')
--
--ifneq '$(_CC_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
--
--ifneq ($(MAKECMDGOALS),help)
--define _CXX_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
--$(info _CXX_VERSION = '$(_CXX_VERSION)')
--
--ifneq '$(_CXX_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
- 
- 
- all\
-diff --git a/make/python3.CLANG12-LINUX-X86_64.makefile b/make/python3.CLANG12-LINUX-X86_64.makefile
-index 1c5d646ae4..6c091fbe17 100644
---- a/make/python3.CLANG12-LINUX-X86_64.makefile
-+++ b/make/python3.CLANG12-LINUX-X86_64.makefile
-@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
- SOURCE_ROOT = $(shell pwd)
- PYTHON = $(shell which python)
- 
--ifneq ($(MAKECMDGOALS),help)
--define _CC_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
--$(info _CC_VERSION = '$(_CC_VERSION)')
--
--ifneq '$(_CC_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
--
--ifneq ($(MAKECMDGOALS),help)
--define _CXX_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
--$(info _CXX_VERSION = '$(_CXX_VERSION)')
--
--ifneq '$(_CXX_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
- 
- 
- all\
-diff --git a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile
-index fcdb75a719..4e1dbc3cd7 100644
---- a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile
-+++ b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile
-@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)
- SOURCE_ROOT = $(shell pwd)
- PYTHON = $(shell which python)
- 
--ifneq ($(MAKECMDGOALS),help)
--define _CC_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)
--$(info _CC_VERSION = '$(_CC_VERSION)')
--
--ifneq '$(_CC_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
--
--ifneq ($(MAKECMDGOALS),help)
--define _CXX_TEST
--__clang_major__ __clang_minor__
--endef
--
--_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)
--$(info _CXX_VERSION = '$(_CXX_VERSION)')
--
--ifneq '$(_CXX_VERSION)' '12 0'
--    $(error clang 12.0 is required)
--endif
--endif
- 
- 
- all\

pkgs/top-level/all-packages.nix

@@ -20755,6 +20755,12 @@ with pkgs;
 
   captive-browser = callPackage ../applications/networking/browsers/captive-browser { };
 
+  catboost = callPackage ../development/libraries/catboost {
+    # catboost requires clang 12+ for build
+    # after bumping the default version of llvm, check for compatibility with the cuda backend and pin it.
+    inherit (llvmPackages_12) stdenv;
+  };
+
   ndn-cxx = callPackage ../development/libraries/ndn-cxx { };
 
   ndn-tools = callPackage ../tools/networking/ndn-tools { };

pkgs/top-level/python-packages.nix

@@ -1803,7 +1803,12 @@ self: super: with self; {
 
   catalogue = callPackage ../development/python-modules/catalogue { };
 
-  catboost = callPackage ../development/python-modules/catboost { };
+  catboost = callPackage ../development/python-modules/catboost {
+    catboost = pkgs.catboost.override {
+      pythonSupport = true;
+      python3Packages = self;
+    };
+  };
 
   catppuccin = callPackage ../development/python-modules/catppuccin { };