Merge pull request #211683 from lilyinstarlight/fix/ibis-framework

python3Packages.ibis-framework: 3.2.0 -> 4.0.0

pkgs/development/python-modules/datafusion/Cargo.lock.patch

···
-diff --git a/Cargo.lock b/Cargo.lock
-index fa84a54c..3d790e1c 100644
---- a/Cargo.lock
-+++ b/Cargo.lock
-@@ -57,9 +57,9 @@ checksum = "be4dc07131ffa69b8072d35f5007352af944213cde02545e2103680baed38fcd"
- 
- [[package]]
- name = "arrow"
--version = "6.0.0"
-+version = "6.5.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "337e668497751234149fd607f5cb41a6ae7b286b6329589126fe67f0ac55d637"
-+checksum = "216c6846a292bdd93c2b93c1baab58c32ff50e2ab5e8d50db333ab518535dd8b"
- dependencies = [
-  "bitflags",
-  "chrono",
-@@ -212,9 +212,9 @@ dependencies = [
- 
- [[package]]
- name = "comfy-table"
--version = "4.1.1"
-+version = "5.0.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "11e95a3e867422fd8d04049041f5671f94d53c32a9dcd82e2be268714942f3f3"
-+checksum = "c42350b81f044f576ff88ac750419f914abb46a03831bb1747134344ee7a4e64"
- dependencies = [
-  "strum",
-  "strum_macros",
-@@ -279,7 +279,7 @@ dependencies = [
- 
- [[package]]
- name = "datafusion"
--version = "5.1.0"
-+version = "6.0.0"
- dependencies = [
-  "ahash",
-  "arrow",
-@@ -310,7 +310,7 @@ dependencies = [
- 
- [[package]]
- name = "datafusion-python"
--version = "0.3.0"
-+version = "0.4.0"
- dependencies = [
-  "datafusion",
-  "pyo3",
-@@ -877,9 +877,9 @@ dependencies = [
- 
- [[package]]
- name = "parquet"
--version = "6.0.0"
-+version = "6.5.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "d263b9b59ba260518de9e57bd65931c3f765fea0fabacfe84f40d6fde38e841a"
-+checksum = "788d9953f4cfbe9db1beff7bebd54299d105e34680d78b82b1ddc85d432cac9d"
- dependencies = [
-  "arrow",
-  "base64",
-@@ -1228,15 +1228,15 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
- 
- [[package]]
- name = "strum"
--version = "0.21.0"
-+version = "0.22.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2"
-+checksum = "f7ac893c7d471c8a21f31cfe213ec4f6d9afeed25537c772e08ef3f005f8729e"
- 
- [[package]]
- name = "strum_macros"
--version = "0.21.1"
-+version = "0.22.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "d06aaeeee809dbc59eb4556183dd927df67db1540de5be8d3ec0b6636358a5ec"
-+checksum = "339f799d8b549e3744c7ac7feb216383e4005d94bdb22561b3ab8f3b808ae9fb"
- dependencies = [
-  "heck",
-  "proc-macro2",

pkgs/development/python-modules/datafusion/default.nix

···
 , fetchPypi
 , fetchFromGitHub
 , rustPlatform
-, maturin
 , pytestCheckHook
 , libiconv
 , numpy
 , pandas
 , pyarrow
-, pytest
 }:
+
 let
-  # le sigh, the perils of unrelated versions of software living in the same
-  # repo: there's no obvious way to map the top level source repo
-  # (arrow-datafusion) version to the version of contained repo
-  # (arrow-datafusion/python)
-  #
-  # A commit hash will do in a pinch, and ultimately the sha256 has the final
-  # say of what the content is when building
-  cargoLock = fetchurl {
-    url = "https://raw.githubusercontent.com/apache/arrow-datafusion/6.0.0/python/Cargo.lock";
-    sha256 = "sha256-xiv3drEU5jOGsEIh0U01ZQ1NBKobxO2ctp4mxy9iigw=";
+  arrow-testing = fetchFromGitHub {
+    owner = "apache";
+    repo = "arrow-testing";
+    rev = "5bab2f264a23f5af68f69ea93d24ef1e8e77fc88";
+    hash = "sha256-Pxx8ohUpXb5u1995IvXmxQMqWiDJ+7LAll/AjQP7ph8=";
   };
 
-  postUnpack = ''
-    cp "${cargoLock}" $sourceRoot/Cargo.lock
-    chmod u+w $sourceRoot/Cargo.lock
-  '';
+  parquet-testing = fetchFromGitHub {
+    owner = "apache";
+    repo = "parquet-testing";
+    rev = "5b82793ef7196f7b3583e85669ced211cd8b5ff2";
+    hash = "sha256-gcOvk7qFHZgJWE9CpucC8zwayYw47VbC3lmSRu4JQFg=";
+  };
 in
+
 buildPythonPackage rec {
   pname = "datafusion";
-  version = "0.4.0";
+  version = "0.7.0";
   format = "pyproject";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-+YqogteKfNhtI2QbVXv/5CIWm3PcOH653dwONm5ZcL8=";
+    sha256 = "sha256-XYXZMorPs2Ue7E38DASd4rmxvX0wlx8A6sCpAbYUh4I=";
   };
 
-  inherit postUnpack;
-
-  # TODO: remove the patch hacking and postUnpack hooks after
-  # https://github.com/apache/arrow-datafusion/pull/1508 is merged
-  #
-  # the lock file isn't up to date as of 6.0.0 so we need to patch the source
-  # lockfile and the vendored cargo deps lockfile
-  patches = [ ./Cargo.lock.patch ];
   cargoDeps = rustPlatform.fetchCargoTarball {
-    inherit src pname version postUnpack;
-    sha256 = "sha256-JGyDxpfBXzduJaMF1sbmRm7KJajHYdVSj+WbiSETiY0=";
-    patches = [ ./Cargo.lock.patch ];
+    inherit src pname version;
+    sha256 = "sha256-6mPdKwsEN09Gf4eNsd/v3EBHVezHmff/KYB2lsXgzcA=";
   };
 
   nativeBuildInputs = with rustPlatform; [
···
     pyarrow
   ];
 
-  nativeCheckInputs = [ pytest ];
+  nativeCheckInputs = [ pytestCheckHook ];
   pythonImportsCheck = [ "datafusion" ];
+  pytestFlagsArray = [ "--pyargs" pname ];
 
-  checkPhase = ''
-    runHook preCheck
-    pytest --pyargs "${pname}"
-    runHook postCheck
+  preCheck = ''
+    pushd $TMPDIR
+    ln -s ${arrow-testing} ./testing
+    ln -s ${parquet-testing} ./parquet
+  '';
+
+  postCheck = ''
+    popd
   '';
 
   meta = with lib; {

pkgs/development/python-modules/ibis-framework/default.nix

···
 , geoalchemy2
 , geopandas
 , graphviz-nox
+, hypothesis
 , lz4
 , multipledispatch
 , numpy
···
 , pymysql
 , pyspark
 , pytest-benchmark
+, pytest-mock
 , pytest-randomly
-, pytest-mock
+, pytest-snapshot
 , pytest-xdist
 , python
 , pytz
···
 
 buildPythonPackage rec {
   pname = "ibis-framework";
-  version = "3.2.0";
+  version = "4.0.0";
   format = "pyproject";
 
   disabled = pythonOlder "3.8";
···
     repo = "ibis";
     owner = "ibis-project";
     rev = version;
-    hash = "sha256-YRP1nGJs4btqXQirm0GfEDKNPCVXexVrwQ6sE8JtD2o=";
+    hash = "sha256-G3kMd6Jyib7ZXHFP6t2CEPlDD5n5zHE2jq/0he3U4Nk=";
   };
 
-  nativeBuildInputs = [ poetry-core ];
+  nativeBuildInputs = [
+    poetry-core
+  ];
 
   propagatedBuildInputs = [
     atpublic
···
     pytestCheckHook
     click
     filelock
+    hypothesis
     pytest-benchmark
     pytest-mock
     pytest-randomly
+    pytest-snapshot
     pytest-xdist
     rsync
   ] ++ lib.concatMap (name: passthru.optional-dependencies.${name}) testBackends;
 
-  preBuild = ''
-    # setup.py exists only for developer convenience and is automatically generated
-    # it gets in the way in nixpkgs so we remove it
-    rm setup.py
-  '';
-
   pytestFlagsArray = [
     "--dist=loadgroup"
     "-m"
     "'${lib.concatStringsSep " or " testBackends} or core'"
-    # this test fails on nixpkgs datafusion version (0.4.0), but works on
-    # datafusion 0.6.0
-    "-k"
-    "'not datafusion-no_op'"
   ];
+
+  # remove when sqlalchemy backend no longer uses deprecated methods
+  SQLALCHEMY_SILENCE_UBER_WARNING = 1;
+
+  # patch out tests that check formatting with black
+  postPatch = ''
+    find ibis/tests -type f -name '*.py' -exec sed -i \
+      -e '/^ *assert_decompile_roundtrip/d' \
+      -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
+  '';
 
   preCheck = ''
     set -eo pipefail

pkgs/development/python-modules/pyarrow/default.nix

···
 , stdenv
 , buildPythonPackage
 , python
+, pythonAtLeast
 , pythonOlder
 , arrow-cpp
 , cffi
···
     "--deselect=pyarrow/tests/test_flight.py::test_large_descriptor"
     "--deselect=pyarrow/tests/test_flight.py::test_large_metadata_client"
     "--deselect=pyarrow/tests/test_flight.py::test_none_action_side_effect"
+  ] ++ lib.optionals (pythonAtLeast "3.11") [
+    # Repr output is printing number instead of enum name so these tests fail
+    "--deselect=pyarrow/tests/test_fs.py::test_get_file_info"
   ];
 
   dontUseSetuptoolsCheck = true;

pkgs/development/python-modules/sqlglot/default.nix

···
 , pytestCheckHook
 , python-dateutil
 , duckdb
+, pyspark
 }:
 buildPythonPackage rec {
   pname = "sqlglot";
-  version = "6.0.7";
+  version = "10.5.2";
 
   disabled = pythonOlder "3.8";
 
···
     repo = "sqlglot";
     owner = "tobymao";
     rev = "v${version}";
-    hash = "sha256-7PBhf9NN/mCi92xSkB9ygfmfxTyOYaEyrNvL309sG5Y=";
+    hash = "sha256-ZFc2aOhCTRFlrzgnYDSdIZxRqKZ8FvkYSZRU0OMHI34=";
   };
 
   propagatedBuildInputs = [ python-dateutil ];
 
-  nativeCheckInputs = [ pytestCheckHook duckdb ];
+  nativeCheckInputs = [ pytestCheckHook duckdb pyspark ];
+
+  # these integration tests assume a running Spark instance
+  disabledTestPaths = [ "tests/dataframe/integration" ];
 
   pythonImportsCheck = [ "sqlglot" ];