python311Packages.tensorflow-datasets: 4.9.7 -> 4.9.8 (#376040)

pkgs/development/python-modules/array-record/default.nix

···
   fetchPypi,
   absl-py,
   etils,
-  importlib-resources,
-  typing-extensions,
-  zipp,
 }:
 
 buildPythonPackage rec {
   pname = "array-record";
-  version = "0.5.0";
+  version = "0.7.1";
   format = "wheel";
 
-  # As of 2023-10-31, PyPI includes wheels for Python 3.9, 3.10, and 3.11.
-  disabled = pythonOlder "3.9" || pythonAtLeast "3.12";
+  disabled = pythonOlder "3.10" || pythonAtLeast "3.13";
 
   src =
     let
···
       platform = "manylinux_2_17_x86_64.manylinux2014_x86_64";
       hash =
         {
-          cp39 = "sha256-BzMOVue7E1S1+5+XTcPELko81ujc9MbmqLhNsU7pqO0=";
-          cp310 = "sha256-eUD9pQu9GsbV8MPD1MiF3Ihr+zYioSOo6P15hYIwPYo=";
-          cp311 = "sha256-rAmkI3EIZPYiXrxFowfDC0Gf3kRw0uX0i6Kx6Zu+hNM=";
+          cp310 = "sha256-JDaj1iJy1BQ7fHjmCbGQkNqG5rIRuwTwENbanM9a8hg=";
+          cp311 = "sha256-QVynMK9t0BnEtgdfbJ5T3s7N02i0XD2siUSRxKtrI+M=";
+          cp312 = "sha256-xJJGm6kLQ2/TzVYTrBtQ1Hqky1odHfbhe/g+PSSYt1c=";
         }
         .${pyShortVersion} or (throw "${pname} is missing hash for ${pyShortVersion}");
     };
 
-  propagatedBuildInputs = [
+  dependencies = [
     absl-py
     etils
-    importlib-resources
-    typing-extensions
-    zipp
-  ];
+  ] ++ etils.optional-dependencies.epath;
 
   pythonImportsCheck = [ "array_record" ];
 
-  meta = with lib; {
+  meta = {
     description = "ArrayRecord is a new file format derived from Riegeli, achieving a new frontier of IO efficiency";
     homepage = "https://github.com/google/array_record";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ GaetanLepage ];
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ GaetanLepage ];
     platforms = [ "x86_64-linux" ];
   };
 }

pkgs/development/python-modules/dm-sonnet/default.nix

···
 {
   lib,
   buildPythonPackage,
-  click,
+  fetchFromGitHub,
+
+  # build-system
+  setuptools,
+
+  # dependencies
   dm-tree,
-  docutils,
   etils,
-  fetchFromGitHub,
-  fetchpatch,
   numpy,
-  pythonOlder,
   tabulate,
+  wrapt,
+
+  # tests
+  click,
+  docutils,
+  keras,
+  pytestCheckHook,
   tensorflow,
   tensorflow-datasets,
-  wrapt,
+  tf-keras,
 }:
 
 buildPythonPackage rec {
   pname = "dm-sonnet";
   version = "2.0.2";
-  format = "setuptools";
-
-  disabled = pythonOlder "3.6";
+  pyproject = true;
 
   src = fetchFromGitHub {
     owner = "deepmind";
     repo = "sonnet";
-    rev = "refs/tags/v${version}";
+    tag = "v${version}";
     hash = "sha256-WkloUbqSyPG3cbLG8ktsjdluACkCbUZ7t6rYWst8rs8=";
   };
 
-  patches = [
-    (fetchpatch {
-      name = "replace-np-bool-with-np-bool_.patch";
-      url = "https://github.com/deepmind/sonnet/commit/df5d099d4557a9a81a0eb969e5a81ed917bcd612.patch";
-      hash = "sha256-s7abl83osD4wa0ZhqgDyjqQ3gagwGYCdQifwFqhNp34=";
-    })
+  build-system = [
+    setuptools
   ];
 
-  propagatedBuildInputs = [
+  dependencies = [
     dm-tree
     etils
     numpy
···
   nativeCheckInputs = [
     click
     docutils
+    keras
+    pytestCheckHook
     tensorflow
     tensorflow-datasets
+    tf-keras
+  ];
+
+  # ImportError: `keras.optimizers.legacy` is not supported in Keras 3
+  preCheck = ''
+    export TF_USE_LEGACY_KERAS=True
+  '';
+
+  disabledTests = [
+    # AssertionError: 2 != 0 : 2 doctests failed
+    "test_doctest_sonnet.functional"
+
+    # AssertionError: Not equal to tolerance
+    "testComputationAgainstNumPy1"
+
+    # tensorflow.python.framework.errors_impl.InvalidArgumentError: cannot compute MatMul as input #1(zero-based) was expected to be a float tensor but is a half tensor [Op:MatMul]
+    "testComputationAgainstNumPy0"
+    "testComputationAgainstNumPy1"
   ];
 
   pythonImportsCheck = [ "sonnet" ];
 
-  meta = with lib; {
+  meta = {
     description = "Library for building neural networks in TensorFlow";
     homepage = "https://github.com/deepmind/sonnet";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ onny ];
+    changelog = "https://github.com/google-deepmind/sonnet/releases/tag/v${version}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ onny ];
   };
 }

pkgs/development/python-modules/tensorflow-datasets/corruptions.patch

···
-diff --git a/tensorflow_datasets/image_classification/corruptions.py b/tensorflow_datasets/image_classification/corruptions.py
-index 066c4460..cb9a6667 100644
---- a/tensorflow_datasets/image_classification/corruptions.py
-+++ b/tensorflow_datasets/image_classification/corruptions.py
-@@ -35,7 +35,7 @@ FROST_FILENAMES = []
- 
- 
- def _imagemagick_bin():
--  return 'imagemagick'  # pylint: disable=unreachable
-+  return 'convert'  # pylint: disable=unreachable
- 
- 
- # /////////////// Corruption Helpers ///////////////
-@@ -675,7 +675,7 @@ def spatter(x, severity=1):
-     #     ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32)
-     #     ker -= np.mean(ker)
-     ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]])
--    dist = cv2.filter2D(dist, cv2.CVX_8U, ker)
-+    dist = cv2.filter2D(dist, cv2.CV_8U, ker)
-     dist = cv2.blur(dist, (3, 3)).astype(np.float32)
- 
-     m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA)

pkgs/development/python-modules/tensorflow-datasets/default.nix

···
 {
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  fetchpatch,
+
+  # dependencies
+  array-record,
+  dill,
+  dm-tree,
+  future,
+  immutabledict,
+  importlib-resources,
+  numpy,
+  promise,
+  protobuf,
+  psutil,
+  requests,
+  simple-parsing,
+  six,
+  tensorflow-metadata,
+  termcolor,
+  tqdm,
+
+  # tests
   apache-beam,
-  array-record,
-  attrs,
   beautifulsoup4,
-  buildPythonPackage,
   click,
   datasets,
-  dill,
-  dm-tree,
-  fetchFromGitHub,
   ffmpeg,
-  future,
   imagemagick,
-  importlib-resources,
   jax,
   jaxlib,
   jinja2,
   langdetect,
-  lib,
   lxml,
   matplotlib,
+  mlcroissant,
   mwparserfromhell,
   mwxml,
   networkx,
   nltk,
-  numpy,
   opencv4,
   pandas,
   pillow,
-  promise,
-  protobuf,
-  psutil,
   pycocotools,
   pydub,
   pytest-xdist,
   pytestCheckHook,
-  requests,
   scikit-image,
   scipy,
-  six,
+  sortedcontainers,
   tensorflow,
-  tensorflow-metadata,
-  termcolor,
   tifffile,
-  tqdm,
   zarr,
 }:
 
 buildPythonPackage rec {
   pname = "tensorflow-datasets";
-  version = "4.9.7";
-  format = "setuptools";
+  version = "4.9.8";
+  pyproject = true;
 
   src = fetchFromGitHub {
     owner = "tensorflow";
     repo = "datasets";
     tag = "v${version}";
-    hash = "sha256-pnut5z3rEYIOOhsQT6uVjPdy+SqftKptSQMwxbMXoKA=";
+    hash = "sha256-nqveZ+8b0f5sGIn6WufKeA37yEsZjzhCIbCfwMZ9JOM=";
   };
 
   patches = [
-    # addresses https://github.com/tensorflow/datasets/issues/3673
-    ./corruptions.patch
+    # mlmlcroissant uses encoding_formats, not encoding_formats.
+    # Backport https://github.com/tensorflow/datasets/pull/11037 until released.
+    (fetchpatch {
+      url = "https://github.com/tensorflow/datasets/commit/92cbcff725a1036569a515cc3356aa8480740451.patch";
+      hash = "sha256-2hnMvQP83+eAJllce19aHujcoWQzUz3+LsasWCo4BtM=";
+    })
   ];
 
-  propagatedBuildInputs = [
+  dependencies = [
     array-record
-    attrs
     dill
     dm-tree
     future
+    immutabledict
     importlib-resources
     numpy
     promise
     protobuf
     psutil
     requests
+    simple-parsing
     six
     tensorflow-metadata
     termcolor
···
     langdetect
     lxml
     matplotlib
+    mlcroissant
     mwparserfromhell
     mwxml
     networkx
···
     pytestCheckHook
     scikit-image
     scipy
+    sortedcontainers
     tensorflow
     tifffile
     zarr
   ];
 
+  pytestFlagsArray = [
+    # AttributeError: 'NoneType' object has no attribute 'Table'
+    "--deselect=tensorflow_datasets/core/file_adapters_test.py::test_read_write"
+    "--deselect=tensorflow_datasets/text/c4_wsrs/c4_wsrs_test.py::C4WSRSTest"
+  ];
+
   disabledTestPaths = [
     # Sandbox violations: network access, filesystem write attempts outside of build dir, ...
     "tensorflow_datasets/core/dataset_builder_test.py"
···
     "tensorflow_datasets/import_without_tf_test.py"
     "tensorflow_datasets/proto/build_tf_proto_test.py"
     "tensorflow_datasets/scripts/cli/build_test.py"
+    "tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder_test.py"
 
     # Requires `pretty_midi` which is not packaged in `nixpkgs`.
     "tensorflow_datasets/audio/groove.py"
···
 
     # Requires `gcld3` and `pretty_midi` which are not packaged in `nixpkgs`.
     "tensorflow_datasets/core/lazy_imports_lib_test.py"
+
+    # AttributeError: 'NoneType' object has no attribute 'Table'
+    "tensorflow_datasets/core/dataset_builder_beam_test.py"
+    "tensorflow_datasets/core/dataset_builders/adhoc_builder_test.py"
+    "tensorflow_datasets/core/split_builder_test.py"
+    "tensorflow_datasets/core/writer_test.py"
 
     # Requires `tensorflow_io` which is not packaged in `nixpkgs`.
     "tensorflow_datasets/core/features/audio_feature_test.py"
     "tensorflow_datasets/image/lsun_test.py"
 
-    # Requires `envlogger` which is not packaged in `nixpkgs`.
-    "tensorflow_datasets/rlds/locomotion/locomotion_test.py"
-    "tensorflow_datasets/rlds/robosuite_panda_pick_place_can/robosuite_panda_pick_place_can_test.py"
-
     # Fails with `TypeError: Constant constructor takes either 0 or 2 positional arguments`
     # deep in TF AutoGraph. Doesn't reproduce in Docker with Ubuntu 22.04 => might be related
     # to the differences in some of the dependencies?
···
     "tensorflow_datasets/text/c4_utils_test.py"
   ];
 
-  meta = with lib; {
+  meta = {
     description = "Library of datasets ready to use with TensorFlow";
     homepage = "https://www.tensorflow.org/datasets/overview";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ ndl ];
+    changelog = "https://github.com/tensorflow/datasets/releases/tag/v${version}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ ndl ];
   };
 }

pkgs/development/python-modules/tensorflow-metadata/build.patch

···
 diff --git a/setup.py b/setup.py
-index 7a09b2f..94c5aa6 100644
+index 30ac370..a05812d 100644
 --- a/setup.py
 +++ b/setup.py
 @@ -141,8 +141,4 @@ setup(

pkgs/development/python-modules/tensorflow-metadata/default.nix

···
   # has no tests
   doCheck = false;
 
-  pythonImportsCheck = [ "tensorflow_metadata" ];
+  pythonImportsCheck = [
+    "tensorflow_metadata"
+    "tensorflow_metadata.proto.v0"
+    "google.protobuf.runtime_version"
+  ];
 
   meta = {
     description = "Standard representations for metadata that are useful when training machine learning models with TensorFlow";
     homepage = "https://github.com/tensorflow/metadata";
+    changelog = "https://github.com/tensorflow/metadata/releases/tag/v${version}";
     license = lib.licenses.asl20;
     maintainers = with lib.maintainers; [ ndl ];
   };