




import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch


def    A						(__A   :    List[Any]       )   ->      Tuple:  # picklable for multiprocessing







     """simple docstring"""


     return x.sum()


def    A						(__A   :    Optional[Any]       )   ->      Optional[Any]:  # picklable for multiprocessing







     """simple docstring"""


     return i + 1







@dataclass
class __snake_case			:
   UpperCAmelCase__  :    int
   UpperCAmelCase__  :    str







class __snake_case			(			a		):







   def lowerCamelCase						(      self		:						List[str]):

        """simple docstring"""


        UpperCAmelCase_          =							{}
        UpperCAmelCase_          =							[]
        UpperCAmelCase_          =							1
        UpperCAmelCase_          =							[1, 2]
        UpperCAmelCase_          =							{'''a''': 1, '''b''': 2}
        UpperCAmelCase_          =							{'''a''': [1, 2], '''b''': [3, 4]}
        UpperCAmelCase_          =							{'''a''': {'''1''': 1}, '''b''': 2}
        UpperCAmelCase_          =							{'''a''': 1, '''b''': 2, '''c''': 3, '''d''': 4}
        UpperCAmelCase_          =							{}
        UpperCAmelCase_          =							[]
        UpperCAmelCase_          =							2
        UpperCAmelCase_          =							[2, 3]
        UpperCAmelCase_          =							{'''a''': 2, '''b''': 3}
        UpperCAmelCase_          =							{'''a''': [2, 3], '''b''': [4, 5]}
        UpperCAmelCase_          =							{'''a''': {'''1''': 2}, '''b''': 3}
        UpperCAmelCase_          =							{'''a''': 2, '''b''': 3, '''c''': 4, '''d''': 5}
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case)					,						_snake_case)

        UpperCAmelCase_          =							2
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						num_proc=_snake_case)					,						_snake_case)

        UpperCAmelCase_          =							{'''a''': np.eye(2), '''b''': np.zeros(3), '''c''': np.ones(2)}
        UpperCAmelCase_          =							{'''a''': 2, '''b''': 0, '''c''': 2}
        UpperCAmelCase_          =							{
            '''a''': np.eye(2).astype(_snake_case),
            '''b''': np.zeros(3).astype(_snake_case),
            '''c''': np.ones(2).astype(_snake_case),
        }
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						map_numpy=_snake_case)					,						_snake_case)
        self.assertEqual(
            {k: v.tolist() for k, v in map_nested(_snake_case					,						_snake_case					,						map_numpy=_snake_case).items()}					,						{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}					,						)
        self.assertEqual(map_nested(_snake_case					,						_snake_case					,						map_numpy=_snake_case					,						num_proc=_snake_case)					,						_snake_case)
        self.assertEqual(
            {k: v.tolist() for k, v in map_nested(_snake_case					,						_snake_case					,						map_numpy=_snake_case					,						num_proc=_snake_case).items()}					,						{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}					,						)
        with self.assertRaises(_snake_case):  # can't pickle a local lambda
             map_nested(lambda _snake_case: x + 1					,						_snake_case					,						num_proc=_snake_case)







   def lowerCamelCase						(      self		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							{'''a''': 1, '''b''': 2}
        UpperCAmelCase_          =							{'''a''': 3, '''b''': 4}
        UpperCAmelCase_          =							{'''a''': 5, '''b''': 6}
        UpperCAmelCase_          =							sorted([('''a''', (1, 3, 5)), ('''b''', (2, 4, 6))])
        self.assertEqual(sorted(zip_dict(_snake_case					,						_snake_case					,						_snake_case))					,						_snake_case)







   def lowerCamelCase						(      self		:						Dict):

        """simple docstring"""


        class __snake_case			:
           UpperCAmelCase__  :    int          =  '''bar'''







        UpperCAmelCase_          =							Foo()
        self.assertEqual(foo.my_attr					,						'''bar''')
        with temporary_assignment(_snake_case					,						'''my_attr'''					,						'''BAR'''):
             self.assertEqual(foo.my_attr					,						'''BAR''')
        self.assertEqual(foo.my_attr					,						'''bar''')







@pytest.mark.parametrize(
    '''iterable_length, num_proc, expected_num_proc'''   ,				[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]   ,				)
def    A						(__A   :    Tuple   ,				__A   :    List[Any]   ,				__A   :    Union[str, Any]       )   ->      Optional[Any]:







     """simple docstring"""


     with patch('''datasets.utils.py_utils._single_map_nested'''       ) as mock_single_map_nested, patch(
         '''datasets.parallel.parallel.Pool'''       ) as mock_multiprocessing_pool:
          UpperCAmelCase_          =							{F"""{i}""": i for i in range(__A       )}
          UpperCAmelCase_          =							map_nested(lambda __A       : x + 10   ,				__A   ,				num_proc=__A   ,				parallel_min_length=16       )
          if expected_num_proc == 1:
               assert mock_single_map_nested.called
               assert not mock_multiprocessing_pool.called
          else:
               assert not mock_single_map_nested.called
               assert mock_multiprocessing_pool.called
               assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc







class __snake_case			(			a		):







   @require_tf
   def lowerCamelCase						(      self		:						List[str]):

        """simple docstring"""


        import tensorflow as tf
        from tensorflow.keras import layers

        UpperCAmelCase_          =							layers.Dense(2)

        def gen_random_output():
             UpperCAmelCase_          =							tf.random.uniform((1, 3))
             return model(_snake_case).numpy()

        with temp_seed(42					,						set_tensorflow=_snake_case):
             UpperCAmelCase_          =							gen_random_output()
        with temp_seed(42					,						set_tensorflow=_snake_case):
             UpperCAmelCase_          =							gen_random_output()
        UpperCAmelCase_          =							gen_random_output()

        np.testing.assert_equal(_snake_case					,						_snake_case)
        self.assertGreater(np.abs(outa - outa).sum()					,						0)







   @require_torch
   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        import torch

        def gen_random_output():
             UpperCAmelCase_          =							torch.nn.Linear(3					,						2)
             UpperCAmelCase_          =							torch.rand(1					,						3)
             return model(_snake_case).detach().numpy()

        with temp_seed(42					,						set_pytorch=_snake_case):
             UpperCAmelCase_          =							gen_random_output()
        with temp_seed(42					,						set_pytorch=_snake_case):
             UpperCAmelCase_          =							gen_random_output()
        UpperCAmelCase_          =							gen_random_output()

        np.testing.assert_equal(_snake_case					,						_snake_case)
        self.assertGreater(np.abs(outa - outa).sum()					,						0)







   def lowerCamelCase						(      self		:						List[str]):

        """simple docstring"""


        def gen_random_output():
             return np.random.rand(1					,						3)

        with temp_seed(42):
             UpperCAmelCase_          =							gen_random_output()
        with temp_seed(42):
             UpperCAmelCase_          =							gen_random_output()
        UpperCAmelCase_          =							gen_random_output()

        np.testing.assert_equal(_snake_case					,						_snake_case)
        self.assertGreater(np.abs(outa - outa).sum()					,						0)







@pytest.mark.parametrize('''input_data'''   ,				[{}]       )
def    A						(__A   :    List[Any]       )   ->      Dict:







     """simple docstring"""


     UpperCAmelCase_          =							NestedDataStructure(__A       ).data
     assert output_data == input_data


@pytest.mark.parametrize(
    '''data, expected_output'''   ,				[
        ({}, []),
        ([], []),
        ('''foo''', ['''foo''']),
        (['''foo''', '''bar'''], ['''foo''', '''bar''']),
        ([['''foo''', '''bar''']], ['''foo''', '''bar''']),
        ([[['''foo'''], ['''bar''']]], ['''foo''', '''bar''']),
        ([[['''foo'''], '''bar''']], ['''foo''', '''bar''']),
        ({'''a''': 1, '''b''': 2}, [1, 2]),
        ({'''a''': [1, 2], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, [4]]}, [1, 2, 3, 4]),
        ({'''a''': {'''1''': 1}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': [2]}, [1, 2]),
    ]   ,				)
def    A						(__A   :    Tuple   ,				__A   :    Dict       )   ->      Dict:







     """simple docstring"""


     UpperCAmelCase_          =							NestedDataStructure(__A       ).flatten()
     assert output == expected_output


def    A						()   ->      Any:







     """simple docstring"""


     UpperCAmelCase_          =							A(x=1   ,				y='''foobar'''       )
     UpperCAmelCase_          =							{'''x''': 1, '''y''': '''foobar'''}
     assert asdict(__A       ) == expected_output

     UpperCAmelCase_          =							{'''a''': {'''b''': A(x=10   ,				y='''foo'''       )}, '''c''': [A(x=20   ,				y='''bar'''       )]}
     UpperCAmelCase_          =							{'''a''': {'''b''': {'''x''': 10, '''y''': '''foo'''}}, '''c''': [{'''x''': 20, '''y''': '''bar'''}]}
     assert asdict(__A       ) == expected_output

     with pytest.raises(__A       ):
          asdict([1, A(x=10   ,				y='''foo'''       )]       )


def    A						(__A   :    str       )   ->      Optional[int]:







     """simple docstring"""


     return text.split()


def    A						(__A   :    Optional[Any]       )   ->      Optional[Any]:







     """simple docstring"""


     yield (time.time(), content)
     time.sleep(2       )
     yield (time.time(), content)


def    A						()   ->      Tuple:







     """simple docstring"""


     with Pool(2       ) as pool:
          UpperCAmelCase_          =							list(iflatmap_unordered(__A   ,				_split_text   ,				kwargs_iterable=[{'''text''': '''hello there'''}] * 10       )       )
          assert out.count('''hello'''       ) == 10
          assert out.count('''there'''       ) == 10
          assert len(__A       ) == 20

     # check multiprocess from pathos (uses dill for pickling)
     with multiprocess.Pool(2       ) as pool:
          UpperCAmelCase_          =							list(iflatmap_unordered(__A   ,				_split_text   ,				kwargs_iterable=[{'''text''': '''hello there'''}] * 10       )       )
          assert out.count('''hello'''       ) == 10
          assert out.count('''there'''       ) == 10
          assert len(__A       ) == 20

     # check that we get items as fast as possible
     with Pool(2       ) as pool:
          UpperCAmelCase_          =							[]
          for yield_time, content in iflatmap_unordered(
              __A   ,				_aseconds_generator_of_aitems_with_timing   ,				kwargs_iterable=[{'''content''': '''a'''}, {'''content''': '''b'''}]       ):
               assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
               out.append(__A       )
          assert out.count('''a'''       ) == 2
          assert out.count('''b'''       ) == 2
          assert len(__A       ) == 4












snake_case_ : Dict    		=       {
    "Pillow": "Pillow",
    "accelerate": "accelerate>=0.11.0",
    "compel": "compel==0.1.8",
    "black": "black~=23.1",
    "datasets": "datasets",
    "filelock": "filelock",
    "flax": "flax>=0.4.1",
    "hf-doc-builder": "hf-doc-builder>=0.3.0",
    "huggingface-hub": "huggingface-hub>=0.13.2",
    "requests-mock": "requests-mock==1.10.0",
    "importlib_metadata": "importlib_metadata",
    "invisible-watermark": "invisible-watermark",
    "isort": "isort>=5.5.4",
    "jax": "jax>=0.2.8,!=0.3.2",
    "jaxlib": "jaxlib>=0.1.65",
    "Jinja2": "Jinja2",
    "k-diffusion": "k-diffusion>=0.0.12",
    "torchsde": "torchsde",
    "note_seq": "note_seq",
    "librosa": "librosa",
    "numpy": "numpy",
    "omegaconf": "omegaconf",
    "parameterized": "parameterized",
    "protobuf": "protobuf>=3.20.3,<4",
    "pytest": "pytest",
    "pytest-timeout": "pytest-timeout",
    "pytest-xdist": "pytest-xdist",
    "ruff": "ruff>=0.0.241",
    "safetensors": "safetensors",
    "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92",
    "scipy": "scipy",
    "onnx": "onnx",
    "regex": "regex!=2019.12.17",
    "requests": "requests",
    "tensorboard": "tensorboard",
    "torch": "torch>=1.4",
    "torchvision": "torchvision",
    "transformers": "transformers>=4.25.1",
    "urllib3": "urllib3<=2.0.0",
}






