


import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class   lowerCamelCase						:

       def __init__(      self						,    a__       ):






        '''simple docstring'''






        snake_case :			int							=   foo



       def __call__(      self       ):






        '''simple docstring'''






        return self.foo







class   lowerCamelCase						(	datasets.Dataset  ):

       @fingerprint_transform(inplace=SCREAMING_SNAKE_CASE_       )
       def 					UpperCAmelCase					(      self						,    a__						,    *a__						,    **a__       ):






        '''simple docstring'''






        return DatasetChild(self.data						,    fingerprint=SCREAMING_SNAKE_CASE_       )



       @fingerprint_transform(inplace=SCREAMING_SNAKE_CASE_       )
       def 					UpperCAmelCase					(      self						,    a__						,    *a__						,    **a__       ):






        '''simple docstring'''






        return DatasetChild(self.data						,    fingerprint=SCREAMING_SNAKE_CASE_       )







class   lowerCamelCase						:

       def __init__(      self						,    a__       ):






        '''simple docstring'''






        snake_case :			Union[str, Any]							=   callable

       def __call__(      self						,    *a__						,    **a__       ):






        '''simple docstring'''






        if self.callable is not None:
         return self.callable(*SCREAMING_SNAKE_CASE_						,    **SCREAMING_SNAKE_CASE_       )



       def __getstate__(      self       ):






        '''simple docstring'''






        raise pickle.PicklingError()







class   lowerCamelCase						(	SCREAMING_SNAKE_CASE  ):

       @require_transformers
       @pytest.mark.integration
       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        from transformers import AutoTokenizer

        def encode(a__       ):
         return tokenizer(SCREAMING_SNAKE_CASE_       )

        # TODO: add hash consistency tests across sessions
        snake_case :			List[str]							=   AutoTokenizer.from_pretrained('''bert-base-uncased'''       )
        snake_case :			Union[str, Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			List[str]							=   mda(datasets.utils.py_utils.dumps(lambda a__       : tokenizer(SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        snake_case :			Any							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Dict							=   AutoTokenizer.from_pretrained('''bert-base-cased'''       )
        snake_case :			List[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Union[str, Any]							=   mda(datasets.utils.py_utils.dumps(lambda a__       : tokenizer(SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        snake_case :			List[str]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			int							=   AutoTokenizer.from_pretrained('''bert-base-uncased'''       )
        snake_case :			str							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			int							=   mda(datasets.utils.py_utils.dumps(lambda a__       : tokenizer(SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        snake_case :			List[str]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       @require_transformers
       @pytest.mark.integration
       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        from transformers import AutoTokenizer

        snake_case :			int							=   AutoTokenizer.from_pretrained('''gpt2'''       )
        snake_case :			Optional[int]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        tokenizer('''Hello world !'''       )  # call once to change the tokenizer's cache
        snake_case :			str							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )



       @require_regex
       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        import regex

        snake_case :			str							=   regex.Regex('''foo'''       )
        snake_case :			Tuple							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Optional[int]							=   regex.Regex('''bar'''       )
        snake_case :			Optional[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			List[str]							=   regex.Regex('''foo'''       )
        snake_case :			Tuple							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )







class   lowerCamelCase						(	SCREAMING_SNAKE_CASE  ):

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        def func():
         return foo

        snake_case :			int							=   [0]
        snake_case :			List[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Union[str, Any]							=   [1]
        snake_case :			int							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Union[str, Any]							=   [0]
        snake_case :			Optional[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        def func():
         pass

        snake_case :			List[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()

        def func():
         pass

        snake_case :			Dict							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case :			Optional[Any]							=   mda(datasets.utils.py_utils.dumps(Foo([0]       )       )       ).hexdigest()
        snake_case :			Union[str, Any]							=   mda(datasets.utils.py_utils.dumps(Foo([1]       )       )       ).hexdigest()
        snake_case :			str							=   mda(datasets.utils.py_utils.dumps(Foo([0]       )       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case :			List[str]							=   mda(datasets.utils.py_utils.dumps(Foo([0]       ).__call__       )       ).hexdigest()
        snake_case :			Optional[Any]							=   mda(datasets.utils.py_utils.dumps(Foo([1]       ).__call__       )       ).hexdigest()
        snake_case :			Optional[Any]							=   mda(datasets.utils.py_utils.dumps(Foo([0]       ).__call__       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case :			Dict							=   (
            '''co_argcount''',
            '''co_kwonlyargcount''',
            '''co_nlocals''',
            '''co_stacksize''',
            '''co_flags''',
            '''co_code''',
            '''co_consts''',
            '''co_names''',
            '''co_varnames''',
            '''co_filename''',
            '''co_name''',
            '''co_firstlineno''',
            '''co_lnotab''',
            '''co_freevars''',
            '''co_cellvars''',
        )

        def _create_code_pyaa(*a__       ):
         if hasattr(args[-3]						,    '''encode'''       ):
          snake_case :			Optional[int]							=   list(SCREAMING_SNAKE_CASE_       )
          snake_case :			str							=   args[-3].encode()  # co_lnotab
          snake_case :			Optional[Any]							=   args[-10].encode()  # co_code
         if hasattr(SCREAMING_SNAKE_CASE_						,    '''co_posonlyargcount'''       ):
          if len(SCREAMING_SNAKE_CASE_       ) == 16:
           return CodeType(*SCREAMING_SNAKE_CASE_       )
          elif len(SCREAMING_SNAKE_CASE_       ) == 15:
           return CodeType(args[0]						,    0						,    *args[1:]       )
          return CodeType(args[0]						,    0						,    0						,    *args[1:]       )
         elif hasattr(SCREAMING_SNAKE_CASE_						,    '''co_kwonlyargcount'''       ):
          if len(SCREAMING_SNAKE_CASE_       ) == 16:
           return CodeType(args[0]						,    *args[2:]       )
          elif len(SCREAMING_SNAKE_CASE_       ) == 15:
           return CodeType(*SCREAMING_SNAKE_CASE_       )
          return CodeType(args[0]						,    0						,    *args[1:]       )
         if len(SCREAMING_SNAKE_CASE_       ) == 16:
          return CodeType(args[0]						,    *args[3:]       )
         elif len(SCREAMING_SNAKE_CASE_       ) == 15:
          return CodeType(args[0]						,    *args[2:]       )
         return CodeType(*SCREAMING_SNAKE_CASE_       )

        def create_ipython_func(a__						,    a__       ):
         def func():
          return returned_obj

         snake_case :			List[Any]							=   func.__code__
         # Use _create_code from dill in order to make it work for different python versions
         if datasets.config.PY_VERSION < version.parse('''3.8'''       ):
          snake_case :			List[str]							=   _create_code_pyaa(
              *[getattr(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       ) if k != '''co_filename''' else co_filename for k in code_args_pyaa]       )
         else:
          snake_case :			Optional[Any]							=   code.replace(co_filename=SCREAMING_SNAKE_CASE_       )
         return FunctionType(SCREAMING_SNAKE_CASE_						,    func.__globals__						,    func.__name__						,    func.__defaults__						,    func.__closure__       )

        snake_case	,       snake_case :			Dict							=   '''<ipython-input-2-e0383a102aae>''', [0]
        snake_case :			List[Any]							=   mda(datasets.utils.py_utils.dumps(create_ipython_func(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        snake_case	,       snake_case :			int							=   '''<ipython-input-2-e0383a102aae>''', [1]
        snake_case :			Optional[int]							=   mda(datasets.utils.py_utils.dumps(create_ipython_func(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        snake_case	,       snake_case :			Dict							=   '''<ipython-input-5-713f6613acf3>''', [0]
        snake_case :			int							=   mda(datasets.utils.py_utils.dumps(create_ipython_func(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

        snake_case	,       snake_case :			Tuple							=   os.path.join(gettempdir()						,    '''ipykernel_12345'''						,    '''321456789.py'''       ), [0]
        snake_case :			str							=   mda(datasets.utils.py_utils.dumps(create_ipython_func(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        snake_case	,       snake_case :			Optional[Any]							=   os.path.join(gettempdir()						,    '''ipykernel_12345'''						,    '''321456789.py'''       ), [1]
        snake_case :			int							=   mda(datasets.utils.py_utils.dumps(create_ipython_func(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        snake_case	,       snake_case :			int							=   os.path.join(gettempdir()						,    '''ipykernel_12345'''						,    '''654123987.py'''       ), [0]
        snake_case :			Tuple							=   mda(datasets.utils.py_utils.dumps(create_ipython_func(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )



       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case	,       snake_case :			str							=   [0], [1]

        def func():
         return foo, bar

        snake_case :			Optional[int]							=   '''__main__'''

        def globalvars_mocka_side_effect(a__						,    *a__						,    **a__       ):
         return {"foo": foo, "bar": bar}

        def globalvars_mocka_side_effect(a__						,    *a__						,    **a__       ):
         return {"bar": bar, "foo": foo}

        with patch('''dill.detect.globalvars'''						,    side_effect=SCREAMING_SNAKE_CASE_       ) as globalvars_mocka:
         snake_case :			Dict							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
         self.assertGreater(globalvars_mocka.call_count						,    0       )
        with patch('''dill.detect.globalvars'''						,    side_effect=SCREAMING_SNAKE_CASE_       ) as globalvars_mocka:
         snake_case :			Any							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
         self.assertGreater(globalvars_mocka.call_count						,    0       )
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )







class   lowerCamelCase						(	SCREAMING_SNAKE_CASE  ):

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case :			int							=   Hasher.hash('''hello'''       )
        snake_case :			Optional[Any]							=   Hasher.hash('''hello'''       )
        snake_case :			str							=   Hasher.hash('''there'''       )
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case :			Union[str, Any]							=   Hasher.hash(Foo('''hello'''       )       )
        snake_case :			Tuple							=   Hasher.hash(Foo('''hello'''       )       )
        snake_case :			Any							=   Hasher.hash(Foo('''there'''       )       )
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case :			List[Any]							=   Hasher()
        for x in ["hello", Foo('''hello'''       )]:
         hasher.update(SCREAMING_SNAKE_CASE_       )
        snake_case :			Any							=   hasher.hexdigest()
        snake_case :			Union[str, Any]							=   Hasher()
        for x in ["hello", Foo('''hello'''       )]:
         hasher.update(SCREAMING_SNAKE_CASE_       )
        snake_case :			int							=   hasher.hexdigest()
        snake_case :			List[Any]							=   Hasher()
        for x in ["there", Foo('''there'''       )]:
         hasher.update(SCREAMING_SNAKE_CASE_       )
        snake_case :			Any							=   hasher.hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        with self.assertRaises(pickle.PicklingError       ):
         Hasher.hash(UnpicklableCallable(Foo('''hello'''       )       )       )

       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        snake_case :			Optional[Any]							=   '''abc'''
        snake_case :			Optional[Any]							=   [string, string]  # two strings have the same ids
        snake_case :			Optional[Any]							=   [string, string]
        snake_case :			str							=   json.loads(f'''["{string}", "{string}"]'''       )  # two strings have different ids
        self.assertIs(obja[0]						,    SCREAMING_SNAKE_CASE_       )
        self.assertIs(obja[0]						,    obja[1]       )
        self.assertIs(obja[0]						,    SCREAMING_SNAKE_CASE_       )
        self.assertIs(obja[0]						,    obja[1]       )
        self.assertIsNot(obja[0]						,    SCREAMING_SNAKE_CASE_       )
        self.assertIsNot(obja[0]						,    obja[1]       )
        snake_case :			Tuple							=   Hasher.hash(SCREAMING_SNAKE_CASE_       )
        snake_case :			str							=   Hasher.hash(SCREAMING_SNAKE_CASE_       )
        snake_case :			List[str]							=   Hasher.hash(SCREAMING_SNAKE_CASE_       )
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       @require_tiktoken
       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        import tiktoken

        snake_case :			Optional[int]							=   tiktoken.get_encoding('''gpt2'''       )
        snake_case :			int							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Union[str, Any]							=   tiktoken.get_encoding('''r50k_base'''       )
        snake_case :			List[str]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Tuple							=   tiktoken.get_encoding('''gpt2'''       )
        snake_case :			Optional[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )

       @require_torch
       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        import torch

        snake_case :			Any							=   torch.tensor([1.0]       )
        snake_case :			Dict							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Union[str, Any]							=   torch.tensor([2.0]       )
        snake_case :			Dict							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Any							=   torch.tensor([1.0]       )
        snake_case :			List[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )



       @require_spacy
       @require_spacy_model('''en_core_web_sm'''       )
       @require_spacy_model('''fr_core_news_sm'''       )
       @pytest.mark.integration
       def 					UpperCAmelCase					(      self       ):






        '''simple docstring'''






        import spacy

        snake_case :			Optional[Any]							=   spacy.load('''en_core_web_sm'''       )
        snake_case :			Optional[int]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			List[Any]							=   spacy.load('''fr_core_news_sm'''       )
        snake_case :			Optional[Any]							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        snake_case :			Optional[Any]							=   spacy.load('''en_core_web_sm'''       )
        snake_case :			Any							=   mda(datasets.utils.py_utils.dumps(SCREAMING_SNAKE_CASE_       )       ).hexdigest()
        self.assertEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )
        self.assertNotEqual(SCREAMING_SNAKE_CASE_						,    SCREAMING_SNAKE_CASE_       )







@pytest.mark.integration
def  _UpperCAmelCase						( __lowerCamelCase							)		->int:


 '''simple docstring'''
 snake_case :			Optional[Any]							=   tmp_path / '''dir1'''
 snake_case :			Optional[int]							=   tmp_path / '''dir2'''
 dira.mkdir()
 dira.mkdir()
 snake_case :			Tuple							=   '''script.py'''
 snake_case :			Union[str, Any]							=   dedent(
     '''\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    '''							)
 snake_case :			Dict							=   dira / script_filename
 snake_case :			Any							=   dira / script_filename
 with script_patha.open('''w'''							) as f:
  f.write(__snake_case							)
 with script_patha.open('''w'''							) as f:
  f.write(__snake_case							)
 snake_case :			Optional[Any]							=   subprocess.check_output(['''python''', str(__snake_case							)]							)
 snake_case :			int							=   subprocess.check_output(['''python''', str(__snake_case							)]							)
 assert fingerprinta == fingerprinta






def  _UpperCAmelCase						( )		->Any:


 '''simple docstring'''
 snake_case :			Optional[Any]							=   {'''a''': [0, 1, 2]}
 snake_case :			List[Any]							=   DatasetChild(InMemoryTable.from_pydict(__snake_case							)							)
 snake_case :			str							=   dataset.funca()._fingerprint
 assert expected_fingerprint == dataset.funca()._fingerprint
 assert expected_fingerprint != dataset.funca()._fingerprint

 with Pool(2							) as p:
  assert expected_fingerprint == p.apply_async(dataset.funca							).get()._fingerprint
  assert expected_fingerprint != p.apply_async(dataset.funca							).get()._fingerprint






def  _UpperCAmelCase						( )		->Any:


 '''simple docstring'''
 snake_case :			Optional[Any]							=   {'''a''': [0, 1, 2]}

 class   lowerCamelCase						(	datasets.Dataset  ):

        @fingerprint_transform(inplace=SCREAMING_SNAKE_CASE_       )
        def 					UpperCAmelCase					(      self						,    a__       ):






         '''simple docstring'''






         return DummyDatasetChild(self.data						,    fingerprint=SCREAMING_SNAKE_CASE_       )







 snake_case :			Optional[Any]							=   DummyDatasetChild(InMemoryTable.from_pydict(__snake_case							)							).func()

 class   lowerCamelCase						(	datasets.Dataset  ):

        @fingerprint_transform(inplace=SCREAMING_SNAKE_CASE_						,    version='''1.0.0'''       )
        def 					UpperCAmelCase					(      self						,    a__       ):






         '''simple docstring'''






         return DummyDatasetChild(self.data						,    fingerprint=SCREAMING_SNAKE_CASE_       )







 snake_case :			Union[str, Any]							=   DummyDatasetChild(InMemoryTable.from_pydict(__snake_case							)							).func()

 class   lowerCamelCase						(	datasets.Dataset  ):

        @fingerprint_transform(inplace=SCREAMING_SNAKE_CASE_						,    version='''2.0.0'''       )
        def 					UpperCAmelCase					(      self						,    a__       ):






         '''simple docstring'''






         return DummyDatasetChild(self.data						,    fingerprint=SCREAMING_SNAKE_CASE_       )







 snake_case :			List[str]							=   DummyDatasetChild(InMemoryTable.from_pydict(__snake_case							)							).func()

 assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}							) == 3






def  _UpperCAmelCase						( )		->Tuple:


 '''simple docstring'''
 # AttributeError: module 'dill._dill' has no attribute 'stack'
 snake_case :			Any							=   Hasher()
 hasher.update(lambda __lowerCamelCase							: x							)








import os
import pathlib
import tempfile
import uuid

import numpy as np

from ..utils import is_soundfile_availble, is_torch_available, is_vision_available, logging


lowerCamelCase_				:							Any       =    logging.get_logger(__name__)

if is_vision_available():
 import PIL.Image
 from PIL import Image
 from PIL.Image import Image as ImageType
else:
 lowerCamelCase_				:							Union[str, Any]       =    object

if is_torch_available():
 import torch

if is_soundfile_availble():
 import soundfile as sf







class  _lowerCamelCase			:

     def __init__(  self							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   value

     def __str__(  self    ):
            return self.to_string()

     def 						__lowerCamelCase				(  self    ):
            logger.error(
                'This is a raw AgentType of unknown type. Display in notebooks and string conversion will be unreliable'    )
            return self._value

     def 						__lowerCamelCase				(  self    ):
            logger.error(
                'This is a raw AgentType of unknown type. Display in notebooks and string conversion will be unreliable'    )
            return str(self._value    )







class  _lowerCamelCase			(lowerCamelCase							,					lowerCamelCase					):

     def 						__lowerCamelCase				(  self    ):
            return self._value

     def 						__lowerCamelCase				(  self    ):
            return self._value







class  _lowerCamelCase			(lowerCamelCase							,					lowerCamelCase					):

     def __init__(  self							,       SCREAMING_SNAKE_CASE_    ):
            super().__init__(SCREAMING_SNAKE_CASE_    )

            if not is_vision_available():
                   raise ImportError('PIL must be installed in order to handle images.'    )

            __snake_case						       =   None
            __snake_case						       =   None
            __snake_case						       =   None

            if isinstance(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
                   __snake_case						       =   value
            elif isinstance(SCREAMING_SNAKE_CASE_							,       (str, pathlib.Path)    ):
                   __snake_case						       =   value
            elif isinstance(SCREAMING_SNAKE_CASE_							,       torch.Tensor    ):
                   __snake_case						       =   value
            else:
                   raise ValueError(f'''Unsupported type for {self.__class__.__name__}: {type(SCREAMING_SNAKE_CASE_    )}'''    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_=None							,       SCREAMING_SNAKE_CASE_=None    ):
            from IPython.display import Image, display

            display(Image(self.to_string()    )    )

     def 						__lowerCamelCase				(  self    ):
            if self._raw is not None:
                   return self._raw

            if self._path is not None:
                   __snake_case						       =   Image.open(self._path    )
                   return self._raw

     def 						__lowerCamelCase				(  self    ):
            if self._path is not None:
                   return self._path

            if self._raw is not None:
                   __snake_case						       =   tempfile.mkdtemp()
                   __snake_case						       =   os.path.join(SCREAMING_SNAKE_CASE_							,       str(uuid.uuida()    ) + '.png'    )
                   self._raw.save(self._path    )

                   return self._path

            if self._tensor is not None:
                   __snake_case						       =   self._tensor.cpu().detach().numpy()

                   # There is likely simpler than load into image into save
                   __snake_case						       =   Image.fromarray((array * 255).astype(np.uinta    )    )

                   __snake_case						       =   tempfile.mkdtemp()
                   __snake_case						       =   os.path.join(SCREAMING_SNAKE_CASE_							,       str(uuid.uuida()    ) + '.png'    )

                   img.save(self._path    )

                   return self._path







class  _lowerCamelCase			(lowerCamelCase					):

     def __init__(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_=16_000    ):
            super().__init__(SCREAMING_SNAKE_CASE_    )

            if not is_soundfile_availble():
                   raise ImportError('soundfile must be installed in order to handle audio.'    )

            __snake_case						       =   None
            __snake_case						       =   None

            __snake_case						       =   samplerate

            if isinstance(SCREAMING_SNAKE_CASE_							,       (str, pathlib.Path)    ):
                   __snake_case						       =   value
            elif isinstance(SCREAMING_SNAKE_CASE_							,       torch.Tensor    ):
                   __snake_case						       =   value
            else:
                   raise ValueError(f'''Unsupported audio type: {type(SCREAMING_SNAKE_CASE_    )}'''    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_=None							,       SCREAMING_SNAKE_CASE_=None    ):
            from IPython.display import Audio, display

            display(Audio(self.to_string()							,       rate=self.samplerate    )    )

     def 						__lowerCamelCase				(  self    ):
            if self._tensor is not None:
                   return self._tensor

            if self._path is not None:
                   __snake_case       ,       __snake_case						       =   sf.read(self._path    )
                   __snake_case						       =   torch.tensor(SCREAMING_SNAKE_CASE_    )
                   return self._tensor

     def 						__lowerCamelCase				(  self    ):
            if self._path is not None:
                   return self._path

            if self._tensor is not None:
                   __snake_case						       =   tempfile.mkdtemp()
                   __snake_case						       =   os.path.join(SCREAMING_SNAKE_CASE_							,       str(uuid.uuida()    ) + '.wav'    )
                   sf.write(self._path							,       self._tensor							,       samplerate=self.samplerate    )
                   return self._path





lowerCamelCase_				:							Optional[Any]       =    {"text": AgentText, "image": AgentImage, "audio": AgentAudio}
lowerCamelCase_				:							Any       =    {str: AgentText}

if is_vision_available():
 lowerCamelCase_				:							Optional[int]       =    AgentImage
def        __lowercase(							*__snake_case				:		Optional[Any]    ,**__snake_case				:		str					)       ->   Tuple:
       __snake_case						       =   [(arg.to_raw() if isinstance(__snake_case    ,__snake_case					) else arg) for arg in args]
       __snake_case						       =   {k: (v.to_raw() if isinstance(__snake_case    ,__snake_case					) else v) for k, v in kwargs.items()}
       return args, kwargs
def        __lowercase(							__snake_case				:		Optional[int]    ,__snake_case				:		List[str]=None					)       ->   Tuple:
       if isinstance(__snake_case    ,__snake_case					):
              __snake_case						       =   {}
              for i, (k, v) in enumerate(outputs.items()					):
                     if output_types is not None:
                            # If the class has defined outputs, we can map directly according to the class definition
                            if output_types[i] in AGENT_TYPE_MAPPING:
                                   __snake_case						       =   AGENT_TYPE_MAPPING[output_types[i]](__snake_case					)
                            else:
                                   __snake_case						       =   AgentType(__snake_case					)

                     else:
                            # If the class does not have defined output, then we map according to the type
                            for _k, _v in INSTANCE_TYPE_MAPPING.items():
                                   if isinstance(__snake_case    ,_k					):
                                          __snake_case						       =   _v(__snake_case					)
                            if k not in decoded_outputs:
                                   __snake_case						       =   AgentType[v]

       elif isinstance(__snake_case    ,(list, tuple)					):
              __snake_case						       =   type(__snake_case					)()
              for i, v in enumerate(__snake_case					):
                     if output_types is not None:
                            # If the class has defined outputs, we can map directly according to the class definition
                            if output_types[i] in AGENT_TYPE_MAPPING:
                                   decoded_outputs.append(AGENT_TYPE_MAPPING[output_types[i]](__snake_case					)					)
                            else:
                                   decoded_outputs.append(AgentType(__snake_case					)					)
                     else:
                            # If the class does not have defined output, then we map according to the type
                            __snake_case						       =   False
                            for _k, _v in INSTANCE_TYPE_MAPPING.items():
                                   if isinstance(__snake_case    ,_k					):
                                          decoded_outputs.append(_v(__snake_case					)					)
                                          __snake_case						       =   True

                            if not found:
                                   decoded_outputs.append(AgentType(__snake_case					)					)

       else:
              if output_types[0] in AGENT_TYPE_MAPPING:
                     # If the class has defined outputs, we can map directly according to the class definition
                     __snake_case						       =   AGENT_TYPE_MAPPING[output_types[0]](__snake_case					)

              else:
                     # If the class does not have defined output, then we map according to the type
                     for _k, _v in INSTANCE_TYPE_MAPPING.items():
                            if isinstance(__snake_case    ,_k					):
                                   return _v(__snake_case					)
                     return AgentType(__snake_case					)

       return decoded_outputs
