



"""simple docstring"""






import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class    __snake_case:




    def __init__(   self    ,							A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			foo




    def __call__(   self			):





      '''simple docstring'''



      return self.foo




class    __snake_case(					datasets.Dataset       ):




    @fingerprint_transform(inplace=A_			)
    def   A   (   self    ,							A_    ,							*A_    ,							**A_			):





      '''simple docstring'''



      return DatasetChild(self.data    ,							fingerprint=A_			)




    @fingerprint_transform(inplace=A_			)
    def   A   (   self    ,							A_    ,							*A_    ,							**A_			):





      '''simple docstring'''



      return DatasetChild(self.data    ,							fingerprint=A_			)




class    __snake_case:




    def __init__(   self    ,							A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			callable




    def __call__(   self    ,							*A_    ,							**A_			):





      '''simple docstring'''



      if self.callable is not None:
        return self.callable(*A_    ,							**A_			)




    def __getstate__(   self			):





      '''simple docstring'''



      raise pickle.PicklingError()




class    __snake_case(					__A       ):




    @require_transformers
    @pytest.mark.integration
    def   A   (   self			):





      '''simple docstring'''



      from transformers import AutoTokenizer

      def encode(A_			):
        return tokenizer(A_			)

      # TODO: add hash consistency tests across sessions
      _SCREAMING_SNAKE_CASE        =			AutoTokenizer.from_pretrained('''bert-base-uncased'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(lambda A_			: tokenizer(A_			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			AutoTokenizer.from_pretrained('''bert-base-cased'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(lambda A_			: tokenizer(A_			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			AutoTokenizer.from_pretrained('''bert-base-uncased'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(lambda A_			: tokenizer(A_			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    @require_transformers
    @pytest.mark.integration
    def   A   (   self			):





      '''simple docstring'''



      from transformers import AutoTokenizer

      _SCREAMING_SNAKE_CASE        =			AutoTokenizer.from_pretrained('''gpt2'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      tokenizer('''Hello world !'''			)  # call once to change the tokenizer's cache
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)




    @require_regex
    def   A   (   self			):





      '''simple docstring'''



      import regex

      _SCREAMING_SNAKE_CASE        =			regex.Regex('''foo'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			regex.Regex('''bar'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			regex.Regex('''foo'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




class    __snake_case(					__A       ):




    def   A   (   self			):





      '''simple docstring'''



      def func():
        return foo

      _SCREAMING_SNAKE_CASE        =			[0]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			[1]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			[0]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      def func():
        pass

      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()

      def func():
        pass

      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			(
          '''co_argcount''',
          '''co_kwonlyargcount''',
          '''co_nlocals''',
          '''co_stacksize''',
          '''co_flags''',
          '''co_code''',
          '''co_consts''',
          '''co_names''',
          '''co_varnames''',
          '''co_filename''',
          '''co_name''',
          '''co_firstlineno''',
          '''co_lnotab''',
          '''co_freevars''',
          '''co_cellvars''',
      )

      def _create_code_pyaa(*A_			):
        if hasattr(args[-3]    ,							'''encode'''			):
          _SCREAMING_SNAKE_CASE        =			list(A_			)
          _SCREAMING_SNAKE_CASE        =			args[-3].encode()  # co_lnotab
          _SCREAMING_SNAKE_CASE        =			args[-10].encode()  # co_code
        if hasattr(A_    ,							'''co_posonlyargcount'''			):
          if len(A_			) == 16:
            return CodeType(*A_			)
          elif len(A_			) == 15:
            return CodeType(args[0]    ,							0    ,							*args[1:]			)
          return CodeType(args[0]    ,							0    ,							0    ,							*args[1:]			)
        elif hasattr(A_    ,							'''co_kwonlyargcount'''			):
          if len(A_			) == 16:
            return CodeType(args[0]    ,							*args[2:]			)
          elif len(A_			) == 15:
            return CodeType(*A_			)
          return CodeType(args[0]    ,							0    ,							*args[1:]			)
        if len(A_			) == 16:
          return CodeType(args[0]    ,							*args[3:]			)
        elif len(A_			) == 15:
          return CodeType(args[0]    ,							*args[2:]			)
        return CodeType(*A_			)

      def create_ipython_func(A_    ,							A_			):
        def func():
          return returned_obj

        _SCREAMING_SNAKE_CASE        =			func.__code__
        # Use _create_code from dill in order to make it work for different python versions
        if datasets.config.PY_VERSION < version.parse('''3.8'''			):
          _SCREAMING_SNAKE_CASE        =			_create_code_pyaa(
              *[getattr(A_    ,							A_			) if k != '''co_filename''' else co_filename for k in code_args_pyaa]			)
        else:
          _SCREAMING_SNAKE_CASE        =			code.replace(co_filename=A_			)
        return FunctionType(A_    ,							func.__globals__    ,							func.__name__    ,							func.__defaults__    ,							func.__closure__			)

      _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			'''<ipython-input-2-e0383a102aae>''', [0]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(create_ipython_func(A_    ,							A_			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			'''<ipython-input-2-e0383a102aae>''', [1]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(create_ipython_func(A_    ,							A_			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			'''<ipython-input-5-713f6613acf3>''', [0]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(create_ipython_func(A_    ,							A_			)			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)

      _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			os.path.join(gettempdir()    ,							'''ipykernel_12345'''    ,							'''321456789.py'''			), [0]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(create_ipython_func(A_    ,							A_			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			os.path.join(gettempdir()    ,							'''ipykernel_12345'''    ,							'''321456789.py'''			), [1]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(create_ipython_func(A_    ,							A_			)			)			).hexdigest()
      _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			os.path.join(gettempdir()    ,							'''ipykernel_12345'''    ,							'''654123987.py'''			), [0]
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(create_ipython_func(A_    ,							A_			)			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			[0], [1]

      def func():
        return foo, bar

      _SCREAMING_SNAKE_CASE        =			'''__main__'''

      def globalvars_mocka_side_effect(A_    ,							*A_    ,							**A_			):
        return {"foo": foo, "bar": bar}

      def globalvars_mocka_side_effect(A_    ,							*A_    ,							**A_			):
        return {"bar": bar, "foo": foo}

      with patch('''dill.detect.globalvars'''    ,							side_effect=A_			) as globalvars_mocka:
        _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
        self.assertGreater(globalvars_mocka.call_count    ,							0			)
      with patch('''dill.detect.globalvars'''    ,							side_effect=A_			) as globalvars_mocka:
        _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
        self.assertGreater(globalvars_mocka.call_count    ,							0			)
      self.assertEqual(A_    ,							A_			)




class    __snake_case(					__A       ):




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			Hasher.hash('''hello'''			)
      _SCREAMING_SNAKE_CASE        =			Hasher.hash('''hello'''			)
      _SCREAMING_SNAKE_CASE        =			Hasher.hash('''there'''			)
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			Hasher.hash(Foo('''hello'''			)			)
      _SCREAMING_SNAKE_CASE        =			Hasher.hash(Foo('''hello'''			)			)
      _SCREAMING_SNAKE_CASE        =			Hasher.hash(Foo('''there'''			)			)
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			Hasher()
      for x in ["hello", Foo('''hello'''			)]:
        hasher.update(A_			)
      _SCREAMING_SNAKE_CASE        =			hasher.hexdigest()
      _SCREAMING_SNAKE_CASE        =			Hasher()
      for x in ["hello", Foo('''hello'''			)]:
        hasher.update(A_			)
      _SCREAMING_SNAKE_CASE        =			hasher.hexdigest()
      _SCREAMING_SNAKE_CASE        =			Hasher()
      for x in ["there", Foo('''there'''			)]:
        hasher.update(A_			)
      _SCREAMING_SNAKE_CASE        =			hasher.hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    def   A   (   self			):





      '''simple docstring'''



      with self.assertRaises(pickle.PicklingError			):
        Hasher.hash(UnpicklableCallable(Foo('''hello'''			)			)			)




    def   A   (   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			'''abc'''
      _SCREAMING_SNAKE_CASE        =			[string, string]  # two strings have the same ids
      _SCREAMING_SNAKE_CASE        =			[string, string]
      _SCREAMING_SNAKE_CASE        =			json.loads(F'''["{string}", "{string}"]'''			)  # two strings have different ids
      self.assertIs(obja[0]    ,							A_			)
      self.assertIs(obja[0]    ,							obja[1]			)
      self.assertIs(obja[0]    ,							A_			)
      self.assertIs(obja[0]    ,							obja[1]			)
      self.assertIsNot(obja[0]    ,							A_			)
      self.assertIsNot(obja[0]    ,							obja[1]			)
      _SCREAMING_SNAKE_CASE        =			Hasher.hash(A_			)
      _SCREAMING_SNAKE_CASE        =			Hasher.hash(A_			)
      _SCREAMING_SNAKE_CASE        =			Hasher.hash(A_			)
      self.assertEqual(A_    ,							A_			)
      self.assertEqual(A_    ,							A_			)




    @require_tiktoken
    def   A   (   self			):





      '''simple docstring'''



      import tiktoken

      _SCREAMING_SNAKE_CASE        =			tiktoken.get_encoding('''gpt2'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			tiktoken.get_encoding('''r50k_base'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			tiktoken.get_encoding('''gpt2'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    @require_torch
    def   A   (   self			):





      '''simple docstring'''



      import torch

      _SCREAMING_SNAKE_CASE        =			torch.tensor([1.0]			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			torch.tensor([2.0]			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			torch.tensor([1.0]			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)




    @require_spacy
    @require_spacy_model('''en_core_web_sm'''			)
    @require_spacy_model('''fr_core_news_sm'''			)
    @pytest.mark.integration
    def   A   (   self			):





      '''simple docstring'''



      import spacy

      _SCREAMING_SNAKE_CASE        =			spacy.load('''en_core_web_sm'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			spacy.load('''fr_core_news_sm'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      _SCREAMING_SNAKE_CASE        =			spacy.load('''en_core_web_sm'''			)
      _SCREAMING_SNAKE_CASE        =			mda(datasets.utils.py_utils.dumps(A_			)			).hexdigest()
      self.assertEqual(A_    ,							A_			)
      self.assertNotEqual(A_    ,							A_			)



@pytest.mark.integration
def 							A__ (     UpperCamelCase__    ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			tmp_path / '''dir1'''
  _SCREAMING_SNAKE_CASE        =			tmp_path / '''dir2'''
  dira.mkdir()
  dira.mkdir()
  _SCREAMING_SNAKE_CASE        =			'''script.py'''
  _SCREAMING_SNAKE_CASE        =			dedent(
      '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    '''    )
  _SCREAMING_SNAKE_CASE        =			dira / script_filename
  _SCREAMING_SNAKE_CASE        =			dira / script_filename
  with script_patha.open('''w'''    ) as f:
    f.write(UpperCamelCase__    )
  with script_patha.open('''w'''    ) as f:
    f.write(UpperCamelCase__    )
  _SCREAMING_SNAKE_CASE        =			subprocess.check_output(['''python''', str(UpperCamelCase__    )]    )
  _SCREAMING_SNAKE_CASE        =			subprocess.check_output(['''python''', str(UpperCamelCase__    )]    )
  assert fingerprinta == fingerprinta


def 							A__ (     ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			{'''a''': [0, 1, 2]}
  _SCREAMING_SNAKE_CASE        =			DatasetChild(InMemoryTable.from_pydict(UpperCamelCase__    )    )
  _SCREAMING_SNAKE_CASE        =			dataset.funca()._fingerprint
  assert expected_fingerprint == dataset.funca()._fingerprint
  assert expected_fingerprint != dataset.funca()._fingerprint

  with Pool(2    ) as p:
    assert expected_fingerprint == p.apply_async(dataset.funca    ).get()._fingerprint
    assert expected_fingerprint != p.apply_async(dataset.funca    ).get()._fingerprint


def 							A__ (     ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			{'''a''': [0, 1, 2]}




  class    __snake_case(					datasets.Dataset       ):




      @fingerprint_transform(inplace=A_			)
      def   A   (   self    ,							A_			):





        '''simple docstring'''



        return DummyDatasetChild(self.data    ,							fingerprint=A_			)



  _SCREAMING_SNAKE_CASE        =			DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__    )    ).func()




  class    __snake_case(					datasets.Dataset       ):




      @fingerprint_transform(inplace=A_    ,							version='''1.0.0'''			)
      def   A   (   self    ,							A_			):





        '''simple docstring'''



        return DummyDatasetChild(self.data    ,							fingerprint=A_			)



  _SCREAMING_SNAKE_CASE        =			DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__    )    ).func()




  class    __snake_case(					datasets.Dataset       ):




      @fingerprint_transform(inplace=A_    ,							version='''2.0.0'''			)
      def   A   (   self    ,							A_			):





        '''simple docstring'''



        return DummyDatasetChild(self.data    ,							fingerprint=A_			)



  _SCREAMING_SNAKE_CASE        =			DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__    )    ).func()

  assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}    ) == 3







def 							A__ (     ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			Hasher()
  hasher.update(lambda UpperCamelCase__    : x    )





"""simple docstring"""






from transformers import ProcessorMixin




class    __snake_case(					__A       ):
    _A       = '''AutoFeatureExtractor'''
    _A       = '''AutoTokenizer'''
