
"""simple docstring"""






import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)





class 							snake_case       :







   def __init__( self       ,							__UpperCAmelCase)   ->int:
      a_						   =     foo







   def __call__( self)   ->Union[str, Any]:
      return self.foo






class 							snake_case       (     datasets.Dataset		):







   @fingerprint_transform(inplace=__UpperCAmelCase)
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase)   ->Optional[int]:
      return DatasetChild(self.data       ,							fingerprint=__UpperCAmelCase)







   @fingerprint_transform(inplace=__UpperCAmelCase)
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase)   ->Optional[int]:
      return DatasetChild(self.data       ,							fingerprint=__UpperCAmelCase)






class 							snake_case       :







   def __init__( self       ,							__UpperCAmelCase)   ->int:
      a_						   =     callable







   def __call__( self       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase)   ->List[Any]:
      if self.callable is not None:
         return self.callable(*__UpperCAmelCase       ,							**__UpperCAmelCase)







   def __getstate__( self)   ->Union[str, Any]:
      raise pickle.PicklingError()






class 							snake_case       (     SCREAMING_SNAKE_CASE_		):







   @require_transformers
   @pytest.mark.integration
   def    UpperCAmelCase__	( self)   ->Optional[int]:
      from transformers import AutoTokenizer

      def encode(__UpperCAmelCase):
         return tokenizer(__UpperCAmelCase)

      # TODO: add hash consistency tests across sessions
      a_						   =     AutoTokenizer.from_pretrained("bert-base-uncased")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase: tokenizer(__UpperCAmelCase))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     AutoTokenizer.from_pretrained("bert-base-cased")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase: tokenizer(__UpperCAmelCase))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     AutoTokenizer.from_pretrained("bert-base-uncased")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase: tokenizer(__UpperCAmelCase))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @require_transformers
   @pytest.mark.integration
   def    UpperCAmelCase__	( self)   ->List[Any]:
      from transformers import AutoTokenizer

      a_						   =     AutoTokenizer.from_pretrained("gpt2")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      tokenizer("Hello world !")  # call once to change the tokenizer's cache
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @require_regex
   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      import regex

      a_						   =     regex.Regex("foo")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     regex.Regex("bar")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     regex.Regex("foo")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)






class 							snake_case       (     SCREAMING_SNAKE_CASE_		):







   def    UpperCAmelCase__	( self)   ->int:
      def func():
         return foo

      a_						   =     [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     [1]
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->List[Any]:
      def func():
         pass

      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()

      def func():
         pass

      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->str:
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->List[str]:
      a_						   =     (
          "co_argcount",
          "co_kwonlyargcount",
          "co_nlocals",
          "co_stacksize",
          "co_flags",
          "co_code",
          "co_consts",
          "co_names",
          "co_varnames",
          "co_filename",
          "co_name",
          "co_firstlineno",
          "co_lnotab",
          "co_freevars",
          "co_cellvars",
      )

      def _create_code_pyaa(*__UpperCAmelCase):
         if hasattr(args[-3]       ,							"encode"):
            a_						   =     list(__UpperCAmelCase)
            a_						   =     args[-3].encode()  # co_lnotab
            a_						   =     args[-10].encode()  # co_code
         if hasattr(__UpperCAmelCase       ,							"co_posonlyargcount"):
            if len(__UpperCAmelCase) == 16:
               return CodeType(*__UpperCAmelCase)
            elif len(__UpperCAmelCase) == 15:
               return CodeType(args[0]       ,							0       ,							*args[1:])
            return CodeType(args[0]       ,							0       ,							0       ,							*args[1:])
         elif hasattr(__UpperCAmelCase       ,							"co_kwonlyargcount"):
            if len(__UpperCAmelCase) == 16:
               return CodeType(args[0]       ,							*args[2:])
            elif len(__UpperCAmelCase) == 15:
               return CodeType(*__UpperCAmelCase)
            return CodeType(args[0]       ,							0       ,							*args[1:])
         if len(__UpperCAmelCase) == 16:
            return CodeType(args[0]       ,							*args[3:])
         elif len(__UpperCAmelCase) == 15:
            return CodeType(args[0]       ,							*args[2:])
         return CodeType(*__UpperCAmelCase)

      def create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase):
         def func():
            return returned_obj

         a_						   =     func.__code__
         # Use _create_code from dill in order to make it work for different python versions
         if datasets.config.PY_VERSION < version.parse("3.8"):
            a_						   =     _create_code_pyaa(
                *[getattr(__UpperCAmelCase       ,							__UpperCAmelCase) if k != "co_filename" else co_filename for k in code_args_pyaa])
         else:
            a_						   =     code.replace(co_filename=__UpperCAmelCase)
         return FunctionType(__UpperCAmelCase       ,							func.__globals__       ,							func.__name__       ,							func.__defaults__       ,							func.__closure__)

      a_   ,			a_						   =     "<ipython-input-2-e0383a102aae>", [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase))).hexdigest()
      a_   ,			a_						   =     "<ipython-input-2-e0383a102aae>", [1]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase))).hexdigest()
      a_   ,			a_						   =     "<ipython-input-5-713f6613acf3>", [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase))).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      a_   ,			a_						   =     os.path.join(gettempdir()       ,							"ipykernel_12345"       ,							"321456789.py"), [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase))).hexdigest()
      a_   ,			a_						   =     os.path.join(gettempdir()       ,							"ipykernel_12345"       ,							"321456789.py"), [1]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase))).hexdigest()
      a_   ,			a_						   =     os.path.join(gettempdir()       ,							"ipykernel_12345"       ,							"654123987.py"), [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase))).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->List[str]:
      a_   ,			a_						   =     [0], [1]

      def func():
         return foo, bar

      a_						   =     "__main__"

      def globalvars_mocka_side_effect(__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase):
         return {"foo": foo, "bar": bar}

      def globalvars_mocka_side_effect(__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase):
         return {"bar": bar, "foo": foo}

      with patch("dill.detect.globalvars"       ,							side_effect=__UpperCAmelCase) as globalvars_mocka:
         a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count       ,							0)
      with patch("dill.detect.globalvars"       ,							side_effect=__UpperCAmelCase) as globalvars_mocka:
         a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count       ,							0)
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)






class 							snake_case       (     SCREAMING_SNAKE_CASE_		):







   def    UpperCAmelCase__	( self)   ->Optional[Any]:
      a_						   =     Hasher.hash("hello")
      a_						   =     Hasher.hash("hello")
      a_						   =     Hasher.hash("there")
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->Any:
      a_						   =     Hasher.hash(Foo("hello"))
      a_						   =     Hasher.hash(Foo("hello"))
      a_						   =     Hasher.hash(Foo("there"))
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      a_						   =     Hasher()
      for x in ["hello", Foo("hello")]:
         hasher.update(__UpperCAmelCase)
      a_						   =     hasher.hexdigest()
      a_						   =     Hasher()
      for x in ["hello", Foo("hello")]:
         hasher.update(__UpperCAmelCase)
      a_						   =     hasher.hexdigest()
      a_						   =     Hasher()
      for x in ["there", Foo("there")]:
         hasher.update(__UpperCAmelCase)
      a_						   =     hasher.hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->Optional[int]:
      with self.assertRaises(pickle.PicklingError):
         Hasher.hash(UnpicklableCallable(Foo("hello")))







   def    UpperCAmelCase__	( self)   ->int:
      a_						   =     "abc"
      a_						   =     [string, string]  # two strings have the same ids
      a_						   =     [string, string]
      a_						   =     json.loads(F'''["{string}", "{string}"]''')  # two strings have different ids
      self.assertIs(obja[0]       ,							__UpperCAmelCase)
      self.assertIs(obja[0]       ,							obja[1])
      self.assertIs(obja[0]       ,							__UpperCAmelCase)
      self.assertIs(obja[0]       ,							obja[1])
      self.assertIsNot(obja[0]       ,							__UpperCAmelCase)
      self.assertIsNot(obja[0]       ,							obja[1])
      a_						   =     Hasher.hash(__UpperCAmelCase)
      a_						   =     Hasher.hash(__UpperCAmelCase)
      a_						   =     Hasher.hash(__UpperCAmelCase)
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @require_tiktoken
   def    UpperCAmelCase__	( self)   ->int:
      import tiktoken

      a_						   =     tiktoken.get_encoding("gpt2")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     tiktoken.get_encoding("r50k_base")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     tiktoken.get_encoding("gpt2")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @require_torch
   def    UpperCAmelCase__	( self)   ->Any:
      import torch

      a_						   =     torch.tensor([1.0])
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     torch.tensor([2.0])
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     torch.tensor([1.0])
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @require_spacy
   @require_spacy_model("en_core_web_sm")
   @require_spacy_model("fr_core_news_sm")
   @pytest.mark.integration
   def    UpperCAmelCase__	( self)   ->Optional[int]:
      import spacy

      a_						   =     spacy.load("en_core_web_sm")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     spacy.load("fr_core_news_sm")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      a_						   =     spacy.load("en_core_web_sm")
      a_						   =     mda(datasets.utils.py_utils.dumps(__UpperCAmelCase)).hexdigest()
      self.assertEqual(__UpperCAmelCase       ,							__UpperCAmelCase)
      self.assertNotEqual(__UpperCAmelCase       ,							__UpperCAmelCase)






@pytest.mark.integration
def    UpperCamelCase						(					UpperCAmelCase     )						->Any:

   """simple docstring"""



   a_						   =     tmp_path / "dir1"
   a_						   =     tmp_path / "dir2"
   dira.mkdir()
   dira.mkdir()
   a_						   =     "script.py"
   a_						   =     dedent(
       "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "     )
   a_						   =     dira / script_filename
   a_						   =     dira / script_filename
   with script_patha.open("w"     ) as f:
      f.write(UpperCAmelCase     )
   with script_patha.open("w"     ) as f:
      f.write(UpperCAmelCase     )
   a_						   =     subprocess.check_output(["python", str(UpperCAmelCase     )]     )
   a_						   =     subprocess.check_output(["python", str(UpperCAmelCase     )]     )
   assert fingerprinta == fingerprinta
def    UpperCamelCase						(					)						->Union[str, Any]:

   """simple docstring"""



   a_						   =     {"a": [0, 1, 2]}
   a_						   =     DatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     )
   a_						   =     dataset.funca()._fingerprint
   assert expected_fingerprint == dataset.funca()._fingerprint
   assert expected_fingerprint != dataset.funca()._fingerprint

   with Pool(2     ) as p:
      assert expected_fingerprint == p.apply_async(dataset.funca     ).get()._fingerprint
      assert expected_fingerprint != p.apply_async(dataset.funca     ).get()._fingerprint
def    UpperCamelCase						(					)						->Any:

   """simple docstring"""



   a_						   =     {"a": [0, 1, 2]}





   class 							snake_case       (     datasets.Dataset		):







      @fingerprint_transform(inplace=__UpperCAmelCase)
      def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->List[Any]:
         return DummyDatasetChild(self.data       ,							fingerprint=__UpperCAmelCase)






   a_						   =     DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     ).func()





   class 							snake_case       (     datasets.Dataset		):







      @fingerprint_transform(inplace=__UpperCAmelCase       ,							version="1.0.0")
      def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Any:
         return DummyDatasetChild(self.data       ,							fingerprint=__UpperCAmelCase)






   a_						   =     DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     ).func()





   class 							snake_case       (     datasets.Dataset		):







      @fingerprint_transform(inplace=__UpperCAmelCase       ,							version="2.0.0")
      def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->int:
         return DummyDatasetChild(self.data       ,							fingerprint=__UpperCAmelCase)






   a_						   =     DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     ).func()

   assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}     ) == 3



def    UpperCamelCase						(					)						->Any:

   """simple docstring"""



   a_						   =     Hasher()
   hasher.update(lambda UpperCAmelCase     : x     )

"""simple docstring"""






def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase = 1_000     )						->str:

   """simple docstring"""



   if (not isinstance(UpperCAmelCase ,   UpperCAmelCase     )) or (digit_position <= 0):
      raise ValueError("Digit position must be a positive integer"     )
   elif (not isinstance(UpperCAmelCase ,   UpperCAmelCase     )) or (precision < 0):
      raise ValueError("Precision must be a nonnegative integer"     )

   # compute an approximation of (16 ** (n - 1)) * pi whose fractional part is mostly
   # accurate
   a_						   =     (
       4 * _subsum(UpperCAmelCase ,   1 ,   UpperCAmelCase     )
       - 2 * _subsum(UpperCAmelCase ,   4 ,   UpperCAmelCase     )
       - _subsum(UpperCAmelCase ,   5 ,   UpperCAmelCase     )
       - _subsum(UpperCAmelCase ,   6 ,   UpperCAmelCase     )
   )

   # return the first hex digit of the fractional part of the result
   return hex(int((sum_result % 1) * 16     )     )[2:]
def    UpperCamelCase						(					UpperCAmelCase ,   UpperCAmelCase ,   UpperCAmelCase     )						->float:

   """simple docstring"""



   a_						   =     0.0
   for sum_index in range(digit_pos_to_extract + precision     ):
      a_						   =     8 * sum_index + denominator_addend
      if sum_index < digit_pos_to_extract:
         # if the exponential term is an integer and we mod it by the denominator
         # before dividing, only the integer part of the sum will change;
         # the fractional part will not
         a_						   =     pow(
             16 ,   digit_pos_to_extract - 1 - sum_index ,   UpperCAmelCase     )
      else:
         a_						   =     pow(16 ,   digit_pos_to_extract - 1 - sum_index     )
      total += exponential_term / denominator
   return total


if __name__ == "__main__":
      import doctest

      doctest.testmod()