

import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)



class 						a_			:



      """simple docstring"""







      def __init__( self	:				Optional[int]     ,snake_case	:				Any							):
           SCREAMING_SNAKE_CASE 		=foo


      def __call__( self	:				Union[str, Any]							):
           return self.foo







class 						a_			(				datasets.Dataset	):



      """simple docstring"""







      @fingerprint_transform(inplace=A_							)
      def 			_lowerCAmelCase	( self	:				Optional[Any]     ,snake_case	:				int     ,*snake_case	:				Tuple     ,**snake_case	:				Dict							):
           return DatasetChild(self.data     ,fingerprint=A_							)


      @fingerprint_transform(inplace=A_							)
      def 			_lowerCAmelCase	( self	:				List[str]     ,snake_case	:				int     ,*snake_case	:				List[str]     ,**snake_case	:				Optional[Any]							):
           return DatasetChild(self.data     ,fingerprint=A_							)







class 						a_			:



      """simple docstring"""







      def __init__( self	:				int     ,snake_case	:				Optional[Any]							):
           SCREAMING_SNAKE_CASE 		=callable
      def __call__( self	:				Union[str, Any]     ,*snake_case	:				Optional[Any]     ,**snake_case	:				Optional[Any]							):



           if self.callable is not None:
                return self.callable(*A_     ,**A_							)


      def __getstate__( self	:				Optional[Any]							):
           raise pickle.PicklingError()







class 						a_			(				__A	):



      """simple docstring"""







      @require_transformers
      @pytest.mark.integration
      def 			_lowerCAmelCase	( self	:				Optional[Any]							):
           from transformers import AutoTokenizer

           def encode(snake_case	:				str							):
                return tokenizer(A_							)

           # TODO: add hash consistency tests across sessions
           SCREAMING_SNAKE_CASE 		=AutoTokenizer.from_pretrained('bert-base-uncased'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(lambda snake_case							: tokenizer(A_							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=AutoTokenizer.from_pretrained('bert-base-cased'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(lambda snake_case							: tokenizer(A_							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=AutoTokenizer.from_pretrained('bert-base-uncased'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(lambda snake_case							: tokenizer(A_							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      @require_transformers
      @pytest.mark.integration
      def 			_lowerCAmelCase	( self	:				Optional[int]							):
           from transformers import AutoTokenizer

           SCREAMING_SNAKE_CASE 		=AutoTokenizer.from_pretrained('gpt2'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           tokenizer('Hello world !'							)  # call once to change the tokenizer's cache
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)


      @require_regex
      def 			_lowerCAmelCase	( self	:				List[str]							):
           import regex

           SCREAMING_SNAKE_CASE 		=regex.Regex('foo'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=regex.Regex('bar'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=regex.Regex('foo'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)







class 						a_			(				__A	):



      """simple docstring"""







      def 			_lowerCAmelCase	( self	:				Optional[Any]							):



           def func():
                return foo

           SCREAMING_SNAKE_CASE 		=[0]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=[1]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=[0]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      def 			_lowerCAmelCase	( self	:				Union[str, Any]							):



           def func():
                pass

           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()

           def func():
                pass

           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
      def 			_lowerCAmelCase	( self	:				Tuple							):
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(Foo([0]							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(Foo([1]							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(Foo([0]							)							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      def 			_lowerCAmelCase	( self	:				str							):
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(Foo([0]							).__call__							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(Foo([1]							).__call__							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(Foo([0]							).__call__							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      def 			_lowerCAmelCase	( self	:				str							):
           SCREAMING_SNAKE_CASE 		=(
               'co_argcount',
               'co_kwonlyargcount',
               'co_nlocals',
               'co_stacksize',
               'co_flags',
               'co_code',
               'co_consts',
               'co_names',
               'co_varnames',
               'co_filename',
               'co_name',
               'co_firstlineno',
               'co_lnotab',
               'co_freevars',
               'co_cellvars',
           )

           def _create_code_pyaa(*snake_case	:				Dict							):
                if hasattr(args[-3]     ,'encode'							):
                     SCREAMING_SNAKE_CASE 		=list(A_							)
                     SCREAMING_SNAKE_CASE 		=args[-3].encode()  # co_lnotab
                     SCREAMING_SNAKE_CASE 		=args[-10].encode()  # co_code
                if hasattr(A_     ,'co_posonlyargcount'							):
                     if len(A_							) == 16:
                          return CodeType(*A_							)
                     elif len(A_							) == 15:
                          return CodeType(args[0]     ,0     ,*args[1:]							)
                     return CodeType(args[0]     ,0     ,0     ,*args[1:]							)
                elif hasattr(A_     ,'co_kwonlyargcount'							):
                     if len(A_							) == 16:
                          return CodeType(args[0]     ,*args[2:]							)
                     elif len(A_							) == 15:
                          return CodeType(*A_							)
                     return CodeType(args[0]     ,0     ,*args[1:]							)
                if len(A_							) == 16:
                     return CodeType(args[0]     ,*args[3:]							)
                elif len(A_							) == 15:
                     return CodeType(args[0]     ,*args[2:]							)
                return CodeType(*A_							)

           def create_ipython_func(snake_case	:				Dict     ,snake_case	:				str							):
                def func():
                     return returned_obj

                SCREAMING_SNAKE_CASE 		=func.__code__
                # Use _create_code from dill in order to make it work for different python versions
                if datasets.config.PY_VERSION < version.parse('3.8'							):
                     SCREAMING_SNAKE_CASE 		=_create_code_pyaa(
                         *[getattr(A_     ,A_							) if k != 'co_filename' else co_filename for k in code_args_pyaa]							)
                else:
                     SCREAMING_SNAKE_CASE 		=code.replace(co_filename=A_							)
                return FunctionType(A_     ,func.__globals__     ,func.__name__     ,func.__defaults__     ,func.__closure__							)

           SCREAMING_SNAKE_CASE		,    SCREAMING_SNAKE_CASE 		='<ipython-input-2-e0383a102aae>', [0]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(create_ipython_func(A_     ,A_							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE		,    SCREAMING_SNAKE_CASE 		='<ipython-input-2-e0383a102aae>', [1]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(create_ipython_func(A_     ,A_							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE		,    SCREAMING_SNAKE_CASE 		='<ipython-input-5-713f6613acf3>', [0]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(create_ipython_func(A_     ,A_							)							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)

           SCREAMING_SNAKE_CASE		,    SCREAMING_SNAKE_CASE 		=os.path.join(gettempdir()     ,'ipykernel_12345'     ,'321456789.py'							), [0]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(create_ipython_func(A_     ,A_							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE		,    SCREAMING_SNAKE_CASE 		=os.path.join(gettempdir()     ,'ipykernel_12345'     ,'321456789.py'							), [1]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(create_ipython_func(A_     ,A_							)							)							).hexdigest()
           SCREAMING_SNAKE_CASE		,    SCREAMING_SNAKE_CASE 		=os.path.join(gettempdir()     ,'ipykernel_12345'     ,'654123987.py'							), [0]
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(create_ipython_func(A_     ,A_							)							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)


      def 			_lowerCAmelCase	( self	:				Optional[Any]							):
           SCREAMING_SNAKE_CASE		,    SCREAMING_SNAKE_CASE 		=[0], [1]

           def func():
                return foo, bar

           SCREAMING_SNAKE_CASE 		='__main__'

           def globalvars_mocka_side_effect(snake_case	:				Tuple     ,*snake_case	:				Optional[int]     ,**snake_case	:				List[Any]							):
                return {"foo": foo, "bar": bar}

           def globalvars_mocka_side_effect(snake_case	:				Any     ,*snake_case	:				List[Any]     ,**snake_case	:				int							):
                return {"bar": bar, "foo": foo}

           with patch('dill.detect.globalvars'     ,side_effect=A_							) as globalvars_mocka:
                SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
                self.assertGreater(globalvars_mocka.call_count     ,0							)
           with patch('dill.detect.globalvars'     ,side_effect=A_							) as globalvars_mocka:
                SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
                self.assertGreater(globalvars_mocka.call_count     ,0							)
           self.assertEqual(A_     ,A_							)







class 						a_			(				__A	):



      """simple docstring"""







      def 			_lowerCAmelCase	( self	:				int							):
           SCREAMING_SNAKE_CASE 		=Hasher.hash('hello'							)
           SCREAMING_SNAKE_CASE 		=Hasher.hash('hello'							)
           SCREAMING_SNAKE_CASE 		=Hasher.hash('there'							)
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      def 			_lowerCAmelCase	( self	:				Optional[int]							):
           SCREAMING_SNAKE_CASE 		=Hasher.hash(Foo('hello'							)							)
           SCREAMING_SNAKE_CASE 		=Hasher.hash(Foo('hello'							)							)
           SCREAMING_SNAKE_CASE 		=Hasher.hash(Foo('there'							)							)
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      def 			_lowerCAmelCase	( self	:				Tuple							):
           SCREAMING_SNAKE_CASE 		=Hasher()
           for x in ["hello", Foo('hello'							)]:
                hasher.update(A_							)
           SCREAMING_SNAKE_CASE 		=hasher.hexdigest()
           SCREAMING_SNAKE_CASE 		=Hasher()
           for x in ["hello", Foo('hello'							)]:
                hasher.update(A_							)
           SCREAMING_SNAKE_CASE 		=hasher.hexdigest()
           SCREAMING_SNAKE_CASE 		=Hasher()
           for x in ["there", Foo('there'							)]:
                hasher.update(A_							)
           SCREAMING_SNAKE_CASE 		=hasher.hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      def 			_lowerCAmelCase	( self	:				Dict							):



           with self.assertRaises(pickle.PicklingError							):
                Hasher.hash(UnpicklableCallable(Foo('hello'							)							)							)
      def 			_lowerCAmelCase	( self	:				Tuple							):
           SCREAMING_SNAKE_CASE 		='abc'
           SCREAMING_SNAKE_CASE 		=[string, string]  # two strings have the same ids
           SCREAMING_SNAKE_CASE 		=[string, string]
           SCREAMING_SNAKE_CASE 		=json.loads(f'["{string}", "{string}"]'							)  # two strings have different ids
           self.assertIs(obja[0]     ,A_							)
           self.assertIs(obja[0]     ,obja[1]							)
           self.assertIs(obja[0]     ,A_							)
           self.assertIs(obja[0]     ,obja[1]							)
           self.assertIsNot(obja[0]     ,A_							)
           self.assertIsNot(obja[0]     ,obja[1]							)
           SCREAMING_SNAKE_CASE 		=Hasher.hash(A_							)
           SCREAMING_SNAKE_CASE 		=Hasher.hash(A_							)
           SCREAMING_SNAKE_CASE 		=Hasher.hash(A_							)
           self.assertEqual(A_     ,A_							)
           self.assertEqual(A_     ,A_							)
      @require_tiktoken
      def 			_lowerCAmelCase	( self	:				Any							):
           import tiktoken

           SCREAMING_SNAKE_CASE 		=tiktoken.get_encoding('gpt2'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=tiktoken.get_encoding('r50k_base'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=tiktoken.get_encoding('gpt2'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)
      @require_torch
      def 			_lowerCAmelCase	( self	:				List[str]							):
           import torch

           SCREAMING_SNAKE_CASE 		=torch.tensor([1.0]							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=torch.tensor([2.0]							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=torch.tensor([1.0]							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)


      @require_spacy
      @require_spacy_model('en_core_web_sm'							)
      @require_spacy_model('fr_core_news_sm'							)
      @pytest.mark.integration
      def 			_lowerCAmelCase	( self	:				List[str]							):
           import spacy

           SCREAMING_SNAKE_CASE 		=spacy.load('en_core_web_sm'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=spacy.load('fr_core_news_sm'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           SCREAMING_SNAKE_CASE 		=spacy.load('en_core_web_sm'							)
           SCREAMING_SNAKE_CASE 		=mda(datasets.utils.py_utils.dumps(A_							)							).hexdigest()
           self.assertEqual(A_     ,A_							)
           self.assertNotEqual(A_     ,A_							)







@pytest.mark.integration
def      snake_case__							(  lowerCAmelCase_     ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE 		=tmp_path / 'dir1'
     SCREAMING_SNAKE_CASE 		=tmp_path / 'dir2'
     dira.mkdir()
     dira.mkdir()
     SCREAMING_SNAKE_CASE 		='script.py'
     SCREAMING_SNAKE_CASE 		=dedent(
         '\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    '     )
     SCREAMING_SNAKE_CASE 		=dira / script_filename
     SCREAMING_SNAKE_CASE 		=dira / script_filename
     with script_patha.open('w'     ) as f:
          f.write(UpperCamelCase__     )
     with script_patha.open('w'     ) as f:
          f.write(UpperCamelCase__     )
     SCREAMING_SNAKE_CASE 		=subprocess.check_output(['python', str(UpperCamelCase__     )]     )
     SCREAMING_SNAKE_CASE 		=subprocess.check_output(['python', str(UpperCamelCase__     )]     )
     assert fingerprinta == fingerprinta







def      snake_case__							(  ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE 		={'a': [0, 1, 2]}
     SCREAMING_SNAKE_CASE 		=DatasetChild(InMemoryTable.from_pydict(UpperCamelCase__     )     )
     SCREAMING_SNAKE_CASE 		=dataset.funca()._fingerprint
     assert expected_fingerprint == dataset.funca()._fingerprint
     assert expected_fingerprint != dataset.funca()._fingerprint

     with Pool(2     ) as p:
          assert expected_fingerprint == p.apply_async(dataset.funca     ).get()._fingerprint
          assert expected_fingerprint != p.apply_async(dataset.funca     ).get()._fingerprint







def      snake_case__							(  ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE 		={'a': [0, 1, 2]}



     class 						a_			(				datasets.Dataset	):



           """simple docstring"""







           @fingerprint_transform(inplace=A_							)
           def 			_lowerCAmelCase	( self	:				Union[str, Any]     ,snake_case	:				Optional[Any]							):
                return DummyDatasetChild(self.data     ,fingerprint=A_							)







     SCREAMING_SNAKE_CASE 		=DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__     )     ).func()



     class 						a_			(				datasets.Dataset	):



           """simple docstring"""







           @fingerprint_transform(inplace=A_     ,version='1.0.0'							)
           def 			_lowerCAmelCase	( self	:				Any     ,snake_case	:				List[str]							):
                return DummyDatasetChild(self.data     ,fingerprint=A_							)







     SCREAMING_SNAKE_CASE 		=DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__     )     ).func()



     class 						a_			(				datasets.Dataset	):



           """simple docstring"""







           @fingerprint_transform(inplace=A_     ,version='2.0.0'							)
           def 			_lowerCAmelCase	( self	:				Any     ,snake_case	:				Dict							):
                return DummyDatasetChild(self.data     ,fingerprint=A_							)







     SCREAMING_SNAKE_CASE 		=DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase__     )     ).func()

     assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}     ) == 3







def      snake_case__							(  ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE 		=Hasher()
     hasher.update(lambda lowerCAmelCase_     : x     )






"""simple docstring"""






from math import gcd


def 							A__ (     UpperCamelCase__ = 12_000    ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			0
  for d in range(max_d + 1    ):
    for n in range(d // 3 + 1       ,							(d + 1) // 2    ):
      if gcd(UpperCamelCase__       ,							UpperCamelCase__    ) == 1:
        fractions_number += 1
  return fractions_number


if __name__ == "__main__":
    print(F'''{solution() = }''')
