
'''simple docstring'''







import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)
class  SCREAMING_SNAKE_CASE__       :
	def __init__(				self  ,    __UpperCamelCase			):



					'''simple docstring'''






					__a       :     str								=	foo

	def __call__(				self			):



					'''simple docstring'''






					return self.foo







class  SCREAMING_SNAKE_CASE__       (  datasets.Dataset   ):
	@fingerprint_transform(inplace=__UpperCamelCase			)
	def __lowerCamelCase    (				self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):



					'''simple docstring'''






					return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

	@fingerprint_transform(inplace=__UpperCamelCase			)
	def __lowerCamelCase    (				self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):



					'''simple docstring'''






					return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







class  SCREAMING_SNAKE_CASE__       :
	def __init__(				self  ,    __UpperCamelCase			):



					'''simple docstring'''






					__a       :     Optional[Any]								=	callable
	def __call__(				self  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):



					'''simple docstring'''






					if self.callable is not None:
									return self.callable(*__UpperCamelCase  ,    **__UpperCamelCase			)

	def __getstate__(				self			):



					'''simple docstring'''






					raise pickle.PicklingError()







class  SCREAMING_SNAKE_CASE__       (  __UpperCamelCase   ):
	@require_transformers
	@pytest.mark.integration
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					from transformers import AutoTokenizer

					def encode(__UpperCamelCase			):
									return tokenizer(__UpperCamelCase			)

					# TODO: add hash consistency tests across sessions
					__a       :     Union[str, Any]								=	AutoTokenizer.from_pretrained("""bert-base-uncased"""			)
					__a       :     Any								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Optional[Any]								=	mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
					__a       :     Any								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Optional[Any]								=	AutoTokenizer.from_pretrained("""bert-base-cased"""			)
					__a       :     Tuple								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     List[str]								=	mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
					__a       :     Any								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     List[str]								=	AutoTokenizer.from_pretrained("""bert-base-uncased"""			)
					__a       :     List[str]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Any								=	mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
					__a       :     Any								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	@require_transformers
	@pytest.mark.integration
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					from transformers import AutoTokenizer

					__a       :     Tuple								=	AutoTokenizer.from_pretrained("""gpt2"""			)
					__a       :     Dict								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					tokenizer("""Hello world !"""			)  # call once to change the tokenizer's cache
					__a       :     Optional[int]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)

	@require_regex
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					import regex

					__a       :     Optional[Any]								=	regex.Regex("""foo"""			)
					__a       :     int								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Optional[int]								=	regex.Regex("""bar"""			)
					__a       :     Dict								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Tuple								=	regex.Regex("""foo"""			)
					__a       :     Union[str, Any]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







class  SCREAMING_SNAKE_CASE__       (  __UpperCamelCase   ):
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					def func():
									return foo

					__a       :     Dict								=	[0]
					__a       :     Union[str, Any]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Any								=	[1]
					__a       :     Tuple								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     List[str]								=	[0]
					__a       :     Union[str, Any]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					def func():
									pass

					__a       :     Dict								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()

					def func():
									pass

					__a       :     Tuple								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Union[str, Any]								=	mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
					__a       :     Union[str, Any]								=	mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
					__a       :     Optional[Any]								=	mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Optional[Any]								=	mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
					__a       :     List[Any]								=	mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
					__a       :     int								=	mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	(
					    """co_argcount""",
					    """co_kwonlyargcount""",
					    """co_nlocals""",
					    """co_stacksize""",
					    """co_flags""",
					    """co_code""",
					    """co_consts""",
					    """co_names""",
					    """co_varnames""",
					    """co_filename""",
					    """co_name""",
					    """co_firstlineno""",
					    """co_lnotab""",
					    """co_freevars""",
					    """co_cellvars""",
					)

					def _create_code_pyaa(*__UpperCamelCase			):
									if hasattr(args[-3]  ,    """encode"""			):
													__a       :     Optional[int]								=	list(__UpperCamelCase			)
													__a       :     Optional[int]								=	args[-3].encode()  # co_lnotab
													__a       :     Optional[Any]								=	args[-10].encode()  # co_code
									if hasattr(__UpperCamelCase  ,    """co_posonlyargcount"""			):
													if len(__UpperCamelCase			) == 16:
																	return CodeType(*__UpperCamelCase			)
													elif len(__UpperCamelCase			) == 15:
																	return CodeType(args[0]  ,    0  ,    *args[1:]			)
													return CodeType(args[0]  ,    0  ,    0  ,    *args[1:]			)
									elif hasattr(__UpperCamelCase  ,    """co_kwonlyargcount"""			):
													if len(__UpperCamelCase			) == 16:
																	return CodeType(args[0]  ,    *args[2:]			)
													elif len(__UpperCamelCase			) == 15:
																	return CodeType(*__UpperCamelCase			)
													return CodeType(args[0]  ,    0  ,    *args[1:]			)
									if len(__UpperCamelCase			) == 16:
													return CodeType(args[0]  ,    *args[3:]			)
									elif len(__UpperCamelCase			) == 15:
													return CodeType(args[0]  ,    *args[2:]			)
									return CodeType(*__UpperCamelCase			)

					def create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			):
									def func():
													return returned_obj

									__a       :     Union[str, Any]								=	func.__code__
									# Use _create_code from dill in order to make it work for different python versions
									if datasets.config.PY_VERSION < version.parse("""3.8"""			):
													__a       :     List[Any]								=	_create_code_pyaa(
													    *[getattr(__UpperCamelCase  ,    __UpperCamelCase			) if k != """co_filename""" else co_filename for k in code_args_pyaa]			)
									else:
													__a       :     Union[str, Any]								=	code.replace(co_filename=__UpperCamelCase			)
									return FunctionType(__UpperCamelCase  ,    func.__globals__  ,    func.__name__  ,    func.__defaults__  ,    func.__closure__			)

					__a							,	__a       :     Tuple								=	"""<ipython-input-2-e0383a102aae>""", [0]
					__a       :     Dict								=	mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
					__a							,	__a       :     List[str]								=	"""<ipython-input-2-e0383a102aae>""", [1]
					__a       :     int								=	mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
					__a							,	__a       :     List[Any]								=	"""<ipython-input-5-713f6613acf3>""", [0]
					__a       :     int								=	mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

					__a							,	__a       :     List[str]								=	os.path.join(gettempdir()  ,    """ipykernel_12345"""  ,    """321456789.py"""			), [0]
					__a       :     Any								=	mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
					__a							,	__a       :     List[Any]								=	os.path.join(gettempdir()  ,    """ipykernel_12345"""  ,    """321456789.py"""			), [1]
					__a       :     Optional[Any]								=	mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
					__a							,	__a       :     Union[str, Any]								=	os.path.join(gettempdir()  ,    """ipykernel_12345"""  ,    """654123987.py"""			), [0]
					__a       :     str								=	mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a							,	__a       :     str								=	[0], [1]

					def func():
									return foo, bar

					__a       :     Tuple								=	"""__main__"""

					def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
									return {"foo": foo, "bar": bar}

					def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
									return {"bar": bar, "foo": foo}

					with patch("""dill.detect.globalvars"""  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
									__a       :     Any								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
									self.assertGreater(globalvars_mocka.call_count  ,    0			)
					with patch("""dill.detect.globalvars"""  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
									__a       :     List[str]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
									self.assertGreater(globalvars_mocka.call_count  ,    0			)
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)







class  SCREAMING_SNAKE_CASE__       (  __UpperCamelCase   ):
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Union[str, Any]								=	Hasher.hash("""hello"""			)
					__a       :     str								=	Hasher.hash("""hello"""			)
					__a       :     List[str]								=	Hasher.hash("""there"""			)
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Any								=	Hasher.hash(Foo("""hello"""			)			)
					__a       :     Union[str, Any]								=	Hasher.hash(Foo("""hello"""			)			)
					__a       :     str								=	Hasher.hash(Foo("""there"""			)			)
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Union[str, Any]								=	Hasher()
					for x in ["hello", Foo("""hello"""			)]:
									hasher.update(__UpperCamelCase			)
					__a       :     Union[str, Any]								=	hasher.hexdigest()
					__a       :     Optional[int]								=	Hasher()
					for x in ["hello", Foo("""hello"""			)]:
									hasher.update(__UpperCamelCase			)
					__a       :     Union[str, Any]								=	hasher.hexdigest()
					__a       :     Any								=	Hasher()
					for x in ["there", Foo("""there"""			)]:
									hasher.update(__UpperCamelCase			)
					__a       :     Tuple								=	hasher.hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					with self.assertRaises(pickle.PicklingError			):
									Hasher.hash(UnpicklableCallable(Foo("""hello"""			)			)			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     str								=	"""abc"""
					__a       :     List[Any]								=	[string, string]  # two strings have the same ids
					__a       :     Tuple								=	[string, string]
					__a       :     str								=	json.loads(f"""[\"{string}\", \"{string}\"]"""			)  # two strings have different ids
					self.assertIs(obja[0]  ,    __UpperCamelCase			)
					self.assertIs(obja[0]  ,    obja[1]			)
					self.assertIs(obja[0]  ,    __UpperCamelCase			)
					self.assertIs(obja[0]  ,    obja[1]			)
					self.assertIsNot(obja[0]  ,    __UpperCamelCase			)
					self.assertIsNot(obja[0]  ,    obja[1]			)
					__a       :     Union[str, Any]								=	Hasher.hash(__UpperCamelCase			)
					__a       :     List[Any]								=	Hasher.hash(__UpperCamelCase			)
					__a       :     Any								=	Hasher.hash(__UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	@require_tiktoken
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					import tiktoken

					__a       :     Tuple								=	tiktoken.get_encoding("""gpt2"""			)
					__a       :     Tuple								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Dict								=	tiktoken.get_encoding("""r50k_base"""			)
					__a       :     Optional[int]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Optional[int]								=	tiktoken.get_encoding("""gpt2"""			)
					__a       :     str								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	@require_torch
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					import torch

					__a       :     Optional[Any]								=	torch.tensor([1.0]			)
					__a       :     Dict								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Dict								=	torch.tensor([2.0]			)
					__a       :     Optional[Any]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     List[str]								=	torch.tensor([1.0]			)
					__a       :     Union[str, Any]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

	@require_spacy
	@require_spacy_model("""en_core_web_sm"""			)
	@require_spacy_model("""fr_core_news_sm"""			)
	@pytest.mark.integration
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					import spacy

					__a       :     Any								=	spacy.load("""en_core_web_sm"""			)
					__a       :     Optional[Any]								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Optional[Any]								=	spacy.load("""fr_core_news_sm"""			)
					__a       :     Any								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					__a       :     Union[str, Any]								=	spacy.load("""en_core_web_sm"""			)
					__a       :     Dict								=	mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
					self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
					self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







@pytest.mark.integration
def 				_snake_case (		lowercase						)       ->       str:
				__a       :     List[str]								=	tmp_path / """dir1"""
				__a       :     Tuple								=	tmp_path / """dir2"""
				dira.mkdir()
				dira.mkdir()
				__a       :     List[Any]								=	"""script.py"""
				__a       :     Union[str, Any]								=	dedent(
				    """
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    """						)
				__a       :     Any								=	dira / script_filename
				__a       :     Tuple								=	dira / script_filename
				with script_patha.open("""w"""						) as f:
								f.write(lowercase						)
				with script_patha.open("""w"""						) as f:
								f.write(lowercase						)
				__a       :     Tuple								=	subprocess.check_output(["""python""", str(lowercase						)]						)
				__a       :     Any								=	subprocess.check_output(["""python""", str(lowercase						)]						)
				assert fingerprinta == fingerprinta

def 				_snake_case (		)       ->       Dict:
				__a       :     Any								=	{"""a""": [0, 1, 2]}
				__a       :     Tuple								=	DatasetChild(InMemoryTable.from_pydict(lowercase						)						)
				__a       :     List[str]								=	dataset.funca()._fingerprint
				assert expected_fingerprint == dataset.funca()._fingerprint
				assert expected_fingerprint != dataset.funca()._fingerprint

				with Pool(2						) as p:
								assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
								assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint

def 				_snake_case (		)       ->       Dict:
				__a       :     Optional[Any]								=	{"""a""": [0, 1, 2]}
				class  SCREAMING_SNAKE_CASE__       (  datasets.Dataset   ):
					@fingerprint_transform(inplace=__UpperCamelCase			)
					def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



									'''simple docstring'''






									return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







				__a       :     Dict								=	DummyDatasetChild(InMemoryTable.from_pydict(lowercase						)						).func()
				class  SCREAMING_SNAKE_CASE__       (  datasets.Dataset   ):
					@fingerprint_transform(inplace=__UpperCamelCase  ,    version="""1.0.0"""			)
					def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



									'''simple docstring'''






									return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







				__a       :     Dict								=	DummyDatasetChild(InMemoryTable.from_pydict(lowercase						)						).func()
				class  SCREAMING_SNAKE_CASE__       (  datasets.Dataset   ):
					@fingerprint_transform(inplace=__UpperCamelCase  ,    version="""2.0.0"""			)
					def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



									'''simple docstring'''






									return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







				__a       :     Dict								=	DummyDatasetChild(InMemoryTable.from_pydict(lowercase						)						).func()

				assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3

def 				_snake_case (		)       ->       int:
				# AttributeError: module 'dill._dill' has no attribute 'stack'
				__a       :     Union[str, Any]								=	Hasher()
				hasher.update(lambda lowercase						: x						)

'''simple docstring'''







def 				_snake_case (		lowercase						)       ->       int:
				if not isinstance(lowercase			,    lowercase						):
								__a       :     Union[str, Any]								=	F"""Input value of [number={number}] must be an integer"""
								raise TypeError(lowercase						)
				if number < 1:
								raise ValueError("""Input must be a positive integer"""						)
				return number * (2 * number - 1)


if __name__ == "__main__":
					import doctest

					doctest.testmod()