



import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class 				lowerCAmelCase__	:







					def __init__(							self   :  Optional[Any]      ,		_A   :  List[Any]):
											A__						:				Any         =			foo







					def __call__(							self   :  List[str]):
											return self.foo




class 				lowerCAmelCase__	(						datasets.Dataset    ):







					@fingerprint_transform(inplace=_A)
					def 			_lowercase      (							self   :  List[Any]      ,		_A   :  Dict      ,		*_A   :  Union[str, Any]      ,		**_A   :  Dict):
											return DatasetChild(self.data      ,		fingerprint=_A)







					@fingerprint_transform(inplace=_A)
					def 			_lowercase      (							self   :  Dict      ,		_A   :  int      ,		*_A   :  Tuple      ,		**_A   :  int):
											return DatasetChild(self.data      ,		fingerprint=_A)




class 				lowerCAmelCase__	:







					def __init__(							self   :  List[Any]      ,		_A   :  Union[str, Any]):
											A__						:				Optional[Any]         =			callable







					def __call__(							self   :  Optional[Any]      ,		*_A   :  int      ,		**_A   :  Tuple):
											if self.callable is not None:
																	return self.callable(*_A      ,		**_A)







					def __getstate__(							self   :  Union[str, Any]):
											raise pickle.PicklingError()




class 				lowerCAmelCase__	(						UpperCamelCase    ):







					@require_transformers
					@pytest.mark.integration
					def 			_lowercase      (							self   :  Optional[Any]):
											from transformers import AutoTokenizer

											def encode(_A   :  Optional[Any]):
																	return tokenizer(_A)

											# TODO: add hash consistency tests across sessions
											A__						:				Optional[int]         =			AutoTokenizer.from_pretrained("bert-base-uncased")
											A__						:				Dict         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				List[str]         =			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
											A__						:				Any         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				int         =			AutoTokenizer.from_pretrained("bert-base-cased")
											A__						:				Any         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				Optional[int]         =			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
											A__						:				Dict         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				str         =			AutoTokenizer.from_pretrained("bert-base-uncased")
											A__						:				Union[str, Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				int         =			mda(datasets.utils.py_utils.dumps(lambda _A: tokenizer(_A))).hexdigest()
											A__						:				List[str]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					@require_transformers
					@pytest.mark.integration
					def 			_lowercase      (							self   :  List[Any]):
											from transformers import AutoTokenizer

											A__						:				Dict         =			AutoTokenizer.from_pretrained("gpt2")
											A__						:				int         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											tokenizer("Hello world !")  # call once to change the tokenizer's cache
											A__						:				Optional[Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)







					@require_regex
					def 			_lowercase      (							self   :  Tuple):
											import regex

											A__						:				List[Any]         =			regex.Regex("foo")
											A__						:				str         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				str         =			regex.Regex("bar")
											A__						:				List[str]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				int         =			regex.Regex("foo")
											A__						:				int         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)




class 				lowerCAmelCase__	(						UpperCamelCase    ):







					def 			_lowercase      (							self   :  Union[str, Any]):
											def func():
																	return foo

											A__						:				Union[str, Any]         =			[0]
											A__						:				Any         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				Tuple         =			[1]
											A__						:				Dict         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				Optional[Any]         =			[0]
											A__						:				Any         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  Optional[Any]):
											def func():
																	pass

											A__						:				Dict         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()

											def func():
																	pass

											A__						:				List[str]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  List[str]):
											A__						:				str         =			mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
											A__						:				Optional[Any]         =			mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
											A__						:				Dict         =			mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  str):
											A__						:				Optional[int]         =			mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
											A__						:				int         =			mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
											A__						:				List[Any]         =			mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  int):
											A__						:				str         =			(
											    "co_argcount",
											    "co_kwonlyargcount",
											    "co_nlocals",
											    "co_stacksize",
											    "co_flags",
											    "co_code",
											    "co_consts",
											    "co_names",
											    "co_varnames",
											    "co_filename",
											    "co_name",
											    "co_firstlineno",
											    "co_lnotab",
											    "co_freevars",
											    "co_cellvars",
											)

											def _create_code_pyaa(*_A   :  Dict):
																	if hasattr(args[-3]      ,		"encode"):
																							A__						:				Dict         =			list(_A)
																							A__						:				List[Any]         =			args[-3].encode()  # co_lnotab
																							A__						:				str         =			args[-10].encode()  # co_code
																	if hasattr(_A      ,		"co_posonlyargcount"):
																							if len(_A) == 16:
																													return CodeType(*_A)
																							elif len(_A) == 15:
																													return CodeType(args[0]      ,		0      ,		*args[1:])
																							return CodeType(args[0]      ,		0      ,		0      ,		*args[1:])
																	elif hasattr(_A      ,		"co_kwonlyargcount"):
																							if len(_A) == 16:
																													return CodeType(args[0]      ,		*args[2:])
																							elif len(_A) == 15:
																													return CodeType(*_A)
																							return CodeType(args[0]      ,		0      ,		*args[1:])
																	if len(_A) == 16:
																							return CodeType(args[0]      ,		*args[3:])
																	elif len(_A) == 15:
																							return CodeType(args[0]      ,		*args[2:])
																	return CodeType(*_A)

											def create_ipython_func(_A   :  List[Any]      ,		_A   :  Any):
																	def func():
																							return returned_obj

																	A__						:				Dict         =			func.__code__
																	# Use _create_code from dill in order to make it work for different python versions
																	if datasets.config.PY_VERSION < version.parse("3.8"):
																							A__						:				List[Any]         =			_create_code_pyaa(
																							    *[getattr(_A      ,		_A) if k != "co_filename" else co_filename for k in code_args_pyaa])
																	else:
																							A__						:				Dict         =			code.replace(co_filename=_A)
																	return FunctionType(_A      ,		func.__globals__      ,		func.__name__      ,		func.__defaults__      ,		func.__closure__)

											A__     ,							A__						:				Optional[Any]         =			"<ipython-input-2-e0383a102aae>", [0]
											A__						:				List[Any]         =			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A      ,		_A))).hexdigest()
											A__     ,							A__						:				Tuple         =			"<ipython-input-2-e0383a102aae>", [1]
											A__						:				Dict         =			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A      ,		_A))).hexdigest()
											A__     ,							A__						:				Any         =			"<ipython-input-5-713f6613acf3>", [0]
											A__						:				List[str]         =			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A      ,		_A))).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)

											A__     ,							A__						:				Union[str, Any]         =			os.path.join(gettempdir()      ,		"ipykernel_12345"      ,		"321456789.py"), [0]
											A__						:				Optional[Any]         =			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A      ,		_A))).hexdigest()
											A__     ,							A__						:				List[str]         =			os.path.join(gettempdir()      ,		"ipykernel_12345"      ,		"321456789.py"), [1]
											A__						:				Optional[Any]         =			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A      ,		_A))).hexdigest()
											A__     ,							A__						:				Any         =			os.path.join(gettempdir()      ,		"ipykernel_12345"      ,		"654123987.py"), [0]
											A__						:				Union[str, Any]         =			mda(datasets.utils.py_utils.dumps(create_ipython_func(_A      ,		_A))).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  Dict):
											A__     ,							A__						:				Dict         =			[0], [1]

											def func():
																	return foo, bar

											A__						:				Optional[int]         =			"__main__"

											def globalvars_mocka_side_effect(_A   :  Tuple      ,		*_A   :  str      ,		**_A   :  List[str]):
																	return {"foo": foo, "bar": bar}

											def globalvars_mocka_side_effect(_A   :  int      ,		*_A   :  Optional[int]      ,		**_A   :  Any):
																	return {"bar": bar, "foo": foo}

											with patch("dill.detect.globalvars"      ,		side_effect=_A) as globalvars_mocka:
																	A__						:				Union[str, Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
																	self.assertGreater(globalvars_mocka.call_count      ,		0)
											with patch("dill.detect.globalvars"      ,		side_effect=_A) as globalvars_mocka:
																	A__						:				List[str]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
																	self.assertGreater(globalvars_mocka.call_count      ,		0)
											self.assertEqual(_A      ,		_A)




class 				lowerCAmelCase__	(						UpperCamelCase    ):







					def 			_lowercase      (							self   :  List[str]):
											A__						:				Union[str, Any]         =			Hasher.hash("hello")
											A__						:				Union[str, Any]         =			Hasher.hash("hello")
											A__						:				Dict         =			Hasher.hash("there")
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  Dict):
											A__						:				List[str]         =			Hasher.hash(Foo("hello"))
											A__						:				str         =			Hasher.hash(Foo("hello"))
											A__						:				Tuple         =			Hasher.hash(Foo("there"))
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  Any):
											A__						:				str         =			Hasher()
											for x in ["hello", Foo("hello")]:
																	hasher.update(_A)
											A__						:				List[str]         =			hasher.hexdigest()
											A__						:				Any         =			Hasher()
											for x in ["hello", Foo("hello")]:
																	hasher.update(_A)
											A__						:				str         =			hasher.hexdigest()
											A__						:				Tuple         =			Hasher()
											for x in ["there", Foo("there")]:
																	hasher.update(_A)
											A__						:				Optional[int]         =			hasher.hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					def 			_lowercase      (							self   :  List[str]):
											with self.assertRaises(pickle.PicklingError):
																	Hasher.hash(UnpicklableCallable(Foo("hello")))







					def 			_lowercase      (							self   :  List[Any]):
											A__						:				Dict         =			"abc"
											A__						:				List[str]         =			[string, string]  # two strings have the same ids
											A__						:				Tuple         =			[string, string]
											A__						:				int         =			json.loads(F'["{string}", "{string}"]')  # two strings have different ids
											self.assertIs(obja[0]      ,		_A)
											self.assertIs(obja[0]      ,		obja[1])
											self.assertIs(obja[0]      ,		_A)
											self.assertIs(obja[0]      ,		obja[1])
											self.assertIsNot(obja[0]      ,		_A)
											self.assertIsNot(obja[0]      ,		obja[1])
											A__						:				Optional[int]         =			Hasher.hash(_A)
											A__						:				Dict         =			Hasher.hash(_A)
											A__						:				Optional[Any]         =			Hasher.hash(_A)
											self.assertEqual(_A      ,		_A)
											self.assertEqual(_A      ,		_A)







					@require_tiktoken
					def 			_lowercase      (							self   :  Any):
											import tiktoken

											A__						:				int         =			tiktoken.get_encoding("gpt2")
											A__						:				List[Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				Any         =			tiktoken.get_encoding("r50k_base")
											A__						:				Any         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				List[Any]         =			tiktoken.get_encoding("gpt2")
											A__						:				Optional[Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					@require_torch
					def 			_lowercase      (							self   :  Optional[Any]):
											import torch

											A__						:				Union[str, Any]         =			torch.tensor([1.0])
											A__						:				Union[str, Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				str         =			torch.tensor([2.0])
											A__						:				Optional[Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				Tuple         =			torch.tensor([1.0])
											A__						:				Tuple         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)







					@require_spacy
					@require_spacy_model("en_core_web_sm")
					@require_spacy_model("fr_core_news_sm")
					@pytest.mark.integration
					def 			_lowercase      (							self   :  Optional[Any]):
											import spacy

											A__						:				List[Any]         =			spacy.load("en_core_web_sm")
											A__						:				Optional[Any]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				str         =			spacy.load("fr_core_news_sm")
											A__						:				Dict         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											A__						:				int         =			spacy.load("en_core_web_sm")
											A__						:				List[str]         =			mda(datasets.utils.py_utils.dumps(_A)).hexdigest()
											self.assertEqual(_A      ,		_A)
											self.assertNotEqual(_A      ,		_A)




@pytest.mark.integration
def 						snake_case__							(       __lowercase							)		->		Optional[int]:







						"""simple docstring"""

						A__						:				Dict         =			tmp_path / "dir1"
						A__						:				str         =			tmp_path / "dir2"
						dira.mkdir()
						dira.mkdir()
						A__						:				Dict         =			"script.py"
						A__						:				str         =			dedent(
						    "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "							)
						A__						:				str         =			dira / script_filename
						A__						:				Optional[Any]         =			dira / script_filename
						with script_patha.open("w"							) as f:
												f.write(__lowercase							)
						with script_patha.open("w"							) as f:
												f.write(__lowercase							)
						A__						:				str         =			subprocess.check_output(["python", str(__lowercase							)]							)
						A__						:				Any         =			subprocess.check_output(["python", str(__lowercase							)]							)
						assert fingerprinta == fingerprinta
def 						snake_case__							(       )		->		int:







						"""simple docstring"""

						A__						:				Optional[Any]         =			{"a": [0, 1, 2]}
						A__						:				List[Any]         =			DatasetChild(InMemoryTable.from_pydict(__lowercase							)							)
						A__						:				int         =			dataset.funca()._fingerprint
						assert expected_fingerprint == dataset.funca()._fingerprint
						assert expected_fingerprint != dataset.funca()._fingerprint

						with Pool(2							) as p:
												assert expected_fingerprint == p.apply_async(dataset.funca							).get()._fingerprint
												assert expected_fingerprint != p.apply_async(dataset.funca							).get()._fingerprint
def 						snake_case__							(       )		->		Union[str, Any]:







						"""simple docstring"""

						A__						:				Union[str, Any]         =			{"a": [0, 1, 2]}




						class 				lowerCAmelCase__	(						datasets.Dataset    ):







											@fingerprint_transform(inplace=_A)
											def 			_lowercase      (							self   :  Union[str, Any]      ,		_A   :  Dict):
																	return DummyDatasetChild(self.data      ,		fingerprint=_A)




						A__						:				List[Any]         =			DummyDatasetChild(InMemoryTable.from_pydict(__lowercase							)							).func()




						class 				lowerCAmelCase__	(						datasets.Dataset    ):







											@fingerprint_transform(inplace=_A      ,		version="1.0.0")
											def 			_lowercase      (							self   :  Tuple      ,		_A   :  Dict):
																	return DummyDatasetChild(self.data      ,		fingerprint=_A)




						A__						:				int         =			DummyDatasetChild(InMemoryTable.from_pydict(__lowercase							)							).func()




						class 				lowerCAmelCase__	(						datasets.Dataset    ):







											@fingerprint_transform(inplace=_A      ,		version="2.0.0")
											def 			_lowercase      (							self   :  Dict      ,		_A   :  Any):
																	return DummyDatasetChild(self.data      ,		fingerprint=_A)




						A__						:				Any         =			DummyDatasetChild(InMemoryTable.from_pydict(__lowercase							)							).func()

						assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}							) == 3




def 						snake_case__							(       )		->		int:







						"""simple docstring"""

						A__						:				List[str]         =			Hasher()
						hasher.update(lambda __lowercase							: x							)




from itertools import compress, repeat
from math import ceil, sqrt
def 						snake_case__							(       __lowercase							)		->		list[int]:







						"""simple docstring"""

						if num <= 2:
												return []
						if num == 3:
												return [2]

						# Odd sieve for numbers in range [3, num - 1]
						A__						:				Dict         =			bytearray(b"\x01"							) * ((num >> 1) - 1)

						for i in range(3				,  int(sqrt(__lowercase							)							) + 1				,  2							):
												if sieve[(i >> 1) - 1]:
																		A__						:				int         =			i**2
																		A__						:				List[str]         =			repeat(
																		    0				,  ceil((num - i_squared) / (i << 1)							)							)

						return [2] + list(compress(range(3				,  __lowercase				,  2							)				,  __lowercase							)							)


if __name__ == "__main__":
		import doctest

		doctest.testmod()