






'''simple docstring'''







import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class 					__UpperCAmelCase :

      '''simple docstring'''





      def __init__(					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   List[Any]												=	foo




      def __call__(					self						):
             return self.foo




class 					__UpperCAmelCase (		datasets.Dataset	):

      '''simple docstring'''





      @fingerprint_transform(inplace=_UpperCAmelCase						)
      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		*_UpperCAmelCase  ,		**_UpperCAmelCase						):
             return DatasetChild(self.data  ,		fingerprint=_UpperCAmelCase						)




      @fingerprint_transform(inplace=_UpperCAmelCase						)
      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		*_UpperCAmelCase  ,		**_UpperCAmelCase						):
             return DatasetChild(self.data  ,		fingerprint=_UpperCAmelCase						)




class 					__UpperCAmelCase :

      '''simple docstring'''





      def __init__(					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   Union[str, Any]												=	callable




      def __call__(					self  ,		*_UpperCAmelCase  ,		**_UpperCAmelCase						):
             if self.callable is not None:
                    return self.callable(*_UpperCAmelCase  ,		**_UpperCAmelCase						)




      def __getstate__(					self						):
             raise pickle.PicklingError()




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      @require_transformers
      @pytest.mark.integration
      def 							lowerCamelCase     (					self						):
             from transformers import AutoTokenizer

             def encode(_UpperCAmelCase						):
                    return tokenizer(_UpperCAmelCase						)

             # TODO: add hash consistency tests across sessions
             UpperCAmelCase__	:   str												=	AutoTokenizer.from_pretrained('''bert-base-uncased'''						)
             UpperCAmelCase__	:   Any												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(lambda _UpperCAmelCase						: tokenizer(_UpperCAmelCase						)						)						).hexdigest()
             UpperCAmelCase__	:   Optional[Any]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Dict												=	AutoTokenizer.from_pretrained('''bert-base-cased'''						)
             UpperCAmelCase__	:   Any												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Optional[int]												=	mda(datasets.utils.py_utils.dumps(lambda _UpperCAmelCase						: tokenizer(_UpperCAmelCase						)						)						).hexdigest()
             UpperCAmelCase__	:   List[str]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   List[Any]												=	AutoTokenizer.from_pretrained('''bert-base-uncased'''						)
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   List[Any]												=	mda(datasets.utils.py_utils.dumps(lambda _UpperCAmelCase						: tokenizer(_UpperCAmelCase						)						)						).hexdigest()
             UpperCAmelCase__	:   Tuple												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      @require_transformers
      @pytest.mark.integration
      def 							lowerCamelCase     (					self						):
             from transformers import AutoTokenizer

             UpperCAmelCase__	:   str												=	AutoTokenizer.from_pretrained('''gpt2'''						)
             UpperCAmelCase__	:   Tuple												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             tokenizer('''Hello world !'''						)  # call once to change the tokenizer's cache
             UpperCAmelCase__	:   List[str]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      @require_regex
      def 							lowerCamelCase     (					self						):
             import regex

             UpperCAmelCase__	:   List[Any]												=	regex.Regex('''foo'''						)
             UpperCAmelCase__	:   Optional[Any]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Union[str, Any]												=	regex.Regex('''bar'''						)
             UpperCAmelCase__	:   Any												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Dict												=	regex.Regex('''foo'''						)
             UpperCAmelCase__	:   int												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      def 							lowerCamelCase     (					self						):
             def func():
                    return foo

             UpperCAmelCase__	:   Union[str, Any]												=	[0]
             UpperCAmelCase__	:   List[str]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   List[Any]												=	[1]
             UpperCAmelCase__	:   Optional[Any]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Union[str, Any]												=	[0]
             UpperCAmelCase__	:   List[Any]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             def func():
                    pass

             UpperCAmelCase__	:   List[str]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()

             def func():
                    pass

             UpperCAmelCase__	:   List[Any]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[int]												=	mda(datasets.utils.py_utils.dumps(Foo([0]						)						)						).hexdigest()
             UpperCAmelCase__	:   Optional[Any]												=	mda(datasets.utils.py_utils.dumps(Foo([1]						)						)						).hexdigest()
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(Foo([0]						)						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[Any]												=	mda(datasets.utils.py_utils.dumps(Foo([0]						).__call__						)						).hexdigest()
             UpperCAmelCase__	:   Tuple												=	mda(datasets.utils.py_utils.dumps(Foo([1]						).__call__						)						).hexdigest()
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(Foo([0]						).__call__						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Dict												=	(
                 '''co_argcount''',
                 '''co_kwonlyargcount''',
                 '''co_nlocals''',
                 '''co_stacksize''',
                 '''co_flags''',
                 '''co_code''',
                 '''co_consts''',
                 '''co_names''',
                 '''co_varnames''',
                 '''co_filename''',
                 '''co_name''',
                 '''co_firstlineno''',
                 '''co_lnotab''',
                 '''co_freevars''',
                 '''co_cellvars''',
             )

             def _create_code_pyaa(*_UpperCAmelCase						):
                    if hasattr(args[-3]  ,		'''encode'''						):
                           UpperCAmelCase__	:   str												=	list(_UpperCAmelCase						)
                           UpperCAmelCase__	:   Optional[Any]												=	args[-3].encode()  # co_lnotab
                           UpperCAmelCase__	:   Optional[int]												=	args[-10].encode()  # co_code
                    if hasattr(_UpperCAmelCase  ,		'''co_posonlyargcount'''						):
                           if len(_UpperCAmelCase						) == 16:
                                  return CodeType(*_UpperCAmelCase						)
                           elif len(_UpperCAmelCase						) == 15:
                                  return CodeType(args[0]  ,		0  ,		*args[1:]						)
                           return CodeType(args[0]  ,		0  ,		0  ,		*args[1:]						)
                    elif hasattr(_UpperCAmelCase  ,		'''co_kwonlyargcount'''						):
                           if len(_UpperCAmelCase						) == 16:
                                  return CodeType(args[0]  ,		*args[2:]						)
                           elif len(_UpperCAmelCase						) == 15:
                                  return CodeType(*_UpperCAmelCase						)
                           return CodeType(args[0]  ,		0  ,		*args[1:]						)
                    if len(_UpperCAmelCase						) == 16:
                           return CodeType(args[0]  ,		*args[3:]						)
                    elif len(_UpperCAmelCase						) == 15:
                           return CodeType(args[0]  ,		*args[2:]						)
                    return CodeType(*_UpperCAmelCase						)

             def create_ipython_func(_UpperCAmelCase  ,		_UpperCAmelCase						):
                    def func():
                           return returned_obj

                    UpperCAmelCase__	:   int												=	func.__code__
                    # Use _create_code from dill in order to make it work for different python versions
                    if datasets.config.PY_VERSION < version.parse('''3.8'''						):
                           UpperCAmelCase__	:   str												=	_create_code_pyaa(
                               *[getattr(_UpperCAmelCase  ,		_UpperCAmelCase						) if k != '''co_filename''' else co_filename for k in code_args_pyaa]						)
                    else:
                           UpperCAmelCase__	:   List[str]												=	code.replace(co_filename=_UpperCAmelCase						)
                    return FunctionType(_UpperCAmelCase  ,		func.__globals__  ,		func.__name__  ,		func.__defaults__  ,		func.__closure__						)

             UpperCAmelCase__							,							UpperCAmelCase__	:   Optional[Any]												=	'''<ipython-input-2-e0383a102aae>''', [0]
             UpperCAmelCase__	:   Tuple												=	mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase  ,		_UpperCAmelCase						)						)						).hexdigest()
             UpperCAmelCase__							,							UpperCAmelCase__	:   Any												=	'''<ipython-input-2-e0383a102aae>''', [1]
             UpperCAmelCase__	:   Union[str, Any]												=	mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase  ,		_UpperCAmelCase						)						)						).hexdigest()
             UpperCAmelCase__							,							UpperCAmelCase__	:   Optional[Any]												=	'''<ipython-input-5-713f6613acf3>''', [0]
             UpperCAmelCase__	:   int												=	mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase  ,		_UpperCAmelCase						)						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)

             UpperCAmelCase__							,							UpperCAmelCase__	:   Any												=	os.path.join(gettempdir()  ,		'''ipykernel_12345'''  ,		'''321456789.py'''						), [0]
             UpperCAmelCase__	:   Dict												=	mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase  ,		_UpperCAmelCase						)						)						).hexdigest()
             UpperCAmelCase__							,							UpperCAmelCase__	:   List[Any]												=	os.path.join(gettempdir()  ,		'''ipykernel_12345'''  ,		'''321456789.py'''						), [1]
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase  ,		_UpperCAmelCase						)						)						).hexdigest()
             UpperCAmelCase__							,							UpperCAmelCase__	:   Optional[Any]												=	os.path.join(gettempdir()  ,		'''ipykernel_12345'''  ,		'''654123987.py'''						), [0]
             UpperCAmelCase__	:   Optional[Any]												=	mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase  ,		_UpperCAmelCase						)						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__							,							UpperCAmelCase__	:   int												=	[0], [1]

             def func():
                    return foo, bar

             UpperCAmelCase__	:   Tuple												=	'''__main__'''

             def globalvars_mocka_side_effect(_UpperCAmelCase  ,		*_UpperCAmelCase  ,		**_UpperCAmelCase						):
                    return {"foo": foo, "bar": bar}

             def globalvars_mocka_side_effect(_UpperCAmelCase  ,		*_UpperCAmelCase  ,		**_UpperCAmelCase						):
                    return {"bar": bar, "foo": foo}

             with patch('''dill.detect.globalvars'''  ,		side_effect=_UpperCAmelCase						) as globalvars_mocka:
                    UpperCAmelCase__	:   int												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
                    self.assertGreater(globalvars_mocka.call_count  ,		0						)
             with patch('''dill.detect.globalvars'''  ,		side_effect=_UpperCAmelCase						) as globalvars_mocka:
                    UpperCAmelCase__	:   Optional[int]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
                    self.assertGreater(globalvars_mocka.call_count  ,		0						)
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[Any]												=	Hasher.hash('''hello'''						)
             UpperCAmelCase__	:   str												=	Hasher.hash('''hello'''						)
             UpperCAmelCase__	:   int												=	Hasher.hash('''there'''						)
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[int]												=	Hasher.hash(Foo('''hello'''						)						)
             UpperCAmelCase__	:   List[str]												=	Hasher.hash(Foo('''hello'''						)						)
             UpperCAmelCase__	:   Optional[Any]												=	Hasher.hash(Foo('''there'''						)						)
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[Any]												=	Hasher()
             for x in ["hello", Foo('''hello'''						)]:
                    hasher.update(_UpperCAmelCase						)
             UpperCAmelCase__	:   Tuple												=	hasher.hexdigest()
             UpperCAmelCase__	:   int												=	Hasher()
             for x in ["hello", Foo('''hello'''						)]:
                    hasher.update(_UpperCAmelCase						)
             UpperCAmelCase__	:   str												=	hasher.hexdigest()
             UpperCAmelCase__	:   Union[str, Any]												=	Hasher()
             for x in ["there", Foo('''there'''						)]:
                    hasher.update(_UpperCAmelCase						)
             UpperCAmelCase__	:   Optional[Any]												=	hasher.hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      def 							lowerCamelCase     (					self						):
             with self.assertRaises(pickle.PicklingError						):
                    Hasher.hash(UnpicklableCallable(Foo('''hello'''						)						)						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   List[Any]												=	'''abc'''
             UpperCAmelCase__	:   int												=	[string, string]  # two strings have the same ids
             UpperCAmelCase__	:   Any												=	[string, string]
             UpperCAmelCase__	:   Dict												=	json.loads(F"""[\"{string}\", \"{string}\"]"""						)  # two strings have different ids
             self.assertIs(obja[0]  ,		_UpperCAmelCase						)
             self.assertIs(obja[0]  ,		obja[1]						)
             self.assertIs(obja[0]  ,		_UpperCAmelCase						)
             self.assertIs(obja[0]  ,		obja[1]						)
             self.assertIsNot(obja[0]  ,		_UpperCAmelCase						)
             self.assertIsNot(obja[0]  ,		obja[1]						)
             UpperCAmelCase__	:   Optional[int]												=	Hasher.hash(_UpperCAmelCase						)
             UpperCAmelCase__	:   Tuple												=	Hasher.hash(_UpperCAmelCase						)
             UpperCAmelCase__	:   List[Any]												=	Hasher.hash(_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      @require_tiktoken
      def 							lowerCamelCase     (					self						):
             import tiktoken

             UpperCAmelCase__	:   List[str]												=	tiktoken.get_encoding('''gpt2'''						)
             UpperCAmelCase__	:   Dict												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Optional[int]												=	tiktoken.get_encoding('''r50k_base'''						)
             UpperCAmelCase__	:   Optional[int]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Union[str, Any]												=	tiktoken.get_encoding('''gpt2'''						)
             UpperCAmelCase__	:   Dict												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      @require_torch
      def 							lowerCamelCase     (					self						):
             import torch

             UpperCAmelCase__	:   Any												=	torch.tensor([1.0]						)
             UpperCAmelCase__	:   int												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   str												=	torch.tensor([2.0]						)
             UpperCAmelCase__	:   Union[str, Any]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   Union[str, Any]												=	torch.tensor([1.0]						)
             UpperCAmelCase__	:   List[Any]												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)




      @require_spacy
      @require_spacy_model('''en_core_web_sm'''						)
      @require_spacy_model('''fr_core_news_sm'''						)
      @pytest.mark.integration
      def 							lowerCamelCase     (					self						):
             import spacy

             UpperCAmelCase__	:   int												=	spacy.load('''en_core_web_sm'''						)
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   str												=	spacy.load('''fr_core_news_sm'''						)
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             UpperCAmelCase__	:   str												=	spacy.load('''en_core_web_sm'''						)
             UpperCAmelCase__	:   str												=	mda(datasets.utils.py_utils.dumps(_UpperCAmelCase						)						).hexdigest()
             self.assertEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)
             self.assertNotEqual(_UpperCAmelCase  ,		_UpperCAmelCase						)






@pytest.mark.integration
def  lowerCAmelCase__		(					a_  :     Dict			)   -> Optional[int]:
       UpperCAmelCase__	:   Optional[Any]												=	tmp_path / '''dir1'''
       UpperCAmelCase__	:   Union[str, Any]												=	tmp_path / '''dir2'''
       dira.mkdir()
       dira.mkdir()
       UpperCAmelCase__	:   List[Any]												=	'''script.py'''
       UpperCAmelCase__	:   Any												=	dedent(
           '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    '''			)
       UpperCAmelCase__	:   str												=	dira / script_filename
       UpperCAmelCase__	:   int												=	dira / script_filename
       with script_patha.open('''w'''			) as f:
              f.write(a_			)
       with script_patha.open('''w'''			) as f:
              f.write(a_			)
       UpperCAmelCase__	:   Tuple												=	subprocess.check_output(['''python''', str(a_			)]			)
       UpperCAmelCase__	:   Tuple												=	subprocess.check_output(['''python''', str(a_			)]			)
       assert fingerprinta == fingerprinta






def  lowerCAmelCase__		(					)   -> int:
       UpperCAmelCase__	:   List[str]												=	{'''a''': [0, 1, 2]}
       UpperCAmelCase__	:   Optional[int]												=	DatasetChild(InMemoryTable.from_pydict(a_			)			)
       UpperCAmelCase__	:   Optional[Any]												=	dataset.funca()._fingerprint
       assert expected_fingerprint == dataset.funca()._fingerprint
       assert expected_fingerprint != dataset.funca()._fingerprint

       with Pool(2			) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca			).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca			).get()._fingerprint






def  lowerCAmelCase__		(					)   -> List[str]:
       UpperCAmelCase__	:   List[Any]												=	{'''a''': [0, 1, 2]}




       class 					__UpperCAmelCase (		datasets.Dataset	):

             '''simple docstring'''





             @fingerprint_transform(inplace=_UpperCAmelCase						)
             def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
                    return DummyDatasetChild(self.data  ,		fingerprint=_UpperCAmelCase						)


       UpperCAmelCase__	:   List[str]												=	DummyDatasetChild(InMemoryTable.from_pydict(a_			)			).func()




       class 					__UpperCAmelCase (		datasets.Dataset	):

             '''simple docstring'''





             @fingerprint_transform(inplace=_UpperCAmelCase  ,		version='''1.0.0'''						)
             def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
                    return DummyDatasetChild(self.data  ,		fingerprint=_UpperCAmelCase						)


       UpperCAmelCase__	:   Any												=	DummyDatasetChild(InMemoryTable.from_pydict(a_			)			).func()




       class 					__UpperCAmelCase (		datasets.Dataset	):

             '''simple docstring'''





             @fingerprint_transform(inplace=_UpperCAmelCase  ,		version='''2.0.0'''						)
             def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
                    return DummyDatasetChild(self.data  ,		fingerprint=_UpperCAmelCase						)


       UpperCAmelCase__	:   Optional[int]												=	DummyDatasetChild(InMemoryTable.from_pydict(a_			)			).func()

       assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}			) == 3






def  lowerCAmelCase__		(					)   -> Any:
       UpperCAmelCase__	:   Tuple												=	Hasher()
       hasher.update(lambda a_			: x			)





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A    :				List[str]    						=	logging.get_logger(__name__)

__A    :				List[Any]    						=	{
    """mnaylor/mega-base-wikitext""": """https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json""",
}




class       UpperCAmelCase_   ( A							):




							'''simple docstring'''



							a__    	=  '''mega'''





							def __init__(       self							:						Any							,  a							:						List[str]=30_522							,  a							:						Dict=128							,  a							:						Dict=4							,  a							:						Union[str, Any]=256							,  a							:						Tuple=16							,  a							:						Any=True							,  a							:						Dict=64							,  a							:						Union[str, Any]=False							,  a							:						Tuple=-1							,  a							:						str=None							,  a							:						int=True							,  a							:						Optional[int]="scalenorm"							,  a							:						Optional[int]=True							,  a							:						Dict="silu"							,  a							:						Optional[Any]="softmax"							,  a							:						Any=0.1							,  a							:						List[Any]=0.1							,  a							:						Tuple=0.1							,  a							:						Optional[int]=False							,  a							:						List[str]=True							,  a							:						Optional[int]=256							,  a							:						List[str]=True							,  a							:						int=0.1							,  a							:						str=2_048							,  a							:						Tuple=False							,  a							:						Optional[int]=2							,  a							:						Optional[int]=0.02							,  a							:						Optional[int]=0.2							,  a							:						Union[str, Any]=0.02							,  a							:						str=1.0							,  a							:						Dict=1							,  a							:						Union[str, Any]=0							,  a							:						Dict=2							,  a							:						List[str]="rotary"							,  a							:						Union[str, Any]=None							,  a							:						List[Any]=True							,  a							:						Tuple=True							,  **a							:						Optional[int]							,  )      ->       Dict:
										super().__init__(pad_token_id=a							,  bos_token_id=a							,  eos_token_id=a							,  **a      )

										SCREAMING_SNAKE_CASE     =       vocab_size
										SCREAMING_SNAKE_CASE     =       hidden_size
										SCREAMING_SNAKE_CASE     =       num_hidden_layers
										SCREAMING_SNAKE_CASE     =       activation
										SCREAMING_SNAKE_CASE     =       attention_activation
										SCREAMING_SNAKE_CASE     =       intermediate_size
										SCREAMING_SNAKE_CASE     =       ema_projection_size
										SCREAMING_SNAKE_CASE     =       bidirectional
										SCREAMING_SNAKE_CASE     =       shared_representation_size
										SCREAMING_SNAKE_CASE     =       use_chunking
										SCREAMING_SNAKE_CASE     =       chunk_size
										SCREAMING_SNAKE_CASE     =       truncation
										SCREAMING_SNAKE_CASE     =       normalize_before_mega
										SCREAMING_SNAKE_CASE     =       normalization_type
										SCREAMING_SNAKE_CASE     =       norm_affine
										SCREAMING_SNAKE_CASE     =       dropout_prob
										SCREAMING_SNAKE_CASE     =       hidden_dropout_prob
										SCREAMING_SNAKE_CASE     =       attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE     =       use_feature_dropout
										SCREAMING_SNAKE_CASE     =       use_normalized_ffn
										SCREAMING_SNAKE_CASE     =       nffn_hidden_size
										SCREAMING_SNAKE_CASE     =       normalize_before_ffn
										SCREAMING_SNAKE_CASE     =       nffn_activation_dropout_prob
										SCREAMING_SNAKE_CASE     =       max_positions
										SCREAMING_SNAKE_CASE     =       add_token_type_embeddings
										SCREAMING_SNAKE_CASE     =       type_vocab_size
										SCREAMING_SNAKE_CASE     =       initializer_range
										SCREAMING_SNAKE_CASE     =       ema_delta_alpha_range
										SCREAMING_SNAKE_CASE     =       ema_beta_range
										SCREAMING_SNAKE_CASE     =       ema_gamma_omega_range
										SCREAMING_SNAKE_CASE     =       relative_positional_bias
										SCREAMING_SNAKE_CASE     =       use_cache
										SCREAMING_SNAKE_CASE     =       classifier_dropout
										SCREAMING_SNAKE_CASE     =       add_lm_hidden_dense_layer
										SCREAMING_SNAKE_CASE     =       1  # not used but required by Hugging Face





class       UpperCAmelCase_   ( A							):




							'''simple docstring'''



							@property
							def  _UpperCAmelCase  (       self							:						Union[str, Any]      )      ->       Mapping[str, Mapping[int, str]]:
										if self.task == "multiple-choice":
													SCREAMING_SNAKE_CASE     =       {0: """batch""", 1: """choice""", 2: """sequence"""}
										else:
													SCREAMING_SNAKE_CASE     =       {0: """batch""", 1: """sequence"""}
										return OrderedDict(
										    [
										        ("""input_ids""", dynamic_axis),
										        ("""attention_mask""", dynamic_axis),
										    ]      )






