





import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases






def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase )		->						str:






				"""simple docstring"""


				assert isinstance(_UpperCamelCase ,				_UpperCamelCase )
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize('''keep_in_memory''' ,				[False, True] )
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						Tuple:






				"""simple docstring"""


				snake_case_						:    Optional[int]				=  tmp_path / '''cache'''
				snake_case_						:    List[Any]				=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								snake_case_						:    Tuple				=  JsonDatasetReader(_UpperCamelCase ,				cache_dir=_UpperCamelCase ,				keep_in_memory=_UpperCamelCase ).read()
				_check_json_dataset(_UpperCamelCase ,				_UpperCamelCase )






@pytest.mark.parametrize(
    '''features''' ,				[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,				)
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						Any:






				"""simple docstring"""


				snake_case_						:    int				=  tmp_path / '''cache'''
				snake_case_						:    Dict				=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				snake_case_						:    Optional[Any]				=  features.copy() if features else default_expected_features
				snake_case_						:    Optional[int]				=  (
				    Features({feature: Value(_UpperCamelCase ) for feature, dtype in features.items()} ) if features is not None else None
				)
				snake_case_						:    Optional[Any]				=  JsonDatasetReader(_UpperCamelCase ,				features=_UpperCamelCase ,				cache_dir=_UpperCamelCase ).read()
				_check_json_dataset(_UpperCamelCase ,				_UpperCamelCase )






@pytest.mark.parametrize(
    '''features''' ,				[
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ] ,				)
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						List[str]:






				"""simple docstring"""


				snake_case_						:    List[str]				=  tmp_path / '''cache'''
				snake_case_						:    Dict				=  {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
				snake_case_						:    Union[str, Any]				=  features.copy() if features else default_expected_features
				snake_case_						:    str				=  (
				    Features({feature: Value(_UpperCamelCase ) for feature, dtype in features.items()} ) if features is not None else None
				)
				snake_case_						:    Dict				=  JsonDatasetReader(_UpperCamelCase ,				features=_UpperCamelCase ,				cache_dir=_UpperCamelCase ).read()
				assert isinstance(_UpperCamelCase ,				_UpperCamelCase )
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_3", "col_1", "col_2"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype






def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase )		->						Union[str, Any]:






				"""simple docstring"""


				snake_case_						:    Optional[Any]				=  {'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
				snake_case_						:    str				=  features.copy()
				snake_case_						:    str				=  (
				    Features({feature: Value(_UpperCamelCase ) for feature, dtype in features.items()} ) if features is not None else None
				)
				snake_case_						:    Dict				=  tmp_path / '''cache'''
				snake_case_						:    int				=  JsonDatasetReader(_UpperCamelCase ,				features=_UpperCamelCase ,				cache_dir=_UpperCamelCase ).read()
				assert isinstance(_UpperCamelCase ,				_UpperCamelCase )
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_2", "col_3", "col_1"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize('''split''' ,				[None, NamedSplit('''train''' ), '''train''', '''test'''] )
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						Union[str, Any]:






				"""simple docstring"""


				snake_case_						:    Tuple				=  tmp_path / '''cache'''
				snake_case_						:    Optional[int]				=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				snake_case_						:    Tuple				=  JsonDatasetReader(_UpperCamelCase ,				cache_dir=_UpperCamelCase ,				split=_UpperCamelCase ).read()
				_check_json_dataset(_UpperCamelCase ,				_UpperCamelCase )
				assert dataset.split == split if split else "train"






@pytest.mark.parametrize('''path_type''' ,				[str, list] )
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						Tuple:






				"""simple docstring"""


				if issubclass(_UpperCamelCase ,				_UpperCamelCase ):
								snake_case_						:    Optional[Any]				=  jsonl_path
				elif issubclass(_UpperCamelCase ,				_UpperCamelCase ):
								snake_case_						:    Tuple				=  [jsonl_path]
				snake_case_						:    Any				=  tmp_path / '''cache'''
				snake_case_						:    Any				=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				snake_case_						:    str				=  JsonDatasetReader(_UpperCamelCase ,				cache_dir=_UpperCamelCase ).read()
				_check_json_dataset(_UpperCamelCase ,				_UpperCamelCase )






def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase=("train",) )		->						Tuple:






				"""simple docstring"""


				assert isinstance(_UpperCamelCase ,				_UpperCamelCase )
				for split in splits:
								snake_case_						:    int				=  dataset_dict[split]
								assert dataset.num_rows == 4
								assert dataset.num_columns == 3
								assert dataset.column_names == ["col_1", "col_2", "col_3"]
								for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize('''keep_in_memory''' ,				[False, True] )
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						str:






				"""simple docstring"""


				snake_case_						:    List[str]				=  tmp_path / '''cache'''
				snake_case_						:    Optional[int]				=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								snake_case_						:    List[str]				=  JsonDatasetReader({'''train''': jsonl_path} ,				cache_dir=_UpperCamelCase ,				keep_in_memory=_UpperCamelCase ).read()
				_check_json_datasetdict(_UpperCamelCase ,				_UpperCamelCase )






@pytest.mark.parametrize(
    '''features''' ,				[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,				)
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						List[Any]:






				"""simple docstring"""


				snake_case_						:    Any				=  tmp_path / '''cache'''
				snake_case_						:    Any				=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				snake_case_						:    List[str]				=  features.copy() if features else default_expected_features
				snake_case_						:    Any				=  (
				    Features({feature: Value(_UpperCamelCase ) for feature, dtype in features.items()} ) if features is not None else None
				)
				snake_case_						:    Optional[Any]				=  JsonDatasetReader({'''train''': jsonl_path} ,				features=_UpperCamelCase ,				cache_dir=_UpperCamelCase ).read()
				_check_json_datasetdict(_UpperCamelCase ,				_UpperCamelCase )






@pytest.mark.parametrize('''split''' ,				[None, NamedSplit('''train''' ), '''train''', '''test'''] )
def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						Union[str, Any]:






				"""simple docstring"""


				if split:
								snake_case_						:    Optional[Any]				=  {split: jsonl_path}
				else:
								snake_case_						:    Optional[Any]				=  '''train'''
								snake_case_						:    Any				=  {'''train''': jsonl_path, '''test''': jsonl_path}
				snake_case_						:    Union[str, Any]				=  tmp_path / '''cache'''
				snake_case_						:    Any				=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				snake_case_						:    Any				=  JsonDatasetReader(_UpperCamelCase ,				cache_dir=_UpperCamelCase ).read()
				_check_json_datasetdict(_UpperCamelCase ,				_UpperCamelCase ,				splits=list(path.keys() ) )
				assert all(dataset[split].split == split for split in path.keys() )






def      lowerCamelCase_						(				_UpperCamelCase )		->						Dict:






				"""simple docstring"""


				return json.load(_UpperCamelCase )






def      lowerCamelCase_						(				_UpperCamelCase )		->						int:






				"""simple docstring"""


				return [json.loads(_UpperCamelCase ) for line in buffer]





class 					__lowerCAmelCase       :






			@pytest.mark.parametrize('''lines, load_json_function'''					,							[(True, load_json_lines), (False, load_json)]    )
			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__					,							__magic_name__    )     ->   Tuple:


							'''simple docstring'''



							with io.BytesIO() as buffer:
											JsonDatasetWriter(__magic_name__					,							__magic_name__					,							lines=__magic_name__    ).write()
											buffer.seek(0    )
											snake_case_						:    List[str]				=  load_json_function(__magic_name__    )
							assert isinstance(__magic_name__					,							__magic_name__    )
							assert isinstance(exported_content[0]					,							__magic_name__    )
							assert len(__magic_name__    ) == 10






			@pytest.mark.parametrize(
			    '''orient, container, keys, len_at'''					,							[
			        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
			        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
			        ('''index''', dict, set('''0123456789'''    ), None),
			        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
			        ('''values''', list, None, None),
			        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
			    ]					,							)
			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__					,							__magic_name__					,							__magic_name__					,							__magic_name__    )     ->   List[str]:


							'''simple docstring'''



							with io.BytesIO() as buffer:
											JsonDatasetWriter(__magic_name__					,							__magic_name__					,							lines=__magic_name__					,							orient=__magic_name__    ).write()
											buffer.seek(0    )
											snake_case_						:    Optional[Any]				=  load_json(__magic_name__    )
							assert isinstance(__magic_name__					,							__magic_name__    )
							if keys:
											if container is dict:
															assert exported_content.keys() == keys
											else:
															assert exported_content[0].keys() == keys
							else:
											assert not hasattr(__magic_name__					,							'''keys'''    ) and not hasattr(exported_content[0]					,							'''keys'''    )
							if len_at:
											assert len(exported_content[len_at]    ) == 10
							else:
											assert len(__magic_name__    ) == 10






			@pytest.mark.parametrize('''lines, load_json_function'''					,							[(True, load_json_lines), (False, load_json)]    )
			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__					,							__magic_name__    )     ->   Optional[int]:


							'''simple docstring'''



							with io.BytesIO() as buffer:
											JsonDatasetWriter(__magic_name__					,							__magic_name__					,							lines=__magic_name__					,							num_proc=2    ).write()
											buffer.seek(0    )
											snake_case_						:    Union[str, Any]				=  load_json_function(__magic_name__    )
							assert isinstance(__magic_name__					,							__magic_name__    )
							assert isinstance(exported_content[0]					,							__magic_name__    )
							assert len(__magic_name__    ) == 10






			@pytest.mark.parametrize(
			    '''orient, container, keys, len_at'''					,							[
			        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
			        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
			        ('''index''', dict, set('''0123456789'''    ), None),
			        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
			        ('''values''', list, None, None),
			        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
			    ]					,							)
			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__					,							__magic_name__					,							__magic_name__					,							__magic_name__    )     ->   Union[str, Any]:


							'''simple docstring'''



							with io.BytesIO() as buffer:
											JsonDatasetWriter(__magic_name__					,							__magic_name__					,							lines=__magic_name__					,							orient=__magic_name__					,							num_proc=2    ).write()
											buffer.seek(0    )
											snake_case_						:    List[str]				=  load_json(__magic_name__    )
							assert isinstance(__magic_name__					,							__magic_name__    )
							if keys:
											if container is dict:
															assert exported_content.keys() == keys
											else:
															assert exported_content[0].keys() == keys
							else:
											assert not hasattr(__magic_name__					,							'''keys'''    ) and not hasattr(exported_content[0]					,							'''keys'''    )
							if len_at:
											assert len(exported_content[len_at]    ) == 10
							else:
											assert len(__magic_name__    ) == 10






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   str:


							'''simple docstring'''



							with pytest.raises(__magic_name__    ):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(__magic_name__					,							__magic_name__					,							num_proc=0    )






			@pytest.mark.parametrize('''compression, extension'''					,							[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]    )
			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__					,							__magic_name__					,							__magic_name__					,							__magic_name__    )     ->   List[Any]:


							'''simple docstring'''



							snake_case_						:    Optional[Any]				=  tmp_path_factory.mktemp('''data'''    ) / F'''test.json.{extension}'''
							snake_case_						:    Any				=  str(shared_datadir / F'''test_file.json.{extension}'''    )
							JsonDatasetWriter(__magic_name__					,							__magic_name__					,							compression=__magic_name__    ).write()

							with fsspec.open(__magic_name__					,							'''rb'''					,							compression='''infer'''    ) as f:
											snake_case_						:    Optional[int]				=  f.read()
							with fsspec.open(__magic_name__					,							'''rb'''					,							compression='''infer'''    ) as f:
											snake_case_						:    List[Any]				=  f.read()
							assert exported_content == original_content




'''simple docstring'''

import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase							:   List[str]  						=	get_tests_dir('fixtures/spiece.model')
@require_sentencepiece
@require_tokenizers
class   __lowerCAmelCase						(lowercase_  ,     unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     int  					=							DebertaVaTokenizer
      lowerCAmelCase__					:     List[Any]  					=							DebertaVaTokenizerFast
      lowerCAmelCase__					:     str  					=							True
      lowerCAmelCase__					:     Tuple  					=							True



      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             super().setUp()

             # We have a SentencePiece fixture for testing
             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						unk_token='''<unk>''' )
             tokenizer.save_pretrained(self.tmpdirname )



      def       UpperCamelCase__		(self    :    Union[str, Any]   ,						UpperCamelCase    :    str ):

             '''simple docstring'''







             lowercase__      			= '''this is a test'''
             lowercase__      			= '''this is a test'''
             return input_text, output_text



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= '''<pad>'''
             lowercase__      			= 0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase )   ,						UpperCamelCase )
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase )   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             lowercase__      			= list(self.get_tokenizer().get_vocab().keys() )
             self.assertEqual(vocab_keys[0]   ,						'''<pad>''' )
             self.assertEqual(vocab_keys[1]   ,						'''<unk>''' )
             self.assertEqual(vocab_keys[-1]   ,						'''[PAD]''' )
             self.assertEqual(len(UpperCamelCase )   ,						30001 )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             self.assertEqual(self.get_tokenizer().vocab_size   ,						30000 )



      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= ''' \tHeLLo!how  \n Are yoU?  '''
             lowercase__      			= ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.''' )
      def       UpperCamelCase__		(self    :    List[Any] ):

             '''simple docstring'''







             pass



      @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.''' )
      def       UpperCamelCase__		(self    :    List[str] ):

             '''simple docstring'''







             pass



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             lowercase__      			= ''' \tHeLLo!how  \n Are yoU?  '''
             lowercase__      			= ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             lowercase__      			= self.get_tokenizer()
             lowercase__      			= self.get_rust_tokenizer()

             lowercase__      			= '''I was born in 92000, and this is falsé.'''

             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             lowercase__      			= rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= self.get_rust_tokenizer()
             lowercase__      			= tokenizer.encode(UpperCamelCase )
             lowercase__      			= rust_tokenizer.encode(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= '''This is a test'''
             lowercase__      			= [13, 1, 4398, 25, 21, 1289]
             lowercase__      			= ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
             lowercase__      			= ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						keep_accents=UpperCamelCase )
             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						keep_accents=UpperCamelCase )

             lowercase__      			= tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             # fmt: off
             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
             lowercase__      			= ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
             lowercase__      			= ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
             # fmt: on

             lowercase__      			= tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= DebertaVaTokenizer(UpperCamelCase )

             lowercase__      			= tokenizer.encode('''sequence builders''' )
             lowercase__      			= tokenizer.encode('''multi-sequence build''' )

             lowercase__      			= tokenizer.build_inputs_with_special_tokens(UpperCamelCase )
             lowercase__      			= tokenizer.build_inputs_with_special_tokens(UpperCamelCase   ,						UpperCamelCase )

             self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]   ,						UpperCamelCase )
             self.assertEqual(
                 [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]   ,						UpperCamelCase   ,						)






      @slow
      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= {'''input_ids''': [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=UpperCamelCase   ,						model_name='''microsoft/deberta-v2-xlarge'''   ,						revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''   ,						)




