


'''simple docstring'''

lowerCamelCase							:   List[str]  						=	{
    'Pillow': 'Pillow<10.0.0',
    'accelerate': 'accelerate>=0.20.3',
    'av': 'av==9.2.0',
    'beautifulsoup4': 'beautifulsoup4',
    'black': 'black~=23.1',
    'codecarbon': 'codecarbon==1.2.0',
    'cookiecutter': 'cookiecutter==1.7.3',
    'dataclasses': 'dataclasses',
    'datasets': 'datasets!=2.5.0',
    'decord': 'decord==0.6.0',
    'deepspeed': 'deepspeed>=0.9.3',
    'diffusers': 'diffusers',
    'dill': 'dill<0.3.5',
    'evaluate': 'evaluate>=0.2.0',
    'fairscale': 'fairscale>0.3',
    'faiss-cpu': 'faiss-cpu',
    'fastapi': 'fastapi',
    'filelock': 'filelock',
    'flax': 'flax>=0.4.1,<=0.7.0',
    'ftfy': 'ftfy',
    'fugashi': 'fugashi>=1.0',
    'GitPython': 'GitPython<3.1.19',
    'hf-doc-builder': 'hf-doc-builder>=0.3.0',
    'huggingface-hub': 'huggingface-hub>=0.14.1,<1.0',
    'importlib_metadata': 'importlib_metadata',
    'ipadic': 'ipadic>=1.0.0,<2.0',
    'isort': 'isort>=5.5.4',
    'jax': 'jax>=0.2.8,!=0.3.2,<=0.4.13',
    'jaxlib': 'jaxlib>=0.1.65,<=0.4.13',
    'jieba': 'jieba',
    'kenlm': 'kenlm',
    'keras-nlp': 'keras-nlp>=0.3.1',
    'librosa': 'librosa',
    'nltk': 'nltk',
    'natten': 'natten>=0.14.6',
    'numpy': 'numpy>=1.17',
    'onnxconverter-common': 'onnxconverter-common',
    'onnxruntime-tools': 'onnxruntime-tools>=1.4.2',
    'onnxruntime': 'onnxruntime>=1.4.0',
    'opencv-python': 'opencv-python',
    'optuna': 'optuna',
    'optax': 'optax>=0.0.8,<=0.1.4',
    'packaging': 'packaging>=20.0',
    'parameterized': 'parameterized',
    'phonemizer': 'phonemizer',
    'protobuf': 'protobuf',
    'psutil': 'psutil',
    'pyyaml': 'pyyaml>=5.1',
    'pydantic': 'pydantic<2',
    'pytest': 'pytest>=7.2.0',
    'pytest-timeout': 'pytest-timeout',
    'pytest-xdist': 'pytest-xdist',
    'python': 'python>=3.8.0',
    'ray[tune]': 'ray[tune]',
    'regex': 'regex!=2019.12.17',
    'requests': 'requests',
    'rhoknp': 'rhoknp>=1.1.0,<1.3.1',
    'rjieba': 'rjieba',
    'rouge-score': 'rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1',
    'ruff': 'ruff>=0.0.241,<=0.0.259',
    'sacrebleu': 'sacrebleu>=1.4.12,<2.0.0',
    'sacremoses': 'sacremoses',
    'safetensors': 'safetensors>=0.3.1',
    'sagemaker': 'sagemaker>=2.31.0',
    'scikit-learn': 'scikit-learn',
    'sentencepiece': 'sentencepiece>=0.1.91,!=0.1.92',
    'sigopt': 'sigopt',
    'starlette': 'starlette',
    'sudachipy': 'sudachipy>=0.6.6',
    'sudachidict_core': 'sudachidict_core>=20220729',
    'tensorflow-cpu': 'tensorflow-cpu>=2.6,<2.14',
    'tensorflow': 'tensorflow>=2.6,<2.14',
    'tensorflow-text': 'tensorflow-text<2.14',
    'tf2onnx': 'tf2onnx',
    'timeout-decorator': 'timeout-decorator',
    'timm': 'timm',
    'tokenizers': 'tokenizers>=0.11.1,!=0.11.3,<0.14',
    'torch': 'torch>=1.9,!=1.12.0',
    'torchaudio': 'torchaudio',
    'torchvision': 'torchvision',
    'pyctcdecode': 'pyctcdecode>=0.4.0',
    'tqdm': 'tqdm>=4.27',
    'unidic': 'unidic>=1.0.2',
    'unidic_lite': 'unidic_lite>=1.0.7',
    'urllib3': 'urllib3<2.0.0',
    'uvicorn': 'uvicorn',
}








'''simple docstring'''

import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase							:   List[str]  						=	get_tests_dir('fixtures/spiece.model')
@require_sentencepiece
@require_tokenizers
class   __lowerCAmelCase						(lowercase_  ,     unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     int  					=							DebertaVaTokenizer
      lowerCAmelCase__					:     List[Any]  					=							DebertaVaTokenizerFast
      lowerCAmelCase__					:     str  					=							True
      lowerCAmelCase__					:     Tuple  					=							True



      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             super().setUp()

             # We have a SentencePiece fixture for testing
             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						unk_token='''<unk>''' )
             tokenizer.save_pretrained(self.tmpdirname )



      def       UpperCamelCase__		(self    :    Union[str, Any]   ,						UpperCamelCase    :    str ):

             '''simple docstring'''







             lowercase__      			= '''this is a test'''
             lowercase__      			= '''this is a test'''
             return input_text, output_text



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= '''<pad>'''
             lowercase__      			= 0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase )   ,						UpperCamelCase )
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase )   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             lowercase__      			= list(self.get_tokenizer().get_vocab().keys() )
             self.assertEqual(vocab_keys[0]   ,						'''<pad>''' )
             self.assertEqual(vocab_keys[1]   ,						'''<unk>''' )
             self.assertEqual(vocab_keys[-1]   ,						'''[PAD]''' )
             self.assertEqual(len(UpperCamelCase )   ,						30001 )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             self.assertEqual(self.get_tokenizer().vocab_size   ,						30000 )



      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= ''' \tHeLLo!how  \n Are yoU?  '''
             lowercase__      			= ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.''' )
      def       UpperCamelCase__		(self    :    List[Any] ):

             '''simple docstring'''







             pass



      @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.''' )
      def       UpperCamelCase__		(self    :    List[str] ):

             '''simple docstring'''







             pass



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             lowercase__      			= ''' \tHeLLo!how  \n Are yoU?  '''
             lowercase__      			= ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
             # fmt: on

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						do_lower_case=UpperCamelCase   ,						split_by_punct=UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )

             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             lowercase__      			= self.get_tokenizer()
             lowercase__      			= self.get_rust_tokenizer()

             lowercase__      			= '''I was born in 92000, and this is falsé.'''

             lowercase__      			= tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase ) )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             lowercase__      			= rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= self.get_rust_tokenizer()
             lowercase__      			= tokenizer.encode(UpperCamelCase )
             lowercase__      			= rust_tokenizer.encode(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= '''This is a test'''
             lowercase__      			= [13, 1, 4398, 25, 21, 1289]
             lowercase__      			= ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
             lowercase__      			= ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

             lowercase__      			= DebertaVaTokenizer(UpperCamelCase   ,						keep_accents=UpperCamelCase )
             lowercase__      			= DebertaVaTokenizerFast(UpperCamelCase   ,						keep_accents=UpperCamelCase )

             lowercase__      			= tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             # fmt: off
             lowercase__      			= '''I was born in 92000, and this is falsé.'''
             lowercase__      			= [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
             lowercase__      			= ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
             lowercase__      			= ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
             # fmt: on

             lowercase__      			= tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )

             lowercase__      			= rust_tokenizer.encode(UpperCamelCase   ,						add_special_tokens=UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.tokenize(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )
             lowercase__      			= rust_tokenizer.convert_ids_to_tokens(UpperCamelCase )
             self.assertListEqual(UpperCamelCase   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= DebertaVaTokenizer(UpperCamelCase )

             lowercase__      			= tokenizer.encode('''sequence builders''' )
             lowercase__      			= tokenizer.encode('''multi-sequence build''' )

             lowercase__      			= tokenizer.build_inputs_with_special_tokens(UpperCamelCase )
             lowercase__      			= tokenizer.build_inputs_with_special_tokens(UpperCamelCase   ,						UpperCamelCase )

             self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]   ,						UpperCamelCase )
             self.assertEqual(
                 [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]   ,						UpperCamelCase   ,						)






      @slow
      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= {'''input_ids''': [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=UpperCamelCase   ,						model_name='''microsoft/deberta-v2-xlarge'''   ,						revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''   ,						)




