


import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__   	=							get_tests_dir('''fixtures/spiece.model''')







@require_sentencepiece
@require_tokenizers
class       lowerCamelCase__   (       A__   ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__      			=			DebertaVaTokenizer
     SCREAMING_SNAKE_CASE__      			=			DebertaVaTokenizerFast
     SCREAMING_SNAKE_CASE__      			=			True
     SCREAMING_SNAKE_CASE__      			=			True





     def      __A					(self       )     ->   Dict:
      super().setUp()

      # We have a SentencePiece fixture for testing
      _lowercase							      =DebertaVaTokenizer(_a		,     unk_token='''<unk>'''       )
      tokenizer.save_pretrained(self.tmpdirname       )





     def      __A					(self		,     UpperCAmelCase       )     ->   Optional[Any]:
      _lowercase							      ='''this is a test'''
      _lowercase							      ='''this is a test'''
      return input_text, output_text





     def      __A					(self       )     ->   str:
      _lowercase							      ='''<pad>'''
      _lowercase							      =0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a       )		,     _a       )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a       )		,     _a       )





     def      __A					(self       )     ->   List[str]:
      _lowercase							      =list(self.get_tokenizer().get_vocab().keys()       )
      self.assertEqual(vocab_keys[0]		,     '''<pad>'''       )
      self.assertEqual(vocab_keys[1]		,     '''<unk>'''       )
      self.assertEqual(vocab_keys[-1]		,     '''[PAD]'''       )
      self.assertEqual(len(_a       )		,     3_0_0_0_1       )





     def      __A					(self       )     ->   Any:
      self.assertEqual(self.get_tokenizer().vocab_size		,     3_0_0_0_0       )





     def      __A					(self       )     ->   List[str]:
      _lowercase							      =''' \tHeLLo!how  \n Are yoU?  '''
      _lowercase							      =['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
      # fmt: on

      _lowercase							      =DebertaVaTokenizer(_a		,     do_lower_case=_a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )

      _lowercase							      =DebertaVaTokenizerFast(_a		,     do_lower_case=_a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )





     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''       )
     def      __A					(self       )     ->   Union[str, Any]:
      pass





     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''       )
     def      __A					(self       )     ->   List[Any]:
      pass





     def      __A					(self       )     ->   int:
      _lowercase							      ='''I was born in 92000, and this is falsé.'''
      _lowercase							      =['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
      # fmt: on

      _lowercase							      =DebertaVaTokenizer(_a		,     split_by_punct=_a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )

      _lowercase							      =DebertaVaTokenizerFast(_a		,     split_by_punct=_a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )





     def      __A					(self       )     ->   Union[str, Any]:
      _lowercase							      ='''I was born in 92000, and this is falsé.'''
      _lowercase							      =['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
      # fmt: on

      _lowercase							      =DebertaVaTokenizer(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(tokenizer.encode(_a		,     add_special_tokens=_a       )       )
      self.assertListEqual(_a		,     _a       )

      _lowercase							      =DebertaVaTokenizerFast(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a		,     add_special_tokens=_a       )       )
      self.assertListEqual(_a		,     _a       )





     def      __A					(self       )     ->   Optional[Any]:
      _lowercase							      ='''I was born in 92000, and this is falsé.'''
      _lowercase							      =['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
      # fmt: on

      _lowercase							      =DebertaVaTokenizer(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )

      _lowercase							      =DebertaVaTokenizerFast(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )





     def      __A					(self       )     ->   Any:
      _lowercase							      ='''I was born in 92000, and this is falsé.'''
      _lowercase							      =['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
      # fmt: on

      _lowercase							      =DebertaVaTokenizer(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )

      _lowercase							      =DebertaVaTokenizerFast(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )





     def      __A					(self       )     ->   Tuple:
      _lowercase							      =''' \tHeLLo!how  \n Are yoU?  '''
      _lowercase							      =['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
      # fmt: on

      _lowercase							      =DebertaVaTokenizer(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )

      _lowercase							      =DebertaVaTokenizerFast(_a		,     do_lower_case=_a		,     split_by_punct=_a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a		,     add_special_tokens=_a       )       )

      self.assertListEqual(_a		,     _a       )





     def      __A					(self       )     ->   str:
      _lowercase							      =self.get_tokenizer()
      _lowercase							      =self.get_rust_tokenizer()

      _lowercase							      ='''I was born in 92000, and this is falsé.'''

      _lowercase							      =tokenizer.convert_ids_to_tokens(tokenizer.encode(_a		,     add_special_tokens=_a       )       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a		,     add_special_tokens=_a       )       )
      self.assertListEqual(_a		,     _a       )

      _lowercase							      =tokenizer.encode(_a		,     add_special_tokens=_a       )
      _lowercase							      =rust_tokenizer.encode(_a		,     add_special_tokens=_a       )
      self.assertListEqual(_a		,     _a       )

      _lowercase							      =self.get_rust_tokenizer()
      _lowercase							      =tokenizer.encode(_a       )
      _lowercase							      =rust_tokenizer.encode(_a       )
      self.assertListEqual(_a		,     _a       )





     def      __A					(self       )     ->   str:
      _lowercase							      ='''This is a test'''
      _lowercase							      =[1_3, 1, 4_3_9_8, 2_5, 2_1, 1_2_8_9]
      _lowercase							      =['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
      _lowercase							      =['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

      _lowercase							      =DebertaVaTokenizer(_a		,     keep_accents=_a       )
      _lowercase							      =DebertaVaTokenizerFast(_a		,     keep_accents=_a       )

      _lowercase							      =tokenizer.encode(_a		,     add_special_tokens=_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =tokenizer.tokenize(_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(_a       )
      self.assertListEqual(_a		,     _a       )

      _lowercase							      =rust_tokenizer.encode(_a		,     add_special_tokens=_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =rust_tokenizer.tokenize(_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(_a       )
      self.assertListEqual(_a		,     _a       )

      # fmt: off
      _lowercase							      ='''I was born in 92000, and this is falsé.'''
      _lowercase							      =[1_3, 1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]
      _lowercase							      =['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
      _lowercase							      =['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
      # fmt: on

      _lowercase							      =tokenizer.encode(_a		,     add_special_tokens=_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =tokenizer.tokenize(_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =tokenizer.convert_ids_to_tokens(_a       )
      self.assertListEqual(_a		,     _a       )

      _lowercase							      =rust_tokenizer.encode(_a		,     add_special_tokens=_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =rust_tokenizer.tokenize(_a       )
      self.assertListEqual(_a		,     _a       )
      _lowercase							      =rust_tokenizer.convert_ids_to_tokens(_a       )
      self.assertListEqual(_a		,     _a       )





     def      __A					(self       )     ->   Union[str, Any]:
      _lowercase							      =DebertaVaTokenizer(_a       )

      _lowercase							      =tokenizer.encode('''sequence builders'''       )
      _lowercase							      =tokenizer.encode('''multi-sequence build'''       )

      _lowercase							      =tokenizer.build_inputs_with_special_tokens(_a       )
      _lowercase							      =tokenizer.build_inputs_with_special_tokens(_a		,     _a       )

      self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]		,     _a       )
      self.assertEqual(
          [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]		,     _a		,     )





     @slow
     def      __A					(self       )     ->   Dict:
      _lowercase							      ={'''input_ids''': [[1, 3_9_8_6_7, 3_6, 1_9_3_9_0, 4_8_6, 2_7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 6_0_6_8_5, 1_2_2_5, 7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 9_3_6_7, 1_6_8_9_9, 1_8, 1_5_9_3_7, 5_3, 5_9_4, 7_7_3, 1_8, 1_6_2_8_7, 3_0_4_6_5, 3_6, 1_5_9_3_7, 6, 4_1_1_3_9, 3_8, 3_6_9_7_9, 6_0_7_6_3, 1_9_1, 6, 3_4_1_3_2, 9_9, 6, 5_0_5_3_8, 3_9_0, 4_3_2_3_0, 6, 3_4_1_3_2, 2_7_7_9, 2_0_8_5_0, 1_4, 6_9_9, 1_0_7_2, 1_1_9_4, 3_6, 3_8_2, 1_0_9_0_1, 5_3, 7, 6_9_9, 1_0_7_2, 2_0_8_4, 3_6, 2_0_4_2_2, 6_3_0, 5_3, 1_9, 1_0_5, 3_0_4_9, 1_8_9_6, 1_0_5_3, 1_6_8_9_9, 1_5_0_6, 1_1, 3_7_9_7_8, 4_2_4_3, 7, 1_2_3_7, 3_1_8_6_9, 2_0_0, 1_6_5_6_6, 6_5_4, 6, 3_5_0_5_2, 8_1_4_3_6, 7, 5_5_6_3_0, 1_3_5_9_3, 4, 2], [1, 2_6, 1_5_0_1_1, 1_3, 6_6_7, 8, 1_0_5_3, 1_8, 2_3_6_1_1, 1_2_3_7, 7_2_3_5_6, 1_2_8_2_0, 3_4, 1_0_4_1_3_4, 1_2_0_9, 3_5, 1_3_3_1_3, 6_6_2_7, 2_1, 2_0_2, 3_4_7, 7, 1_6_4, 2_3_9_9, 1_1, 4_6, 4_4_8_5, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_2_3_2, 2_8_6_4, 1_5_7_8_5, 1_4_9_5_1, 1_0_5, 5, 8_5_8_1, 1_2_5_0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=_a		,     model_name='''microsoft/deberta-v2-xlarge'''		,     revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''		,     )




'''simple docstring'''
lowerCamelCase :      Any					 =     "\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

lowerCamelCase :      int					 =     [{"type": "code", "content": INSTALL_CONTENT}]
lowerCamelCase :      str					 =     {
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}
