import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 				_SCREAMING_SNAKE_CASE( snake_case__							,   unittest.TestCase						):
   UpperCamelCase_					:    Union[str, Any]			      = RealmTokenizer
   UpperCamelCase_					:    Union[str, Any]			      = RealmTokenizerFast
   UpperCamelCase_					:    Dict			      = True
   UpperCamelCase_					:    Dict			      = True
   UpperCamelCase_					:    str			      = filter_non_english
   def _lowercase(     self     )						->Any:
        super().setUp()

        a	:     Tuple						      =    [
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        a	:     Optional[int]						      =    os.path.join(self.tmpdirname				,  VOCAB_FILES_NAMES['''vocab_file''']     )
        with open(self.vocab_file				,  '''w'''				,  encoding='''utf-8'''     ) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]     )     )
   def _lowercase(     self				,  lowerCAmelCase_     )						->Optional[Any]:
        a	:     Tuple						      =    '''UNwant\u00E9d,running'''
        a	:     List[str]						      =    '''unwanted, running'''
        return input_text, output_text
   def _lowercase(     self     )						->Tuple:
        a	:     Tuple						      =    self.tokenizer_class(self.vocab_file     )

        a	:     str						      =    tokenizer.tokenize('''UNwant\u00E9d,running'''     )
        self.assertListEqual(lowerCAmelCase_				,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']     )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )				,  [9, 6, 7, 12, 10, 11]     )
   def _lowercase(     self     )						->Optional[int]:
        if not self.test_rust_tokenizer:
             return

        a	:     Optional[int]						      =    self.get_tokenizer()
        a	:     List[str]						      =    self.get_rust_tokenizer()

        a	:     Optional[int]						      =    '''UNwant\u00E9d,running'''

        a	:     Dict						      =    tokenizer.tokenize(lowerCAmelCase_     )
        a	:     List[str]						      =    rust_tokenizer.tokenize(lowerCAmelCase_     )
        self.assertListEqual(lowerCAmelCase_				,  lowerCAmelCase_     )

        a	:     Optional[int]						      =    tokenizer.encode(lowerCAmelCase_				,  add_special_tokens=lowerCAmelCase_     )
        a	:     Optional[int]						      =    rust_tokenizer.encode(lowerCAmelCase_				,  add_special_tokens=lowerCAmelCase_     )
        self.assertListEqual(lowerCAmelCase_				,  lowerCAmelCase_     )

        a	:     List[Any]						      =    self.get_rust_tokenizer()
        a	:     int						      =    tokenizer.encode(lowerCAmelCase_     )
        a	:     List[Any]						      =    rust_tokenizer.encode(lowerCAmelCase_     )
        self.assertListEqual(lowerCAmelCase_				,  lowerCAmelCase_     )

        # With lower casing
        a	:     List[Any]						      =    self.get_tokenizer(do_lower_case=lowerCAmelCase_     )
        a	:     Any						      =    self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_     )

        a	:     Dict						      =    '''UNwant\u00E9d,running'''

        a	:     Optional[Any]						      =    tokenizer.tokenize(lowerCAmelCase_     )
        a	:     Dict						      =    rust_tokenizer.tokenize(lowerCAmelCase_     )
        self.assertListEqual(lowerCAmelCase_				,  lowerCAmelCase_     )

        a	:     int						      =    tokenizer.encode(lowerCAmelCase_				,  add_special_tokens=lowerCAmelCase_     )
        a	:     Tuple						      =    rust_tokenizer.encode(lowerCAmelCase_				,  add_special_tokens=lowerCAmelCase_     )
        self.assertListEqual(lowerCAmelCase_				,  lowerCAmelCase_     )

        a	:     Optional[int]						      =    self.get_rust_tokenizer()
        a	:     List[str]						      =    tokenizer.encode(lowerCAmelCase_     )
        a	:     int						      =    rust_tokenizer.encode(lowerCAmelCase_     )
        self.assertListEqual(lowerCAmelCase_				,  lowerCAmelCase_     )
   def _lowercase(     self     )						->Tuple:
        a	:     Any						      =    BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''     )				,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']     )
   def _lowercase(     self     )						->Optional[Any]:
        a	:     List[str]						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''     )				,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )				,  ['''hello''']     )
   def _lowercase(     self     )						->Union[str, Any]:
        a	:     List[Any]						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_				,  strip_accents=lowerCAmelCase_     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )				,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )				,  ['''h\u00E9llo''']     )
   def _lowercase(     self     )						->List[str]:
        a	:     str						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_				,  strip_accents=lowerCAmelCase_     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )				,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )				,  ['''hello''']     )
   def _lowercase(     self     )						->Dict:
        a	:     Any						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )				,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )				,  ['''hello''']     )
   def _lowercase(     self     )						->str:
        a	:     List[str]						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''     )				,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']     )
   def _lowercase(     self     )						->Union[str, Any]:
        a	:     Tuple						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_				,  strip_accents=lowerCAmelCase_     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )				,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']     )
   def _lowercase(     self     )						->int:
        a	:     List[Any]						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_				,  strip_accents=lowerCAmelCase_     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )				,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']     )
   def _lowercase(     self     )						->Dict:
        a	:     Union[str, Any]						      =    BasicTokenizer(do_lower_case=lowerCAmelCase_				,  never_split=['''[UNK]''']     )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''     )				,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']     )
   def _lowercase(     self     )						->Tuple:
        a	:     str						      =    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

        a	:     Union[str, Any]						      =    {}
        for i, token in enumerate(lowerCAmelCase_     ):
             a	:     List[Any]						      =    i
        a	:     Dict						      =    WordpieceTokenizer(vocab=lowerCAmelCase_				,  unk_token='''[UNK]'''     )

        self.assertListEqual(tokenizer.tokenize(''''''     )				,  []     )

        self.assertListEqual(tokenizer.tokenize('''unwanted running'''     )				,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']     )

        self.assertListEqual(tokenizer.tokenize('''unwantedX running'''     )				,  ['''[UNK]''', '''runn''', '''##ing''']     )
   def _lowercase(     self     )						->Optional[Any]:
        self.assertTrue(_is_whitespace(''' '''     )     )
        self.assertTrue(_is_whitespace('''\t'''     )     )
        self.assertTrue(_is_whitespace('''\r'''     )     )
        self.assertTrue(_is_whitespace('''\n'''     )     )
        self.assertTrue(_is_whitespace('''\u00A0'''     )     )

        self.assertFalse(_is_whitespace('''A'''     )     )
        self.assertFalse(_is_whitespace('''-'''     )     )
   def _lowercase(     self     )						->Optional[int]:
        self.assertTrue(_is_control('''\u0005'''     )     )

        self.assertFalse(_is_control('''A'''     )     )
        self.assertFalse(_is_control(''' '''     )     )
        self.assertFalse(_is_control('''\t'''     )     )
        self.assertFalse(_is_control('''\r'''     )     )
   def _lowercase(     self     )						->List[str]:
        self.assertTrue(_is_punctuation('''-'''     )     )
        self.assertTrue(_is_punctuation('''$'''     )     )
        self.assertTrue(_is_punctuation('''`'''     )     )
        self.assertTrue(_is_punctuation('''.'''     )     )

        self.assertFalse(_is_punctuation('''A'''     )     )
        self.assertFalse(_is_punctuation(''' '''     )     )
   def _lowercase(     self     )						->str:
        a	:     Dict						      =    self.get_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_     ) for t in ['''Test''', '''\xad''', '''test''']]				,  [['''[UNK]'''], [], ['''[UNK]''']]     )

        if self.test_rust_tokenizer:
             a	:     Tuple						      =    self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(lowerCAmelCase_     ) for t in ['''Test''', '''\xad''', '''test''']]				,  [['''[UNK]'''], [], ['''[UNK]''']]     )
   @slow
   def _lowercase(     self     )						->List[Any]:
        a	:     Optional[int]						      =    self.tokenizer_class.from_pretrained('''bert-base-uncased'''     )

        a	:     Dict						      =    tokenizer.encode('''sequence builders'''				,  add_special_tokens=lowerCAmelCase_     )
        a	:     Tuple						      =    tokenizer.encode('''multi-sequence build'''				,  add_special_tokens=lowerCAmelCase_     )

        a	:     Dict						      =    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
        a	:     Union[str, Any]						      =    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_				,  lowerCAmelCase_     )

        assert encoded_sentence == [1_01] + text + [1_02]
        assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
   def _lowercase(     self     )						->Tuple:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
                  a	:     List[Any]						      =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,  **lowerCAmelCase_     )

                  a	:     Dict						      =    F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                  a	:     Any						      =    tokenizer_r.encode_plus(
                      lowerCAmelCase_				,  return_attention_mask=lowerCAmelCase_				,  return_token_type_ids=lowerCAmelCase_				,  return_offsets_mapping=lowerCAmelCase_				,  add_special_tokens=lowerCAmelCase_				,  )

                  a	:     List[Any]						      =    tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_				,  '''do_lower_case'''     ) else False
                  a	:     int						      =    (
                      [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), '''A'''),
                          ((1, 2), ''','''),
                          ((3, 5), '''na'''),
                          ((5, 6), '''##ï'''),
                          ((6, 8), '''##ve'''),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), '''Allen'''),
                          ((21, 23), '''##NL'''),
                          ((23, 24), '''##P'''),
                          ((25, 33), '''sentence'''),
                          ((33, 34), '''.'''),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                      if not do_lower_case
                      else [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), '''a'''),
                          ((1, 2), ''','''),
                          ((3, 8), '''naive'''),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), '''allen'''),
                          ((21, 23), '''##nl'''),
                          ((23, 24), '''##p'''),
                          ((25, 33), '''sentence'''),
                          ((33, 34), '''.'''),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                  )

                  self.assertEqual(
                      [e[1] for e in expected_results]				,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']     )     )
                  self.assertEqual([e[0] for e in expected_results]				,  tokens['''offset_mapping''']     )


   @slow
   def _lowercase(     self     )						->Any:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
                  a	:     Union[str, Any]						      =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,  **lowerCAmelCase_     )
                  a	:     List[Any]						      =    self.tokenizer_class.from_pretrained(lowerCAmelCase_				,  **lowerCAmelCase_     )
                  a	:     Union[str, Any]						      =    [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                  a	:     List[Any]						      =    tokenizer_r.batch_encode_candidates(lowerCAmelCase_				,  max_length=10				,  return_tensors='''np'''     )
                  a	:     Tuple						      =    tokenizer_p.batch_encode_candidates(lowerCAmelCase_				,  max_length=10				,  return_tensors='''np'''     )

                  a	:     Dict						      =    (2, 2, 10)

                  self.assertEqual(encoded_sentence_r['''input_ids'''].shape				,  lowerCAmelCase_     )
                  self.assertEqual(encoded_sentence_r['''attention_mask'''].shape				,  lowerCAmelCase_     )
                  self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape				,  lowerCAmelCase_     )

                  self.assertEqual(encoded_sentence_p['''input_ids'''].shape				,  lowerCAmelCase_     )
                  self.assertEqual(encoded_sentence_p['''attention_mask'''].shape				,  lowerCAmelCase_     )
                  self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape				,  lowerCAmelCase_     )
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase				 =   {"""configuration_pegasus""": ["""PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP""", """PegasusConfig"""]}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase				 =   ["""PegasusTokenizer"""]

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase				 =   ["""PegasusTokenizerFast"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase				 =   [
      """PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """PegasusForCausalLM""",
      """PegasusForConditionalGeneration""",
      """PegasusModel""",
      """PegasusPreTrainedModel""",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase				 =   [
      """TFPegasusForConditionalGeneration""",
      """TFPegasusModel""",
      """TFPegasusPreTrainedModel""",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase				 =   [
      """FlaxPegasusForConditionalGeneration""",
      """FlaxPegasusModel""",
      """FlaxPegasusPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_pegasus import PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusConfig

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_pegasus import PegasusTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_pegasus_fast import PegasusTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_pegasus import (
        PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST,
        PegasusForCausalLM,
        PegasusForConditionalGeneration,
        PegasusModel,
        PegasusPreTrainedModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_pegasus import TFPegasusForConditionalGeneration, TFPegasusModel, TFPegasusPreTrainedModel

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_pegasus import (
        FlaxPegasusForConditionalGeneration,
        FlaxPegasusModel,
        FlaxPegasusPreTrainedModel,
    )

else:
  import sys

  UpperCAmelCase				 =   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)