

'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase       =     {
    '''configuration_bert''': ['''BERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BertConfig''', '''BertOnnxConfig'''],
    '''tokenization_bert''': ['''BasicTokenizer''', '''BertTokenizer''', '''WordpieceTokenizer'''],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase       =     ['''BertTokenizerFast''']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase       =     [
           '''BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''BertForMaskedLM''',
           '''BertForMultipleChoice''',
           '''BertForNextSentencePrediction''',
           '''BertForPreTraining''',
           '''BertForQuestionAnswering''',
           '''BertForSequenceClassification''',
           '''BertForTokenClassification''',
           '''BertLayer''',
           '''BertLMHeadModel''',
           '''BertModel''',
           '''BertPreTrainedModel''',
           '''load_tf_weights_in_bert''',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase       =     [
           '''TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''TFBertEmbeddings''',
           '''TFBertForMaskedLM''',
           '''TFBertForMultipleChoice''',
           '''TFBertForNextSentencePrediction''',
           '''TFBertForPreTraining''',
           '''TFBertForQuestionAnswering''',
           '''TFBertForSequenceClassification''',
           '''TFBertForTokenClassification''',
           '''TFBertLMHeadModel''',
           '''TFBertMainLayer''',
           '''TFBertModel''',
           '''TFBertPreTrainedModel''',
       ]
try:
       if not is_tensorflow_text_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase       =     ['''TFBertTokenizer''']

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase       =     [
           '''FlaxBertForCausalLM''',
           '''FlaxBertForMaskedLM''',
           '''FlaxBertForMultipleChoice''',
           '''FlaxBertForNextSentencePrediction''',
           '''FlaxBertForPreTraining''',
           '''FlaxBertForQuestionAnswering''',
           '''FlaxBertForSequenceClassification''',
           '''FlaxBertForTokenClassification''',
           '''FlaxBertModel''',
           '''FlaxBertPreTrainedModel''',
       ]

if TYPE_CHECKING:
       from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
       from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_bert_fast import BertTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_bert import (
                  BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  BertForMaskedLM,
                  BertForMultipleChoice,
                  BertForNextSentencePrediction,
                  BertForPreTraining,
                  BertForQuestionAnswering,
                  BertForSequenceClassification,
                  BertForTokenClassification,
                  BertLayer,
                  BertLMHeadModel,
                  BertModel,
                  BertPreTrainedModel,
                  load_tf_weights_in_bert,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_bert import (
                  TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFBertEmbeddings,
                  TFBertForMaskedLM,
                  TFBertForMultipleChoice,
                  TFBertForNextSentencePrediction,
                  TFBertForPreTraining,
                  TFBertForQuestionAnswering,
                  TFBertForSequenceClassification,
                  TFBertForTokenClassification,
                  TFBertLMHeadModel,
                  TFBertMainLayer,
                  TFBertModel,
                  TFBertPreTrainedModel,
              )

       try:
              if not is_tensorflow_text_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_bert_tf import TFBertTokenizer

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_bert import (
                  FlaxBertForCausalLM,
                  FlaxBertForMaskedLM,
                  FlaxBertForMultipleChoice,
                  FlaxBertForNextSentencePrediction,
                  FlaxBertForPreTraining,
                  FlaxBertForQuestionAnswering,
                  FlaxBertForSequenceClassification,
                  FlaxBertForTokenClassification,
                  FlaxBertModel,
                  FlaxBertPreTrainedModel,
              )

else:
       import sys

       UpperCAmelCase       =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)









'''simple docstring'''




import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE     :int       =				get_tests_dir('''fixtures/test_sentencepiece_no_bos.model''')







@require_sentencepiece
@require_tokenizers
class   A_	(  lowerCAmelCase_							,						unittest.TestCase			):
 _lowerCamelCase  : List[str]           =							PegasusTokenizer
 _lowerCamelCase  : int           =							PegasusTokenizerFast
 _lowerCamelCase  : Union[str, Any]           =							True
 _lowerCamelCase  : List[str]           =							True




 def      lowercase	(					self       :    Optional[int]    ):
      super().setUp()

      # We have a SentencePiece fixture for testing
      _UpperCAmelCase     							=   PegasusTokenizer(snake_case_    )
      tokenizer.save_pretrained(self.tmpdirname    )




 @cached_property
 def      lowercase	(					self       :    Tuple    ):
      return PegasusTokenizer.from_pretrained("google/pegasus-large"    )




 def      lowercase	(					self       :    Union[str, Any]	,			**snake_case_       :    Union[str, Any]    ):
      return PegasusTokenizer.from_pretrained(self.tmpdirname	,			**snake_case_    )




 def      lowercase	(					self       :    Tuple	,			snake_case_       :    Any    ):
      return ("This is a test", "This is a test")




 def      lowercase	(					self       :    Optional[int]    ):
      _UpperCAmelCase     							=   "</s>"
      _UpperCAmelCase     							=   1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_    )	,			snake_case_    )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_    )	,			snake_case_    )




 def      lowercase	(					self       :    Union[str, Any]    ):
      _UpperCAmelCase     							=   list(self.get_tokenizer().get_vocab().keys()    )

      self.assertEqual(vocab_keys[0]	,			"<pad>"    )
      self.assertEqual(vocab_keys[1]	,			"</s>"    )
      self.assertEqual(vocab_keys[-1]	,			"v"    )
      self.assertEqual(len(snake_case_    )	,			1_1_0_3    )




 def      lowercase	(					self       :    Any    ):
      self.assertEqual(self.get_tokenizer().vocab_size	,			1_1_0_3    )




 def      lowercase	(					self       :    List[Any]    ):
      _UpperCAmelCase     							=   self.rust_tokenizer_class.from_pretrained(self.tmpdirname    )
      _UpperCAmelCase     							=   self.tokenizer_class.from_pretrained(self.tmpdirname    )
      _UpperCAmelCase     							=   (
          "Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"
          " </s> <pad> <pad> <pad>"
      )
      _UpperCAmelCase     							=   rust_tokenizer([raw_input_str]	,			return_tensors=snake_case_	,			add_special_tokens=snake_case_    ).input_ids[0]
      _UpperCAmelCase     							=   py_tokenizer([raw_input_str]	,			return_tensors=snake_case_	,			add_special_tokens=snake_case_    ).input_ids[0]
      self.assertListEqual(snake_case_	,			snake_case_    )




 def      lowercase	(					self       :    Tuple    ):
      _UpperCAmelCase     							=   self._large_tokenizer
      # <mask_1> masks whole sentence while <mask_2> masks single word
      _UpperCAmelCase     							=   "<mask_1> To ensure a <mask_2> flow of bank resolutions."
      _UpperCAmelCase     							=   [2, 4_1_3, 6_1_5, 1_1_4, 3, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
      _UpperCAmelCase     							=   tokenizer([raw_input_str]	,			return_tensors=snake_case_    ).input_ids[0]
      self.assertListEqual(snake_case_	,			snake_case_    )




 def      lowercase	(					self       :    Union[str, Any]    ):
      _UpperCAmelCase     							=   self._large_tokenizer
      # The tracebacks for the following asserts are **better** without messages or self.assertEqual
      assert tokenizer.vocab_size == 9_6_1_0_3
      assert tokenizer.pad_token_id == 0
      assert tokenizer.eos_token_id == 1
      assert tokenizer.offset == 1_0_3
      assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_0_5
      assert tokenizer.unk_token == "<unk>"
      assert tokenizer.model_max_length == 1_0_2_4
      _UpperCAmelCase     							=   "To ensure a smooth flow of bank resolutions."
      _UpperCAmelCase     							=   [4_1_3, 6_1_5, 1_1_4, 2_2_9_1, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
      _UpperCAmelCase     							=   tokenizer([raw_input_str]	,			return_tensors=snake_case_    ).input_ids[0]
      self.assertListEqual(snake_case_	,			snake_case_    )
      assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]    ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]




 @require_torch
 def      lowercase	(					self       :    int    ):
      _UpperCAmelCase     							=   ["This is going to be way too long." * 1_5_0, "short example"]
      _UpperCAmelCase     							=   ["not super long but more than 5 tokens", "tiny"]
      _UpperCAmelCase     							=   self._large_tokenizer(snake_case_	,			padding=snake_case_	,			truncation=snake_case_	,			return_tensors="pt"    )
      _UpperCAmelCase     							=   self._large_tokenizer(
          text_target=snake_case_	,			max_length=5	,			padding=snake_case_	,			truncation=snake_case_	,			return_tensors="pt"    )

      assert batch.input_ids.shape == (2, 1_0_2_4)
      assert batch.attention_mask.shape == (2, 1_0_2_4)
      assert targets["input_ids"].shape == (2, 5)
      assert len(snake_case_    ) == 2  # input_ids, attention_mask.




 @slow
 def      lowercase	(					self       :    Dict    ):
      # fmt: off
      _UpperCAmelCase     							=   {"input_ids": [[3_8_9_7_9, 1_4_3, 1_8_4_8_5, 6_0_6, 1_3_0, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 5_4_1_8_9, 1_1_2_9, 1_1_1, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 9_1_1_4, 1_4_7_8_7, 1_2_1, 1_3_2_4_9, 1_5_8, 5_9_2, 9_5_6, 1_2_1, 1_4_6_2_1, 3_1_5_7_6, 1_4_3, 6_2_6_1_3, 1_0_8, 9_6_8_8, 9_3_0, 4_3_4_3_0, 1_1_5_6_2, 6_2_6_1_3, 3_0_4, 1_0_8, 1_1_4_4_3, 8_9_7, 1_0_8, 9_3_1_4, 1_7_4_1_5, 6_3_3_9_9, 1_0_8, 1_1_4_4_3, 7_6_1_4, 1_8_3_1_6, 1_1_8, 4_2_8_4, 7_1_4_8, 1_2_4_3_0, 1_4_3, 1_4_0_0, 2_5_7_0_3, 1_5_8, 1_1_1, 4_2_8_4, 7_1_4_8, 1_1_7_7_2, 1_4_3, 2_1_2_9_7, 1_0_6_4, 1_5_8, 1_2_2, 2_0_4, 3_5_0_6, 1_7_5_4, 1_1_3_3, 1_4_7_8_7, 1_5_8_1, 1_1_5, 3_3_2_2_4, 4_4_8_2, 1_1_1, 1_3_5_5, 1_1_0, 2_9_1_7_3, 3_1_7, 5_0_8_3_3, 1_0_8, 2_0_1_4_7, 9_4_6_6_5, 1_1_1, 7_7_1_9_8, 1_0_7, 1], [1_1_0, 6_2_6_1_3, 1_1_7, 6_3_8, 1_1_2, 1_1_3_3, 1_2_1, 2_0_0_9_8, 1_3_5_5, 7_9_0_5_0, 1_3_8_7_2, 1_3_5, 1_5_9_6, 5_3_5_4_1, 1_3_5_2, 1_4_1, 1_3_0_3_9, 5_5_4_2, 1_2_4, 3_0_2, 5_1_8, 1_1_1, 2_6_8, 2_9_5_6, 1_1_5, 1_4_9, 4_4_2_7, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_3_9, 1_2_3_5, 2_7_9_9, 1_8_2_8_9, 1_7_7_8_0, 2_0_4, 1_0_9, 9_4_7_4, 1_2_9_6, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=snake_case_	,			model_name="google/bigbird-pegasus-large-arxiv"	,			revision="ba85d0851d708441f91440d509690f1ab6353415"	,			)







@require_sentencepiece
@require_tokenizers
class   A_	(  lowerCAmelCase_							,						unittest.TestCase			):
 _lowerCamelCase  : List[str]           =							PegasusTokenizer
 _lowerCamelCase  : List[Any]           =							PegasusTokenizerFast
 _lowerCamelCase  : int           =							True
 _lowerCamelCase  : Union[str, Any]           =							True




 def      lowercase	(					self       :    Any    ):
      super().setUp()

      # We have a SentencePiece fixture for testing
      _UpperCAmelCase     							=   PegasusTokenizer(snake_case_	,			offset=0	,			mask_token_sent=snake_case_	,			mask_token="[MASK]"    )
      tokenizer.save_pretrained(self.tmpdirname    )




 @cached_property
 def      lowercase	(					self       :    Tuple    ):
      return PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv"    )




 def      lowercase	(					self       :    Optional[Any]	,			**snake_case_       :    Dict    ):
      return PegasusTokenizer.from_pretrained(self.tmpdirname	,			**snake_case_    )




 def      lowercase	(					self       :    Union[str, Any]	,			snake_case_       :    str    ):
      return ("This is a test", "This is a test")




 def      lowercase	(					self       :    List[str]    ):
      _UpperCAmelCase     							=   self.rust_tokenizer_class.from_pretrained(self.tmpdirname    )
      _UpperCAmelCase     							=   self.tokenizer_class.from_pretrained(self.tmpdirname    )
      _UpperCAmelCase     							=   (
          "Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"
          " <pad> <pad> <pad>"
      )
      _UpperCAmelCase     							=   rust_tokenizer([raw_input_str]	,			return_tensors=snake_case_	,			add_special_tokens=snake_case_    ).input_ids[0]
      _UpperCAmelCase     							=   py_tokenizer([raw_input_str]	,			return_tensors=snake_case_	,			add_special_tokens=snake_case_    ).input_ids[0]
      self.assertListEqual(snake_case_	,			snake_case_    )




 @require_torch
 def      lowercase	(					self       :    Tuple    ):
      _UpperCAmelCase     							=   ["This is going to be way too long." * 1_0_0_0, "short example"]
      _UpperCAmelCase     							=   ["not super long but more than 5 tokens", "tiny"]
      _UpperCAmelCase     							=   self._large_tokenizer(snake_case_	,			padding=snake_case_	,			truncation=snake_case_	,			return_tensors="pt"    )
      _UpperCAmelCase     							=   self._large_tokenizer(
          text_target=snake_case_	,			max_length=5	,			padding=snake_case_	,			truncation=snake_case_	,			return_tensors="pt"    )

      assert batch.input_ids.shape == (2, 4_0_9_6)
      assert batch.attention_mask.shape == (2, 4_0_9_6)
      assert targets["input_ids"].shape == (2, 5)
      assert len(snake_case_    ) == 2  # input_ids, attention_mask.




 def      lowercase	(					self       :    Union[str, Any]    ):
      _UpperCAmelCase     							=   (
          "This is an example string that is used to test the original TF implementation against the HF"
          " implementation"
      )

      _UpperCAmelCase     							=   self._large_tokenizer(snake_case_    ).input_ids

      self.assertListEqual(
          snake_case_	,			[1_8_2, 1_1_7, 1_4_2, 5_8_7, 4_2_1_1, 1_2_0, 1_1_7, 2_6_3, 1_1_2, 8_0_4, 1_0_9, 8_5_6, 2_5_0_1_6, 3_1_3_7, 4_6_4, 1_0_9, 2_6_9_5_5, 3_1_3_7, 1]	,			)




