


"""simple docstring"""







import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase    					=      """▁"""

__lowercase    					=      get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
@require_tokenizers
class 						_A  (  _a    ,unittest.TestCase  ):







    """simple docstring"""




    UpperCAmelCase       :							int	=       BigBirdTokenizer
    UpperCAmelCase       :							int	=       BigBirdTokenizerFast
    UpperCAmelCase       :							List[Any]	=       True
    UpperCAmelCase       :							List[str]	=       True






    def 	__snake_case							(					self				:    List[str]):
      super().setUp()

      a						:		int      							=			self.tokenizer_class(__UpperCAmelCase				, keep_accents=__UpperCAmelCase)
      tokenizer.save_pretrained(self.tmpdirname)






    def 	__snake_case							(					self				:    Tuple):
      a						:		Optional[int]      							=			"<s>"
      a						:		Optional[Any]      							=			1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase)				, __UpperCAmelCase)
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase)				, __UpperCAmelCase)






    def 	__snake_case							(					self				:    Tuple):
      a						:		Tuple      							=			list(self.get_tokenizer().get_vocab().keys())

      self.assertEqual(vocab_keys[0]				, "<unk>")
      self.assertEqual(vocab_keys[1]				, "<s>")
      self.assertEqual(vocab_keys[-1]				, "[MASK]")
      self.assertEqual(len(__UpperCAmelCase)				, 1004)






    def 	__snake_case							(					self				:    Any):
      self.assertEqual(self.get_tokenizer().vocab_size				, 1000)






    def 	__snake_case							(					self				:    int):
      if not self.test_rust_tokenizer:
        return

      a						:		Dict      							=			self.get_tokenizer()
      a						:		int      							=			self.get_rust_tokenizer()

      a						:		Optional[Any]      							=			"I was born in 92000, and this is falsé."

      a						:		str      							=			tokenizer.tokenize(__UpperCAmelCase)
      a						:		Dict      							=			rust_tokenizer.tokenize(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase				, __UpperCAmelCase)

      a						:		Dict      							=			tokenizer.encode(__UpperCAmelCase				, add_special_tokens=__UpperCAmelCase)
      a						:		List[str]      							=			rust_tokenizer.encode(__UpperCAmelCase				, add_special_tokens=__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase				, __UpperCAmelCase)

      a						:		List[str]      							=			self.get_rust_tokenizer()
      a						:		Any      							=			tokenizer.encode(__UpperCAmelCase)
      a						:		List[Any]      							=			rust_tokenizer.encode(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase				, __UpperCAmelCase)






    def 	__snake_case							(					self				:    List[str]):
      a						:		Optional[int]      							=			BigBirdTokenizer(__UpperCAmelCase				, keep_accents=__UpperCAmelCase)

      a						:		int      							=			tokenizer.tokenize("This is a test")
      self.assertListEqual(__UpperCAmelCase				, ["▁This", "▁is", "▁a", "▁t", "est"])

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__UpperCAmelCase)				, [285, 46, 10, 170, 382]				, )

      a						:		Optional[int]      							=			tokenizer.tokenize("I was born in 92000, and this is falsé.")
      self.assertListEqual(
          __UpperCAmelCase				, [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]				, )
      a						:		Tuple      							=			tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(
          __UpperCAmelCase				, [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]				, )

      a						:		Dict      							=			tokenizer.convert_ids_to_tokens(__UpperCAmelCase)
      self.assertListEqual(
          __UpperCAmelCase				, [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "<unk>",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "<unk>",
              ".",
          ]				, )






    @cached_property
    def 	__snake_case							(					self				:    Any):
      return BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")






    @slow
    def 	__snake_case							(					self				:    Tuple):
      a						:		Optional[int]      							=			"Hello World!"
      a						:		int      							=			[65, 18536, 2260, 101, 66]

      self.assertListEqual(__UpperCAmelCase				, self.big_tokenizer.encode(__UpperCAmelCase))






    @slow
    def 	__snake_case							(					self				:    Union[str, Any]):
      a						:		Optional[Any]      							=			(
          "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
          " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
      )
      # fmt: off
      a						:		int      							=			[65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 34324, 497, 391, 408, 11342, 1244, 385, 100, 938, 985, 456, 574, 362, 12597, 3200, 3129, 1172, 66]  # noqa: E231
      # fmt: on
      self.assertListEqual(__UpperCAmelCase				, self.big_tokenizer.encode(__UpperCAmelCase))






    @require_torch
    @slow
    def 	__snake_case							(					self				:    str):
      import torch

      from transformers import BigBirdConfig, BigBirdModel

      # Build sequence
      a						:		Dict      							=			list(self.big_tokenizer.get_vocab().keys())[:10]
      a						:		Optional[int]      							=			" ".join(__UpperCAmelCase)
      a						:		Optional[int]      							=			self.big_tokenizer.encode_plus(__UpperCAmelCase				, return_tensors="pt"				, return_token_type_ids=__UpperCAmelCase)
      a						:		Tuple      							=			self.big_tokenizer.batch_encode_plus(
          [sequence + " " + sequence]				, return_tensors="pt"				, return_token_type_ids=__UpperCAmelCase)

      a						:		Dict      							=			BigBirdConfig(attention_type="original_full")
      a						:		List[str]      							=			BigBirdModel(__UpperCAmelCase)

      assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

      with torch.no_grad():
        model(**__UpperCAmelCase)
        model(**__UpperCAmelCase)






    @slow
    def 	__snake_case							(					self				:    List[Any]):
      a						:		Any      							=			BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")
      a						:		Optional[int]      							=			tokenizer.decode(tokenizer("Paris is the [MASK].").input_ids)

      self.assertTrue(decoded_text == "[CLS] Paris is the[MASK].[SEP]")






    @slow
    def 	__snake_case							(					self				:    Optional[int]):
      # fmt: off
      a						:		Any      							=			{"input_ids": [[65, 39286, 458, 36335, 2001, 456, 13073, 13266, 455, 113, 7746, 1741, 11157, 391, 13073, 13266, 455, 113, 3967, 35412, 113, 4936, 109, 3870, 2377, 113, 30084, 45720, 458, 134, 17496, 112, 503, 11672, 113, 118, 112, 5665, 13347, 38687, 112, 1496, 31389, 112, 3268, 47264, 134, 962, 112, 16377, 8035, 23130, 430, 12169, 15518, 28592, 458, 146, 41697, 109, 391, 12169, 15518, 16689, 458, 146, 41358, 109, 452, 726, 4034, 111, 763, 35412, 5082, 388, 1903, 111, 9051, 391, 2870, 48918, 1900, 1123, 550, 998, 112, 9586, 15985, 455, 391, 410, 22955, 37636, 114, 66], [65, 448, 17496, 419, 3663, 385, 763, 113, 27533, 2870, 3283, 13043, 1639, 24713, 523, 656, 24013, 18550, 2521, 517, 27014, 21244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 11786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 21932, 18146, 726, 363, 17032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=__UpperCAmelCase				, model_name="google/bigbird-roberta-base"				, revision="215c99f1600e06f83acce68422f2035b2b5c3510"				, )










"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowercase    					=      {
    """configuration_bert""": ["""BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BertConfig""", """BertOnnxConfig"""],
    """tokenization_bert""": ["""BasicTokenizer""", """BertTokenizer""", """WordpieceTokenizer"""],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    					=      ["""BertTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    					=      [
     """BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """BertForMaskedLM""",
     """BertForMultipleChoice""",
     """BertForNextSentencePrediction""",
     """BertForPreTraining""",
     """BertForQuestionAnswering""",
     """BertForSequenceClassification""",
     """BertForTokenClassification""",
     """BertLayer""",
     """BertLMHeadModel""",
     """BertModel""",
     """BertPreTrainedModel""",
     """load_tf_weights_in_bert""",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    					=      [
     """TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFBertEmbeddings""",
     """TFBertForMaskedLM""",
     """TFBertForMultipleChoice""",
     """TFBertForNextSentencePrediction""",
     """TFBertForPreTraining""",
     """TFBertForQuestionAnswering""",
     """TFBertForSequenceClassification""",
     """TFBertForTokenClassification""",
     """TFBertLMHeadModel""",
     """TFBertMainLayer""",
     """TFBertModel""",
     """TFBertPreTrainedModel""",
 ]
try:
 if not is_tensorflow_text_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    					=      ["""TFBertTokenizer"""]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    					=      [
     """FlaxBertForCausalLM""",
     """FlaxBertForMaskedLM""",
     """FlaxBertForMultipleChoice""",
     """FlaxBertForNextSentencePrediction""",
     """FlaxBertForPreTraining""",
     """FlaxBertForQuestionAnswering""",
     """FlaxBertForSequenceClassification""",
     """FlaxBertForTokenClassification""",
     """FlaxBertModel""",
     """FlaxBertPreTrainedModel""",
 ]

if TYPE_CHECKING:
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
 from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_bert_fast import BertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_bert import (
      BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      BertForMaskedLM,
      BertForMultipleChoice,
      BertForNextSentencePrediction,
      BertForPreTraining,
      BertForQuestionAnswering,
      BertForSequenceClassification,
      BertForTokenClassification,
      BertLayer,
      BertLMHeadModel,
      BertModel,
      BertPreTrainedModel,
      load_tf_weights_in_bert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_bert import (
      TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFBertEmbeddings,
      TFBertForMaskedLM,
      TFBertForMultipleChoice,
      TFBertForNextSentencePrediction,
      TFBertForPreTraining,
      TFBertForQuestionAnswering,
      TFBertForSequenceClassification,
      TFBertForTokenClassification,
      TFBertLMHeadModel,
      TFBertMainLayer,
      TFBertModel,
      TFBertPreTrainedModel,
  )

 try:
  if not is_tensorflow_text_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_bert_tf import TFBertTokenizer

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_bert import (
      FlaxBertForCausalLM,
      FlaxBertForMaskedLM,
      FlaxBertForMultipleChoice,
      FlaxBertForNextSentencePrediction,
      FlaxBertForPreTraining,
      FlaxBertForQuestionAnswering,
      FlaxBertForSequenceClassification,
      FlaxBertForTokenClassification,
      FlaxBertModel,
      FlaxBertPreTrainedModel,
  )

else:
 import sys

 __lowercase    					=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






