
import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase			:     Any           =   get_tests_dir('fixtures/test_sentencepiece.model')
__lowerCAmelCase			:     Optional[int]           =   get_tests_dir('fixtures/test_sentencepiece_bpe.model')

__lowerCAmelCase			:     Tuple           =   'pt' if is_torch_available() else 'tf'




@require_sentencepiece
@require_tokenizers
class        UpperCAmelCase_			( __lowerCamelCase  ,       unittest.TestCase     ):


  '''simple docstring'''







  a__           =		CamembertTokenizer
  a__           =		CamembertTokenizerFast
  a__           =		True
  a__           =		True
  def 							_lowercase							( self			: int    )			->	List[str]:







    """simple docstring"""






    super().setUp()

    # We have a SentencePiece fixture for testing
    __magic_name__				     =       CamembertTokenizer(UpperCamelCase_    )
    tokenizer.save_pretrained(self.tmpdirname    )
  def 							_lowercase							( self			: Union[str, Any]    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       """<pad>"""
    __magic_name__				     =       1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_    )			,							UpperCamelCase_    )
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_    )			,							UpperCamelCase_    )
  def 							_lowercase							( self			: Dict    )			->	Optional[Any]:







    """simple docstring"""






    __magic_name__				     =       list(self.get_tokenizer().get_vocab().keys()    )

    self.assertEqual(vocab_keys[0]			,							"""<s>NOTUSED"""    )
    self.assertEqual(vocab_keys[1]			,							"""<pad>"""    )
    self.assertEqual(vocab_keys[-1]			,							"""<mask>"""    )
    self.assertEqual(len(UpperCamelCase_    )			,							1004    )
  def 							_lowercase							( self			: Any    )			->	str:







    """simple docstring"""






    self.assertEqual(self.get_tokenizer().vocab_size			,							1005    )
  def 							_lowercase							( self			: str    )			->	Dict:







    """simple docstring"""






    __magic_name__				     =       CamembertTokenizer(UpperCamelCase_    )
    tokenizer.save_pretrained(self.tmpdirname    )
    __magic_name__				     =       CamembertTokenizerFast.from_pretrained(self.tmpdirname    )

    __magic_name__				     =       """I was born in 92000, and this is falsé."""

    __magic_name__				     =       tokenizer.encode(UpperCamelCase_    )
    __magic_name__				     =       rust_tokenizer.encode(UpperCamelCase_    )
    self.assertListEqual(UpperCamelCase_			,							UpperCamelCase_    )

    __magic_name__				     =       tokenizer.encode(UpperCamelCase_			,							add_special_tokens=UpperCamelCase_    )
    __magic_name__				     =       rust_tokenizer.encode(UpperCamelCase_			,							add_special_tokens=UpperCamelCase_    )
    self.assertListEqual(UpperCamelCase_			,							UpperCamelCase_    )

    # <unk> tokens are not the same for `rust` than for `slow`.
    # Because spm gives back raw token instead of `unk` in EncodeAsPieces
    # tokens = tokenizer.tokenize(sequence)
    __magic_name__				     =       tokenizer.convert_ids_to_tokens(UpperCamelCase_    )
    __magic_name__				     =       rust_tokenizer.tokenize(UpperCamelCase_    )
    self.assertListEqual(UpperCamelCase_			,							UpperCamelCase_    )
  def 							_lowercase							( self			: Tuple    )			->	int:







    """simple docstring"""






    if not self.test_rust_tokenizer:
      return

    __magic_name__				     =       self.get_tokenizer()
    __magic_name__				     =       self.get_rust_tokenizer()

    __magic_name__				     =       """I was born in 92000, and this is falsé."""

    __magic_name__				     =       tokenizer.tokenize(UpperCamelCase_    )
    __magic_name__				     =       rust_tokenizer.tokenize(UpperCamelCase_    )
    self.assertListEqual(UpperCamelCase_			,							UpperCamelCase_    )

    __magic_name__				     =       tokenizer.encode(UpperCamelCase_			,							add_special_tokens=UpperCamelCase_    )
    __magic_name__				     =       rust_tokenizer.encode(UpperCamelCase_			,							add_special_tokens=UpperCamelCase_    )
    self.assertListEqual(UpperCamelCase_			,							UpperCamelCase_    )

    __magic_name__				     =       self.get_rust_tokenizer()
    __magic_name__				     =       tokenizer.encode(UpperCamelCase_    )
    __magic_name__				     =       rust_tokenizer.encode(UpperCamelCase_    )
    self.assertListEqual(UpperCamelCase_			,							UpperCamelCase_    )





  @slow
  def 							_lowercase							( self			: Dict    )			->	Optional[int]:







    """simple docstring"""






    __magic_name__				     =       {"""input_ids""": [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 2_7575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 2_2804, 1_8818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 1_0326, 24, 2267, 20, 416, 5072, 1_5612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
    # fmt: on

    # camembert is a french model. So we also use french texts.
    __magic_name__				     =       [
        """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
        """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
        """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
        """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
        """telles que la traduction et la synthèse de texte.""",
    ]

    self.tokenizer_integration_test_util(
        expected_encoding=UpperCamelCase_			,							model_name="""camembert-base"""			,							revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""			,							sequences=UpperCamelCase_			,							)







import unittest

from transformers import BigBirdConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
       import jax

       from transformers.models.big_bird.modeling_flax_big_bird import (
           FlaxBigBirdForCausalLM,
           FlaxBigBirdForMaskedLM,
           FlaxBigBirdForMultipleChoice,
           FlaxBigBirdForPreTraining,
           FlaxBigBirdForQuestionAnswering,
           FlaxBigBirdForSequenceClassification,
           FlaxBigBirdForTokenClassification,
           FlaxBigBirdModel,
       )





class     lowerCamelCase__(					unittest.TestCase):
      def __init__(				self: Optional[int]    ,       UpperCamelCase_: Any    ,       UpperCamelCase_: int=2    ,       UpperCamelCase_: Optional[Any]=56    ,       UpperCamelCase_: Tuple=True    ,       UpperCamelCase_: Union[str, Any]=True    ,       UpperCamelCase_: Optional[Any]=True    ,       UpperCamelCase_: str=True    ,       UpperCamelCase_: str=99    ,       UpperCamelCase_: Tuple=32    ,       UpperCamelCase_: int=2    ,       UpperCamelCase_: Optional[int]=2    ,       UpperCamelCase_: Tuple=7    ,       UpperCamelCase_: Optional[int]="gelu_new"    ,       UpperCamelCase_: Any=0.1    ,       UpperCamelCase_: List[str]=0.1    ,       UpperCamelCase_: List[Any]=5_12    ,       UpperCamelCase_: Union[str, Any]=16    ,       UpperCamelCase_: int=2    ,       UpperCamelCase_: Dict=0.02    ,       UpperCamelCase_: Tuple=4    ,       UpperCamelCase_: Union[str, Any]="block_sparse"    ,       UpperCamelCase_: Optional[Any]=True    ,       UpperCamelCase_: Any=False    ,       UpperCamelCase_: Any=2    ,       UpperCamelCase_: int=3    ,       ):
             __lowerCamelCase   =						parent
             __lowerCamelCase   =						batch_size
             __lowerCamelCase   =						seq_length
             __lowerCamelCase   =						is_training
             __lowerCamelCase   =						use_attention_mask
             __lowerCamelCase   =						use_token_type_ids
             __lowerCamelCase   =						use_labels
             __lowerCamelCase   =						vocab_size
             __lowerCamelCase   =						hidden_size
             __lowerCamelCase   =						num_hidden_layers
             __lowerCamelCase   =						num_attention_heads
             __lowerCamelCase   =						intermediate_size
             __lowerCamelCase   =						hidden_act
             __lowerCamelCase   =						hidden_dropout_prob
             __lowerCamelCase   =						attention_probs_dropout_prob
             __lowerCamelCase   =						max_position_embeddings
             __lowerCamelCase   =						type_vocab_size
             __lowerCamelCase   =						type_sequence_label_size
             __lowerCamelCase   =						initializer_range
             __lowerCamelCase   =						num_choices

             __lowerCamelCase   =						rescale_embeddings
             __lowerCamelCase   =						attention_type
             __lowerCamelCase   =						use_bias
             __lowerCamelCase   =						block_size
             __lowerCamelCase   =						num_random_blocks
      def 			lowerCAmelCase__						(				self: int   ):
             __lowerCamelCase   =						ids_tensor([self.batch_size, self.seq_length]    ,       self.vocab_size   )

             __lowerCamelCase   =						None
             if self.use_attention_mask:
                    __lowerCamelCase   =						random_attention_mask([self.batch_size, self.seq_length]   )

             __lowerCamelCase   =						None
             if self.use_token_type_ids:
                    __lowerCamelCase   =						ids_tensor([self.batch_size, self.seq_length]    ,       self.type_vocab_size   )

             __lowerCamelCase   =						BigBirdConfig(
                 vocab_size=self.vocab_size    ,       hidden_size=self.hidden_size    ,       num_hidden_layers=self.num_hidden_layers    ,       num_attention_heads=self.num_attention_heads    ,       intermediate_size=self.intermediate_size    ,       hidden_act=self.hidden_act    ,       hidden_dropout_prob=self.hidden_dropout_prob    ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,       max_position_embeddings=self.max_position_embeddings    ,       type_vocab_size=self.type_vocab_size    ,       is_decoder=UpperCamelCase_    ,       initializer_range=self.initializer_range    ,       attention_type=self.attention_type    ,       block_size=self.block_size    ,       num_random_blocks=self.num_random_blocks    ,       use_bias=self.use_bias    ,       rescale_embeddings=self.rescale_embeddings    ,       )

             return config, input_ids, token_type_ids, attention_mask




      def 			lowerCAmelCase__						(				self: Dict   ):
             __lowerCamelCase   =						self.prepare_config_and_inputs()
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						config_and_inputs
             __lowerCamelCase   =						{
                 """input_ids""": input_ids,
                 """token_type_ids""": token_type_ids,
                 """attention_mask""": attention_mask,
             }
             return config, inputs_dict





@require_flax
class     lowerCamelCase__(					__lowerCamelCase ,     unittest.TestCase):
      UpperCAmelCase__      :	Optional[int]    		=   (
          (
              FlaxBigBirdForCausalLM,
              FlaxBigBirdModel,
              FlaxBigBirdForPreTraining,
              FlaxBigBirdForMaskedLM,
              FlaxBigBirdForMultipleChoice,
              FlaxBigBirdForQuestionAnswering,
              FlaxBigBirdForSequenceClassification,
              FlaxBigBirdForTokenClassification,
          )
          if is_flax_available()
          else ()
      )

      UpperCAmelCase__      :	Optional[int]    		=   False
      UpperCAmelCase__      :	Optional[int]    		=   False
      def 			lowerCAmelCase__						(				self: List[str]   ):
             __lowerCamelCase   =						FlaxBigBirdModelTester(self   )
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: Optional[Any]   ):
             super().test_from_pretrained_save_pretrained()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[Any]   ):
             super().test_from_pretrained_with_no_automatic_init()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[Any]   ):
             super().test_no_automatic_init()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[str]   ):
             super().test_hidden_states_output()
      @slow
      def 			lowerCAmelCase__						(				self: Optional[Any]   ):
             for model_class_name in self.all_model_classes:
                    __lowerCamelCase   =						model_class_name.from_pretrained("""google/bigbird-roberta-base"""   )
                    self.assertIsNotNone(UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Dict   ):
             if self.test_attn_probs:
                    super().test_attention_outputs()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[Any]   ):
             __lowerCamelCase,   __lowerCamelCase   =						self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                    with self.subTest(model_class.__name__   ):
                           __lowerCamelCase   =						self._prepare_for_class(UpperCamelCase_    ,       UpperCamelCase_   )
                           __lowerCamelCase   =						model_class(UpperCamelCase_   )

                           @jax.jit
                           def model_jitted(UpperCamelCase_: Tuple    ,       UpperCamelCase_: List[Any]=None    ,       **UpperCamelCase_: Union[str, Any]   ):
                                  return model(input_ids=UpperCamelCase_    ,       attention_mask=UpperCamelCase_    ,       **UpperCamelCase_   )

                           with self.subTest("""JIT Enabled"""   ):
                                  __lowerCamelCase   =						model_jitted(**UpperCamelCase_   ).to_tuple()

                           with self.subTest("""JIT Disabled"""   ):
                                  with jax.disable_jit():
                                         __lowerCamelCase   =						model_jitted(**UpperCamelCase_   ).to_tuple()

                           self.assertEqual(len(UpperCamelCase_   )    ,       len(UpperCamelCase_   )   )
                           for jitted_output, output in zip(UpperCamelCase_    ,       UpperCamelCase_   ):
                                  self.assertEqual(jitted_output.shape    ,       output.shape   )




      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: int    ,       UpperCamelCase_: List[str]    ,       UpperCamelCase_: Any    ,       UpperCamelCase_: Dict=1E-5    ,       UpperCamelCase_: List[str]="outputs"    ,       UpperCamelCase_: List[str]=None   ):
             # `bigbird_block_sparse_attention` in `FlaxBigBird` returns `attention_probs = None`, while in PyTorch version,
             # an effort was done to return `attention_probs` (yet to be verified).
             if name.startswith("""outputs.attentions"""   ):
                    return
             else:
                    super().check_pt_flax_outputs(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_   )

