




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        RemBertConfig,
        RemBertForCausalLM,
        RemBertForMaskedLM,
        RemBertForMultipleChoice,
        RemBertForQuestionAnswering,
        RemBertForSequenceClassification,
        RemBertForTokenClassification,
        RemBertModel,
    )
    from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class        UpperCAmelCase__				:



     '''simple docstring'''
     def __init__(				self ,    _lowerCAmelCase ,    _lowerCAmelCase=13 ,    _lowerCAmelCase=7 ,    _lowerCAmelCase=True ,    _lowerCAmelCase=True ,    _lowerCAmelCase=True ,    _lowerCAmelCase=True ,    _lowerCAmelCase=99 ,    _lowerCAmelCase=32 ,    _lowerCAmelCase=18 ,    _lowerCAmelCase=43 ,    _lowerCAmelCase=5 ,    _lowerCAmelCase=4 ,    _lowerCAmelCase=37 ,    _lowerCAmelCase="gelu" ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=512 ,    _lowerCAmelCase=16 ,    _lowerCAmelCase=2 ,    _lowerCAmelCase=0.02 ,    _lowerCAmelCase=3 ,    _lowerCAmelCase=4 ,    _lowerCAmelCase=None ,    ):
            a			   =parent
            a			   =batch_size
            a			   =seq_length
            a			   =is_training
            a			   =use_input_mask
            a			   =use_token_type_ids
            a			   =use_labels
            a			   =vocab_size
            a			   =hidden_size
            a			   =input_embedding_size
            a			   =output_embedding_size
            a			   =num_hidden_layers
            a			   =num_attention_heads
            a			   =intermediate_size
            a			   =hidden_act
            a			   =hidden_dropout_prob
            a			   =attention_probs_dropout_prob
            a			   =max_position_embeddings
            a			   =type_vocab_size
            a			   =type_sequence_label_size
            a			   =initializer_range
            a			   =num_labels
            a			   =num_choices
            a			   =scope
     def        lowerCAmelCase__  (				self	):
            a			   =ids_tensor([self.batch_size, self.seq_length] ,    self.vocab_size	)

            a			   =None
            if self.use_input_mask:
                   a			   =random_attention_mask([self.batch_size, self.seq_length]	)

            a			   =None
            if self.use_token_type_ids:
                   a			   =ids_tensor([self.batch_size, self.seq_length] ,    self.type_vocab_size	)

            a			   =None
            a			   =None
            a			   =None
            if self.use_labels:
                   a			   =ids_tensor([self.batch_size] ,    self.type_sequence_label_size	)
                   a			   =ids_tensor([self.batch_size, self.seq_length] ,    self.num_labels	)
                   a			   =ids_tensor([self.batch_size] ,    self.num_choices	)

            a			   =RemBertConfig(
                vocab_size=self.vocab_size ,    hidden_size=self.hidden_size ,    input_embedding_size=self.input_embedding_size ,    output_embedding_size=self.output_embedding_size ,    num_hidden_layers=self.num_hidden_layers ,    num_attention_heads=self.num_attention_heads ,    intermediate_size=self.intermediate_size ,    hidden_act=self.hidden_act ,    hidden_dropout_prob=self.hidden_dropout_prob ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob ,    max_position_embeddings=self.max_position_embeddings ,    type_vocab_size=self.type_vocab_size ,    is_decoder=__UpperCamelCase ,    initializer_range=self.initializer_range ,    )

            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     def        lowerCAmelCase__  (				self	):
            (
                (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      			
            )			   =self.prepare_config_and_inputs()

            a			   =True
            a			   =floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
            a			   =ids_tensor([self.batch_size, self.seq_length] ,    vocab_size=2	)

            return (
                config,
                input_ids,
                token_type_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                encoder_hidden_states,
                encoder_attention_mask,
            )
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	):
            a			   =RemBertModel(config=__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =model(__UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase	)
            a			   =model(__UpperCamelCase ,    token_type_ids=__UpperCamelCase	)
            a			   =model(__UpperCamelCase	)
            self.parent.assertEqual(result.last_hidden_state.shape ,    (self.batch_size, self.seq_length, self.hidden_size)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    ):
            a			   =True
            a			   =RemBertModel(__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =model(
                __UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    encoder_hidden_states=__UpperCamelCase ,    encoder_attention_mask=__UpperCamelCase ,    )
            a			   =model(
                __UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    encoder_hidden_states=__UpperCamelCase ,    )
            a			   =model(__UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase	)
            self.parent.assertEqual(result.last_hidden_state.shape ,    (self.batch_size, self.seq_length, self.hidden_size)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    ):
            a			   =RemBertForCausalLM(config=__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =model(__UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    labels=__UpperCamelCase	)
            self.parent.assertEqual(result.logits.shape ,    (self.batch_size, self.seq_length, self.vocab_size)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	):
            a			   =RemBertForMaskedLM(config=__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =model(__UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    labels=__UpperCamelCase	)
            self.parent.assertEqual(result.logits.shape ,    (self.batch_size, self.seq_length, self.vocab_size)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    ):
            a			   =True
            a			   =True
            a			   =RemBertForCausalLM(config=__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()

            # first forward pass
            a			   =model(
                __UpperCamelCase ,    attention_mask=__UpperCamelCase ,    encoder_hidden_states=__UpperCamelCase ,    encoder_attention_mask=__UpperCamelCase ,    use_cache=__UpperCamelCase ,    )
            a			   =outputs.past_key_values

            # create hypothetical multiple next token and extent to next_input_ids
            a			   =ids_tensor((self.batch_size, 3) ,    config.vocab_size	)
            a			   =ids_tensor((self.batch_size, 3) ,    vocab_size=2	)

            # append to next input_ids and
            a			   =torch.cat([input_ids, next_tokens] ,    dim=-1	)
            a			   =torch.cat([input_mask, next_mask] ,    dim=-1	)

            a			   =model(
                __UpperCamelCase ,    attention_mask=__UpperCamelCase ,    encoder_hidden_states=__UpperCamelCase ,    encoder_attention_mask=__UpperCamelCase ,    output_hidden_states=__UpperCamelCase ,    )["""hidden_states"""][0]
            a			   =model(
                __UpperCamelCase ,    attention_mask=__UpperCamelCase ,    encoder_hidden_states=__UpperCamelCase ,    encoder_attention_mask=__UpperCamelCase ,    past_key_values=__UpperCamelCase ,    output_hidden_states=__UpperCamelCase ,    )["""hidden_states"""][0]

            # select random slice
            a			   =ids_tensor((1,) ,    output_from_past.shape[-1]	).item()
            a			   =output_from_no_past[:, -3:, random_slice_idx].detach()
            a			   =output_from_past[:, :, random_slice_idx].detach()

            self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

            # test that outputs are equal for slice
            self.parent.assertTrue(torch.allclose(__UpperCamelCase ,    __UpperCamelCase ,    atol=1E-3	)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	):
            a			   =RemBertForQuestionAnswering(config=__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =model(
                __UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    start_positions=__UpperCamelCase ,    end_positions=__UpperCamelCase ,    )
            self.parent.assertEqual(result.start_logits.shape ,    (self.batch_size, self.seq_length)	)
            self.parent.assertEqual(result.end_logits.shape ,    (self.batch_size, self.seq_length)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	):
            a			   =self.num_labels
            a			   =RemBertForSequenceClassification(__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =model(__UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    labels=__UpperCamelCase	)
            self.parent.assertEqual(result.logits.shape ,    (self.batch_size, self.num_labels)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	):
            a			   =self.num_labels
            a			   =RemBertForTokenClassification(config=__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =model(__UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    labels=__UpperCamelCase	)
            self.parent.assertEqual(result.logits.shape ,    (self.batch_size, self.seq_length, self.num_labels)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	):
            a			   =self.num_choices
            a			   =RemBertForMultipleChoice(config=__UpperCamelCase	)
            model.to(__UpperCamelCase	)
            model.eval()
            a			   =input_ids.unsqueeze(1	).expand(-1 ,    self.num_choices ,    -1	).contiguous()
            a			   =token_type_ids.unsqueeze(1	).expand(-1 ,    self.num_choices ,    -1	).contiguous()
            a			   =input_mask.unsqueeze(1	).expand(-1 ,    self.num_choices ,    -1	).contiguous()
            a			   =model(
                __UpperCamelCase ,    attention_mask=__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    labels=__UpperCamelCase ,    )
            self.parent.assertEqual(result.logits.shape ,    (self.batch_size, self.num_choices)	)






     def        lowerCAmelCase__  (				self	):
            a			   =self.prepare_config_and_inputs()
            (
                (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      			
            )			   =config_and_inputs
            a			   ={"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
            return config, inputs_dict



@require_torch
class        UpperCAmelCase__				(							__lowerCAmelCase      ,		__lowerCAmelCase      ,		unittest.TestCase				):



     '''simple docstring'''
     _SCREAMING_SNAKE_CASE      :					Tuple     					=    (
         (
             RemBertModel,
             RemBertForMaskedLM,
             RemBertForCausalLM,
             RemBertForMultipleChoice,
             RemBertForQuestionAnswering,
             RemBertForSequenceClassification,
             RemBertForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     _SCREAMING_SNAKE_CASE      :					Dict     					=    (RemBertForCausalLM,) if is_torch_available() else ()
     _SCREAMING_SNAKE_CASE      :					Dict     					=    (
         {
             """feature-extraction""": RemBertModel,
             """fill-mask""": RemBertForMaskedLM,
             """question-answering""": RemBertForQuestionAnswering,
             """text-classification""": RemBertForSequenceClassification,
             """text-generation""": RemBertForCausalLM,
             """token-classification""": RemBertForTokenClassification,
             """zero-shot""": RemBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     def        lowerCAmelCase__  (				self	):
            a			   =RemBertModelTester(self	)
            a			   =ConfigTester(self ,    config_class=__UpperCamelCase ,    hidden_size=37	)
     def        lowerCAmelCase__  (				self	):
            self.config_tester.run_common_tests()
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs()
            for type in ["absolute", "relative_key", "relative_key_query"]:
                   a			   =type
                   self.model_tester.create_and_check_model(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_lm(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_multiple_choice(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs_for_decoder()
            self.model_tester.create_and_check_decoder_model_past_large_inputs(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_question_answering(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_sequence_classification(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_token_classification(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.model_tester.prepare_config_and_inputs_for_decoder()
            self.model_tester.create_and_check_model_as_decoder(*__UpperCamelCase	)
     def        lowerCAmelCase__  (				self	):
            (
                (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      (
                a			
            )   ,      			
            )			   =self.model_tester.prepare_config_and_inputs_for_decoder()

            a			   =None

            self.model_tester.create_and_check_model_as_decoder(
                __UpperCamelCase ,    __UpperCamelCase ,    __UpperCamelCase ,    __UpperCamelCase ,    __UpperCamelCase ,    __UpperCamelCase ,    __UpperCamelCase ,    __UpperCamelCase ,    __UpperCamelCase ,    )






     @slow
     def        lowerCAmelCase__  (				self	):






            for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   a			   =RemBertModel.from_pretrained(__UpperCamelCase	)
                   self.assertIsNotNone(__UpperCamelCase	)



@require_torch
class        UpperCAmelCase__				(							unittest.TestCase				):



     '''simple docstring'''
     @slow
     def        lowerCAmelCase__  (				self	):
            a			   =RemBertModel.from_pretrained("""google/rembert"""	)
            a			   =torch.tensor([[312, 56_498, 313, 2_125, 313]]	)
            a			   =torch.tensor([[0, 0, 0, 1, 1]]	)
            with torch.no_grad():
                   a			   =model(__UpperCamelCase ,    token_type_ids=__UpperCamelCase ,    output_hidden_states=__UpperCamelCase	)

            a			   =1_152

            a			   =torch.Size((1, 5, hidden_size)	)
            self.assertEqual(output["""last_hidden_state"""].shape ,    __UpperCamelCase	)

            a			   =torch.tensor(
                [
                    [
                        [0.07_54, -0.20_22, 0.19_04],
                        [-0.33_54, -0.36_92, -0.47_91],
                        [-0.23_14, -0.67_29, -0.07_49],
                        [-0.03_96, -0.31_05, -0.42_34],
                        [-0.15_71, -0.05_25, 0.53_53],
                    ]
                ]	)

            # Running on the original tf implementation gives slightly different results here.
            # Not clear why this variations is present
            # TODO: Find reason for discrepancy
            # expected_original_implementation = [[
            #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
            #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
            #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
            #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
            #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
            # ]]

            self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3] ,    __UpperCamelCase ,    atol=1E-4	)	)






'''simple docstring'''

import warnings

from ...utils import logging
from .image_processing_vit import ViTImageProcessor


a__         =     logging.get_logger(__name__)





class __magic_name__(						__lowerCAmelCase      ):






						def __init__(   self						:						List[str]					,   *__UpperCamelCase						:						Tuple					,   **__UpperCamelCase						:						Tuple       ):



									'''simple docstring'''






									warnings.warn(
									    """The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"""
									    """ use ViTImageProcessor instead."""					,   __UpperCamelCase					,   )
									super().__init__(*__UpperCamelCase					,   **__UpperCamelCase       )