

import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
					import torch

					from transformers import (
					    MraForMaskedLM,
					    MraForMultipleChoice,
					    MraForQuestionAnswering,
					    MraForSequenceClassification,
					    MraForTokenClassification,
					    MraModel,
					)
					from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST







class  lowercase				:




				def __init__(					self      ,   snake_case      ,   snake_case=2      ,   snake_case=8      ,   snake_case=True      ,   snake_case=True      ,   snake_case=True      ,   snake_case=True      ,   snake_case=99      ,   snake_case=16      ,   snake_case=5      ,   snake_case=2      ,   snake_case=36      ,   snake_case="gelu"      ,   snake_case=0.0      ,   snake_case=0.0      ,   snake_case=512      ,   snake_case=16      ,   snake_case=2      ,   snake_case=0.02      ,   snake_case=3      ,   snake_case=4      ,   snake_case=None      ,   ):
											snake_case_							=    parent
											snake_case_							=    batch_size
											snake_case_							=    seq_length
											snake_case_							=    is_training
											snake_case_							=    use_input_mask
											snake_case_							=    use_token_type_ids
											snake_case_							=    use_labels
											snake_case_							=    vocab_size
											snake_case_							=    hidden_size
											snake_case_							=    num_hidden_layers
											snake_case_							=    num_attention_heads
											snake_case_							=    intermediate_size
											snake_case_							=    hidden_act
											snake_case_							=    hidden_dropout_prob
											snake_case_							=    attention_probs_dropout_prob
											snake_case_							=    max_position_embeddings
											snake_case_							=    type_vocab_size
											snake_case_							=    type_sequence_label_size
											snake_case_							=    initializer_range
											snake_case_							=    num_labels
											snake_case_							=    num_choices
											snake_case_							=    scope




				def     a							(					self       ):
											snake_case_							=    ids_tensor([self.batch_size, self.seq_length]      ,   self.vocab_size       )

											snake_case_							=    None
											if self.use_input_mask:
																		snake_case_							=    random_attention_mask([self.batch_size, self.seq_length]       )

											snake_case_							=    None
											if self.use_token_type_ids:
																		snake_case_							=    ids_tensor([self.batch_size, self.seq_length]      ,   self.type_vocab_size       )

											snake_case_							=    None
											snake_case_							=    None
											snake_case_							=    None
											if self.use_labels:
																		snake_case_							=    ids_tensor([self.batch_size]      ,   self.type_sequence_label_size       )
																		snake_case_							=    ids_tensor([self.batch_size, self.seq_length]      ,   self.num_labels       )
																		snake_case_							=    ids_tensor([self.batch_size]      ,   self.num_choices       )

											snake_case_							=    self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def     a							(					self       ):
											return MraConfig(
											    vocab_size=self.vocab_size      ,   hidden_size=self.hidden_size      ,   num_hidden_layers=self.num_hidden_layers      ,   num_attention_heads=self.num_attention_heads      ,   intermediate_size=self.intermediate_size      ,   hidden_act=self.hidden_act      ,   hidden_dropout_prob=self.hidden_dropout_prob      ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,   max_position_embeddings=self.max_position_embeddings      ,   type_vocab_size=self.type_vocab_size      ,   is_decoder=__UpperCAmelCase      ,   initializer_range=self.initializer_range      ,   )




				def     a							(					self       ):
											snake_case_							=    self.get_config()
											snake_case_							=    300
											return config




				def     a							(					self       ):
											(
											    snake_case_   
											)							=    self.prepare_config_and_inputs()

											snake_case_							=    True
											snake_case_							=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
											snake_case_							=    ids_tensor([self.batch_size, self.seq_length]      ,   vocab_size=2       )

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    MraModel(config=__UpperCAmelCase       )
											model.to(__UpperCAmelCase       )
											model.eval()
											snake_case_							=    model(__UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase       )
											snake_case_							=    model(__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase       )
											snake_case_							=    model(__UpperCAmelCase       )
											self.parent.assertEqual(result.last_hidden_state.shape      ,   (self.batch_size, self.seq_length, self.hidden_size)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   ):
											snake_case_							=    True
											snake_case_							=    MraModel(__UpperCAmelCase       )
											model.to(__UpperCAmelCase       )
											model.eval()
											snake_case_							=    model(
											    __UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase      ,   encoder_hidden_states=__UpperCAmelCase      ,   encoder_attention_mask=__UpperCAmelCase      ,   )
											snake_case_							=    model(
											    __UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase      ,   encoder_hidden_states=__UpperCAmelCase      ,   )
											snake_case_							=    model(__UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase       )
											self.parent.assertEqual(result.last_hidden_state.shape      ,   (self.batch_size, self.seq_length, self.hidden_size)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    MraForMaskedLM(config=__UpperCAmelCase       )
											model.to(__UpperCAmelCase       )
											model.eval()
											snake_case_							=    model(__UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase      ,   labels=__UpperCAmelCase       )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    MraForQuestionAnswering(config=__UpperCAmelCase       )
											model.to(__UpperCAmelCase       )
											model.eval()
											snake_case_							=    model(
											    __UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase      ,   start_positions=__UpperCAmelCase      ,   end_positions=__UpperCAmelCase      ,   )
											self.parent.assertEqual(result.start_logits.shape      ,   (self.batch_size, self.seq_length)       )
											self.parent.assertEqual(result.end_logits.shape      ,   (self.batch_size, self.seq_length)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    self.num_labels
											snake_case_							=    MraForSequenceClassification(__UpperCAmelCase       )
											model.to(__UpperCAmelCase       )
											model.eval()
											snake_case_							=    model(__UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase      ,   labels=__UpperCAmelCase       )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_labels)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    self.num_labels
											snake_case_							=    MraForTokenClassification(config=__UpperCAmelCase       )
											model.to(__UpperCAmelCase       )
											model.eval()
											snake_case_							=    model(__UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase      ,   labels=__UpperCAmelCase       )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.num_labels)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    self.num_choices
											snake_case_							=    MraForMultipleChoice(config=__UpperCAmelCase       )
											model.to(__UpperCAmelCase       )
											model.eval()
											snake_case_							=    input_ids.unsqueeze(1       ).expand(-1      ,   self.num_choices      ,   -1       ).contiguous()
											snake_case_							=    token_type_ids.unsqueeze(1       ).expand(-1      ,   self.num_choices      ,   -1       ).contiguous()
											snake_case_							=    input_mask.unsqueeze(1       ).expand(-1      ,   self.num_choices      ,   -1       ).contiguous()
											snake_case_							=    model(
											    __UpperCAmelCase      ,   attention_mask=__UpperCAmelCase      ,   token_type_ids=__UpperCAmelCase      ,   labels=__UpperCAmelCase      ,   )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_choices)       )




				def     a							(					self       ):
											snake_case_							=    self.prepare_config_and_inputs()
											(
											    snake_case_   
											)							=    config_and_inputs
											snake_case_							=    {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
											return config, inputs_dict







@require_torch
class  lowercase				(       SCREAMING_SNAKE_CASE_	,  unittest.TestCase			):
				__SCREAMING_SNAKE_CASE					: Dict       			=			(
				    (
				        MraModel,
				        MraForMaskedLM,
				        MraForMultipleChoice,
				        MraForQuestionAnswering,
				        MraForSequenceClassification,
				        MraForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				__SCREAMING_SNAKE_CASE					: str       			=			False
				__SCREAMING_SNAKE_CASE					: Union[str, Any]       			=			False
				__SCREAMING_SNAKE_CASE					: Optional[Any]       			=			False
				__SCREAMING_SNAKE_CASE					: int       			=			False

				__SCREAMING_SNAKE_CASE					: int       			=			()




				def     a							(					self       ):
											snake_case_							=    MraModelTester(self       )
											snake_case_							=    ConfigTester(self      ,   config_class=__UpperCAmelCase      ,   hidden_size=37       )




				def     a							(					self       ):
											self.config_tester.run_common_tests()




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__UpperCAmelCase       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																		snake_case_							=    type
																		self.model_tester.create_and_check_model(*__UpperCAmelCase       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*__UpperCAmelCase       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase       )




				@slow
				def     a							(					self       ):
											for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		snake_case_							=    MraModel.from_pretrained(__UpperCAmelCase       )
																		self.assertIsNotNone(__UpperCAmelCase       )




				@unittest.skip(reason='MRA does not output attentions'       )
				def     a							(					self       ):
											return







@require_torch
class  lowercase				(       unittest.TestCase			):




				@slow
				def     a							(					self       ):
											snake_case_							=    MraModel.from_pretrained('uw-madison/mra-base-512-4'       )
											snake_case_							=    torch.arange(256       ).unsqueeze(0       )

											with torch.no_grad():
																		snake_case_							=    model(__UpperCAmelCase       )[0]

											snake_case_							=    torch.Size((1, 256, 768)       )
											self.assertEqual(output.shape      ,   __UpperCAmelCase       )

											snake_case_							=    torch.tensor(
											    [[[-0.01_40, 0.08_30, -0.03_81], [0.15_46, 0.14_02, 0.02_20], [0.11_62, 0.08_51, 0.01_65]]]       )

											self.assertTrue(torch.allclose(output[:, :3, :3]      ,   __UpperCAmelCase      ,   atol=1e-4       )       )




				@slow
				def     a							(					self       ):
											snake_case_							=    MraForMaskedLM.from_pretrained('uw-madison/mra-base-512-4'       )
											snake_case_							=    torch.arange(256       ).unsqueeze(0       )

											with torch.no_grad():
																		snake_case_							=    model(__UpperCAmelCase       )[0]

											snake_case_							=    5_0265

											snake_case_							=    torch.Size((1, 256, vocab_size)       )
											self.assertEqual(output.shape      ,   __UpperCAmelCase       )

											snake_case_							=    torch.tensor(
											    [[[9.25_95, -3.60_38, 11.8819], [9.38_69, -3.26_93, 11.0956], [11.8524, -3.49_38, 13.1210]]]       )

											self.assertTrue(torch.allclose(output[:, :3, :3]      ,   __UpperCAmelCase      ,   atol=1e-4       )       )




				@slow
				def     a							(					self       ):
											snake_case_							=    MraForMaskedLM.from_pretrained('uw-madison/mra-base-4096-8-d3'       )
											snake_case_							=    torch.arange(4096       ).unsqueeze(0       )

											with torch.no_grad():
																		snake_case_							=    model(__UpperCAmelCase       )[0]

											snake_case_							=    5_0265

											snake_case_							=    torch.Size((1, 4096, vocab_size)       )
											self.assertEqual(output.shape      ,   __UpperCAmelCase       )

											snake_case_							=    torch.tensor(
											    [[[5.47_89, -2.35_64, 7.50_64], [7.90_67, -1.33_69, 9.96_68], [9.07_12, -1.81_06, 7.03_80]]]       )

											self.assertTrue(torch.allclose(output[:, :3, :3]      ,   __UpperCAmelCase      ,   atol=1e-4       )       )









'''simple docstring'''





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
 import torch

 from transformers import XLMRobertaModel






@require_sentencepiece
@require_tokenizers
@require_torch
class        lowerCAmelCase_(				unittest.TestCase     ):
  '''simple docstring'''




  @slow
  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      lowerCAmelCase__ :  Tuple							      =					XLMRobertaModel.from_pretrained("""xlm-roberta-base"""      )
      lowerCAmelCase__ :  Optional[int]							      =					torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]]      )
      # The dog is cute and lives in the garden house

      lowerCAmelCase__ :  str							      =					torch.Size((1, 12, 768)      )  # batch_size, sequence_length, embedding_vector_dim
      lowerCAmelCase__ :  Dict							      =					torch.tensor(
          [[-0.0_1_0_1, 0.1_2_1_8, -0.0_8_0_3, 0.0_8_0_1, 0.1_3_2_7, 0.0_7_7_6, -0.1_2_1_5, 0.2_3_8_3, 0.3_3_3_8, 0.3_1_0_6, 0.0_3_0_0, 0.0_2_5_2]]      )
      #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
      #  xlmr.eval()
      #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
      with torch.no_grad():
          lowerCAmelCase__ :  Optional[int]							      =					model(__UpperCAmelCase      )["""last_hidden_state"""].detach()
      self.assertEqual(output.shape  ,__UpperCAmelCase      )
      # compare the actual values for a slice of last dim
      self.assertTrue(torch.allclose(output[:, :, -1]  ,__UpperCAmelCase  ,atol=1E-3      )      )






  @slow
  def 							UpperCAmelCase_	(    self      )    ->		int:
      lowerCAmelCase__ :  Union[str, Any]							      =					XLMRobertaModel.from_pretrained("""xlm-roberta-large"""      )
      lowerCAmelCase__ :  Optional[Any]							      =					torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]]      )
      # The dog is cute and lives in the garden house

      lowerCAmelCase__ :  Dict							      =					torch.Size((1, 12, 1024)      )  # batch_size, sequence_length, embedding_vector_dim
      lowerCAmelCase__ :  Union[str, Any]							      =					torch.tensor(
          [[-0.0_6_9_9, -0.0_3_1_8, 0.0_7_0_5, -0.1_2_4_1, 0.0_9_9_9, -0.0_5_2_0, 0.1_0_0_4, -0.1_8_3_8, -0.4_7_0_4, 0.1_4_3_7, 0.0_8_2_1, 0.0_1_2_6]]      )
      #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
      #  xlmr.eval()
      #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
      with torch.no_grad():
          lowerCAmelCase__ :  Union[str, Any]							      =					model(__UpperCAmelCase      )["""last_hidden_state"""].detach()
      self.assertEqual(output.shape  ,__UpperCAmelCase      )
      # compare the actual values for a slice of last dim
      self.assertTrue(torch.allclose(output[:, :, -1]  ,__UpperCAmelCase  ,atol=1E-3      )      )


