



'''simple docstring'''





import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
 import torch

 from transformers import (
     MraForMaskedLM,
     MraForMultipleChoice,
     MraForQuestionAnswering,
     MraForSequenceClassification,
     MraForTokenClassification,
     MraModel,
 )
 from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST






class        lowerCAmelCase_:
  '''simple docstring'''




  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase=2  ,__UpperCAmelCase=8  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=99  ,__UpperCAmelCase=16  ,__UpperCAmelCase=5  ,__UpperCAmelCase=2  ,__UpperCAmelCase=36  ,__UpperCAmelCase="gelu"  ,__UpperCAmelCase=0.0  ,__UpperCAmelCase=0.0  ,__UpperCAmelCase=512  ,__UpperCAmelCase=16  ,__UpperCAmelCase=2  ,__UpperCAmelCase=0.0_2  ,__UpperCAmelCase=3  ,__UpperCAmelCase=4  ,__UpperCAmelCase=None  ,)    ->		Dict:
      lowerCAmelCase__ :  Dict							      =					parent
      lowerCAmelCase__ :  Optional[int]							      =					batch_size
      lowerCAmelCase__ :  Optional[int]							      =					seq_length
      lowerCAmelCase__ :  Any							      =					is_training
      lowerCAmelCase__ :  str							      =					use_input_mask
      lowerCAmelCase__ :  Any							      =					use_token_type_ids
      lowerCAmelCase__ :  Union[str, Any]							      =					use_labels
      lowerCAmelCase__ :  List[Any]							      =					vocab_size
      lowerCAmelCase__ :  str							      =					hidden_size
      lowerCAmelCase__ :  Tuple							      =					num_hidden_layers
      lowerCAmelCase__ :  Any							      =					num_attention_heads
      lowerCAmelCase__ :  Union[str, Any]							      =					intermediate_size
      lowerCAmelCase__ :  List[Any]							      =					hidden_act
      lowerCAmelCase__ :  Tuple							      =					hidden_dropout_prob
      lowerCAmelCase__ :  int							      =					attention_probs_dropout_prob
      lowerCAmelCase__ :  Dict							      =					max_position_embeddings
      lowerCAmelCase__ :  Optional[int]							      =					type_vocab_size
      lowerCAmelCase__ :  Optional[int]							      =					type_sequence_label_size
      lowerCAmelCase__ :  int							      =					initializer_range
      lowerCAmelCase__ :  Dict							      =					num_labels
      lowerCAmelCase__ :  List[str]							      =					num_choices
      lowerCAmelCase__ :  str							      =					scope




  def 							UpperCAmelCase_	(    self      )    ->		int:
      lowerCAmelCase__ :  int							      =					ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size      )

      lowerCAmelCase__ :  Union[str, Any]							      =					None
      if self.use_input_mask:
          lowerCAmelCase__ :  Optional[int]							      =					random_attention_mask([self.batch_size, self.seq_length]      )

      lowerCAmelCase__ :  Tuple							      =					None
      if self.use_token_type_ids:
          lowerCAmelCase__ :  List[Any]							      =					ids_tensor([self.batch_size, self.seq_length]  ,self.type_vocab_size      )

      lowerCAmelCase__ :  int							      =					None
      lowerCAmelCase__ :  str							      =					None
      lowerCAmelCase__ :  int							      =					None
      if self.use_labels:
          lowerCAmelCase__ :  Union[str, Any]							      =					ids_tensor([self.batch_size]  ,self.type_sequence_label_size      )
          lowerCAmelCase__ :  int							      =					ids_tensor([self.batch_size, self.seq_length]  ,self.num_labels      )
          lowerCAmelCase__ :  Any							      =					ids_tensor([self.batch_size]  ,self.num_choices      )

      lowerCAmelCase__ :  Any							      =					self.get_config()

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




  def 							UpperCAmelCase_	(    self      )    ->		int:
      return MraConfig(
          vocab_size=self.vocab_size  ,hidden_size=self.hidden_size  ,num_hidden_layers=self.num_hidden_layers  ,num_attention_heads=self.num_attention_heads  ,intermediate_size=self.intermediate_size  ,hidden_act=self.hidden_act  ,hidden_dropout_prob=self.hidden_dropout_prob  ,attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,max_position_embeddings=self.max_position_embeddings  ,type_vocab_size=self.type_vocab_size  ,is_decoder=__UpperCAmelCase  ,initializer_range=self.initializer_range  ,)




  def 							UpperCAmelCase_	(    self      )    ->		Optional[int]:
      lowerCAmelCase__ :  Dict							      =					self.get_config()
      lowerCAmelCase__ :  Union[str, Any]							      =					300
      return config




  def 							UpperCAmelCase_	(    self      )    ->		Optional[int]:
      (
          (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   
      ) :  int							      =					self.prepare_config_and_inputs()

      lowerCAmelCase__ :  int							      =					True
      lowerCAmelCase__ :  Any							      =					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]      )
      lowerCAmelCase__ :  List[Any]							      =					ids_tensor([self.batch_size, self.seq_length]  ,vocab_size=2      )

      return (
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Optional[int]:
      lowerCAmelCase__ :  Optional[int]							      =					MraModel(config=__UpperCAmelCase      )
      model.to(__UpperCAmelCase      )
      model.eval()
      lowerCAmelCase__ :  Optional[int]							      =					model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase      )
      lowerCAmelCase__ :  List[Any]							      =					model(__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase      )
      lowerCAmelCase__ :  List[Any]							      =					model(__UpperCAmelCase      )
      self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,)    ->		int:
      lowerCAmelCase__ :  List[str]							      =					True
      lowerCAmelCase__ :  Union[str, Any]							      =					MraModel(__UpperCAmelCase      )
      model.to(__UpperCAmelCase      )
      model.eval()
      lowerCAmelCase__ :  Optional[Any]							      =					model(
          __UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,encoder_hidden_states=__UpperCAmelCase  ,encoder_attention_mask=__UpperCAmelCase  ,)
      lowerCAmelCase__ :  str							      =					model(
          __UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,encoder_hidden_states=__UpperCAmelCase  ,)
      lowerCAmelCase__ :  Any							      =					model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase      )
      self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		int:
      lowerCAmelCase__ :  Tuple							      =					MraForMaskedLM(config=__UpperCAmelCase      )
      model.to(__UpperCAmelCase      )
      model.eval()
      lowerCAmelCase__ :  Any							      =					model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase      )
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.vocab_size)      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		List[str]:
      lowerCAmelCase__ :  List[Any]							      =					MraForQuestionAnswering(config=__UpperCAmelCase      )
      model.to(__UpperCAmelCase      )
      model.eval()
      lowerCAmelCase__ :  int							      =					model(
          __UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,start_positions=__UpperCAmelCase  ,end_positions=__UpperCAmelCase  ,)
      self.parent.assertEqual(result.start_logits.shape  ,(self.batch_size, self.seq_length)      )
      self.parent.assertEqual(result.end_logits.shape  ,(self.batch_size, self.seq_length)      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		str:
      lowerCAmelCase__ :  Optional[Any]							      =					self.num_labels
      lowerCAmelCase__ :  Union[str, Any]							      =					MraForSequenceClassification(__UpperCAmelCase      )
      model.to(__UpperCAmelCase      )
      model.eval()
      lowerCAmelCase__ :  int							      =					model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase      )
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_labels)      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Optional[int]:
      lowerCAmelCase__ :  List[Any]							      =					self.num_labels
      lowerCAmelCase__ :  Optional[Any]							      =					MraForTokenClassification(config=__UpperCAmelCase      )
      model.to(__UpperCAmelCase      )
      model.eval()
      lowerCAmelCase__ :  Any							      =					model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase      )
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.num_labels)      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Optional[Any]:
      lowerCAmelCase__ :  Any							      =					self.num_choices
      lowerCAmelCase__ :  Optional[Any]							      =					MraForMultipleChoice(config=__UpperCAmelCase      )
      model.to(__UpperCAmelCase      )
      model.eval()
      lowerCAmelCase__ :  int							      =					input_ids.unsqueeze(1      ).expand(-1  ,self.num_choices  ,-1      ).contiguous()
      lowerCAmelCase__ :  List[Any]							      =					token_type_ids.unsqueeze(1      ).expand(-1  ,self.num_choices  ,-1      ).contiguous()
      lowerCAmelCase__ :  List[Any]							      =					input_mask.unsqueeze(1      ).expand(-1  ,self.num_choices  ,-1      ).contiguous()
      lowerCAmelCase__ :  Tuple							      =					model(
          __UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase  ,)
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_choices)      )






  def 							UpperCAmelCase_	(    self      )    ->		List[str]:
      lowerCAmelCase__ :  Optional[int]							      =					self.prepare_config_and_inputs()
      (
          (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   (
          lowerCAmelCase__
      )      ,   
      ) :  Optional[Any]							      =					config_and_inputs
      lowerCAmelCase__ :  Tuple							      =					{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
      return config, inputs_dict







@require_torch
class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_						, unittest.TestCase     ):
  '''simple docstring'''




  __lowercase							:      Dict				=      (
      (
          MraModel,
          MraForMaskedLM,
          MraForMultipleChoice,
          MraForQuestionAnswering,
          MraForSequenceClassification,
          MraForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  __lowercase							:      str				=      False
  __lowercase							:      Union[str, Any]				=      False
  __lowercase							:      Optional[Any]				=      False
  __lowercase							:      int				=      False

  __lowercase							:      int				=      ()




  def 							UpperCAmelCase_	(    self      )    ->		Tuple:
      lowerCAmelCase__ :  List[str]							      =					MraModelTester(self      )
      lowerCAmelCase__ :  Dict							      =					ConfigTester(self  ,config_class=__UpperCAmelCase  ,hidden_size=37      )




  def 							UpperCAmelCase_	(    self      )    ->		Tuple:
      self.config_tester.run_common_tests()




  def 							UpperCAmelCase_	(    self      )    ->		List[Any]:
      lowerCAmelCase__ :  int							      =					self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      lowerCAmelCase__ :  Dict							      =					self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
          lowerCAmelCase__ :  str							      =					type
          self.model_tester.create_and_check_model(*__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		List[Any]:
      lowerCAmelCase__ :  int							      =					self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		str:
      lowerCAmelCase__ :  Dict							      =					self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		str:
      lowerCAmelCase__ :  Union[str, Any]							      =					self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  Dict							      =					self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  Optional[Any]							      =					self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase      )




  @slow
  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase__ :  Any							      =					MraModel.from_pretrained(__UpperCAmelCase      )
          self.assertIsNotNone(__UpperCAmelCase      )






  @unittest.skip(reason="""MRA does not output attentions"""      )
  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      return







@require_torch
class        lowerCAmelCase_(				unittest.TestCase     ):
  '''simple docstring'''




  @slow
  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  str							      =					MraModel.from_pretrained("""uw-madison/mra-base-512-4"""      )
      lowerCAmelCase__ :  List[Any]							      =					torch.arange(256      ).unsqueeze(0      )

      with torch.no_grad():
          lowerCAmelCase__ :  int							      =					model(__UpperCAmelCase      )[0]

      lowerCAmelCase__ :  Optional[int]							      =					torch.Size((1, 256, 768)      )
      self.assertEqual(output.shape  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  str							      =					torch.tensor(
          [[[-0.0_1_4_0, 0.0_8_3_0, -0.0_3_8_1], [0.1_5_4_6, 0.1_4_0_2, 0.0_2_2_0], [0.1_1_6_2, 0.0_8_5_1, 0.0_1_6_5]]]      )

      self.assertTrue(torch.allclose(output[:, :3, :3]  ,__UpperCAmelCase  ,atol=1E-4      )      )




  @slow
  def 							UpperCAmelCase_	(    self      )    ->		Tuple:
      lowerCAmelCase__ :  Any							      =					MraForMaskedLM.from_pretrained("""uw-madison/mra-base-512-4"""      )
      lowerCAmelCase__ :  Tuple							      =					torch.arange(256      ).unsqueeze(0      )

      with torch.no_grad():
          lowerCAmelCase__ :  List[Any]							      =					model(__UpperCAmelCase      )[0]

      lowerCAmelCase__ :  List[str]							      =					5_0265

      lowerCAmelCase__ :  int							      =					torch.Size((1, 256, vocab_size)      )
      self.assertEqual(output.shape  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[int]							      =					torch.tensor(
          [[[9.2_5_9_5, -3.6_0_3_8, 1_1.8_8_1_9], [9.3_8_6_9, -3.2_6_9_3, 1_1.0_9_5_6], [1_1.8_5_2_4, -3.4_9_3_8, 1_3.1_2_1_0]]]      )

      self.assertTrue(torch.allclose(output[:, :3, :3]  ,__UpperCAmelCase  ,atol=1E-4      )      )






  @slow
  def 							UpperCAmelCase_	(    self      )    ->		Tuple:
      lowerCAmelCase__ :  Union[str, Any]							      =					MraForMaskedLM.from_pretrained("""uw-madison/mra-base-4096-8-d3"""      )
      lowerCAmelCase__ :  Optional[int]							      =					torch.arange(4096      ).unsqueeze(0      )

      with torch.no_grad():
          lowerCAmelCase__ :  Optional[int]							      =					model(__UpperCAmelCase      )[0]

      lowerCAmelCase__ :  Optional[Any]							      =					5_0265

      lowerCAmelCase__ :  Tuple							      =					torch.Size((1, 4096, vocab_size)      )
      self.assertEqual(output.shape  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[int]							      =					torch.tensor(
          [[[5.4_7_8_9, -2.3_5_6_4, 7.5_0_6_4], [7.9_0_6_7, -1.3_3_6_9, 9.9_6_6_8], [9.0_7_1_2, -1.8_1_0_6, 7.0_3_8_0]]]      )

      self.assertTrue(torch.allclose(output[:, :3, :3]  ,__UpperCAmelCase  ,atol=1E-4      )      )







'''simple docstring'''





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
 import torch

 from transformers import XLMRobertaModel






@require_sentencepiece
@require_tokenizers
@require_torch
class        lowerCAmelCase_(				unittest.TestCase     ):
  '''simple docstring'''




  @slow
  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      lowerCAmelCase__ :  Tuple							      =					XLMRobertaModel.from_pretrained("""xlm-roberta-base"""      )
      lowerCAmelCase__ :  Optional[int]							      =					torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]]      )
      # The dog is cute and lives in the garden house

      lowerCAmelCase__ :  str							      =					torch.Size((1, 12, 768)      )  # batch_size, sequence_length, embedding_vector_dim
      lowerCAmelCase__ :  Dict							      =					torch.tensor(
          [[-0.0_1_0_1, 0.1_2_1_8, -0.0_8_0_3, 0.0_8_0_1, 0.1_3_2_7, 0.0_7_7_6, -0.1_2_1_5, 0.2_3_8_3, 0.3_3_3_8, 0.3_1_0_6, 0.0_3_0_0, 0.0_2_5_2]]      )
      #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
      #  xlmr.eval()
      #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
      with torch.no_grad():
          lowerCAmelCase__ :  Optional[int]							      =					model(__UpperCAmelCase      )["""last_hidden_state"""].detach()
      self.assertEqual(output.shape  ,__UpperCAmelCase      )
      # compare the actual values for a slice of last dim
      self.assertTrue(torch.allclose(output[:, :, -1]  ,__UpperCAmelCase  ,atol=1E-3      )      )






  @slow
  def 							UpperCAmelCase_	(    self      )    ->		int:
      lowerCAmelCase__ :  Union[str, Any]							      =					XLMRobertaModel.from_pretrained("""xlm-roberta-large"""      )
      lowerCAmelCase__ :  Optional[Any]							      =					torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]]      )
      # The dog is cute and lives in the garden house

      lowerCAmelCase__ :  Dict							      =					torch.Size((1, 12, 1024)      )  # batch_size, sequence_length, embedding_vector_dim
      lowerCAmelCase__ :  Union[str, Any]							      =					torch.tensor(
          [[-0.0_6_9_9, -0.0_3_1_8, 0.0_7_0_5, -0.1_2_4_1, 0.0_9_9_9, -0.0_5_2_0, 0.1_0_0_4, -0.1_8_3_8, -0.4_7_0_4, 0.1_4_3_7, 0.0_8_2_1, 0.0_1_2_6]]      )
      #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
      #  xlmr.eval()
      #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
      with torch.no_grad():
          lowerCAmelCase__ :  Union[str, Any]							      =					model(__UpperCAmelCase      )["""last_hidden_state"""].detach()
      self.assertEqual(output.shape  ,__UpperCAmelCase      )
      # compare the actual values for a slice of last dim
      self.assertTrue(torch.allclose(output[:, :, -1]  ,__UpperCAmelCase  ,atol=1E-3      )      )


