'''simple docstring'''
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RemBertConfig,
      RemBertForCausalLM,
      RemBertForMaskedLM,
      RemBertForMultipleChoice,
      RemBertForQuestionAnswering,
      RemBertForSequenceClassification,
      RemBertForTokenClassification,
      RemBertModel,
  )
  from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class       __lowercase		:







     """simple docstring"""






     def __init__(		self  ,      _lowerCamelCase  ,      _lowerCamelCase=1_3  ,      _lowerCamelCase=7  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=9_9  ,      _lowerCamelCase=3_2  ,      _lowerCamelCase=1_8  ,      _lowerCamelCase=4_3  ,      _lowerCamelCase=5  ,      _lowerCamelCase=4  ,      _lowerCamelCase=3_7  ,      _lowerCamelCase="gelu"  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=5_1_2  ,      _lowerCamelCase=1_6  ,      _lowerCamelCase=2  ,      _lowerCamelCase=0.0_2  ,      _lowerCamelCase=3  ,      _lowerCamelCase=4  ,      _lowerCamelCase=None  ,      ):
         __UpperCamelCase						:						Any		     =	parent
         __UpperCamelCase						:						str		     =	batch_size
         __UpperCamelCase						:						List[str]		     =	seq_length
         __UpperCamelCase						:						Dict		     =	is_training
         __UpperCamelCase						:						List[Any]		     =	use_input_mask
         __UpperCamelCase						:						Optional[Any]		     =	use_token_type_ids
         __UpperCamelCase						:						int		     =	use_labels
         __UpperCamelCase						:						Optional[Any]		     =	vocab_size
         __UpperCamelCase						:						Tuple		     =	hidden_size
         __UpperCamelCase						:						List[str]		     =	input_embedding_size
         __UpperCamelCase						:						Union[str, Any]		     =	output_embedding_size
         __UpperCamelCase						:						Tuple		     =	num_hidden_layers
         __UpperCamelCase						:						str		     =	num_attention_heads
         __UpperCamelCase						:						Dict		     =	intermediate_size
         __UpperCamelCase						:						List[str]		     =	hidden_act
         __UpperCamelCase						:						int		     =	hidden_dropout_prob
         __UpperCamelCase						:						Union[str, Any]		     =	attention_probs_dropout_prob
         __UpperCamelCase						:						List[str]		     =	max_position_embeddings
         __UpperCamelCase						:						Any		     =	type_vocab_size
         __UpperCamelCase						:						int		     =	type_sequence_label_size
         __UpperCamelCase						:						Dict		     =	initializer_range
         __UpperCamelCase						:						List[str]		     =	num_labels
         __UpperCamelCase						:						List[str]		     =	num_choices
         __UpperCamelCase						:						Tuple		     =	scope







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.vocab_size     )

         __UpperCamelCase						:						Union[str, Any]		     =	None
         if self.use_input_mask:
             __UpperCamelCase						:						Dict		     =	random_attention_mask([self.batch_size, self.seq_length]     )

         __UpperCamelCase						:						Any		     =	None
         if self.use_token_type_ids:
             __UpperCamelCase						:						List[str]		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.type_vocab_size     )

         __UpperCamelCase						:						str		     =	None
         __UpperCamelCase						:						str		     =	None
         __UpperCamelCase						:						Optional[int]		     =	None
         if self.use_labels:
             __UpperCamelCase						:						List[str]		     =	ids_tensor([self.batch_size]  ,      self.type_sequence_label_size     )
             __UpperCamelCase						:						List[str]		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.num_labels     )
             __UpperCamelCase						:						Optional[int]		     =	ids_tensor([self.batch_size]  ,      self.num_choices     )

         __UpperCamelCase						:						List[Any]		     =	RemBertConfig(
             vocab_size=self.vocab_size  ,      hidden_size=self.hidden_size  ,      input_embedding_size=self.input_embedding_size  ,      output_embedding_size=self.output_embedding_size  ,      num_hidden_layers=self.num_hidden_layers  ,      num_attention_heads=self.num_attention_heads  ,      intermediate_size=self.intermediate_size  ,      hidden_act=self.hidden_act  ,      hidden_dropout_prob=self.hidden_dropout_prob  ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,      max_position_embeddings=self.max_position_embeddings  ,      type_vocab_size=self.type_vocab_size  ,      is_decoder=lowerCAmelCase_  ,      initializer_range=self.initializer_range  ,      )

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







     def lowerCAmelCase  (		self     ):
         (
             (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       						
         )						:						Tuple		     =	self.prepare_config_and_inputs()

         __UpperCamelCase						:						Optional[Any]		     =	True
         __UpperCamelCase						:						int		     =	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
         __UpperCamelCase						:						int		     =	ids_tensor([self.batch_size, self.seq_length]  ,      vocab_size=2     )

         return (
             config,
             input_ids,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         __UpperCamelCase						:						Optional[int]		     =	RemBertModel(config=lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						Optional[int]		     =	model(lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_     )
         __UpperCamelCase						:						int		     =	model(lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_     )
         __UpperCamelCase						:						List[str]		     =	model(lowerCAmelCase_     )
         self.parent.assertEqual(result.last_hidden_state.shape  ,      (self.batch_size, self.seq_length, self.hidden_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						List[str]		     =	True
         __UpperCamelCase						:						Optional[int]		     =	RemBertModel(lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						Dict		     =	model(
             lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      encoder_hidden_states=lowerCAmelCase_  ,      encoder_attention_mask=lowerCAmelCase_  ,      )
         __UpperCamelCase						:						Optional[Any]		     =	model(
             lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      encoder_hidden_states=lowerCAmelCase_  ,      )
         __UpperCamelCase						:						Union[str, Any]		     =	model(lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_     )
         self.parent.assertEqual(result.last_hidden_state.shape  ,      (self.batch_size, self.seq_length, self.hidden_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						str		     =	RemBertForCausalLM(config=lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						Optional[Any]		     =	model(lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      labels=lowerCAmelCase_     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.vocab_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         __UpperCamelCase						:						Tuple		     =	RemBertForMaskedLM(config=lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						Union[str, Any]		     =	model(lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      labels=lowerCAmelCase_     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.vocab_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						Tuple		     =	True
         __UpperCamelCase						:						List[str]		     =	True
         __UpperCamelCase						:						Dict		     =	RemBertForCausalLM(config=lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()

         # first forward pass
         __UpperCamelCase						:						Optional[Any]		     =	model(
             lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      encoder_hidden_states=lowerCAmelCase_  ,      encoder_attention_mask=lowerCAmelCase_  ,      use_cache=lowerCAmelCase_  ,      )
         __UpperCamelCase						:						Optional[Any]		     =	outputs.past_key_values

         # create hypothetical multiple next token and extent to next_input_ids
         __UpperCamelCase						:						Tuple		     =	ids_tensor((self.batch_size, 3)  ,      config.vocab_size     )
         __UpperCamelCase						:						Optional[int]		     =	ids_tensor((self.batch_size, 3)  ,      vocab_size=2     )

         # append to next input_ids and
         __UpperCamelCase						:						Any		     =	torch.cat([input_ids, next_tokens]  ,      dim=-1     )
         __UpperCamelCase						:						str		     =	torch.cat([input_mask, next_mask]  ,      dim=-1     )

         __UpperCamelCase						:						Tuple		     =	model(
             lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      encoder_hidden_states=lowerCAmelCase_  ,      encoder_attention_mask=lowerCAmelCase_  ,      output_hidden_states=lowerCAmelCase_  ,      )['hidden_states'][0]
         __UpperCamelCase						:						Optional[Any]		     =	model(
             lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      encoder_hidden_states=lowerCAmelCase_  ,      encoder_attention_mask=lowerCAmelCase_  ,      past_key_values=lowerCAmelCase_  ,      output_hidden_states=lowerCAmelCase_  ,      )['hidden_states'][0]

         # select random slice
         __UpperCamelCase						:						int		     =	ids_tensor((1,)  ,      output_from_past.shape[-1]     ).item()
         __UpperCamelCase						:						Optional[Any]		     =	output_from_no_past[:, -3:, random_slice_idx].detach()
         __UpperCamelCase						:						Dict		     =	output_from_past[:, :, random_slice_idx].detach()

         self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(lowerCAmelCase_  ,      lowerCAmelCase_  ,      atol=1E-3     )     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         __UpperCamelCase						:						str		     =	RemBertForQuestionAnswering(config=lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						Optional[int]		     =	model(
             lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      start_positions=lowerCAmelCase_  ,      end_positions=lowerCAmelCase_  ,      )
         self.parent.assertEqual(result.start_logits.shape  ,      (self.batch_size, self.seq_length)     )
         self.parent.assertEqual(result.end_logits.shape  ,      (self.batch_size, self.seq_length)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         __UpperCamelCase						:						List[str]		     =	self.num_labels
         __UpperCamelCase						:						List[str]		     =	RemBertForSequenceClassification(lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						int		     =	model(lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      labels=lowerCAmelCase_     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.num_labels)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         __UpperCamelCase						:						Dict		     =	self.num_labels
         __UpperCamelCase						:						int		     =	RemBertForTokenClassification(config=lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						Union[str, Any]		     =	model(lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      labels=lowerCAmelCase_     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.num_labels)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         __UpperCamelCase						:						Union[str, Any]		     =	self.num_choices
         __UpperCamelCase						:						int		     =	RemBertForMultipleChoice(config=lowerCAmelCase_     )
         model.to(lowerCAmelCase_     )
         model.eval()
         __UpperCamelCase						:						List[Any]		     =	input_ids.unsqueeze(1     ).expand(-1  ,      self.num_choices  ,      -1     ).contiguous()
         __UpperCamelCase						:						List[str]		     =	token_type_ids.unsqueeze(1     ).expand(-1  ,      self.num_choices  ,      -1     ).contiguous()
         __UpperCamelCase						:						str		     =	input_mask.unsqueeze(1     ).expand(-1  ,      self.num_choices  ,      -1     ).contiguous()
         __UpperCamelCase						:						Any		     =	model(
             lowerCAmelCase_  ,      attention_mask=lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      labels=lowerCAmelCase_  ,      )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.num_choices)     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Dict		     =	self.prepare_config_and_inputs()
         (
             (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       						
         )						:						Any		     =	config_and_inputs
         __UpperCamelCase						:						Optional[Any]		     =	{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
         return config, inputs_dict






@require_torch
class       __lowercase		(    _lowerCamelCase      ,				_lowerCamelCase      ,				unittest.TestCase    ):







     """simple docstring"""






     SCREAMING_SNAKE_CASE__		 =    (
         (
             RemBertModel,
             RemBertForMaskedLM,
             RemBertForCausalLM,
             RemBertForMultipleChoice,
             RemBertForQuestionAnswering,
             RemBertForSequenceClassification,
             RemBertForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     SCREAMING_SNAKE_CASE__		 =    (RemBertForCausalLM,) if is_torch_available() else ()
     SCREAMING_SNAKE_CASE__		 =    (
         {
             '''feature-extraction''': RemBertModel,
             '''fill-mask''': RemBertForMaskedLM,
             '''question-answering''': RemBertForQuestionAnswering,
             '''text-classification''': RemBertForSequenceClassification,
             '''text-generation''': RemBertForCausalLM,
             '''token-classification''': RemBertForTokenClassification,
             '''zero-shot''': RemBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Dict		     =	RemBertModelTester(self     )
         __UpperCamelCase						:						Optional[int]		     =	ConfigTester(self  ,      config_class=lowerCAmelCase_  ,      hidden_size=3_7     )







     def lowerCAmelCase  (		self     ):
         self.config_tester.run_common_tests()







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Dict		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
             __UpperCamelCase						:						Optional[Any]		     =	type
             self.model_tester.create_and_check_model(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[Any]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[Any]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[str]		     =	self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Tuple		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Tuple		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[int]		     =	self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase_     )







     def lowerCAmelCase  (		self     ):
         # This regression test was failing with PyTorch < 1.3
         (
             (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       (
             __UpperCamelCase						
         )     ,       						
         )						:						Dict		     =	self.model_tester.prepare_config_and_inputs_for_decoder()

         __UpperCamelCase						:						Optional[Any]		     =	None

         self.model_tester.create_and_check_model_as_decoder(
             lowerCAmelCase_  ,      lowerCAmelCase_  ,      lowerCAmelCase_  ,      lowerCAmelCase_  ,      lowerCAmelCase_  ,      lowerCAmelCase_  ,      lowerCAmelCase_  ,      lowerCAmelCase_  ,      lowerCAmelCase_  ,      )







     @slow
     def lowerCAmelCase  (		self     ):
         for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __UpperCamelCase						:						List[Any]		     =	RemBertModel.from_pretrained(lowerCAmelCase_     )
             self.assertIsNotNone(lowerCAmelCase_     )






@require_torch
class       __lowercase		(    unittest.TestCase    ):







     """simple docstring"""






     @slow
     def lowerCAmelCase  (		self     ):
         # Test exact values at the last hidden layer
         __UpperCamelCase						:						Optional[int]		     =	RemBertModel.from_pretrained('google/rembert'     )
         __UpperCamelCase						:						Tuple		     =	torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]     )
         __UpperCamelCase						:						int		     =	torch.tensor([[0, 0, 0, 1, 1]]     )
         with torch.no_grad():
             __UpperCamelCase						:						List[Any]		     =	model(lowerCAmelCase_  ,      token_type_ids=lowerCAmelCase_  ,      output_hidden_states=lowerCAmelCase_     )

         __UpperCamelCase						:						Union[str, Any]		     =	1_1_5_2

         __UpperCamelCase						:						Tuple		     =	torch.Size((1, 5, hidden_size)     )
         self.assertEqual(output['last_hidden_state'].shape  ,      lowerCAmelCase_     )

         __UpperCamelCase						:						Optional[Any]		     =	torch.tensor(
             [
                 [
                     [0.0_7_5_4, -0.2_0_2_2, 0.1_9_0_4],
                     [-0.3_3_5_4, -0.3_6_9_2, -0.4_7_9_1],
                     [-0.2_3_1_4, -0.6_7_2_9, -0.0_7_4_9],
                     [-0.0_3_9_6, -0.3_1_0_5, -0.4_2_3_4],
                     [-0.1_5_7_1, -0.0_5_2_5, 0.5_3_5_3],
                 ]
             ]     )

         # Running on the original tf implementation gives slightly different results here.
         # Not clear why this variations is present
         # TODO: Find reason for discrepancy
         # expected_original_implementation = [[
         #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
         #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
         #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
         #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
         #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
         # ]]

         self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]  ,      lowerCAmelCase_  ,      atol=1E-4     )     )





'''simple docstring'''





import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available


if is_torch_available():
     from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
     A							     = False

if is_vision_available():
     from PIL import Image

     from transformers import (
         AutoProcessor,
         PixaStructImageProcessor,
         PixaStructProcessor,
         PreTrainedTokenizerFast,
         TaTokenizer,
     )



@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11       ,				reason='`Pix2StructImageProcessor` requires `torch>=1.11.0`.'       ,				)
@require_vision
@require_torch
class 			lowerCAmelCase__ (							unittest.TestCase							):



  '''simple docstring'''






  def 		_lowerCamelCase						(self	:			Tuple     ) ->							Tuple:
        snake_case_    							=				tempfile.mkdtemp()

        snake_case_    							=				PixaStructImageProcessor()
        snake_case_    							=				TaTokenizer.from_pretrained("t5-small"     )

        snake_case_    							=				PixaStructProcessor(lowerCAmelCase_						,			lowerCAmelCase_     )

        processor.save_pretrained(self.tmpdirname     )

  def 		_lowerCamelCase						(self	:			List[str]						,			**lowerCAmelCase_	:			int     ) ->							Optional[Any]:
        return AutoProcessor.from_pretrained(self.tmpdirname						,			**lowerCAmelCase_     ).tokenizer

  def 		_lowerCamelCase						(self	:			Union[str, Any]						,			**lowerCAmelCase_	:			List[str]     ) ->							List[Any]:
        return AutoProcessor.from_pretrained(self.tmpdirname						,			**lowerCAmelCase_     ).image_processor

  def 		_lowerCamelCase						(self	:			int     ) ->							List[str]:
        shutil.rmtree(self.tmpdirname     )

  def 		_lowerCamelCase						(self	:			Any     ) ->							Optional[Any]:
        snake_case_    							=				[np.random.randint(2_55						,			size=(3, 30, 4_00)						,			dtype=np.uinta     )]

        snake_case_    							=				[Image.fromarray(np.moveaxis(lowerCAmelCase_						,			0						,			-1     )     ) for x in image_inputs]

        return image_inputs

  def 		_lowerCamelCase						(self	:			Union[str, Any]     ) ->							Dict:
        snake_case_    							=				PixaStructProcessor(tokenizer=self.get_tokenizer()						,			image_processor=self.get_image_processor()     )
        processor.save_pretrained(self.tmpdirname     )

        snake_case_    							=				self.get_tokenizer(bos_token="(BOS)"						,			eos_token="(EOS)"     )
        snake_case_    							=				self.get_image_processor(do_normalize=lowerCAmelCase_						,			padding_value=1.0     )

        snake_case_    							=				PixaStructProcessor.from_pretrained(
            self.tmpdirname						,			bos_token="(BOS)"						,			eos_token="(EOS)"						,			do_normalize=lowerCAmelCase_						,			padding_value=1.0     )

        self.assertEqual(processor.tokenizer.get_vocab()						,			tokenizer_add_kwargs.get_vocab()     )
        self.assertIsInstance(processor.tokenizer						,			lowerCAmelCase_     )

        self.assertEqual(processor.image_processor.to_json_string()						,			image_processor_add_kwargs.to_json_string()     )
        self.assertIsInstance(processor.image_processor						,			lowerCAmelCase_     )

  def 		_lowerCamelCase						(self	:			Dict     ) ->							Optional[int]:
        snake_case_    							=				self.get_image_processor()
        snake_case_    							=				self.get_tokenizer()

        snake_case_    							=				PixaStructProcessor(tokenizer=lowerCAmelCase_						,			image_processor=lowerCAmelCase_     )

        snake_case_    							=				self.prepare_image_inputs()

        snake_case_    							=				image_processor(lowerCAmelCase_						,			return_tensors="np"     )
        snake_case_    							=				processor(images=lowerCAmelCase_						,			return_tensors="np"     )

        for key in input_feat_extract.keys():
              self.assertAlmostEqual(input_feat_extract[key].sum()						,			input_processor[key].sum()						,			delta=1E-2     )

  def 		_lowerCamelCase						(self	:			Tuple     ) ->							List[str]:
        snake_case_    							=				self.get_image_processor()
        snake_case_    							=				self.get_tokenizer()

        snake_case_    							=				PixaStructProcessor(tokenizer=lowerCAmelCase_						,			image_processor=lowerCAmelCase_     )

        snake_case_    							=				"lower newer"

        snake_case_    							=				processor(text=lowerCAmelCase_     )

        snake_case_    							=				tokenizer(lowerCAmelCase_						,			return_token_type_ids=lowerCAmelCase_						,			add_special_tokens=lowerCAmelCase_     )

        for key in encoded_tok.keys():
              self.assertListEqual(encoded_tok[key]						,			encoded_processor[key]     )

  def 		_lowerCamelCase						(self	:			str     ) ->							List[str]:
        snake_case_    							=				self.get_image_processor()
        snake_case_    							=				self.get_tokenizer()

        snake_case_    							=				PixaStructProcessor(tokenizer=lowerCAmelCase_						,			image_processor=lowerCAmelCase_     )

        snake_case_    							=				"lower newer"
        snake_case_    							=				self.prepare_image_inputs()

        snake_case_    							=				processor(text=lowerCAmelCase_						,			images=lowerCAmelCase_     )

        self.assertListEqual(
            list(inputs.keys()     )						,			["flattened_patches", "attention_mask", "decoder_attention_mask", "decoder_input_ids"]     )

        # test if it raises when no input is passed
        with pytest.raises(lowerCAmelCase_     ):
              processor()

  def 		_lowerCamelCase						(self	:			Optional[Any]     ) ->							Any:
        snake_case_    							=				self.get_image_processor()
        snake_case_    							=				self.get_tokenizer()

        snake_case_    							=				PixaStructProcessor(tokenizer=lowerCAmelCase_						,			image_processor=lowerCAmelCase_     )

        snake_case_    							=				"lower newer"
        snake_case_    							=				self.prepare_image_inputs()

        snake_case_    							=				processor(text=lowerCAmelCase_						,			images=lowerCAmelCase_     )

        snake_case_    							=				[5_12, 10_24, 20_48, 40_96]
        snake_case_    							=				[7_70, 7_70, 7_70, 7_70]
        # with text
        for i, max_patch in enumerate(lowerCAmelCase_     ):
              snake_case_    							=				processor(text=lowerCAmelCase_						,			images=lowerCAmelCase_						,			max_patches=lowerCAmelCase_     )
              self.assertEqual(inputs["flattened_patches"][0].shape[0]						,			lowerCAmelCase_     )
              self.assertEqual(inputs["flattened_patches"][0].shape[1]						,			expected_hidden_size[i]     )

        # without text input
        for i, max_patch in enumerate(lowerCAmelCase_     ):
              snake_case_    							=				processor(images=lowerCAmelCase_						,			max_patches=lowerCAmelCase_     )
              self.assertEqual(inputs["flattened_patches"][0].shape[0]						,			lowerCAmelCase_     )
              self.assertEqual(inputs["flattened_patches"][0].shape[1]						,			expected_hidden_size[i]     )

  def 		_lowerCamelCase						(self	:			int     ) ->							Any:
        snake_case_    							=				self.get_image_processor()
        snake_case_    							=				self.get_tokenizer()

        snake_case_    							=				PixaStructProcessor(tokenizer=lowerCAmelCase_						,			image_processor=lowerCAmelCase_     )

        snake_case_    							=				[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        snake_case_    							=				processor.batch_decode(lowerCAmelCase_     )
        snake_case_    							=				tokenizer.batch_decode(lowerCAmelCase_     )

        self.assertListEqual(lowerCAmelCase_						,			lowerCAmelCase_     )


  def 		_lowerCamelCase						(self	:			Optional[int]     ) ->							str:
        snake_case_    							=				self.get_image_processor()
        snake_case_    							=				self.get_tokenizer()

        snake_case_    							=				PixaStructProcessor(tokenizer=lowerCAmelCase_						,			image_processor=lowerCAmelCase_     )

        snake_case_    							=				"lower newer"
        snake_case_    							=				self.prepare_image_inputs()

        snake_case_    							=				processor(text=lowerCAmelCase_						,			images=lowerCAmelCase_     )

        # For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"]
        self.assertListEqual(
            list(inputs.keys()     )						,			["flattened_patches", "attention_mask", "decoder_attention_mask", "decoder_input_ids"]     )

        snake_case_    							=				processor(text=lowerCAmelCase_     )

        # For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"]
        self.assertListEqual(list(inputs.keys()     )						,			["input_ids", "attention_mask"]     )

