



'''simple docstring'''


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      FunnelBaseModel,
      FunnelForMaskedLM,
      FunnelForMultipleChoice,
      FunnelForPreTraining,
      FunnelForQuestionAnswering,
      FunnelForSequenceClassification,
      FunnelForTokenClassification,
      FunnelModel,
  )







class 						UpperCAmelCase						:


 '''simple docstring'''




 def __init__( self				,	lowerCAmelCase_				,	lowerCAmelCase_=1_3				,	lowerCAmelCase_=7				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=9_9				,	lowerCAmelCase_=[1, 1, 2]				,	lowerCAmelCase_=1				,	lowerCAmelCase_=3_2				,	lowerCAmelCase_=4				,	lowerCAmelCase_=8				,	lowerCAmelCase_=3_7				,	lowerCAmelCase_="gelu_new"				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.0				,	lowerCAmelCase_=5_1_2				,	lowerCAmelCase_=3				,	lowerCAmelCase_=0.0_2				,	lowerCAmelCase_=3				,	lowerCAmelCase_=4				,	lowerCAmelCase_=None				,	lowerCAmelCase_=False				,	)			->   List[str]:



      """simple docstring"""







      a_									=parent
      a_									=batch_size
      a_									=seq_length
      a_									=is_training
      a_									=use_input_mask
      a_									=use_token_type_ids
      a_									=use_labels
      a_									=vocab_size
      a_									=block_sizes
      a_									=num_decoder_layers
      a_									=d_model
      a_									=n_head
      a_									=d_head
      a_									=d_inner
      a_									=hidden_act
      a_									=hidden_dropout
      a_									=attention_dropout
      a_									=activation_dropout
      a_									=max_position_embeddings
      a_									=type_vocab_size
      a_									=2
      a_									=num_labels
      a_									=num_choices
      a_									=scope
      a_									=initializer_std

      # Used in the tests to check the size of the first attention layer
      a_									=n_head
      # Used in the tests to check the size of the first hidden state
      a_									=self.d_model
      # Used in the tests to check the number of output hidden states/attentions
      a_									=sum(self.block_sizes) + (0 if base else self.num_decoder_layers)
      # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
      # the last hidden state of the first block (which is the first hidden state of the decoder).
      if not base:
           a_									=self.num_hidden_layers + 2




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      a_									=ids_tensor([self.batch_size, self.seq_length]				,	self.vocab_size)

      a_									=None
      if self.use_input_mask:
           a_									=ids_tensor([self.batch_size, self.seq_length]				,	vocab_size=2)

      a_									=None
      if self.use_token_type_ids:
           a_									=ids_tensor([self.batch_size, self.seq_length]				,	self.type_vocab_size)

      a_									=None
      a_									=None
      a_									=None
      if self.use_labels:
           a_									=ids_tensor([self.batch_size]				,	self.type_sequence_label_size)
           a_									=ids_tensor([self.batch_size, self.seq_length]				,	self.num_labels)
           a_									=ids_tensor([self.batch_size]				,	self.num_choices)
           a_									=ids_tensor([self.batch_size, self.seq_length]				,	1)

      a_									=self.get_config()

      return (
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          fake_token_labels,
      )




 def    lowercase_	( self)			->   Any:



      """simple docstring"""







      return FunnelConfig(
          vocab_size=self.vocab_size				,	block_sizes=self.block_sizes				,	num_decoder_layers=self.num_decoder_layers				,	d_model=self.d_model				,	n_head=self.n_head				,	d_head=self.d_head				,	d_inner=self.d_inner				,	hidden_act=self.hidden_act				,	hidden_dropout=self.hidden_dropout				,	attention_dropout=self.attention_dropout				,	activation_dropout=self.activation_dropout				,	max_position_embeddings=self.max_position_embeddings				,	type_vocab_size=self.type_vocab_size				,	initializer_std=self.initializer_std				,	)




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   List[str]:



      """simple docstring"""







      a_									=FunnelModel(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=model(lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_)
      a_									=model(lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_)
      a_									=model(lowerCAmelCase_)
      self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, self.seq_length, self.d_model))

      a_									=False
      a_									=model(lowerCAmelCase_)
      self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, self.seq_length, self.d_model))

      a_									=False
      a_									=model(lowerCAmelCase_)
      self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, self.seq_length, self.d_model))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   Optional[Any]:



      """simple docstring"""







      a_									=FunnelBaseModel(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=model(lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_)
      a_									=model(lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_)
      a_									=model(lowerCAmelCase_)
      self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, 2, self.d_model))

      a_									=False
      a_									=model(lowerCAmelCase_)
      self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, 3, self.d_model))

      a_									=False
      a_									=model(lowerCAmelCase_)
      self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, 2, self.d_model))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   Dict:



      """simple docstring"""







      a_									=self.num_labels
      a_									=FunnelForPreTraining(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=model(lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_				,	labels=lowerCAmelCase_)
      self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.seq_length))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   Any:



      """simple docstring"""







      a_									=FunnelForMaskedLM(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=model(lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_				,	labels=lowerCAmelCase_)
      self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.seq_length, self.vocab_size))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   Tuple:



      """simple docstring"""







      a_									=self.num_labels
      a_									=FunnelForSequenceClassification(lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=model(lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_				,	labels=lowerCAmelCase_)
      self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.num_labels))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   List[str]:



      """simple docstring"""







      a_									=self.num_choices
      a_									=FunnelForMultipleChoice(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=input_ids.unsqueeze(1).expand(-1				,	self.num_choices				,	-1).contiguous()
      a_									=token_type_ids.unsqueeze(1).expand(-1				,	self.num_choices				,	-1).contiguous()
      a_									=input_mask.unsqueeze(1).expand(-1				,	self.num_choices				,	-1).contiguous()
      a_									=model(
          lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_				,	labels=lowerCAmelCase_				,	)
      self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.num_choices))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   Optional[Any]:



      """simple docstring"""







      a_									=self.num_labels
      a_									=FunnelForTokenClassification(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=model(lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_				,	labels=lowerCAmelCase_)
      self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.seq_length, self.num_labels))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	)			->   str:



      """simple docstring"""







      a_									=FunnelForQuestionAnswering(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      a_									=model(
          lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_				,	start_positions=lowerCAmelCase_				,	end_positions=lowerCAmelCase_				,	)
      self.parent.assertEqual(result.start_logits.shape				,	(self.batch_size, self.seq_length))
      self.parent.assertEqual(result.end_logits.shape				,	(self.batch_size, self.seq_length))




 def    lowercase_	( self)			->   List[str]:



      """simple docstring"""







      a_									=self.prepare_config_and_inputs()
      (
          (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    
      )									=config_and_inputs
      a_									={"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
      return config, inputs_dict







@require_torch
class 						UpperCAmelCase						(		__a       ,						__a       ,						unittest.TestCase):


 '''simple docstring'''




 __magic_name__       :			List[Any]	=   False
 __magic_name__       :			Optional[int]	=   False
 __magic_name__       :			Tuple	=   (
     (
         FunnelModel,
         FunnelForMaskedLM,
         FunnelForPreTraining,
         FunnelForQuestionAnswering,
         FunnelForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 __magic_name__       :			Optional[int]	=   (
     {
         "feature-extraction": (FunnelBaseModel, FunnelModel),
         "fill-mask": FunnelForMaskedLM,
         "question-answering": FunnelForQuestionAnswering,
         "text-classification": FunnelForSequenceClassification,
         "token-classification": FunnelForTokenClassification,
         "zero-shot": FunnelForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_=False)			->   Union[str, Any]:



      """simple docstring"""







      a_									=super()._prepare_for_class(lowerCAmelCase_				,	lowerCAmelCase_				,	return_labels=lowerCAmelCase_)

      if return_labels:
           if model_class in get_values(lowerCAmelCase_):
                a_									=torch.zeros(
                    (self.model_tester.batch_size, self.model_tester.seq_length)				,	dtype=torch.long				,	device=lowerCAmelCase_)
      return inputs_dict




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      a_									=FunnelModelTester(self)
      a_									=ConfigTester(self				,	config_class=lowerCAmelCase_)




 def    lowercase_	( self)			->   Union[str, Any]:



      """simple docstring"""







      self.config_tester.run_common_tests()




 def    lowercase_	( self)			->   Dict:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*lowerCAmelCase_)




 def    lowercase_	( self)			->   Dict:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase_)




 def    lowercase_	( self)			->   Union[str, Any]:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_)




 def    lowercase_	( self)			->   int:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_)




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_)




 def    lowercase_	( self				,	lowerCAmelCase_)			->   int:



      """simple docstring"""







      if hasattr(lowerCAmelCase_				,	"weight") and module.weight is not None:
           module.weight.data.fill_(3)
      if hasattr(lowerCAmelCase_				,	"bias") and module.bias is not None:
           module.bias.data.fill_(3)

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
           if hasattr(lowerCAmelCase_				,	lowerCAmelCase_) and getattr(lowerCAmelCase_				,	lowerCAmelCase_) is not None:
                a_									=getattr(lowerCAmelCase_				,	lowerCAmelCase_)
                weight.data.fill_(3)







@require_torch
class 						UpperCAmelCase						(		__a       ,						unittest.TestCase):


 '''simple docstring'''




 __magic_name__       :			Tuple	=   False
 __magic_name__       :			int	=   False
 __magic_name__       :			str	=   (
     (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
 )




 def    lowercase_	( self)			->   Any:



      """simple docstring"""







      a_									=FunnelModelTester(self				,	base=lowerCAmelCase_)
      a_									=ConfigTester(self				,	config_class=lowerCAmelCase_)




 def    lowercase_	( self)			->   Optional[int]:



      """simple docstring"""







      self.config_tester.run_common_tests()




 def    lowercase_	( self)			->   List[str]:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_base_model(*lowerCAmelCase_)




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_)




 def    lowercase_	( self)			->   Dict:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_)




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      a_		,    a_									=self.model_tester.prepare_config_and_inputs_for_common()
      a_									=True

      for model_class in self.all_model_classes:
           if model_class.__name__ == "FunnelBaseModel":
                continue
           a_									=model_class(lowerCAmelCase_)
           model.to(lowerCAmelCase_)
           model.train()
           a_									=self._prepare_for_class(lowerCAmelCase_				,	lowerCAmelCase_				,	return_labels=lowerCAmelCase_)
           a_									=model(**lowerCAmelCase_).loss
           loss.backward()




 def    lowercase_	( self				,	lowerCAmelCase_)			->   List[str]:



      """simple docstring"""







      if hasattr(lowerCAmelCase_				,	"weight") and module.weight is not None:
           module.weight.data.fill_(3)
      if hasattr(lowerCAmelCase_				,	"bias") and module.bias is not None:
           module.bias.data.fill_(3)

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
           if hasattr(lowerCAmelCase_				,	lowerCAmelCase_) and getattr(lowerCAmelCase_				,	lowerCAmelCase_) is not None:
                a_									=getattr(lowerCAmelCase_				,	lowerCAmelCase_)
                weight.data.fill_(3)







@require_torch
@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase						(		unittest.TestCase):


 '''simple docstring'''




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      a_									=1_3
      a_									=7
      a_									=torch.arange(0				,	batch_size * sequence_length).long().reshape(lowerCAmelCase_				,	lowerCAmelCase_)
      a_									=[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
      a_									=torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths])

      a_									=FunnelModel.from_pretrained("sgugger/funnel-random-tiny")
      a_									=model(lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_)[0].abs()

      a_									=torch.tensor(2_3_4_4.8_3_5_2)
      a_									=torch.tensor(0.8_0_5_2)
      self.assertTrue(torch.allclose(output.sum()				,	lowerCAmelCase_				,	atol=1e-4))
      self.assertTrue(torch.allclose(output.mean()				,	lowerCAmelCase_				,	atol=1e-4))

      a_									=torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]])
      a_									=model(lowerCAmelCase_				,	attention_mask=lowerCAmelCase_				,	token_type_ids=lowerCAmelCase_)[0].abs()

      a_									=torch.tensor(2_3_4_3.8_4_2_5)
      a_									=torch.tensor(0.8_0_4_9)
      self.assertTrue(torch.allclose(output.sum()				,	lowerCAmelCase_				,	atol=1e-4))
      self.assertTrue(torch.allclose(output.mean()				,	lowerCAmelCase_				,	atol=1e-4))




 @slow
 def    lowercase_	( self)			->   Union[str, Any]:



      """simple docstring"""







      a_									=FunnelTokenizer.from_pretrained("huggingface/funnel-small")
      a_									=FunnelModel.from_pretrained("huggingface/funnel-small")
      a_									=tokenizer("Hello! I am the Funnel Transformer model."				,	return_tensors="pt")
      a_									=model(**lowerCAmelCase_)[0]

      a_									=torch.tensor(2_3_5.7_2_4_6)
      a_									=torch.tensor(0.0_2_5_6)
      self.assertTrue(torch.allclose(output.sum()				,	lowerCAmelCase_				,	atol=1e-4))
      self.assertTrue(torch.allclose(output.mean()				,	lowerCAmelCase_				,	atol=1e-4))







'''simple docstring'''


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowercase											=      logging.get_logger(__name__)

lowercase											=      {
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 __magic_name__       :			str	=   "gpt2"
 __magic_name__       :			str	=   ["past_key_values"]
 __magic_name__       :			Dict	=   {
     "hidden_size": "n_embd",
     "max_position_embeddings": "n_positions",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }




 def __init__( self				,	lowerCAmelCase_=5_0_2_5_7				,	lowerCAmelCase_=1_0_2_4				,	lowerCAmelCase_=7_6_8				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=None				,	lowerCAmelCase_="gelu_new"				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=1e-5				,	lowerCAmelCase_=0.0_2				,	lowerCAmelCase_="cls_index"				,	lowerCAmelCase_=True				,	lowerCAmelCase_=None				,	lowerCAmelCase_=True				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False				,	**lowerCAmelCase_				,	)			->   Dict:



      """simple docstring"""







      a_									=vocab_size
      a_									=n_positions
      a_									=n_embd
      a_									=n_layer
      a_									=n_head
      a_									=n_inner
      a_									=activation_function
      a_									=resid_pdrop
      a_									=embd_pdrop
      a_									=attn_pdrop
      a_									=layer_norm_epsilon
      a_									=initializer_range
      a_									=summary_type
      a_									=summary_use_proj
      a_									=summary_activation
      a_									=summary_first_dropout
      a_									=summary_proj_to_labels
      a_									=scale_attn_weights
      a_									=use_cache
      a_									=scale_attn_by_inverse_layer_idx
      a_									=reorder_and_upcast_attn

      a_									=bos_token_id
      a_									=eos_token_id

      super().__init__(bos_token_id=lowerCAmelCase_				,	eos_token_id=lowerCAmelCase_				,	**lowerCAmelCase_)







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 def __init__( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = "default"				,	lowerCAmelCase_ = None				,	lowerCAmelCase_ = False				,	)			->   List[str]:



      """simple docstring"""







      super().__init__(lowerCAmelCase_				,	task=lowerCAmelCase_				,	patching_specs=lowerCAmelCase_				,	use_past=lowerCAmelCase_)
      if not getattr(self._config				,	"pad_token_id"				,	lowerCAmelCase_):
           # TODO: how to do that better?
           a_									=0




 @property
 def    lowercase_	( self)			->   Mapping[str, Mapping[int, str]]:



      """simple docstring"""







      a_									=OrderedDict({"input_ids": {0: "batch", 1: "sequence"}})
      if self.use_past:
           self.fill_with_past_key_values_(lowerCAmelCase_				,	direction="inputs")
           a_									={0: "batch", 1: "past_sequence + sequence"}
      else:
           a_									={0: "batch", 1: "sequence"}

      return common_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_layer




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_head




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = False				,	lowerCAmelCase_ = None				,	)			->   Mapping[str, Any]:



      """simple docstring"""







      a_									=super(lowerCAmelCase_				,	self).generate_dummy_inputs(
          lowerCAmelCase_				,	batch_size=lowerCAmelCase_				,	seq_length=lowerCAmelCase_				,	is_pair=lowerCAmelCase_				,	framework=lowerCAmelCase_)

      # We need to order the input in the way they appears in the forward()
      a_									=OrderedDict({"input_ids": common_inputs["input_ids"]})

      # Need to add the past_keys
      if self.use_past:
           if not is_torch_available():
                raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.")
           else:
                import torch

                a_		,    a_									=common_inputs["input_ids"].shape
                # Not using the same length for past_key_values
                a_									=seqlen + 2
                a_									=(
                    batch,
                    self.num_attention_heads,
                    past_key_values_length,
                    self._config.hidden_size // self.num_attention_heads,
                )
                a_									=[
                    (torch.zeros(lowerCAmelCase_), torch.zeros(lowerCAmelCase_)) for _ in range(self.num_layers)
                ]

      a_									=common_inputs["attention_mask"]
      if self.use_past:
           a_									=ordered_inputs["attention_mask"].dtype
           a_									=torch.cat(
               [ordered_inputs["attention_mask"], torch.ones(lowerCAmelCase_				,	lowerCAmelCase_				,	dtype=lowerCAmelCase_)]				,	dim=1)

      return ordered_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return 1_3


