






import unittest

from transformers import LayoutLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     LayoutLMForMaskedLM,
     LayoutLMForQuestionAnswering,
     LayoutLMForSequenceClassification,
     LayoutLMForTokenClassification,
     LayoutLMModel,
 )
class 							__snake_case:



 def __init__(						self : List[Any]			,		UpperCAmelCase__ : Tuple			,		UpperCAmelCase__ : int=13			,		UpperCAmelCase__ : int=7			,		UpperCAmelCase__ : int=True			,		UpperCAmelCase__ : Union[str, Any]=True			,		UpperCAmelCase__ : Union[str, Any]=True			,		UpperCAmelCase__ : str=True			,		UpperCAmelCase__ : List[Any]=99			,		UpperCAmelCase__ : Dict=32			,		UpperCAmelCase__ : Optional[int]=5			,		UpperCAmelCase__ : List[str]=4			,		UpperCAmelCase__ : List[str]=37			,		UpperCAmelCase__ : Dict="gelu"			,		UpperCAmelCase__ : Optional[Any]=0.1			,		UpperCAmelCase__ : Optional[Any]=0.1			,		UpperCAmelCase__ : Any=512			,		UpperCAmelCase__ : Tuple=16			,		UpperCAmelCase__ : int=2			,		UpperCAmelCase__ : Union[str, Any]=0.02			,		UpperCAmelCase__ : Dict=3			,		UpperCAmelCase__ : List[Any]=4			,		UpperCAmelCase__ : List[Any]=None			,		UpperCAmelCase__ : Union[str, Any]=1_000			,		):

    """simple docstring"""






    lowercase__    :						Optional[Any]       =			parent
    lowercase__    :						List[Any]       =			batch_size
    lowercase__    :						Optional[int]       =			seq_length
    lowercase__    :						Optional[int]       =			is_training
    lowercase__    :						str       =			use_input_mask
    lowercase__    :						Tuple       =			use_token_type_ids
    lowercase__    :						Dict       =			use_labels
    lowercase__    :						Tuple       =			vocab_size
    lowercase__    :						Union[str, Any]       =			hidden_size
    lowercase__    :						Any       =			num_hidden_layers
    lowercase__    :						Any       =			num_attention_heads
    lowercase__    :						List[str]       =			intermediate_size
    lowercase__    :						Tuple       =			hidden_act
    lowercase__    :						Optional[Any]       =			hidden_dropout_prob
    lowercase__    :						Optional[Any]       =			attention_probs_dropout_prob
    lowercase__    :						str       =			max_position_embeddings
    lowercase__    :						Optional[int]       =			type_vocab_size
    lowercase__    :						List[Any]       =			type_sequence_label_size
    lowercase__    :						Tuple       =			initializer_range
    lowercase__    :						Dict       =			num_labels
    lowercase__    :						Optional[Any]       =			num_choices
    lowercase__    :						List[str]       =			scope
    lowercase__    :						Tuple       =			range_bbox



 def     UpperCAmelCase     (						self : Tuple						):

    """simple docstring"""






    lowercase__    :						Union[str, Any]       =			ids_tensor([self.batch_size, self.seq_length]			,		self.vocab_size						)

    lowercase__    :						List[str]       =			ids_tensor([self.batch_size, self.seq_length, 4]			,		self.range_bbox						)
    # Ensure that bbox is legal
    for i in range(bbox.shape[0]						):
       for j in range(bbox.shape[1]						):
          if bbox[i, j, 3] < bbox[i, j, 1]:
             lowercase__    :						List[str]       =			bbox[i, j, 3]
             lowercase__    :						List[str]       =			bbox[i, j, 1]
             lowercase__    :						Any       =			t
          if bbox[i, j, 2] < bbox[i, j, 0]:
             lowercase__    :						Tuple       =			bbox[i, j, 2]
             lowercase__    :						Any       =			bbox[i, j, 0]
             lowercase__    :						Optional[int]       =			t

    lowercase__    :						Union[str, Any]       =			None
    if self.use_input_mask:
       lowercase__    :						Tuple       =			ids_tensor([self.batch_size, self.seq_length]			,		vocab_size=2						)

    lowercase__    :						Any       =			None
    if self.use_token_type_ids:
       lowercase__    :						str       =			ids_tensor([self.batch_size, self.seq_length]			,		self.type_vocab_size						)

    lowercase__    :						int       =			None
    lowercase__    :						Dict       =			None
    lowercase__    :						Tuple       =			None
    if self.use_labels:
       lowercase__    :						Dict       =			ids_tensor([self.batch_size]			,		self.type_sequence_label_size						)
       lowercase__    :						List[str]       =			ids_tensor([self.batch_size, self.seq_length]			,		self.num_labels						)
       lowercase__    :						int       =			ids_tensor([self.batch_size]			,		self.num_choices						)

    lowercase__    :						Tuple       =			self.get_config()

    return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



 def     UpperCAmelCase     (						self : List[Any]						):

    """simple docstring"""






    return LayoutLMConfig(
        vocab_size=self.vocab_size			,		hidden_size=self.hidden_size			,		num_hidden_layers=self.num_hidden_layers			,		num_attention_heads=self.num_attention_heads			,		intermediate_size=self.intermediate_size			,		hidden_act=self.hidden_act			,		hidden_dropout_prob=self.hidden_dropout_prob			,		attention_probs_dropout_prob=self.attention_probs_dropout_prob			,		max_position_embeddings=self.max_position_embeddings			,		type_vocab_size=self.type_vocab_size			,		initializer_range=self.initializer_range			,		)



 def     UpperCAmelCase     (						self : Union[str, Any]			,		UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : List[str]			,		UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : Tuple			,		UpperCAmelCase__ : List[str]			,		UpperCAmelCase__ : Any			,		UpperCAmelCase__ : Dict			,		UpperCAmelCase__ : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						List[str]       =			LayoutLMModel(config=lowerCAmelCase_						)
    model.to(lowerCAmelCase_						)
    model.eval()
    lowercase__    :						List[str]       =			model(lowerCAmelCase_			,		lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_						)
    lowercase__    :						Optional[Any]       =			model(lowerCAmelCase_			,		lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_						)
    lowercase__    :						int       =			model(lowerCAmelCase_			,		lowerCAmelCase_						)
    self.parent.assertEqual(result.last_hidden_state.shape			,		(self.batch_size, self.seq_length, self.hidden_size)						)
    self.parent.assertEqual(result.pooler_output.shape			,		(self.batch_size, self.hidden_size)						)



 def     UpperCAmelCase     (						self : int			,		UpperCAmelCase__ : Optional[Any]			,		UpperCAmelCase__ : List[Any]			,		UpperCAmelCase__ : Optional[int]			,		UpperCAmelCase__ : Dict			,		UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : int			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : List[Any]						):

    """simple docstring"""






    lowercase__    :						int       =			LayoutLMForMaskedLM(config=lowerCAmelCase_						)
    model.to(lowerCAmelCase_						)
    model.eval()
    lowercase__    :						List[Any]       =			model(lowerCAmelCase_			,		lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_			,		labels=lowerCAmelCase_						)
    self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.seq_length, self.vocab_size)						)



 def     UpperCAmelCase     (						self : List[str]			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : Tuple			,		UpperCAmelCase__ : List[Any]			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : Any			,		UpperCAmelCase__ : Dict			,		UpperCAmelCase__ : Optional[Any]			,		UpperCAmelCase__ : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						List[Any]       =			self.num_labels
    lowercase__    :						List[str]       =			LayoutLMForSequenceClassification(lowerCAmelCase_						)
    model.to(lowerCAmelCase_						)
    model.eval()
    lowercase__    :						Union[str, Any]       =			model(
        lowerCAmelCase_			,		lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_			,		labels=lowerCAmelCase_						)
    self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.num_labels)						)



 def     UpperCAmelCase     (						self : List[str]			,		UpperCAmelCase__ : List[Any]			,		UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : List[str]			,		UpperCAmelCase__ : Optional[Any]			,		UpperCAmelCase__ : List[str]			,		UpperCAmelCase__ : Optional[Any]			,		UpperCAmelCase__ : Tuple			,		UpperCAmelCase__ : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						Optional[int]       =			self.num_labels
    lowercase__    :						Optional[int]       =			LayoutLMForTokenClassification(config=lowerCAmelCase_						)
    model.to(lowerCAmelCase_						)
    model.eval()
    lowercase__    :						Dict       =			model(lowerCAmelCase_			,		lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_			,		labels=lowerCAmelCase_						)
    self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.seq_length, self.num_labels)						)



 def     UpperCAmelCase     (						self : Optional[int]			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : Optional[int]			,		UpperCAmelCase__ : List[Any]			,		UpperCAmelCase__ : Tuple			,		UpperCAmelCase__ : Optional[Any]			,		UpperCAmelCase__ : List[Any]			,		UpperCAmelCase__ : Optional[int]			,		UpperCAmelCase__ : Tuple						):

    """simple docstring"""






    lowercase__    :						Tuple       =			LayoutLMForQuestionAnswering(config=lowerCAmelCase_						)
    model.to(lowerCAmelCase_						)
    model.eval()
    lowercase__    :						Tuple       =			model(
        lowerCAmelCase_			,		bbox=lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_			,		start_positions=lowerCAmelCase_			,		end_positions=lowerCAmelCase_			,		)
    self.parent.assertEqual(result.start_logits.shape			,		(self.batch_size, self.seq_length)						)
    self.parent.assertEqual(result.end_logits.shape			,		(self.batch_size, self.seq_length)						)







 def     UpperCAmelCase     (						self : Any						):

    """simple docstring"""






    lowercase__    :						Union[str, Any]       =			self.prepare_config_and_inputs()
    (
        (
        lowercase__    
    )  ,       (
        lowercase__    
    )  ,       (
        lowercase__    
    )  ,       (
        lowercase__    
    )  ,       (
        lowercase__    
    )  ,       (
        lowercase__    
    )  ,       (
        lowercase__    
    )  ,       (
        lowercase__    
    )  ,           
    )    :						Optional[int]       =			config_and_inputs
    lowercase__    :						Dict       =			{
        """input_ids""": input_ids,
        """bbox""": bbox,
        """token_type_ids""": token_type_ids,
        """attention_mask""": input_mask,
    }
    return config, inputs_dict
@require_torch
class 							__snake_case(       __a  ,							__a  ,							unittest.TestCase   ):
 lowercase			:							Tuple		   =		(
     (
         LayoutLMModel,
         LayoutLMForMaskedLM,
         LayoutLMForSequenceClassification,
         LayoutLMForTokenClassification,
         LayoutLMForQuestionAnswering,
     )
     if is_torch_available()
     else None
 )
 lowercase			:							str		   =		(
     {
         "document-question-answering": LayoutLMForQuestionAnswering,
         "feature-extraction": LayoutLMModel,
         "fill-mask": LayoutLMForMaskedLM,
         "text-classification": LayoutLMForSequenceClassification,
         "token-classification": LayoutLMForTokenClassification,
         "zero-shot": LayoutLMForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 lowercase			:							Optional[int]		   =		True



 def     UpperCAmelCase     (						self : Tuple						):

    """simple docstring"""






    lowercase__    :						List[str]       =			LayoutLMModelTester(self						)
    lowercase__    :						Union[str, Any]       =			ConfigTester(self			,		config_class=lowerCAmelCase_			,		hidden_size=37						)



 def     UpperCAmelCase     (						self : List[str]						):

    """simple docstring"""






    self.config_tester.run_common_tests()



 def     UpperCAmelCase     (						self : Union[str, Any]						):

    """simple docstring"""






    lowercase__    :						Tuple       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*lowerCAmelCase_						)



 def     UpperCAmelCase     (						self : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						Optional[Any]       =			self.model_tester.prepare_config_and_inputs()
    for type in ["absolute", "relative_key", "relative_key_query"]:
       lowercase__    :						Tuple       =			type
       self.model_tester.create_and_check_model(*lowerCAmelCase_						)



 def     UpperCAmelCase     (						self : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						Union[str, Any]       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_						)



 def     UpperCAmelCase     (						self : int						):

    """simple docstring"""






    lowercase__    :						Tuple       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_						)



 def     UpperCAmelCase     (						self : Optional[int]						):

    """simple docstring"""






    lowercase__    :						Optional[Any]       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_						)



 def     UpperCAmelCase     (						self : Tuple						):

    """simple docstring"""






    lowercase__    :						Optional[Any]       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_						)







 @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""						)
 def     UpperCAmelCase     (						self : str						):

    """simple docstring"""






    pass
def    a__    (	):
   lowercase__    :						str       =			torch.tensor([[101,1_019,1_014,1_016,1_037,12_849,4_747,1_004,14_246,2_278,5_439,4_524,5_002,2_930,2_193,2_930,4_341,3_208,1_005,1_055,2_171,2_848,11_300,3_531,102],[101,4_070,4_034,7_020,1_024,3_058,1_015,1_013,2_861,1_013,6_070,19_274,2_772,6_205,27_814,16_147,16_147,4_343,2_047,10_283,10_969,14_389,1_012,2_338,102]]		,					device=lowercase__       )  # noqa: E231
   lowercase__    :						List[str]       =			torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]		,					device=lowercase__       )  # noqa: E231
   lowercase__    :						str       =			torch.tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1_000,1_000,1_000,1_000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1_000,1_000,1_000,1_000]]]		,					device=lowercase__       )  # noqa: E231
   lowercase__    :						str       =			torch.tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]		,					device=lowercase__       )  # noqa: E231
   # these are sequence labels (i.e. at the token level)
   lowercase__    :						Dict       =			torch.tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]		,					device=lowercase__       )  # noqa: E231
   # fmt: on

   return input_ids, attention_mask, bbox, token_type_ids, labels






@require_torch
class 							__snake_case(       unittest.TestCase   ):



 @slow
 def     UpperCAmelCase     (						self : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						Dict       =			LayoutLMModel.from_pretrained("""microsoft/layoutlm-base-uncased"""						).to(lowerCAmelCase_						)

    lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__    :						List[Any]       =			prepare_layoutlm_batch_inputs()

    # forward pass
    lowercase__    :						List[Any]       =			model(input_ids=lowerCAmelCase_			,		bbox=lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_						)

    # test the sequence output on [0, :3, :3]
    lowercase__    :						str       =			torch.tensor(
        [[0.1_785, -0.1_947, -0.0_425], [-0.3_254, -0.2_807, 0.2_553], [-0.5_391, -0.3_322, 0.3_364]]			,		device=lowerCAmelCase_			,		)

    self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]			,		lowerCAmelCase_			,		atol=1e-3						)						)

    # test the pooled output on [1, :3]
    lowercase__    :						Tuple       =			torch.tensor([-0.6_580, -0.0_214, 0.8_552]			,		device=lowerCAmelCase_						)

    self.assertTrue(torch.allclose(outputs.pooler_output[1, :3]			,		lowerCAmelCase_			,		atol=1e-3						)						)



 @slow
 def     UpperCAmelCase     (						self : Dict						):

    """simple docstring"""






    lowercase__    :						Optional[int]       =			LayoutLMForSequenceClassification.from_pretrained("""microsoft/layoutlm-base-uncased"""			,		num_labels=2						).to(
        lowerCAmelCase_						)

    lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__    :						int       =			prepare_layoutlm_batch_inputs()

    # forward pass
    lowercase__    :						Any       =			model(
        input_ids=lowerCAmelCase_			,		bbox=lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_			,		labels=torch.tensor([1, 1]			,		device=lowerCAmelCase_						)			,		)

    # test whether we get a loss as a scalar
    lowercase__    :						Any       =			outputs.loss
    lowercase__    :						List[Any]       =			torch.Size([]						)
    self.assertEqual(loss.shape			,		lowerCAmelCase_						)

    # test the shape of the logits
    lowercase__    :						Optional[Any]       =			outputs.logits
    lowercase__    :						Any       =			torch.Size((2, 2)						)
    self.assertEqual(logits.shape			,		lowerCAmelCase_						)



 @slow
 def     UpperCAmelCase     (						self : Any						):

    """simple docstring"""






    lowercase__    :						List[str]       =			LayoutLMForTokenClassification.from_pretrained("""microsoft/layoutlm-base-uncased"""			,		num_labels=13						).to(
        lowerCAmelCase_						)

    lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__    :						Optional[int]       =			prepare_layoutlm_batch_inputs()

    # forward pass
    lowercase__    :						Any       =			model(
        input_ids=lowerCAmelCase_			,		bbox=lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_			,		labels=lowerCAmelCase_						)

    # test the loss calculation to be around 2.65
    # expected_loss = torch.tensor(2.65, device=torch_device)

    # The loss is currently somewhat random and can vary between 0.1-0.3 atol.
    # self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=0.1))

    # test the shape of the logits
    lowercase__    :						List[str]       =			outputs.logits
    lowercase__    :						Optional[Any]       =			torch.Size((2, 25, 13)						)
    self.assertEqual(logits.shape			,		lowerCAmelCase_						)







 @slow
 def     UpperCAmelCase     (						self : Union[str, Any]						):

    """simple docstring"""






    lowercase__    :						Optional[int]       =			LayoutLMForQuestionAnswering.from_pretrained("""microsoft/layoutlm-base-uncased"""						).to(lowerCAmelCase_						)

    lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__    :						int       =			prepare_layoutlm_batch_inputs()

    # forward pass
    lowercase__    :						List[str]       =			model(input_ids=lowerCAmelCase_			,		bbox=lowerCAmelCase_			,		attention_mask=lowerCAmelCase_			,		token_type_ids=lowerCAmelCase_						)

    # test the shape of the logits
    lowercase__    :						List[str]       =			torch.Size((2, 25)						)
    self.assertEqual(outputs.start_logits.shape			,		lowerCAmelCase_						)
    self.assertEqual(outputs.end_logits.shape			,		lowerCAmelCase_						)







'''simple docstring'''


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowercase											=      logging.get_logger(__name__)

lowercase											=      {
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 __magic_name__       :			str	=   "gpt2"
 __magic_name__       :			str	=   ["past_key_values"]
 __magic_name__       :			Dict	=   {
     "hidden_size": "n_embd",
     "max_position_embeddings": "n_positions",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }




 def __init__( self				,	lowerCAmelCase_=5_0_2_5_7				,	lowerCAmelCase_=1_0_2_4				,	lowerCAmelCase_=7_6_8				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=None				,	lowerCAmelCase_="gelu_new"				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=1e-5				,	lowerCAmelCase_=0.0_2				,	lowerCAmelCase_="cls_index"				,	lowerCAmelCase_=True				,	lowerCAmelCase_=None				,	lowerCAmelCase_=True				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False				,	**lowerCAmelCase_				,	)			->   Dict:



      """simple docstring"""







      a_									=vocab_size
      a_									=n_positions
      a_									=n_embd
      a_									=n_layer
      a_									=n_head
      a_									=n_inner
      a_									=activation_function
      a_									=resid_pdrop
      a_									=embd_pdrop
      a_									=attn_pdrop
      a_									=layer_norm_epsilon
      a_									=initializer_range
      a_									=summary_type
      a_									=summary_use_proj
      a_									=summary_activation
      a_									=summary_first_dropout
      a_									=summary_proj_to_labels
      a_									=scale_attn_weights
      a_									=use_cache
      a_									=scale_attn_by_inverse_layer_idx
      a_									=reorder_and_upcast_attn

      a_									=bos_token_id
      a_									=eos_token_id

      super().__init__(bos_token_id=lowerCAmelCase_				,	eos_token_id=lowerCAmelCase_				,	**lowerCAmelCase_)







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 def __init__( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = "default"				,	lowerCAmelCase_ = None				,	lowerCAmelCase_ = False				,	)			->   List[str]:



      """simple docstring"""







      super().__init__(lowerCAmelCase_				,	task=lowerCAmelCase_				,	patching_specs=lowerCAmelCase_				,	use_past=lowerCAmelCase_)
      if not getattr(self._config				,	"pad_token_id"				,	lowerCAmelCase_):
           # TODO: how to do that better?
           a_									=0




 @property
 def    lowercase_	( self)			->   Mapping[str, Mapping[int, str]]:



      """simple docstring"""







      a_									=OrderedDict({"input_ids": {0: "batch", 1: "sequence"}})
      if self.use_past:
           self.fill_with_past_key_values_(lowerCAmelCase_				,	direction="inputs")
           a_									={0: "batch", 1: "past_sequence + sequence"}
      else:
           a_									={0: "batch", 1: "sequence"}

      return common_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_layer




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_head




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = False				,	lowerCAmelCase_ = None				,	)			->   Mapping[str, Any]:



      """simple docstring"""







      a_									=super(lowerCAmelCase_				,	self).generate_dummy_inputs(
          lowerCAmelCase_				,	batch_size=lowerCAmelCase_				,	seq_length=lowerCAmelCase_				,	is_pair=lowerCAmelCase_				,	framework=lowerCAmelCase_)

      # We need to order the input in the way they appears in the forward()
      a_									=OrderedDict({"input_ids": common_inputs["input_ids"]})

      # Need to add the past_keys
      if self.use_past:
           if not is_torch_available():
                raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.")
           else:
                import torch

                a_		,    a_									=common_inputs["input_ids"].shape
                # Not using the same length for past_key_values
                a_									=seqlen + 2
                a_									=(
                    batch,
                    self.num_attention_heads,
                    past_key_values_length,
                    self._config.hidden_size // self.num_attention_heads,
                )
                a_									=[
                    (torch.zeros(lowerCAmelCase_), torch.zeros(lowerCAmelCase_)) for _ in range(self.num_layers)
                ]

      a_									=common_inputs["attention_mask"]
      if self.use_past:
           a_									=ordered_inputs["attention_mask"].dtype
           a_									=torch.cat(
               [ordered_inputs["attention_mask"], torch.ones(lowerCAmelCase_				,	lowerCAmelCase_				,	dtype=lowerCAmelCase_)]				,	dim=1)

      return ordered_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return 1_3


