





'''simple docstring'''







from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
					import numpy as np
					import tensorflow as tf

					from transformers import (
					    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
					    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
					    TF_MODEL_FOR_MASKED_LM_MAPPING,
					    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
					    TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
					    TF_MODEL_FOR_PRETRAINING_MAPPING,
					    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
					    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
					    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
					    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
					    TFSharedEmbeddings,
					)

					if _tf_gpu_memory_limit is not None:
										snake_case_  :							Tuple            =	tf.config.list_physical_devices("GPU")
										for gpu in gpus:
															# Restrict TensorFlow to only allocate x GB of memory on the GPUs
															try:
																				tf.config.set_logical_device_configuration(
																				    gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
																				)
																				snake_case_  :							int            =	tf.config.list_logical_devices("GPU")
																				print("Logical GPUs", logical_gpus)
															except RuntimeError as e:
																				# Virtual devices must be set before GPUs have been initialized
																				print(e)


@require_tf
class        __a   :
							__a       : Optional[Any]          =							None
							__a       : str          =							()
							__a       : Union[str, Any]          =							()
							__a       : Tuple          =							True
							__a       : Tuple          =							True
							__a       : int          =							True
							__a       : Optional[Any]          =							False

							def      UpperCAmelCase__						(				self				:  Tuple							,  __magic_name__				:  List[str]							,  __magic_name__				:  Union[str, Any]							,  __magic_name__				:  Dict=False )       ->							dict:







													"""simple docstring"""
													UpperCAmelCase_      :   int      =	copy.deepcopy(__magic_name__ )

													if model_class in get_values(__magic_name__ ):
																			UpperCAmelCase_      :   List[str]      =	{
																			    k: tf.tile(tf.expand_dims(__magic_name__							,  1 )							,  (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1) )
																			    if isinstance(__magic_name__							,  tf.Tensor ) and v.ndim > 0
																			    else v
																			    for k, v in inputs_dict.items()
																			}

													if return_labels:
																			if model_class in get_values(__magic_name__ ):
																									UpperCAmelCase_      :   Any      =	tf.ones(self.model_tester.batch_size							,  dtype=tf.intaa )
																			elif model_class in get_values(__magic_name__ ):
																									UpperCAmelCase_      :   Dict      =	tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa )
																									UpperCAmelCase_      :   Dict      =	tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa )
																			elif model_class in [
																			    *get_values(__magic_name__ ),
																			    *get_values(__magic_name__ ),
																			]:
																									UpperCAmelCase_      :   Optional[int]      =	tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa )
																			elif model_class in get_values(__magic_name__ ):
																									UpperCAmelCase_      :   List[str]      =	tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa )
																			elif model_class in [
																			    *get_values(__magic_name__ ),
																			    *get_values(__magic_name__ ),
																			    *get_values(__magic_name__ ),
																			    *get_values(__magic_name__ ),
																			    *get_values(__magic_name__ ),
																			]:
																									UpperCAmelCase_      :   Optional[Any]      =	tf.zeros(
																									    (self.model_tester.batch_size, self.model_tester.seq_length)							,  dtype=tf.intaa )
													return inputs_dict

							@slow
							def      UpperCAmelCase__						(				self				:  Tuple )       ->							str:







													"""simple docstring"""
													UpperCAmelCase_    ,							UpperCAmelCase_      :   str      =	self.model_tester.prepare_config_and_inputs_for_common()
													for model_class in self.all_model_classes[:2]:
																			UpperCAmelCase_      :   Union[str, Any]      =	self._prepare_for_class(__magic_name__							,  __magic_name__ )
																			UpperCAmelCase_      :   Dict      =	model_class(__magic_name__ )

																			@tf.function
																			def run_in_graph_mode():
																									return model(__magic_name__ )

																			UpperCAmelCase_      :   Any      =	run_in_graph_mode()
																			self.assertIsNotNone(__magic_name__ )

							@slow
							def      UpperCAmelCase__						(				self				:  int )       ->							List[Any]:







													"""simple docstring"""
													UpperCAmelCase_    ,							UpperCAmelCase_      :   int      =	self.model_tester.prepare_config_and_inputs_for_common()
													for model_class in self.all_model_classes[:2]:
																			UpperCAmelCase_      :   Union[str, Any]      =	self._prepare_for_class(__magic_name__							,  __magic_name__ )
																			UpperCAmelCase_      :   List[str]      =	model_class(__magic_name__ )

																			@tf.function(experimental_compile=__magic_name__ )
																			def run_in_graph_mode():
																									return model(__magic_name__ )

																			UpperCAmelCase_      :   int      =	run_in_graph_mode()
																			self.assertIsNotNone(__magic_name__ )

							@slow
							def      UpperCAmelCase__						(				self				:  Union[str, Any] )       ->							Optional[int]:







													"""simple docstring"""
													# This is a copy of the test_keras_fit method, but we use XLA compilation instead of eager
													UpperCAmelCase_    ,							UpperCAmelCase_      :   Optional[Any]      =	self.model_tester.prepare_config_and_inputs_for_common()
													for model_class in self.all_model_classes[:2]:
																			UpperCAmelCase_      :   str      =	model_class(__magic_name__ )
																			if getattr(__magic_name__							,  '''hf_compute_loss'''							,  __magic_name__ ):
																									# Test that model correctly compute the loss with kwargs
																									UpperCAmelCase_      :   Dict      =	self._prepare_for_class(inputs_dict.copy()							,  __magic_name__							,  return_labels=__magic_name__ )
																									# Is there a better way to remove these decoder inputs?
																									UpperCAmelCase_      :   Optional[Any]      =	{
																									    key: val
																									    for key, val in prepared_for_class.items()
																									    if key not in ('''head_mask''', '''decoder_head_mask''', '''cross_attn_head_mask''', '''decoder_input_ids''')
																									}

																									UpperCAmelCase_      :   Any      =	{
																									    '''labels''',
																									    '''label''',
																									    '''label_ids''',
																									    '''start_positions''',
																									    '''start_position''',
																									    '''end_positions''',
																									    '''end_position''',
																									    '''next_sentence_label''',
																									}
																									UpperCAmelCase_      :   int      =	possible_label_cols.intersection(set(__magic_name__ ) )
																									self.assertGreater(len(__magic_name__ )							,  0							,  msg='''No matching label names found!''' )
																									UpperCAmelCase_      :   Optional[Any]      =	{key: val for key, val in prepared_for_class.items() if key in label_names}
																									UpperCAmelCase_      :   Dict      =	{key: val for key, val in prepared_for_class.items() if key not in label_names}
																									self.assertGreater(len(__magic_name__ )							,  0 )

																									# Make sure it works with XLA!
																									model.compile(optimizer=tf.keras.optimizers.SGD(0.0 )							,  jit_compile=__magic_name__ )
																									# Make sure the model fits without crashing regardless of where we pass the labels
																									UpperCAmelCase_      :   Union[str, Any]      =	model.fit(
																									    __magic_name__							,  validation_data=__magic_name__							,  steps_per_epoch=1							,  validation_steps=1							,  shuffle=__magic_name__							,  verbose=0							,  )
																									UpperCAmelCase_      :   Union[str, Any]      =	history.history['''loss'''][0]
																									self.assertTrue(not isnan(__magic_name__ ) )
																									UpperCAmelCase_      :   Union[str, Any]      =	history.history['''val_loss'''][0]
																									self.assertTrue(not isnan(__magic_name__ ) )

																									# Now test it with separate labels, to make sure that path works in XLA too.
																									UpperCAmelCase_      :   List[str]      =	model_class(__magic_name__ )
																									model.compile(optimizer=tf.keras.optimizers.SGD(0.0 )							,  jit_compile=__magic_name__ )
																									UpperCAmelCase_      :   int      =	model.fit(
																									    __magic_name__							,  __magic_name__							,  validation_data=(inputs_minus_labels, labels)							,  steps_per_epoch=1							,  validation_steps=1							,  shuffle=__magic_name__							,  verbose=0							,  )

																									UpperCAmelCase_      :   List[Any]      =	history.history['''loss'''][0]
																									self.assertTrue(not isnan(__magic_name__ ) )
																									UpperCAmelCase_      :   Union[str, Any]      =	history.history['''val_loss'''][0]
																									self.assertTrue(not isnan(__magic_name__ ) )

							@slow
							def      UpperCAmelCase__						(				self				:  Tuple )       ->							Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_    ,							UpperCAmelCase_      :   List[Any]      =	self.model_tester.prepare_config_and_inputs_for_common()
													UpperCAmelCase_      :   str      =	True
													UpperCAmelCase_      :   Dict      =	True

													if hasattr(__magic_name__							,  '''use_cache''' ):
																			UpperCAmelCase_      :   int      =	True

													UpperCAmelCase_      :   Optional[Any]      =	getattr(self.model_tester							,  '''encoder_seq_length'''							,  self.model_tester.seq_length )
													UpperCAmelCase_      :   Any      =	getattr(self.model_tester							,  '''key_length'''							,  __magic_name__ )

													for model_class in self.all_model_classes[:2]:
																			UpperCAmelCase_      :   int      =	self._prepare_for_class(__magic_name__							,  __magic_name__ )
																			UpperCAmelCase_      :   Any      =	model_class(__magic_name__ )
																			model.build()
																			UpperCAmelCase_      :   Dict      =	len(model(__magic_name__ ) )

																			for key in list(class_inputs_dict.keys() ):
																									# Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
																									if key not in model.input_signature:
																															del class_inputs_dict[key]
																									# Check it's a tensor, in case the inputs dict has some bools in it too
																									elif isinstance(class_inputs_dict[key]							,  tf.Tensor ) and class_inputs_dict[key].dtype.is_integer:
																															UpperCAmelCase_      :   Optional[Any]      =	tf.cast(class_inputs_dict[key]							,  tf.intaa )

																			if set(class_inputs_dict.keys() ) != set(model.input_signature.keys() ):
																									continue  # Some models have inputs that the preparation functions don't create, we skip those

																			with tempfile.TemporaryDirectory() as tmpdirname:
																									model.save_pretrained(__magic_name__							,  saved_model=__magic_name__ )
																									UpperCAmelCase_      :   List[str]      =	os.path.join(__magic_name__							,  '''saved_model'''							,  '''1''' )
																									UpperCAmelCase_      :   Union[str, Any]      =	tf.keras.models.load_model(__magic_name__ )
																									UpperCAmelCase_      :   Union[str, Any]      =	model(__magic_name__ )

																									if self.is_encoder_decoder:
																															UpperCAmelCase_      :   Optional[Any]      =	outputs['''encoder_hidden_states''']
																															UpperCAmelCase_      :   Optional[int]      =	outputs['''encoder_attentions''']
																									else:
																															UpperCAmelCase_      :   Optional[int]      =	outputs['''hidden_states''']
																															UpperCAmelCase_      :   int      =	outputs['''attentions''']

																									self.assertEqual(len(__magic_name__ )							,  __magic_name__ )

																									UpperCAmelCase_      :   List[Any]      =	getattr(
																									    self.model_tester							,  '''expected_num_hidden_layers'''							,  self.model_tester.num_hidden_layers + 1 )

																									self.assertEqual(len(__magic_name__ )							,  __magic_name__ )
																									self.assertListEqual(
																									    list(output_hidden_states[0].shape[-2:] )							,  [self.model_tester.seq_length, self.model_tester.hidden_size]							,  )

																									self.assertEqual(len(__magic_name__ )							,  self.model_tester.num_hidden_layers )
																									self.assertListEqual(
																									    list(output_attentions[0].shape[-3:] )							,  [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]							,  )

							@slow
							def      UpperCAmelCase__						(				self				:  Dict )       ->							Tuple:







													"""simple docstring"""
													tf.keras.mixed_precision.set_global_policy('''mixed_float16''' )

													# try/finally block to ensure subsequent tests run in float32
													try:
																			UpperCAmelCase_    ,							UpperCAmelCase_      :   str      =	self.model_tester.prepare_config_and_inputs_for_common()
																			for model_class in self.all_model_classes[:2]:
																									UpperCAmelCase_      :   Optional[int]      =	self._prepare_for_class(__magic_name__							,  __magic_name__ )
																									UpperCAmelCase_      :   Any      =	model_class(__magic_name__ )
																									UpperCAmelCase_      :   Union[str, Any]      =	model(__magic_name__ )

																									self.assertIsNotNone(__magic_name__ )
													finally:
																			tf.keras.mixed_precision.set_global_policy('''float32''' )

							@slow
							def      UpperCAmelCase__						(				self				:  Tuple )       ->							Any:







													"""simple docstring"""
													UpperCAmelCase_    ,							UpperCAmelCase_      :   Union[str, Any]      =	self.model_tester.prepare_config_and_inputs_for_common()
													# head_mask and decoder_head_mask has different shapes than other input args
													if "head_mask" in inputs_dict:
																			del inputs_dict["head_mask"]
													if "decoder_head_mask" in inputs_dict:
																			del inputs_dict["decoder_head_mask"]
													if "cross_attn_head_mask" in inputs_dict:
																			del inputs_dict["cross_attn_head_mask"]
													UpperCAmelCase_      :   Optional[int]      =	{
													    module_member
													    for model_class in self.all_model_classes
													    for module in (import_module(model_class.__module__ ),)
													    for module_member_name in dir(__magic_name__ )
													    if module_member_name.endswith('''MainLayer''' )
													    for module_member in (getattr(__magic_name__							,  __magic_name__ ),)
													    if isinstance(__magic_name__							,  __magic_name__ )
													    and tf.keras.layers.Layer in module_member.__bases__
													    and getattr(__magic_name__							,  '''_keras_serializable'''							,  __magic_name__ )
													}

													for main_layer_class in tf_main_layer_classes:
																			# T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
																			if "T5" in main_layer_class.__name__:
																									# Take the same values than in TFT5ModelTester for this shared layer
																									UpperCAmelCase_      :   Tuple      =	TFSharedEmbeddings(self.model_tester.vocab_size							,  self.model_tester.hidden_size							,  name='''shared''' )
																									UpperCAmelCase_      :   Dict      =	False
																									UpperCAmelCase_      :   Optional[int]      =	main_layer_class(__magic_name__							,  embed_tokens=__magic_name__ )
																			else:
																									UpperCAmelCase_      :   List[str]      =	main_layer_class(__magic_name__ )

																			UpperCAmelCase_      :   Union[str, Any]      =	{
																			    name: tf.keras.Input(tensor.shape[1:]							,  dtype=tensor.dtype ) for name, tensor in inputs_dict.items()
																			}

																			if hasattr(self.model_tester							,  '''num_labels''' ):
																									UpperCAmelCase_      :   List[str]      =	self.model_tester.num_labels
																			else:
																									UpperCAmelCase_      :   Optional[int]      =	2

																			UpperCAmelCase_      :   Any      =	tf.data.Dataset.from_tensor_slices(
																			    (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1) )) ).batch(1 )

																			UpperCAmelCase_      :   Optional[Any]      =	main_layer(__magic_name__ )[0]
																			UpperCAmelCase_      :   List[str]      =	tf.keras.layers.Dense(__magic_name__							,  activation='''softmax'''							,  name='''outputs''' )(__magic_name__ )
																			UpperCAmelCase_      :   int      =	tf.keras.models.Model(inputs=__magic_name__							,  outputs=[outputs] )

																			model.compile(loss='''binary_crossentropy'''							,  optimizer='''adam'''							,  metrics=['''binary_accuracy'''] )
																			model.fit(__magic_name__							,  epochs=1 )

																			with tempfile.TemporaryDirectory() as tmpdirname:
																									UpperCAmelCase_      :   List[str]      =	os.path.join(__magic_name__							,  '''keras_model.h5''' )
																									model.save(__magic_name__ )
																									if "T5" in main_layer_class.__name__:
																															UpperCAmelCase_      :   Any      =	tf.keras.models.load_model(
																															    __magic_name__							,  custom_objects={
																															        main_layer_class.__name__: main_layer_class,
																															        '''TFSharedEmbeddings''': TFSharedEmbeddings,
																															    }							,  )
																									else:
																															UpperCAmelCase_      :   Dict      =	tf.keras.models.load_model(
																															    __magic_name__							,  custom_objects={main_layer_class.__name__: main_layer_class} )
																									assert isinstance(__magic_name__							,  tf.keras.Model )
																									model(__magic_name__ )

							@slow
							def      UpperCAmelCase__						(				self				:  List[Any] )       ->							Any:







													"""simple docstring"""
													UpperCAmelCase_    ,							UpperCAmelCase_      :   Any      =	self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes[:2]:
																			UpperCAmelCase_      :   Optional[Any]      =	model_class(__magic_name__ )

																			UpperCAmelCase_      :   Optional[Any]      =	copy.deepcopy(__magic_name__ )

																			if not self.is_encoder_decoder:
																									UpperCAmelCase_      :   Tuple      =	inputs['''input_ids''']
																									del inputs["input_ids"]
																			else:
																									UpperCAmelCase_      :   Dict      =	inputs['''input_ids''']
																									UpperCAmelCase_      :   str      =	inputs.get('''decoder_input_ids'''							,  __magic_name__ )
																									del inputs["input_ids"]
																									inputs.pop('''decoder_input_ids'''							,  __magic_name__ )

																			if not self.is_encoder_decoder:
																									UpperCAmelCase_      :   Optional[int]      =	model.get_input_embeddings()(__magic_name__ )
																			else:
																									UpperCAmelCase_      :   Tuple      =	model.get_input_embeddings()(__magic_name__ )
																									UpperCAmelCase_      :   List[str]      =	model.get_input_embeddings()(__magic_name__ )

																			UpperCAmelCase_      :   List[Any]      =	self._prepare_for_class(__magic_name__							,  __magic_name__ )

																			@tf.function
																			def run_in_graph_mode():
																									return model(__magic_name__ )

																			UpperCAmelCase_      :   List[str]      =	run_in_graph_mode()
																			self.assertIsNotNone(__magic_name__ )

							def      UpperCAmelCase__						(				self				:  int							,  __magic_name__				:  Any							,  __magic_name__				:  List[Any] )       ->							List[Any]:







													"""simple docstring"""
													# special tokens cannot be bad tokens
													UpperCAmelCase_      :   Any      =	[]
													if model.config.bos_token_id is not None:
																			special_tokens.append(model.config.bos_token_id )
													if model.config.pad_token_id is not None:
																			special_tokens.append(model.config.pad_token_id )
													if model.config.eos_token_id is not None:
																			special_tokens.append(model.config.eos_token_id )

													# create random bad tokens that are not special tokens
													UpperCAmelCase_      :   Optional[int]      =	[]
													while len(__magic_name__ ) < num_bad_tokens:
																			UpperCAmelCase_      :   Optional[Any]      =	tf.squeeze(ids_tensor((1, 1)							,  self.model_tester.vocab_size )							,  0 ).numpy()[0]
																			if token not in special_tokens:
																									bad_tokens.append(__magic_name__ )
													return bad_tokens

							def      UpperCAmelCase__						(				self				:  Optional[int]							,  __magic_name__				:  str )       ->							int:







													"""simple docstring"""
													for token_id in output_ids[0].numpy().tolist():
																			self.assertGreaterEqual(__magic_name__							,  0 )
																			self.assertLess(__magic_name__							,  self.model_tester.vocab_size )



							def      UpperCAmelCase__						(				self				:  Any							,  __magic_name__				:  Dict							,  __magic_name__				:  int )       ->							str:







													"""simple docstring"""
													# for all bad word tokens
													for bad_word_ids in bad_words_ids:
																			# for all slices in batch
																			for generated_ids_slice in generated_ids:
																									# for all word idx
																									for i in range(len(__magic_name__ )							,  len(__magic_name__ ) ):
																															# if tokens match
																															if generated_ids_slice[i - len(__magic_name__ ) : i] == bad_word_ids:
																																					return True
													return False







'''simple docstring'''

from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__UpperCAmelCase												=   logging.get_logger(__name__)

__UpperCAmelCase												=   {
    'gpt2': 'https://huggingface.co/gpt2/resolve/main/config.json',
    'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/config.json',
    'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/config.json',
    'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/config.json',
    'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/config.json',
}


class 		_a     (							SCREAMING_SNAKE_CASE  ):







   """simple docstring"""




   A		= 'gpt2'
   A		= ['past_key_values']
   A		= {
       'hidden_size': 'n_embd',
       'max_position_embeddings': 'n_positions',
       'num_attention_heads': 'n_head',
       'num_hidden_layers': 'n_layer',
   }






   def __init__(       self    ,__SCREAMING_SNAKE_CASE=50257    ,__SCREAMING_SNAKE_CASE=1024    ,__SCREAMING_SNAKE_CASE=768    ,__SCREAMING_SNAKE_CASE=12    ,__SCREAMING_SNAKE_CASE=12    ,__SCREAMING_SNAKE_CASE=None    ,__SCREAMING_SNAKE_CASE="gelu_new"    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=1e-5    ,__SCREAMING_SNAKE_CASE=0.02    ,__SCREAMING_SNAKE_CASE="cls_index"    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=None    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=0.1    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=True    ,__SCREAMING_SNAKE_CASE=50256    ,__SCREAMING_SNAKE_CASE=50256    ,__SCREAMING_SNAKE_CASE=False    ,__SCREAMING_SNAKE_CASE=False    ,**__SCREAMING_SNAKE_CASE    ,):
    SCREAMING_SNAKE_CASE  :  Optional[Any]								=		vocab_size
    SCREAMING_SNAKE_CASE  :  Tuple								=		n_positions
    SCREAMING_SNAKE_CASE  :  int								=		n_embd
    SCREAMING_SNAKE_CASE  :  Dict								=		n_layer
    SCREAMING_SNAKE_CASE  :  Dict								=		n_head
    SCREAMING_SNAKE_CASE  :  List[Any]								=		n_inner
    SCREAMING_SNAKE_CASE  :  Dict								=		activation_function
    SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		resid_pdrop
    SCREAMING_SNAKE_CASE  :  str								=		embd_pdrop
    SCREAMING_SNAKE_CASE  :  int								=		attn_pdrop
    SCREAMING_SNAKE_CASE  :  Tuple								=		layer_norm_epsilon
    SCREAMING_SNAKE_CASE  :  Tuple								=		initializer_range
    SCREAMING_SNAKE_CASE  :  Any								=		summary_type
    SCREAMING_SNAKE_CASE  :  List[Any]								=		summary_use_proj
    SCREAMING_SNAKE_CASE  :  int								=		summary_activation
    SCREAMING_SNAKE_CASE  :  int								=		summary_first_dropout
    SCREAMING_SNAKE_CASE  :  List[Any]								=		summary_proj_to_labels
    SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		scale_attn_weights
    SCREAMING_SNAKE_CASE  :  Any								=		use_cache
    SCREAMING_SNAKE_CASE  :  List[str]								=		scale_attn_by_inverse_layer_idx
    SCREAMING_SNAKE_CASE  :  Any								=		reorder_and_upcast_attn

    SCREAMING_SNAKE_CASE  :  Tuple								=		bos_token_id
    SCREAMING_SNAKE_CASE  :  Optional[int]								=		eos_token_id

    super().__init__(bos_token_id=__SCREAMING_SNAKE_CASE    ,eos_token_id=__SCREAMING_SNAKE_CASE    ,**__SCREAMING_SNAKE_CASE      )



class 		_a     (							SCREAMING_SNAKE_CASE  ):







   """simple docstring"""




   def __init__(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE = "default"    ,__SCREAMING_SNAKE_CASE = None    ,__SCREAMING_SNAKE_CASE = False    ,):
    super().__init__(__SCREAMING_SNAKE_CASE    ,task=__SCREAMING_SNAKE_CASE    ,patching_specs=__SCREAMING_SNAKE_CASE    ,use_past=__SCREAMING_SNAKE_CASE      )
    if not getattr(self._config    ,'pad_token_id'    ,__SCREAMING_SNAKE_CASE      ):
     # TODO: how to do that better?
     SCREAMING_SNAKE_CASE  :  int								=		0






   @property
   def 					__a			(       self      ):
    SCREAMING_SNAKE_CASE  :  Dict								=		OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}      )
    if self.use_past:
     self.fill_with_past_key_values_(__SCREAMING_SNAKE_CASE    ,direction='inputs'      )
     SCREAMING_SNAKE_CASE  :  Dict								=		{0: 'batch', 1: 'past_sequence + sequence'}
    else:
     SCREAMING_SNAKE_CASE  :  Optional[Any]								=		{0: 'batch', 1: 'sequence'}

    return common_inputs






   @property
   def 					__a			(       self      ):
    return self._config.n_layer






   @property
   def 					__a			(       self      ):
    return self._config.n_head






   def 					__a			(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE = -1    ,__SCREAMING_SNAKE_CASE = -1    ,__SCREAMING_SNAKE_CASE = False    ,__SCREAMING_SNAKE_CASE = None    ,):
    SCREAMING_SNAKE_CASE  :  List[str]								=		super(__SCREAMING_SNAKE_CASE    ,self      ).generate_dummy_inputs(
        __SCREAMING_SNAKE_CASE    ,batch_size=__SCREAMING_SNAKE_CASE    ,seq_length=__SCREAMING_SNAKE_CASE    ,is_pair=__SCREAMING_SNAKE_CASE    ,framework=__SCREAMING_SNAKE_CASE      )

    # We need to order the input in the way they appears in the forward()
    SCREAMING_SNAKE_CASE  :  int								=		OrderedDict({'input_ids': common_inputs['input_ids']}      )

    # Need to add the past_keys
    if self.use_past:
     if not is_torch_available():
      raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
     else:
      import torch

      SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE  :  List[str]								=		common_inputs['input_ids'].shape
      # Not using the same length for past_key_values
      SCREAMING_SNAKE_CASE  :  str								=		seqlen + 2
      SCREAMING_SNAKE_CASE  :  Dict								=		(
          batch,
          self.num_attention_heads,
          past_key_values_length,
          self._config.hidden_size // self.num_attention_heads,
      )
      SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		[
          (torch.zeros(__SCREAMING_SNAKE_CASE      ), torch.zeros(__SCREAMING_SNAKE_CASE      )) for _ in range(self.num_layers      )
      ]

    SCREAMING_SNAKE_CASE  :  Optional[int]								=		common_inputs['attention_mask']
    if self.use_past:
     SCREAMING_SNAKE_CASE  :  Tuple								=		ordered_inputs['attention_mask'].dtype
     SCREAMING_SNAKE_CASE  :  Any								=		torch.cat(
         [ordered_inputs['attention_mask'], torch.ones(__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE    ,dtype=__SCREAMING_SNAKE_CASE      )]    ,dim=1      )

    return ordered_inputs






   @property
   def 					__a			(       self      ):
    return 13


