






from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


UpperCAmelCase__				   =		logging.get_logger(__name__)

UpperCAmelCase__				   =		{
    "EleutherAI/gpt-neo-1.3B": "https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json",
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  '''gpt_neo'''
      __snake_case       =  ['''past_key_values''']
      __snake_case       =  {'''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}







      def __init__(      self      :						Dict ,					__UpperCAmelCase      :						str=50_257 ,					__UpperCAmelCase      :						Optional[int]=2_048 ,					__UpperCAmelCase      :						Optional[Any]=2_048 ,					__UpperCAmelCase      :						Optional[int]=24 ,					__UpperCAmelCase      :						Optional[int]=[[["global", "local"], 12]] ,					__UpperCAmelCase      :						str=16 ,					__UpperCAmelCase      :						Any=None ,					__UpperCAmelCase      :						List[str]=256 ,					__UpperCAmelCase      :						Optional[Any]="gelu_new" ,					__UpperCAmelCase      :						Tuple=0.0 ,					__UpperCAmelCase      :						Optional[int]=0.0 ,					__UpperCAmelCase      :						Union[str, Any]=0.0 ,					__UpperCAmelCase      :						Optional[Any]=0.1 ,					__UpperCAmelCase      :						Any=1e-5 ,					__UpperCAmelCase      :						Dict=0.02 ,					__UpperCAmelCase      :						Optional[int]=True ,					__UpperCAmelCase      :						Tuple=50_256 ,					__UpperCAmelCase      :						Optional[int]=50_256 ,					**__UpperCAmelCase      :						Optional[int] ,					)    ->Union[str, Any]:




       """simple docstring"""


       a		  =      vocab_size
       a		  =      max_position_embeddings
       a		  =      hidden_size
       a		  =      num_layers
       a		  =      num_heads
       a		  =      intermediate_size
       a		  =      window_size
       a		  =      activation_function
       a		  =      resid_dropout
       a		  =      embed_dropout
       a		  =      attention_dropout
       a		  =      classifier_dropout
       a		  =      layer_norm_epsilon
       a		  =      initializer_range
       a		  =      use_cache

       a		  =      bos_token_id
       a		  =      eos_token_id

       a		  =      attention_types
       a		  =      self.expand_attention_types_params(__UpperCAmelCase  )

       if len(self.attention_layers  ) != self.num_layers:
        raise ValueError(
            '''Configuration for convolutional module is incorrect. '''
            '''It is required that `len(config.attention_layers)` == `config.num_layers` '''
            F"""but is `len(config.attention_layers) = {len(self.attention_layers  )}`, """
            F"""`config.num_layers = {self.num_layers}`. """
            '''`config.attention_layers` is prepared using `config.attention_types`. '''
            '''Please verify the value of `config.attention_types` argument.'''  )

       super().__init__(bos_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					**__UpperCAmelCase  )







      @staticmethod
      def       __lowerCAmelCase					(      __UpperCAmelCase      :						List[str]  )    ->Optional[int]:




       """simple docstring"""


       a		  =      []
       for item in attention_types:
        for _ in range(item[1]  ):
         attentions.extend(item[0]  )
       return attentions
def 						_a    (      a :str	,							a :Any	,							a :Union[str, Any]	,							a :Optional[int] )							->			List[Any]:
 import torch

 a		  =      input.size()
 a		  =      len(a )
 a		  =      shape[dimension]

 a		  =      torch.arange(0	,							a	,							a )
 a		  =      torch.div(sizedim - size	,							a	,							rounding_mode='''floor''' ) + 1
 a		  =      torch.arange(a ) + low_indices[:min_length][:, None]

 a		  =      [slice(a )] * rank
 a		  =      indices
 a		  =      input[s]

 a		  =      list(range(0	,							rank + 1 ) )
 perm.append(perm.pop(dimension + 1 ) )

 return sliced.permute(a )
def 						_a    (      a :List[str]	,							a :List[str] )							->			Any:
 import torch

 a		  =      torch.arange(1	,							a )
 a		  =      torch.remainder(a	,							a )
 a		  =      remainders == 0
 a		  =      candidates[divisor_indices]
 a		  =      torch.max(a )
 return largest_divisor, torch.div(a	,							a	,							rounding_mode='''floor''' )






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      @property
      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Mapping[str, Mapping[int, str]]:




       """simple docstring"""


       a		  =      OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}  )
       if self.use_past:
        self.fill_with_past_key_values_(__UpperCAmelCase ,					direction='''inputs'''  )
        a		  =      {0: '''batch''', 1: '''past_sequence + sequence'''}
       else:
        a		  =      {0: '''batch''', 1: '''sequence'''}

       return common_inputs







      @property
      def       __lowerCAmelCase					(      self      :						int  )    ->int:




       """simple docstring"""


       return self._config.num_heads







      def       __lowerCAmelCase					(      self      :						Optional[int] ,					__UpperCAmelCase      :						PreTrainedTokenizer ,					__UpperCAmelCase      :						int = -1 ,					__UpperCAmelCase      :						int = -1 ,					__UpperCAmelCase      :						bool = False ,					__UpperCAmelCase      :						Optional[TensorType] = None ,					)    ->Mapping[str, Any]:




       """simple docstring"""


       a		  =      super(__UpperCAmelCase ,					self  ).generate_dummy_inputs(
           __UpperCAmelCase ,					batch_size=__UpperCAmelCase ,					seq_length=__UpperCAmelCase ,					is_pair=__UpperCAmelCase ,					framework=__UpperCAmelCase  )

       # We need to order the input in the way they appears in the forward()
       a		  =      OrderedDict({'''input_ids''': common_inputs['''input_ids''']}  )

       # Need to add the past_keys
       if self.use_past:
        if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''  )
        else:
         import torch

         a  ,     a		  =      common_inputs['''input_ids'''].shape
         # Not using the same length for past_key_values
         a		  =      seqlen + 2
         a		  =      (
             batch,
             self.num_attention_heads,
             past_key_values_length,
             self._config.hidden_size // self.num_attention_heads,
         )
         a		  =      [
             (torch.zeros(__UpperCAmelCase  ), torch.zeros(__UpperCAmelCase  )) for _ in range(self.num_layers  )
         ]

       a		  =      common_inputs['''attention_mask''']
       if self.use_past:
        a		  =      ordered_inputs['''attention_mask'''].dtype
        a		  =      torch.cat(
            [ordered_inputs['''attention_mask'''], torch.ones(__UpperCAmelCase ,					__UpperCAmelCase ,					dtype=__UpperCAmelCase  )] ,					dim=1  )

       return ordered_inputs







      @property
      def       __lowerCAmelCase					(      self      :						Dict  )    ->int:




       """simple docstring"""


       return 13










from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


UpperCAmelCase__				   =		logging.get_logger(__name__)

UpperCAmelCase__				   =		{
    "t5-small": "https://huggingface.co/t5-small/resolve/main/config.json",
    "t5-base": "https://huggingface.co/t5-base/resolve/main/config.json",
    "t5-large": "https://huggingface.co/t5-large/resolve/main/config.json",
    "t5-3b": "https://huggingface.co/t5-3b/resolve/main/config.json",
    "t5-11b": "https://huggingface.co/t5-11b/resolve/main/config.json",
}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  '''t5'''
      __snake_case       =  ['''past_key_values''']
      __snake_case       =  {'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}







      def __init__(      self      :						Optional[Any] ,					__UpperCAmelCase      :						Optional[Any]=32_128 ,					__UpperCAmelCase      :						List[Any]=512 ,					__UpperCAmelCase      :						Dict=64 ,					__UpperCAmelCase      :						Tuple=2_048 ,					__UpperCAmelCase      :						int=6 ,					__UpperCAmelCase      :						Optional[int]=None ,					__UpperCAmelCase      :						Optional[int]=8 ,					__UpperCAmelCase      :						str=32 ,					__UpperCAmelCase      :						Tuple=128 ,					__UpperCAmelCase      :						Optional[Any]=0.1 ,					__UpperCAmelCase      :						int=1e-6 ,					__UpperCAmelCase      :						int=1.0 ,					__UpperCAmelCase      :						List[str]="relu" ,					__UpperCAmelCase      :						int=True ,					__UpperCAmelCase      :						int=True ,					__UpperCAmelCase      :						List[Any]=0 ,					__UpperCAmelCase      :						int=1 ,					**__UpperCAmelCase      :						str ,					)    ->Optional[Any]:




       """simple docstring"""


       a		  =      vocab_size
       a		  =      d_model
       a		  =      d_kv
       a		  =      d_ff
       a		  =      num_layers
       a		  =      (
           num_decoder_layers if num_decoder_layers is not None else self.num_layers
       )  # default = symmetry
       a		  =      num_heads
       a		  =      relative_attention_num_buckets
       a		  =      relative_attention_max_distance
       a		  =      dropout_rate
       a		  =      layer_norm_epsilon
       a		  =      initializer_factor
       a		  =      feed_forward_proj
       a		  =      use_cache

       a		  =      self.feed_forward_proj.split('''-'''  )
       a		  =      act_info[-1]
       a		  =      act_info[0] == '''gated'''

       if len(__UpperCAmelCase  ) > 1 and act_info[0] != "gated" or len(__UpperCAmelCase  ) > 2:
        raise ValueError(
            F"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
            '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
            '''\'gated-gelu\' or \'relu\''''  )

       # for backwards compatibility
       if feed_forward_proj == "gated-gelu":
        a		  =      '''gelu_new'''

       super().__init__(
           pad_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					is_encoder_decoder=__UpperCAmelCase ,					**__UpperCAmelCase ,					)






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      @property
      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->Mapping[str, Mapping[int, str]]:




       """simple docstring"""


       a		  =      {
           '''input_ids''': {0: '''batch''', 1: '''encoder_sequence'''},
           '''attention_mask''': {0: '''batch''', 1: '''encoder_sequence'''},
       }
       if self.use_past:
        a		  =      '''past_encoder_sequence + sequence'''
        a		  =      {0: '''batch'''}
        a		  =      {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
       else:
        a		  =      {0: '''batch''', 1: '''decoder_sequence'''}
        a		  =      {0: '''batch''', 1: '''decoder_sequence'''}

       if self.use_past:
        self.fill_with_past_key_values_(__UpperCAmelCase ,					direction='''inputs'''  )

       return common_inputs







      @property
      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->int:




       """simple docstring"""


       return 13


