
from typing import Optional, Union

import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models.modeling_utils import ModelMixin






class   UpperCAmelCase		(						A_       ,A_ ):



   @register_to_config
   def __init__(self		:					int    ,						snake_case__		:					int = 7_68    ,						)		->	Dict:


          '''simple docstring'''


          super().__init__()

          snake_case       :   int							=      nn.Parameter(torch.zeros(1    ,						snake_case__ ) )
          snake_case       :   Any							=      nn.Parameter(torch.ones(1    ,						snake_case__ ) )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any]    ,						snake_case__		:					Optional[Union[str, torch.device]] = None    ,						snake_case__		:					Optional[torch.dtype] = None    ,						)		->	int:


          '''simple docstring'''


          snake_case       :   Dict							=      nn.Parameter(self.mean.to(snake_case__ ).to(snake_case__ ) )
          snake_case       :   List[str]							=      nn.Parameter(self.std.to(snake_case__ ).to(snake_case__ ) )
          return self



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[Any]    ,						snake_case__		:					List[str] )		->	Union[str, Any]:


          '''simple docstring'''


          snake_case       :   str							=      (embeds - self.mean) * 1.0 / self.std
          return embeds



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict    ,						snake_case__		:					int )		->	Dict:


          '''simple docstring'''


          snake_case       :   List[str]							=      (embeds * self.std) + self.mean
          return embeds







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase							=       logging.get_logger(__name__)

__lowerCamelCase							=       {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}






class   UpperCAmelCase		(						A_ ):
   A__					:		List[str]						     =     "megatron-bert"



   def __init__(self		:					Optional[int]    ,						snake_case__		:					List[str]=2_90_56    ,						snake_case__		:					List[Any]=10_24    ,						snake_case__		:					str=24    ,						snake_case__		:					Tuple=16    ,						snake_case__		:					Union[str, Any]=40_96    ,						snake_case__		:					str="gelu"    ,						snake_case__		:					str=0.1    ,						snake_case__		:					Optional[int]=0.1    ,						snake_case__		:					Tuple=5_12    ,						snake_case__		:					Union[str, Any]=2    ,						snake_case__		:					Dict=0.02    ,						snake_case__		:					List[Any]=1e-12    ,						snake_case__		:					int=0    ,						snake_case__		:					Tuple="absolute"    ,						snake_case__		:					Any=True    ,						**snake_case__		:					Union[str, Any]    ,						)		->	Optional[Any]:


          '''simple docstring'''


          super().__init__(pad_token_id=snake_case__    ,						**snake_case__ )

          snake_case       :   Tuple							=      vocab_size
          snake_case       :   str							=      hidden_size
          snake_case       :   str							=      num_hidden_layers
          snake_case       :   str							=      num_attention_heads
          snake_case       :   Optional[int]							=      hidden_act
          snake_case       :   int							=      intermediate_size
          snake_case       :   List[str]							=      hidden_dropout_prob
          snake_case       :   Union[str, Any]							=      attention_probs_dropout_prob
          snake_case       :   Dict							=      max_position_embeddings
          snake_case       :   List[str]							=      type_vocab_size
          snake_case       :   List[str]							=      initializer_range
          snake_case       :   Tuple							=      layer_norm_eps
          snake_case       :   int							=      position_embedding_type
          snake_case       :   str							=      use_cache





