


"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE__				:  str						= {
    "configuration_switch_transformers": [
        "SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "SwitchTransformersConfig",
        "SwitchTransformersOnnxConfig",
    ]
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE__				:  Union[str, Any]						= [
     "SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST",
     "SwitchTransformersEncoderModel",
     "SwitchTransformersForConditionalGeneration",
     "SwitchTransformersModel",
     "SwitchTransformersPreTrainedModel",
     "SwitchTransformersTop1Router",
     "SwitchTransformersSparseMLP",
 ]


if TYPE_CHECKING:
 from .configuration_switch_transformers import (
     SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP,
     SwitchTransformersConfig,
     SwitchTransformersOnnxConfig,
 )

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_switch_transformers import (
      SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST,
      SwitchTransformersEncoderModel,
      SwitchTransformersForConditionalGeneration,
      SwitchTransformersModel,
      SwitchTransformersPreTrainedModel,
      SwitchTransformersSparseMLP,
      SwitchTransformersTopaRouter,
  )


else:
 import sys

 SCREAMING_SNAKE_CASE__				:  Dict						= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase				 =   logging.get_logger(__name__)

UpperCAmelCase				 =   {
    """facebook/opt-125m""": """https://huggingface.co/facebook/opt-125m/blob/main/config.json""",
    """facebook/opt-350m""": """https://huggingface.co/facebook/opt-350m/blob/main/config.json""",
    """facebook/opt-1.3b""": """https://huggingface.co/facebook/opt-1.3b/blob/main/config.json""",
    """facebook/opt-2.7b""": """https://huggingface.co/facebook/opt-2.7b/blob/main/config.json""",
    """facebook/opt-6.7b""": """https://huggingface.co/facebook/opt-6.7b/blob/main/config.json""",
    """facebook/opt-13b""": """https://huggingface.co/facebook/opt-13b/blob/main/config.json""",
}






class 				_SCREAMING_SNAKE_CASE( snake_case__						):
   UpperCamelCase_					:    List[str]			      = 'opt'
   UpperCamelCase_					:    List[str]			      = ['past_key_values']
   def __init__(     self				,  lowerCAmelCase_=5_02_72				,  lowerCAmelCase_=7_68				,  lowerCAmelCase_=12				,  lowerCAmelCase_=30_72				,  lowerCAmelCase_=20_48				,  lowerCAmelCase_=True				,  lowerCAmelCase_=False				,  lowerCAmelCase_=None				,  lowerCAmelCase_=0.1				,  lowerCAmelCase_=0.0				,  lowerCAmelCase_=12				,  lowerCAmelCase_="relu"				,  lowerCAmelCase_=0.0				,  lowerCAmelCase_=0.02				,  lowerCAmelCase_=True				,  lowerCAmelCase_=1				,  lowerCAmelCase_=2				,  lowerCAmelCase_=2				,  lowerCAmelCase_=True				,  lowerCAmelCase_=True				,  **lowerCAmelCase_				,  )						->Any:
        super().__init__(
            pad_token_id=lowerCAmelCase_				,  bos_token_id=lowerCAmelCase_				,  eos_token_id=lowerCAmelCase_				,  **lowerCAmelCase_				,  )
        a	:     Dict						      =    vocab_size
        a	:     int						      =    max_position_embeddings
        a	:     List[Any]						      =    num_attention_heads
        a	:     Union[str, Any]						      =    word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
        a	:     Any						      =    ffn_dim
        a	:     Optional[int]						      =    hidden_size
        a	:     Optional[Any]						      =    num_hidden_layers
        a	:     Optional[int]						      =    dropout
        a	:     Optional[Any]						      =    attention_dropout
        a	:     Union[str, Any]						      =    activation_function
        a	:     Any						      =    init_std
        a	:     List[Any]						      =    layerdrop
        a	:     str						      =    use_cache
        a	:     Any						      =    do_layer_norm_before
        # We keep these variables at `True` for backward compatibility.
        a	:     List[Any]						      =    enable_bias
        a	:     List[str]						      =    layer_norm_elementwise_affine

        # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
        # with checkpoints that have been fine-tuned before transformers v4.20.1
        # see https://github.com/facebookresearch/metaseq/pull/164
        a	:     Dict						      =    _remove_final_layer_norm