





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A									=				logging.get_logger(__name__)

__A									=				{
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			"open-llama"

 def __init__(self				:   Dict       , UpperCAmelCase_				:   Optional[Any]=100_000       , UpperCAmelCase_				:   Union[str, Any]=4_096       , UpperCAmelCase_				:   Union[str, Any]=11_008       , UpperCAmelCase_				:   Optional[Any]=32       , UpperCAmelCase_				:   Any=32       , UpperCAmelCase_				:   str="silu"       , UpperCAmelCase_				:   Any=2_048       , UpperCAmelCase_				:   Union[str, Any]=0.02       , UpperCAmelCase_				:   int=1E-6       , UpperCAmelCase_				:   Dict=True       , UpperCAmelCase_				:   List[Any]=0       , UpperCAmelCase_				:   List[str]=1       , UpperCAmelCase_				:   Tuple=2       , UpperCAmelCase_				:   Tuple=False       , UpperCAmelCase_				:   List[Any]=True       , UpperCAmelCase_				:   List[Any]=0.1       , UpperCAmelCase_				:   Any=0.1       , UpperCAmelCase_				:   Tuple=True       , UpperCAmelCase_				:   Optional[Any]=True       , UpperCAmelCase_				:   List[str]=None       , **UpperCAmelCase_				:   int       , )       ->Union[str, Any]:



   '''simple docstring'''



   lowerCamelCase__:      int			  =vocab_size
   lowerCamelCase__:      Dict			  =max_position_embeddings
   lowerCamelCase__:      str			  =hidden_size
   lowerCamelCase__:      Union[str, Any]			  =intermediate_size
   lowerCamelCase__:      Any			  =num_hidden_layers
   lowerCamelCase__:      Union[str, Any]			  =num_attention_heads
   lowerCamelCase__:      List[str]			  =hidden_act
   lowerCamelCase__:      Optional[Any]			  =initializer_range
   lowerCamelCase__:      str			  =rms_norm_eps
   lowerCamelCase__:      Tuple			  =use_cache
   lowerCamelCase__:      str			  =kwargs.pop(
       "use_memorry_efficient_attention"       , UpperCAmelCase_)
   lowerCamelCase__:      str			  =hidden_dropout_prob
   lowerCamelCase__:      Optional[Any]			  =attention_dropout_prob
   lowerCamelCase__:      Dict			  =use_stable_embedding
   lowerCamelCase__:      int			  =shared_input_output_embedding
   lowerCamelCase__:      Dict			  =rope_scaling
   self._rope_scaling_validation()

   super().__init__(
       pad_token_id=UpperCAmelCase_       , bos_token_id=UpperCAmelCase_       , eos_token_id=UpperCAmelCase_       , tie_word_embeddings=UpperCAmelCase_       , **UpperCAmelCase_       , )







 def 							SCREAMING_SNAKE_CASE_					(self				:   int)       ->Any:



   '''simple docstring'''



   if self.rope_scaling is None:
     return

   if not isinstance(self.rope_scaling       , UpperCAmelCase_) or len(self.rope_scaling) != 2:
     raise ValueError(
         "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
         F"""got {self.rope_scaling}""")
   lowerCamelCase__:      Any			  =self.rope_scaling.get("type"       , UpperCAmelCase_)
   lowerCamelCase__:      Tuple			  =self.rope_scaling.get("factor"       , UpperCAmelCase_)
   if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
     raise ValueError(
         F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}""")
   if rope_scaling_factor is None or not isinstance(UpperCAmelCase_       , UpperCAmelCase_) or rope_scaling_factor <= 1.0:
     raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}""")








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__A									=				{"configuration_ibert": ["IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "IBertConfig", "IBertOnnxConfig"]}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __A									=				[
         "IBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
         "IBertForMaskedLM",
         "IBertForMultipleChoice",
         "IBertForQuestionAnswering",
         "IBertForSequenceClassification",
         "IBertForTokenClassification",
         "IBertModel",
         "IBertPreTrainedModel",
     ]

if TYPE_CHECKING:
     from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_ibert import (
              IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              IBertForMaskedLM,
              IBertForMultipleChoice,
              IBertForQuestionAnswering,
              IBertForSequenceClassification,
              IBertForTokenClassification,
              IBertModel,
              IBertPreTrainedModel,
          )

else:
     import sys

     __A									=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

