


"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case__						:   str			 = logging.get_logger(__name__)

snake_case__						:   str			 = {
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}



class 						snake_case_(				a__	):
     __UpperCamelCase     =    '''falcon'''
     __UpperCamelCase     =    ['''past_key_values''']






     def __init__(		self     :    Tuple    , UpperCamelCase_     :    Tuple=6_5_0_2_4    , UpperCamelCase_     :    List[Any]=4_5_4_4    , UpperCamelCase_     :    Optional[Any]=3_2    , UpperCamelCase_     :    Dict=7_1    , UpperCamelCase_     :    int=1E-5    , UpperCamelCase_     :    str=0.02    , UpperCamelCase_     :    Optional[int]=True    , UpperCamelCase_     :    Optional[Any]=0.0    , UpperCamelCase_     :    List[str]=0.0    , UpperCamelCase_     :    Dict=None    , UpperCamelCase_     :    str=False    , UpperCamelCase_     :    List[Any]=False    , UpperCamelCase_     :    int=True    , UpperCamelCase_     :    Optional[int]=True    , UpperCamelCase_     :    int=False    , UpperCamelCase_     :    Optional[int]=1_1    , UpperCamelCase_     :    Dict=1_1    , **UpperCamelCase_     :    Any    , ):
           lowerCAmelCase    :		List[Any]				=      vocab_size
           # Backward compatibility with n_embed kwarg
           lowerCAmelCase    :		Union[str, Any]				=      kwargs.pop('''n_embed'''    , UpperCamelCase_ )
           lowerCAmelCase    :		str				=      hidden_size if n_embed is None else n_embed
           lowerCAmelCase    :		List[Any]				=      num_hidden_layers
           lowerCAmelCase    :		Optional[Any]				=      num_attention_heads
           lowerCAmelCase    :		List[str]				=      layer_norm_epsilon
           lowerCAmelCase    :		Tuple				=      initializer_range
           lowerCAmelCase    :		str				=      use_cache
           lowerCAmelCase    :		Optional[Any]				=      hidden_dropout
           lowerCAmelCase    :		int				=      attention_dropout

           lowerCAmelCase    :		Dict				=      bos_token_id
           lowerCAmelCase    :		str				=      eos_token_id
           lowerCAmelCase    :		Any				=      num_attention_heads if num_kv_heads is None else num_kv_heads
           lowerCAmelCase    :		int				=      alibi
           lowerCAmelCase    :		Tuple				=      new_decoder_architecture
           lowerCAmelCase    :		int				=      multi_query  # Ignored when new_decoder_architecture is True
           lowerCAmelCase    :		Dict				=      parallel_attn
           lowerCAmelCase    :		Any				=      bias

           super().__init__(bos_token_id=UpperCamelCase_    , eos_token_id=UpperCamelCase_    , **UpperCamelCase_ )






     @property
     def    lowerCamelCase__   (		self     :    Optional[Any] ):
           return self.hidden_size // self.num_attention_heads






     @property
     def    lowerCamelCase__   (		self     :    Optional[Any] ):
           return not self.alibi






"""simple docstring"""





import numpy as np


def    _snake_case  (  _snake_case :	np.array	):
      return 1 / (1 + np.exp(-vector	))


if __name__ == "__main__":
 import doctest

 doctest.testmod()


