


UpperCAmelCase_      : Optional[int]  					=       {
    'Pillow': 'Pillow<10.0.0',
    'accelerate': 'accelerate>=0.20.3',
    'av': 'av==9.2.0',
    'beautifulsoup4': 'beautifulsoup4',
    'black': 'black~=23.1',
    'codecarbon': 'codecarbon==1.2.0',
    'cookiecutter': 'cookiecutter==1.7.3',
    'dataclasses': 'dataclasses',
    'datasets': 'datasets!=2.5.0',
    'decord': 'decord==0.6.0',
    'deepspeed': 'deepspeed>=0.9.3',
    'diffusers': 'diffusers',
    'dill': 'dill<0.3.5',
    'evaluate': 'evaluate>=0.2.0',
    'fairscale': 'fairscale>0.3',
    'faiss-cpu': 'faiss-cpu',
    'fastapi': 'fastapi',
    'filelock': 'filelock',
    'flax': 'flax>=0.4.1,<=0.7.0',
    'ftfy': 'ftfy',
    'fugashi': 'fugashi>=1.0',
    'GitPython': 'GitPython<3.1.19',
    'hf-doc-builder': 'hf-doc-builder>=0.3.0',
    'huggingface-hub': 'huggingface-hub>=0.14.1,<1.0',
    'importlib_metadata': 'importlib_metadata',
    'ipadic': 'ipadic>=1.0.0,<2.0',
    'isort': 'isort>=5.5.4',
    'jax': 'jax>=0.2.8,!=0.3.2,<=0.4.13',
    'jaxlib': 'jaxlib>=0.1.65,<=0.4.13',
    'jieba': 'jieba',
    'kenlm': 'kenlm',
    'keras-nlp': 'keras-nlp>=0.3.1',
    'librosa': 'librosa',
    'nltk': 'nltk',
    'natten': 'natten>=0.14.6',
    'numpy': 'numpy>=1.17',
    'onnxconverter-common': 'onnxconverter-common',
    'onnxruntime-tools': 'onnxruntime-tools>=1.4.2',
    'onnxruntime': 'onnxruntime>=1.4.0',
    'opencv-python': 'opencv-python',
    'optuna': 'optuna',
    'optax': 'optax>=0.0.8,<=0.1.4',
    'packaging': 'packaging>=20.0',
    'parameterized': 'parameterized',
    'phonemizer': 'phonemizer',
    'protobuf': 'protobuf',
    'psutil': 'psutil',
    'pyyaml': 'pyyaml>=5.1',
    'pydantic': 'pydantic<2',
    'pytest': 'pytest>=7.2.0',
    'pytest-timeout': 'pytest-timeout',
    'pytest-xdist': 'pytest-xdist',
    'python': 'python>=3.8.0',
    'ray[tune]': 'ray[tune]',
    'regex': 'regex!=2019.12.17',
    'requests': 'requests',
    'rhoknp': 'rhoknp>=1.1.0,<1.3.1',
    'rjieba': 'rjieba',
    'rouge-score': 'rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1',
    'ruff': 'ruff>=0.0.241,<=0.0.259',
    'sacrebleu': 'sacrebleu>=1.4.12,<2.0.0',
    'sacremoses': 'sacremoses',
    'safetensors': 'safetensors>=0.3.1',
    'sagemaker': 'sagemaker>=2.31.0',
    'scikit-learn': 'scikit-learn',
    'sentencepiece': 'sentencepiece>=0.1.91,!=0.1.92',
    'sigopt': 'sigopt',
    'starlette': 'starlette',
    'sudachipy': 'sudachipy>=0.6.6',
    'sudachidict_core': 'sudachidict_core>=20220729',
    'tensorflow-cpu': 'tensorflow-cpu>=2.6,<2.14',
    'tensorflow': 'tensorflow>=2.6,<2.14',
    'tensorflow-text': 'tensorflow-text<2.14',
    'tf2onnx': 'tf2onnx',
    'timeout-decorator': 'timeout-decorator',
    'timm': 'timm',
    'tokenizers': 'tokenizers>=0.11.1,!=0.11.3,<0.14',
    'torch': 'torch>=1.9,!=1.12.0',
    'torchaudio': 'torchaudio',
    'torchvision': 'torchvision',
    'pyctcdecode': 'pyctcdecode>=0.4.0',
    'tqdm': 'tqdm>=4.27',
    'unidic': 'unidic>=1.0.2',
    'unidic_lite': 'unidic_lite>=1.0.7',
    'urllib3': 'urllib3<2.0.0',
    'uvicorn': 'uvicorn',
}






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_      : Dict  					=       logging.get_logger(__name__)

UpperCAmelCase_      : Union[str, Any]  					=       {
    'google/switch-base-8': 'https://huggingface.co/google/switch-base-8/blob/main/config.json',
}
class  SCREAMING_SNAKE_CASE__  (				lowercase__			):
      snake_case__				:     List[str]        =						'''switch_transformers'''
      snake_case__				:     Optional[int]        =						['''past_key_values''']
      snake_case__				:     Optional[Any]        =						{'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}
      def __init__(       self				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=3_2_1_2_8     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=7_6_8     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=6_4     ,							SCREAMING_SNAKE_CASE__				:       List[str]=2_0_4_8     ,							SCREAMING_SNAKE_CASE__				:       Dict=6_4     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=1_2     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=3     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=1_2     ,							SCREAMING_SNAKE_CASE__				:       str=3     ,							SCREAMING_SNAKE_CASE__				:       Tuple=1_2     ,							SCREAMING_SNAKE_CASE__				:       Tuple=8     ,							SCREAMING_SNAKE_CASE__				:       Dict=False     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=0.01     ,							SCREAMING_SNAKE_CASE__				:       str="float32"     ,							SCREAMING_SNAKE_CASE__				:       str=False     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=3_2     ,							SCREAMING_SNAKE_CASE__				:       Dict=1_2_8     ,							SCREAMING_SNAKE_CASE__				:       Dict=0.1     ,							SCREAMING_SNAKE_CASE__				:       Dict=1E-6     ,							SCREAMING_SNAKE_CASE__				:       Dict=0.001     ,							SCREAMING_SNAKE_CASE__				:       Any=0.001     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=1.0     ,							SCREAMING_SNAKE_CASE__				:       Any="relu"     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=True     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=False     ,							SCREAMING_SNAKE_CASE__				:       Dict=True     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=0     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=1     ,							**SCREAMING_SNAKE_CASE__				:       Dict     ,							)		->			Optional[Any]:
             a_					:		Optional[int]							=  vocab_size
             a_					:		List[str]							=  d_model
             a_					:		Tuple							=  d_kv
             a_					:		Optional[Any]							=  d_ff

             a_					:		List[Any]							=  num_sparse_encoder_layers

             a_					:		Any							=  num_layers
             a_					:		str							=  (
                 num_decoder_layers if num_decoder_layers is not None else self.num_layers
             )  # default = symmetry
             a_					:		List[Any]							=  num_sparse_decoder_layers

             # This tells us, each how many encoder layer we'll have to set a sparse layer.
             if self.num_sparse_encoder_layers > 0:
                    a_					:		Optional[int]							=  self.num_layers // self.num_sparse_encoder_layers
             else:
                    a_					:		List[Any]							=  self.num_layers  # HACK: this will create 0 sparse layers

             # This tells us, each how many encoder layer we'll have to set a sparse layer.
             if self.num_sparse_decoder_layers > 0:
                    a_					:		Union[str, Any]							=  self.num_decoder_layers // self.num_sparse_decoder_layers
             else:
                    a_					:		List[str]							=  self.num_decoder_layers  # HACK: this will create 0 sparse layers

             a_					:		Dict							=  num_heads
             a_					:		str							=  num_experts
             a_					:		Any							=  expert_capacity
             a_					:		List[Any]							=  router_bias
             a_					:		str							=  router_jitter_noise
             if router_dtype not in ["float32", "float16", "bfloat16"]:
                    raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""     )
             a_					:		Optional[int]							=  router_dtype

             a_					:		int							=  router_ignore_padding_tokens
             a_					:		Any							=  relative_attention_num_buckets
             a_					:		List[str]							=  relative_attention_max_distance

             a_					:		Optional[Any]							=  dropout_rate
             a_					:		Tuple							=  layer_norm_epsilon
             a_					:		Dict							=  initializer_factor
             a_					:		Any							=  feed_forward_proj
             a_					:		Tuple							=  use_cache
             a_					:		str							=  add_router_probs

             a_					:		Optional[int]							=  router_z_loss_coef
             a_					:		List[str]							=  router_aux_loss_coef

             a_					:		int							=  self.feed_forward_proj.split('-'     )
             a_					:		int							=  act_info[-1]
             a_					:		Optional[int]							=  act_info[0] == 'gated'

             if len(SCREAMING_SNAKE_CASE__     ) > 1 and act_info[0] != "gated" or len(SCREAMING_SNAKE_CASE__     ) > 2:
                    raise ValueError(
                        F"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
                        'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
                        '\'gated-gelu\' or \'relu\''     )

             # for backwards compatibility
             if feed_forward_proj == "gated-gelu":
                    a_					:		Any							=  'gelu_new'

             super().__init__(
                 pad_token_id=SCREAMING_SNAKE_CASE__     ,							eos_token_id=SCREAMING_SNAKE_CASE__     ,							is_encoder_decoder=SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     ,							)


