






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__				   =		logging.get_logger(__name__)

UpperCAmelCase__				   =		{
    "google/switch-base-8": "https://huggingface.co/google/switch-base-8/blob/main/config.json",
}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  '''switch_transformers'''
      __snake_case       =  ['''past_key_values''']
      __snake_case       =  {'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}







      def __init__(      self      :						Dict ,					__UpperCAmelCase      :						List[Any]=32_128 ,					__UpperCAmelCase      :						List[str]=768 ,					__UpperCAmelCase      :						str=64 ,					__UpperCAmelCase      :						Dict=2_048 ,					__UpperCAmelCase      :						int=64 ,					__UpperCAmelCase      :						str=12 ,					__UpperCAmelCase      :						Tuple=3 ,					__UpperCAmelCase      :						str=12 ,					__UpperCAmelCase      :						List[str]=3 ,					__UpperCAmelCase      :						Dict=12 ,					__UpperCAmelCase      :						List[str]=8 ,					__UpperCAmelCase      :						Union[str, Any]=False ,					__UpperCAmelCase      :						List[Any]=0.01 ,					__UpperCAmelCase      :						Any="float32" ,					__UpperCAmelCase      :						Any=False ,					__UpperCAmelCase      :						int=32 ,					__UpperCAmelCase      :						str=128 ,					__UpperCAmelCase      :						Optional[int]=0.1 ,					__UpperCAmelCase      :						Optional[Any]=1e-6 ,					__UpperCAmelCase      :						Optional[int]=0.001 ,					__UpperCAmelCase      :						Any=0.001 ,					__UpperCAmelCase      :						List[Any]=1.0 ,					__UpperCAmelCase      :						List[Any]="relu" ,					__UpperCAmelCase      :						Optional[int]=True ,					__UpperCAmelCase      :						Tuple=False ,					__UpperCAmelCase      :						str=True ,					__UpperCAmelCase      :						Any=0 ,					__UpperCAmelCase      :						str=1 ,					**__UpperCAmelCase      :						List[Any] ,					)    ->Optional[int]:




       """simple docstring"""


       a		  =      vocab_size
       a		  =      d_model
       a		  =      d_kv
       a		  =      d_ff

       a		  =      num_sparse_encoder_layers

       a		  =      num_layers
       a		  =      (
           num_decoder_layers if num_decoder_layers is not None else self.num_layers
       )  # default = symmetry
       a		  =      num_sparse_decoder_layers

       # This tells us, each how many encoder layer we'll have to set a sparse layer.
       if self.num_sparse_encoder_layers > 0:
        a		  =      self.num_layers // self.num_sparse_encoder_layers
       else:
        a		  =      self.num_layers  # HACK: this will create 0 sparse layers

       # This tells us, each how many encoder layer we'll have to set a sparse layer.
       if self.num_sparse_decoder_layers > 0:
        a		  =      self.num_decoder_layers // self.num_sparse_decoder_layers
       else:
        a		  =      self.num_decoder_layers  # HACK: this will create 0 sparse layers

       a		  =      num_heads
       a		  =      num_experts
       a		  =      expert_capacity
       a		  =      router_bias
       a		  =      router_jitter_noise
       if router_dtype not in ["float32", "float16", "bfloat16"]:
        raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""  )
       a		  =      router_dtype

       a		  =      router_ignore_padding_tokens
       a		  =      relative_attention_num_buckets
       a		  =      relative_attention_max_distance

       a		  =      dropout_rate
       a		  =      layer_norm_epsilon
       a		  =      initializer_factor
       a		  =      feed_forward_proj
       a		  =      use_cache
       a		  =      add_router_probs

       a		  =      router_z_loss_coef
       a		  =      router_aux_loss_coef

       a		  =      self.feed_forward_proj.split('''-'''  )
       a		  =      act_info[-1]
       a		  =      act_info[0] == '''gated'''

       if len(__UpperCAmelCase  ) > 1 and act_info[0] != "gated" or len(__UpperCAmelCase  ) > 2:
        raise ValueError(
            F"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
            '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
            '''\'gated-gelu\' or \'relu\''''  )

       # for backwards compatibility
       if feed_forward_proj == "gated-gelu":
        a		  =      '''gelu_new'''

       super().__init__(
           pad_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					is_encoder_decoder=__UpperCAmelCase ,					**__UpperCAmelCase ,					)










from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch






@require_torch
@require_tf
class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      def       __lowerCAmelCase					(      self      :						str  )    ->int:




       """simple docstring"""


       a		  =      SMALL_MODEL_IDENTIFIER
       a		  =      '''pt'''
       a		  =      '''tf'''







      def       __lowerCAmelCase					(      self      :						Optional[Any] ,					__UpperCAmelCase      :						str  )    ->Union[str, Any]:




       """simple docstring"""


       a		  =      AutoModel.from_pretrained(self.test_model  )
       model_pt.save_pretrained(__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Any ,					__UpperCAmelCase      :						Union[str, Any]  )    ->List[str]:




       """simple docstring"""


       a		  =      TFAutoModel.from_pretrained(self.test_model ,					from_pt=__UpperCAmelCase  )
       model_tf.save_pretrained(__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->int:




       """simple docstring"""


       a		  =      '''mock_framework'''

       # Framework provided - return whatever the user provides
       a		  =      FeaturesManager.determine_framework(self.test_model ,					__UpperCAmelCase  )
       self.assertEqual(__UpperCAmelCase ,					__UpperCAmelCase  )

       # Local checkpoint and framework provided - return provided framework
       # PyTorch checkpoint
       with TemporaryDirectory() as local_pt_ckpt:
        self._setup_pt_ckpt(__UpperCAmelCase  )
        a		  =      FeaturesManager.determine_framework(__UpperCAmelCase ,					__UpperCAmelCase  )
        self.assertEqual(__UpperCAmelCase ,					__UpperCAmelCase  )

       # TensorFlow checkpoint
       with TemporaryDirectory() as local_tf_ckpt:
        self._setup_tf_ckpt(__UpperCAmelCase  )
        a		  =      FeaturesManager.determine_framework(__UpperCAmelCase ,					__UpperCAmelCase  )
        self.assertEqual(__UpperCAmelCase ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						str  )    ->int:




       """simple docstring"""


       with TemporaryDirectory() as local_pt_ckpt:
        self._setup_pt_ckpt(__UpperCAmelCase  )
        a		  =      FeaturesManager.determine_framework(__UpperCAmelCase  )
        self.assertEqual(__UpperCAmelCase ,					self.framework_pt  )

       # TensorFlow checkpoint
       with TemporaryDirectory() as local_tf_ckpt:
        self._setup_tf_ckpt(__UpperCAmelCase  )
        a		  =      FeaturesManager.determine_framework(__UpperCAmelCase  )
        self.assertEqual(__UpperCAmelCase ,					self.framework_tf  )

       # Invalid local checkpoint
       with TemporaryDirectory() as local_invalid_ckpt:
        with self.assertRaises(__UpperCAmelCase  ):
         a		  =      FeaturesManager.determine_framework(__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->List[str]:




       """simple docstring"""


       a		  =      MagicMock(return_value=__UpperCAmelCase  )
       with patch('''transformers.onnx.features.is_tf_available''' ,					__UpperCAmelCase  ):
        a		  =      FeaturesManager.determine_framework(self.test_model  )
        self.assertEqual(__UpperCAmelCase ,					self.framework_pt  )

       # PyTorch not in environment -> use TensorFlow
       a		  =      MagicMock(return_value=__UpperCAmelCase  )
       with patch('''transformers.onnx.features.is_torch_available''' ,					__UpperCAmelCase  ):
        a		  =      FeaturesManager.determine_framework(self.test_model  )
        self.assertEqual(__UpperCAmelCase ,					self.framework_tf  )

       # Both in environment -> use PyTorch
       a		  =      MagicMock(return_value=__UpperCAmelCase  )
       a		  =      MagicMock(return_value=__UpperCAmelCase  )
       with patch('''transformers.onnx.features.is_tf_available''' ,					__UpperCAmelCase  ), patch(
           '''transformers.onnx.features.is_torch_available''' ,					__UpperCAmelCase  ):
        a		  =      FeaturesManager.determine_framework(self.test_model  )
        self.assertEqual(__UpperCAmelCase ,					self.framework_pt  )

       # Both not in environment -> raise error
       a		  =      MagicMock(return_value=__UpperCAmelCase  )
       a		  =      MagicMock(return_value=__UpperCAmelCase  )
       with patch('''transformers.onnx.features.is_tf_available''' ,					__UpperCAmelCase  ), patch(
           '''transformers.onnx.features.is_torch_available''' ,					__UpperCAmelCase  ):
        with self.assertRaises(__UpperCAmelCase  ):
         a		  =      FeaturesManager.determine_framework(self.test_model  )


