import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
				import transformer_engine.pytorch as te
def 					UpperCamelCase__		(							A__     ,					A__=True     ,					A__=True     ,					A__=True       )  ->   Tuple:
				if not is_fpa_available():
								raise ImportError('Using `convert_model` requires transformer_engine to be installed.'       )
				for name, module in model.named_children():
								if isinstance(A__     ,					nn.Linear       ) and to_transformer_engine and _convert_linear:
												# Return early if the linear layer weights are not multiples of 16
												if any(p % 16 != 0 for p in module.weight.shape       ):
																return
												snake_case__   :      Any   				= module.bias is not None
												snake_case__   :      Dict   				= te.Linear(
												    module.in_features     ,					module.out_features     ,					bias=A__     ,					params_dtype=module.weight.dtype       )
												snake_case__   :      Dict   				= module.weight.data.clone()
												if has_bias:
																snake_case__   :      Optional[Any]   				= module.bias.data.clone()

												setattr(A__     ,					A__     ,					A__       )
								elif isinstance(A__     ,					nn.LayerNorm       ) and to_transformer_engine and _convert_ln:
												snake_case__   :      Union[str, Any]   				= te.LayerNorm(module.normalized_shape[0]     ,					eps=module.eps     ,					params_dtype=module.weight.dtype       )
												snake_case__   :      Optional[Any]   				= module.weight.data.clone()
												snake_case__   :      List[str]   				= module.bias.data.clone()

												setattr(A__     ,					A__     ,					A__       )
								elif isinstance(A__     ,					te.Linear       ) and not to_transformer_engine and _convert_linear:
												snake_case__   :      Union[str, Any]   				= module.bias is not None
												snake_case__   :      int   				= nn.Linear(
												    module.in_features     ,					module.out_features     ,					bias=A__     ,					params_dtype=module.weight.dtype       )
												snake_case__   :      Optional[int]   				= module.weight.data.clone()
												if has_bias:
																snake_case__   :      int   				= module.bias.data.clone()

												setattr(A__     ,					A__     ,					A__       )
								elif isinstance(A__     ,					te.LayerNorm       ) and not to_transformer_engine and _convert_ln:
												snake_case__   :      Tuple   				= nn.LayerNorm(module.normalized_shape[0]     ,					eps=module.eps     ,					params_dtype=module.weight.dtype       )
												snake_case__   :      Optional[int]   				= module.weight.data.clone()
												snake_case__   :      Any   				= module.bias.data.clone()

												setattr(A__     ,					A__     ,					A__       )
								else:
												convert_model(
												    A__     ,					to_transformer_engine=A__     ,					_convert_linear=_convert_linear     ,					_convert_ln=_convert_ln     ,					)

def 					UpperCamelCase__		(							A__       )  ->   Union[str, Any]:
				if not is_fpa_available():
								raise ImportError('Using `has_transformer_engine_layers` requires transformer_engine to be installed.'       )
				for m in model.modules():
								if isinstance(A__     ,					(te.LayerNorm, te.Linear)       ):
												return True
				return False



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase__	:  int						=  {
    '''configuration_switch_transformers''': [
        '''SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''SwitchTransformersConfig''',
        '''SwitchTransformersOnnxConfig''',
    ]
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowerCAmelCase__	:  List[Any]						=  [
				    '''SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''SwitchTransformersEncoderModel''',
				    '''SwitchTransformersForConditionalGeneration''',
				    '''SwitchTransformersModel''',
				    '''SwitchTransformersPreTrainedModel''',
				    '''SwitchTransformersTop1Router''',
				    '''SwitchTransformersSparseMLP''',
				]


if TYPE_CHECKING:
				from .configuration_switch_transformers import (
				    SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP,
				    SwitchTransformersConfig,
				    SwitchTransformersOnnxConfig,
				)

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_switch_transformers import (
								    SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST,
								    SwitchTransformersEncoderModel,
								    SwitchTransformersForConditionalGeneration,
								    SwitchTransformersModel,
								    SwitchTransformersPreTrainedModel,
								    SwitchTransformersSparseMLP,
								    SwitchTransformersTopaRouter,
								)


else:
				import sys

				lowerCAmelCase__	:  Union[str, Any]						=  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


