





import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__lowercase							:Union[str, Any]							=	logging.get_logger(__name__)



class  _a  (					nn.Module				):






							"""simple docstring"""

							def __init__(   self					:  Any     )						->Optional[Any]:
													super().__init__()
													if version.parse(torch.__version__     ) < version.parse("1.12.0"     ):
																			raise ImportError(
																			    f"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
																			    "PytorchGELUTanh. Please upgrade torch."     )






							def   A_	(   self					:  Dict				,						a					:  Tensor     )						->Tensor:
													return nn.functional.gelu(UpperCAmelCase_				,						approximate="tanh"     )







class  _a  (					nn.Module				):






							"""simple docstring"""

							def   A_	(   self					:  Optional[Any]				,						a					:  Tensor     )						->Tensor:
													return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi     ) * (input + 0.04_4715 * torch.pow(UpperCAmelCase_				,						3.0     ))     ))







class  _a  (					nn.Module				):






							"""simple docstring"""

							def __init__(   self					:  List[str]				,						a					:  bool = False     )						->List[Any]:
													super().__init__()
													if use_gelu_python:
																			SCREAMING_SNAKE_CASE__  :					Optional[int] =       self._gelu_python
													else:
																			SCREAMING_SNAKE_CASE__  :					Any =       nn.functional.gelu

							def   A_	(   self					:  Any				,						a					:  Tensor     )						->Tensor:
													return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0     )     ))






							def   A_	(   self					:  Optional[Any]				,						a					:  Tensor     )						->Tensor:
													return self.act(UpperCAmelCase_     )







class  _a  (					nn.Module				):






							"""simple docstring"""

							def   A_	(   self					:  Optional[Any]				,						a					:  Tensor     )						->Tensor:
													return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)     ))







class  _a  (					nn.Module				):






							"""simple docstring"""

							def   A_	(   self					:  Optional[Any]				,						a					:  Tensor     )						->Tensor:
													return input * torch.sigmoid(1.702 * input     )







class  _a  (					nn.Module				):






							"""simple docstring"""

							def __init__(   self					:  Dict				,						a					:  float				,						a					:  float     )						->Any:







													if min > max:
																			raise ValueError(f"""min should be < max (got min: {min}, max: {max})"""     )

													super().__init__()
													SCREAMING_SNAKE_CASE__  :					Optional[int] =       min
													SCREAMING_SNAKE_CASE__  :					str =       max






							def   A_	(   self					:  Union[str, Any]				,						a					:  Tensor     )						->Tensor:
													return torch.clip(gelu(UpperCAmelCase_     )				,						self.min				,						self.max     )







class  _a  (					nn.Module				):






							"""simple docstring"""

							def __init__(   self					:  Dict     )						->Optional[int]:
													super().__init__()
													SCREAMING_SNAKE_CASE__  :					Tuple =       math.sqrt(2 / math.pi     )






							def   A_	(   self					:  int				,						a					:  Tensor     )						->Tensor:
													return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(UpperCAmelCase_				,						3     ))     ))







class  _a  (					nn.Module				):






							"""simple docstring"""

							def   A_	(   self					:  Optional[Any]				,						a					:  Tensor     )						->Tensor:
													return nn.functional.silu(UpperCAmelCase_     )







class  _a  (					nn.Module				):






							"""simple docstring"""

							def __init__(   self					:  Tuple     )						->List[Any]:
													super().__init__()
													if version.parse(torch.__version__     ) < version.parse("1.9.0"     ):
																			SCREAMING_SNAKE_CASE__  :					List[Any] =       self._mish_python
													else:
																			SCREAMING_SNAKE_CASE__  :					int =       nn.functional.mish

							def   A_	(   self					:  Optional[Any]				,						a					:  Tensor     )						->Tensor:
													return input * torch.tanh(nn.functional.softplus(UpperCAmelCase_     )     )






							def   A_	(   self					:  Optional[int]				,						a					:  Tensor     )						->Tensor:
													return self.act(UpperCAmelCase_     )







class  _a  (					nn.Module				):






							"""simple docstring"""

							def   A_	(   self					:  Any				,						a					:  Tensor     )						->Tensor:
													return input







class  _a  (					nn.Module				):






							"""simple docstring"""

							def   A_	(   self					:  Optional[Any]				,						a					:  Dict				,						a					:  int=0.70_7107				,						a					:  List[str]=0.28_2095     )						->Union[str, Any]:
													SCREAMING_SNAKE_CASE__  :					int =       (input - mu).div(sigma * math.sqrt(2.0     )     )
													return 0.5 * (1.0 + torch.erf(UpperCAmelCase_     ))







class  _a  (					nn.Module				):






							"""simple docstring"""

							def   A_	(   self					:  List[Any]				,						a					:  int     )						->List[str]:
													SCREAMING_SNAKE_CASE__  :					Optional[Any] =       nn.functional.relu(UpperCAmelCase_     )
													SCREAMING_SNAKE_CASE__  :					Tuple =       torch.square(UpperCAmelCase_     )
													return squared







class  _a  (					lowercase_				):






							"""simple docstring"""

							def __getitem__(   self					:  int				,						a					:  Optional[Any]     )						->int:
													SCREAMING_SNAKE_CASE__  :					Dict =       super().__getitem__(UpperCAmelCase_     )
													SCREAMING_SNAKE_CASE__  :					str =       content if isinstance(UpperCAmelCase_				,						UpperCAmelCase_     ) else (content, {})
													return cls(**UpperCAmelCase_     )







__lowercase							:Union[str, Any]							=	{
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -10, 'max': 10}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
__lowercase							:List[str]							=	ClassInstantier(ACTaCLS)







def    UpperCAmelCase				(  _lowerCamelCase   :	Tuple  ):





						'''simple docstring'''


						if activation_string in ACTaFN:
												return ACTaFN[activation_string]
						else:
												raise KeyError(f"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()  )}"""  )


# For backwards compatibility with: from activations import gelu_python
__lowercase							:Optional[Any]							=	get_activation("gelu_python")
__lowercase							:List[str]							=	get_activation("gelu_new")
__lowercase							:Union[str, Any]							=	get_activation("gelu")
__lowercase							:Union[str, Any]							=	get_activation("gelu_fast")
__lowercase							:str							=	get_activation("quick_gelu")
__lowercase							:Dict							=	get_activation("silu")
__lowercase							:Union[str, Any]							=	get_activation("mish")



__lowercase							:int							=	get_activation("linear")




UpperCAmelCase__			:   Optional[int]      			=    [0.4_85, 0.4_56, 0.4_06]
UpperCAmelCase__			:   Union[str, Any]      			=    [0.2_29, 0.2_24, 0.2_25]
UpperCAmelCase__			:   Union[str, Any]      			=    [0.5, 0.5, 0.5]
UpperCAmelCase__			:   Optional[Any]      			=    [0.5, 0.5, 0.5]
UpperCAmelCase__			:   str      			=    [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]
UpperCAmelCase__			:   Dict      			=    [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]
