






'''simple docstring'''


import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


lowerCamelCase_					:			str     						=				logging.get_logger(__name__)
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def __init__( self				:    Optional[Any]      )				->     Optional[int]:

									'''simple docstring'''







									super().__init__()
									if version.parse(torch.__version__      ) < version.parse("""1.12.0"""      ):
															raise ImportError(
															    f"You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use "
															    """PytorchGELUTanh. Please upgrade torch."""      )







			def 					A   ( self				:    Optional[Any]			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return nn.functional.gelu(lowercase			,   approximate="""tanh"""      )
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def 					A   ( self				:    List[str]			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi      ) * (input + 0.0_4_4_7_1_5 * torch.pow(lowercase			,   3.0      ))      ))
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def __init__( self				:    Optional[int]			,   lowercase				:    bool = False      )				->     Tuple:

									'''simple docstring'''







									super().__init__()
									if use_gelu_python:
															UpperCamelCase__       =			self._gelu_python
									else:
															UpperCamelCase__       =			nn.functional.gelu






			def 					A   ( self				:    Optional[int]			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0      )      ))







			def 					A   ( self				:    Tuple			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return self.act(lowercase      )
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def 					A   ( self				:    int			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return 0.5 * input * (1.0 + torch.tanh(input * 0.7_9_7_8_8_4_5_6_0_8 * (1.0 + 0.0_4_4_7_1_5 * input * input)      ))
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def 					A   ( self				:    Union[str, Any]			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return input * torch.sigmoid(1.7_0_2 * input      )
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def __init__( self				:    Any			,   lowercase				:    float			,   lowercase				:    float      )				->     List[Any]:

									'''simple docstring'''







									if min > max:
															raise ValueError(f"min should be < max (got min: {min}, max: {max})"      )

									super().__init__()
									UpperCamelCase__       =			min
									UpperCamelCase__       =			max







			def 					A   ( self				:    Any			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return torch.clip(gelu(lowercase      )			,   self.min			,   self.max      )
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def __init__( self				:    List[str]      )				->     Optional[int]:

									'''simple docstring'''







									super().__init__()
									UpperCamelCase__       =			math.sqrt(2 / math.pi      )







			def 					A   ( self				:    int			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.0_4_4_7_1_5 * torch.pow(lowercase			,   3      ))      ))
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def 					A   ( self				:    List[str]			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return nn.functional.silu(lowercase      )
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def __init__( self				:    List[str]      )				->     Optional[int]:

									'''simple docstring'''







									super().__init__()
									if version.parse(torch.__version__      ) < version.parse("""1.9.0"""      ):
															UpperCamelCase__       =			self._mish_python
									else:
															UpperCamelCase__       =			nn.functional.mish






			def 					A   ( self				:    Optional[Any]			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return input * torch.tanh(nn.functional.softplus(lowercase      )      )







			def 					A   ( self				:    List[str]			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return self.act(lowercase      )
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def 					A   ( self				:    Tuple			,   lowercase				:    Tensor      )				->     Tensor:

									'''simple docstring'''







									return input
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def 					A   ( self				:    List[str]			,   lowercase				:    Tuple			,   lowercase				:    Union[str, Any]=0.7_0_7_1_0_7			,   lowercase				:    Any=0.2_8_2_0_9_5      )				->     Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__       =			(input - mu).div(sigma * math.sqrt(2.0      )      )
									return 0.5 * (1.0 + torch.erf(lowercase      ))
class  _SCREAMING_SNAKE_CASE				(		nn.Module						):


			'''simple docstring'''





			def 					A   ( self				:    str			,   lowercase				:    List[str]      )				->     List[str]:

									'''simple docstring'''







									UpperCamelCase__       =			nn.functional.relu(lowercase      )
									UpperCamelCase__       =			torch.square(lowercase      )
									return squared
class  _SCREAMING_SNAKE_CASE				(		SCREAMING_SNAKE_CASE						):


			'''simple docstring'''





			def __getitem__( self				:    Optional[Any]			,   lowercase				:    Optional[Any]      )				->     Optional[int]:

									'''simple docstring'''







									UpperCamelCase__       =			super().__getitem__(lowercase      )
									UpperCamelCase__     , UpperCamelCase__       =			content if isinstance(lowercase			,   lowercase      ) else (content, {})
									return cls(**lowercase      )
lowerCamelCase_					:			int     						=				{
    '''gelu''': GELUActivation,
    '''gelu_10''': (ClippedGELUActivation, {'''min''': -10, '''max''': 10}),
    '''gelu_fast''': FastGELUActivation,
    '''gelu_new''': NewGELUActivation,
    '''gelu_python''': (GELUActivation, {'''use_gelu_python''': True}),
    '''gelu_pytorch_tanh''': PytorchGELUTanh,
    '''gelu_accurate''': AccurateGELUActivation,
    '''laplace''': LaplaceActivation,
    '''linear''': LinearActivation,
    '''mish''': MishActivation,
    '''quick_gelu''': QuickGELUActivation,
    '''relu''': nn.ReLU,
    '''relu2''': ReLUSquaredActivation,
    '''relu6''': nn.ReLUa,
    '''sigmoid''': nn.Sigmoid,
    '''silu''': SiLUActivation,
    '''swish''': SiLUActivation,
    '''tanh''': nn.Tanh,
}
lowerCamelCase_					:			Dict     						=				ClassInstantier(ACTaCLS)







def 						__magic_name__(			_A       ):
						'''simple docstring'''





						if activation_string in ACTaFN:
												return ACTaFN[activation_string]
						else:
												raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()       )}"       )


# For backwards compatibility with: from activations import gelu_python
lowerCamelCase_					:			str     						=				get_activation('''gelu_python''')
lowerCamelCase_					:			int     						=				get_activation('''gelu_new''')
lowerCamelCase_					:			Any     						=				get_activation('''gelu''')
lowerCamelCase_					:			int     						=				get_activation('''gelu_fast''')
lowerCamelCase_					:			List[Any]     						=				get_activation('''quick_gelu''')
lowerCamelCase_					:			Union[str, Any]     						=				get_activation('''silu''')
lowerCamelCase_					:			List[str]     						=				get_activation('''mish''')







lowerCamelCase_					:			int     						=				get_activation('''linear''')








'''simple docstring'''


def 						__magic_name__(			_A       ):
						'''simple docstring'''





						UpperCamelCase__       =			len(_A       )

						# Set the gap value to be decreased by a factor of 1.3
						# after each iteration
						UpperCamelCase__       =			1.3

						# Continue sorting until the gap is 1
						while gap > 1:
												# Decrease the gap value
												UpperCamelCase__       =			int(gap / shrink       )

												# Sort the elements using insertion sort
												for i in range(_A    ,   len(_A       )       ):
																		UpperCamelCase__       =			collection[i]
																		UpperCamelCase__       =			i
																		while j >= gap and collection[j - gap] > temp:
																								UpperCamelCase__       =			collection[j - gap]
																								j -= gap
																		UpperCamelCase__       =			temp

						return collection


if __name__ == "__main__":
					import doctest

					doctest.testmod()
