






import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


lowerCamelCase					    =	logging.get_logger(__name__)

class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def __init__(     self:						str						)	->		Optional[Any]:


       """simple docstring"""



       super().__init__()
       if version.parse(torch.__version__						) < version.parse('1.12.0'						):
           raise ImportError(
               f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
               'PytorchGELUTanh. Please upgrade torch.'						)

   def       SCREAMING_SNAKE_CASE__					(     self:						Dict ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return nn.functional.gelu(snake_case__ ,							approximate='tanh'						)






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def       SCREAMING_SNAKE_CASE__					(     self:						Optional[Any] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi						) * (input + 0.04_47_15 * torch.pow(snake_case__ ,							3.0						))						))






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def __init__(     self:						Union[str, Any] ,							snake_case__:						bool = False						)	->		Union[str, Any]:


       """simple docstring"""



       super().__init__()
       if use_gelu_python:
           _lowercase     			=						self._gelu_python
       else:
           _lowercase     			=						nn.functional.gelu

   def       SCREAMING_SNAKE_CASE__					(     self:						Optional[Any] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0						)						))

   def       SCREAMING_SNAKE_CASE__					(     self:						List[Any] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return self.act(snake_case__						)






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def       SCREAMING_SNAKE_CASE__					(     self:						List[Any] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return 0.5 * input * (1.0 + torch.tanh(input * 0.79_78_84_56_08 * (1.0 + 0.04_47_15 * input * input)						))






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def       SCREAMING_SNAKE_CASE__					(     self:						Any ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return input * torch.sigmoid(1.7_02 * input						)






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def __init__(     self:						Union[str, Any] ,							snake_case__:						float ,							snake_case__:						float						)	->		int:


       """simple docstring"""



       if min > max:
           raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''						)

       super().__init__()
       _lowercase     			=						min
       _lowercase     			=						max

   def       SCREAMING_SNAKE_CASE__					(     self:						Any ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return torch.clip(gelu(snake_case__						) ,							self.min ,							self.max						)






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def __init__(     self:						Optional[Any]						)	->		Any:


       """simple docstring"""



       super().__init__()
       _lowercase     			=						math.sqrt(2 / math.pi						)

   def       SCREAMING_SNAKE_CASE__					(     self:						Union[str, Any] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_47_15 * torch.pow(snake_case__ ,							3						))						))






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def       SCREAMING_SNAKE_CASE__					(     self:						Union[str, Any] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return nn.functional.silu(snake_case__						)






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def __init__(     self:						Tuple						)	->		Tuple:


       """simple docstring"""



       super().__init__()
       if version.parse(torch.__version__						) < version.parse('1.9.0'						):
           _lowercase     			=						self._mish_python
       else:
           _lowercase     			=						nn.functional.mish

   def       SCREAMING_SNAKE_CASE__					(     self:						str ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return input * torch.tanh(nn.functional.softplus(snake_case__						)						)

   def       SCREAMING_SNAKE_CASE__					(     self:						List[str] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return self.act(snake_case__						)






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def       SCREAMING_SNAKE_CASE__					(     self:						Union[str, Any] ,							snake_case__:						Tensor						)	->		Tensor:


       """simple docstring"""



       return input






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def       SCREAMING_SNAKE_CASE__					(     self:						Dict ,							snake_case__:						int ,							snake_case__:						Dict=0.70_71_07 ,							snake_case__:						Optional[Any]=0.28_20_95						)	->		Tuple:


       """simple docstring"""



       _lowercase     			=						(input - mu).div(sigma * math.sqrt(2.0						)						)
       return 0.5 * (1.0 + torch.erf(snake_case__						))






class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def       SCREAMING_SNAKE_CASE__					(     self:						Union[str, Any] ,							snake_case__:						str						)	->		int:


       """simple docstring"""



       _lowercase     			=						nn.functional.relu(snake_case__						)
       _lowercase     			=						torch.square(snake_case__						)
       return squared






class        snake_case_				(							lowercase_   ):







   """simple docstring"""



   def __getitem__(     self:						List[Any] ,							snake_case__:						Any						)	->		int:


       """simple docstring"""



       _lowercase     			=						super().__getitem__(snake_case__						)
       _lowercase			,     _lowercase     			=						content if isinstance(snake_case__ ,							snake_case__						) else (content, {})
       return cls(**snake_case__						)






lowerCamelCase					    =	{
    '''gelu''': GELUActivation,
    '''gelu_10''': (ClippedGELUActivation, {'''min''': -10, '''max''': 10}),
    '''gelu_fast''': FastGELUActivation,
    '''gelu_new''': NewGELUActivation,
    '''gelu_python''': (GELUActivation, {'''use_gelu_python''': True}),
    '''gelu_pytorch_tanh''': PytorchGELUTanh,
    '''gelu_accurate''': AccurateGELUActivation,
    '''laplace''': LaplaceActivation,
    '''linear''': LinearActivation,
    '''mish''': MishActivation,
    '''quick_gelu''': QuickGELUActivation,
    '''relu''': nn.ReLU,
    '''relu2''': ReLUSquaredActivation,
    '''relu6''': nn.ReLUa,
    '''sigmoid''': nn.Sigmoid,
    '''silu''': SiLUActivation,
    '''swish''': SiLUActivation,
    '''tanh''': nn.Tanh,
}
lowerCamelCase					    =	ClassInstantier(ACTaCLS)
def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    if activation_string in ACTaFN:
        return ACTaFN[activation_string]
    else:
        raise KeyError(f'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()   )}'''   )


# For backwards compatibility with: from activations import gelu_python
lowerCamelCase					    =	get_activation('''gelu_python''')
lowerCamelCase					    =	get_activation('''gelu_new''')
lowerCamelCase					    =	get_activation('''gelu''')
lowerCamelCase					    =	get_activation('''gelu_fast''')
lowerCamelCase					    =	get_activation('''quick_gelu''')
lowerCamelCase					    =	get_activation('''silu''')
lowerCamelCase					    =	get_activation('''mish''')
lowerCamelCase					    =	get_activation('''linear''')








from __future__ import annotations

from enum import Enum, unique
from typing import TypeVar

# Create a generic variable that can be 'Enum', or any subclass.
lowerCamelCase					    =	TypeVar('''T''', bound='''Enum''')

@unique
class        snake_case_				(							lowercase_   ):







   """simple docstring"""



   __lowerCAmelCase  =    80
   __lowerCAmelCase  =    70
   __lowerCAmelCase  =    60
   __lowerCAmelCase  =    50
   __lowerCAmelCase  =    40
   __lowerCAmelCase  =    30
   __lowerCAmelCase  =    20
   __lowerCAmelCase  =    10






@unique
class        snake_case_				(							lowercase_   ):







   """simple docstring"""



   __lowerCAmelCase  =    24
   __lowerCAmelCase  =    21
   __lowerCAmelCase  =    18
   __lowerCAmelCase  =    15
   __lowerCAmelCase  =    12
   __lowerCAmelCase  =    9
   __lowerCAmelCase  =    6
   __lowerCAmelCase  =    3
   __lowerCAmelCase  =    2
   __lowerCAmelCase  =    1
   __lowerCAmelCase  =    -1
   __lowerCAmelCase  =    -2
   __lowerCAmelCase  =    -3
   __lowerCAmelCase  =    -6
   __lowerCAmelCase  =    -9
   __lowerCAmelCase  =    -12
   __lowerCAmelCase  =    -15
   __lowerCAmelCase  =    -18
   __lowerCAmelCase  =    -21
   __lowerCAmelCase  =    -24

   @classmethod
   def       SCREAMING_SNAKE_CASE__					(     cls:						type[T]						)	->		dict:


       """simple docstring"""



       return {unit.name: unit.value for unit in cls if unit.value > 0}

   @classmethod
   def       SCREAMING_SNAKE_CASE__					(     cls:						type[T]						)	->		dict:


       """simple docstring"""



       return {unit.name: unit.value for unit in cls if unit.value < 0}






def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    _lowercase     			=						SIUnit.get_positive() if value > 0 else SIUnit.get_negative()
    for name_prefix, value_prefix in prefixes.items():
        _lowercase     			=						value / (10**value_prefix)
        if numerical_part > 1:
            return f'''{numerical_part!s} {name_prefix}'''
    return str(_lowercase   )
def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    for prefix in BinaryUnit:
        _lowercase     			=						value / (2**prefix.value)
        if numerical_part > 1:
            return f'''{numerical_part!s} {prefix.name}'''
    return str(_lowercase   )


if __name__ == "__main__":
   import doctest

   doctest.testmod()
