






import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


a 			=							logging.get_logger(__name__)
class 							__snake_case(       nn.Module   ):



 def __init__(						self : List[Any]						):

    """simple docstring"""






    super().__init__()
    if version.parse(torch.__version__						) < version.parse("""1.12.0"""						):
       raise ImportError(
           f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
           """PytorchGELUTanh. Please upgrade torch."""						)







 def     UpperCAmelCase     (						self : Any			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return nn.functional.gelu(UpperCAmelCase__			,		approximate="""tanh"""						)
class 							__snake_case(       nn.Module   ):



 def     UpperCAmelCase     (						self : Union[str, Any]			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi						) * (input + 0.044_715 * torch.pow(UpperCAmelCase__			,		3.0						))						))
class 							__snake_case(       nn.Module   ):



 def __init__(						self : Optional[Any]			,		UpperCAmelCase__ : bool = False						):

    """simple docstring"""






    super().__init__()
    if use_gelu_python:
       lowercase__    :						Any       =			self._gelu_python
    else:
       lowercase__    :						Tuple       =			nn.functional.gelu



 def     UpperCAmelCase     (						self : Tuple			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0						)						))







 def     UpperCAmelCase     (						self : Tuple			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return self.act(UpperCAmelCase__						)
class 							__snake_case(       nn.Module   ):



 def     UpperCAmelCase     (						self : List[str]			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)						))
class 							__snake_case(       nn.Module   ):



 def     UpperCAmelCase     (						self : List[Any]			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return input * torch.sigmoid(1.702 * input						)
class 							__snake_case(       nn.Module   ):



 def __init__(						self : Dict			,		UpperCAmelCase__ : float			,		UpperCAmelCase__ : float						):

    """simple docstring"""






    if min > max:
       raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''						)

    super().__init__()
    lowercase__    :						List[Any]       =			min
    lowercase__    :						List[str]       =			max







 def     UpperCAmelCase     (						self : Union[str, Any]			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return torch.clip(gelu(UpperCAmelCase__						)			,		self.min			,		self.max						)
class 							__snake_case(       nn.Module   ):



 def __init__(						self : int						):

    """simple docstring"""






    super().__init__()
    lowercase__    :						Optional[Any]       =			math.sqrt(2 / math.pi						)







 def     UpperCAmelCase     (						self : str			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(UpperCAmelCase__			,		3						))						))
class 							__snake_case(       nn.Module   ):



 def     UpperCAmelCase     (						self : List[str]			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return nn.functional.silu(UpperCAmelCase__						)
class 							__snake_case(       nn.Module   ):



 def __init__(						self : Union[str, Any]						):

    """simple docstring"""






    super().__init__()
    if version.parse(torch.__version__						) < version.parse("""1.9.0"""						):
       lowercase__    :						Union[str, Any]       =			self._mish_python
    else:
       lowercase__    :						List[Any]       =			nn.functional.mish



 def     UpperCAmelCase     (						self : str			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return input * torch.tanh(nn.functional.softplus(UpperCAmelCase__						)						)







 def     UpperCAmelCase     (						self : Optional[int]			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return self.act(UpperCAmelCase__						)
class 							__snake_case(       nn.Module   ):



 def     UpperCAmelCase     (						self : Any			,		UpperCAmelCase__ : Tensor						):

    """simple docstring"""






    return input
class 							__snake_case(       nn.Module   ):



 def     UpperCAmelCase     (						self : List[Any]			,		UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : Dict=0.707_107			,		UpperCAmelCase__ : Union[str, Any]=0.282_095						):

    """simple docstring"""






    lowercase__    :						Any       =			(input - mu).div(sigma * math.sqrt(2.0						)						)
    return 0.5 * (1.0 + torch.erf(UpperCAmelCase__						))
class 							__snake_case(       nn.Module   ):



 def     UpperCAmelCase     (						self : Dict			,		UpperCAmelCase__ : int						):

    """simple docstring"""






    lowercase__    :						Tuple       =			nn.functional.relu(UpperCAmelCase__						)
    lowercase__    :						Any       =			torch.square(UpperCAmelCase__						)
    return squared
class 							__snake_case(       __UpperCamelCase   ):



 def __getitem__(						self : str			,		UpperCAmelCase__ : Any						):

    """simple docstring"""






    lowercase__    :						int       =			super().__getitem__(UpperCAmelCase__						)
    lowercase__  ,       lowercase__    :						Union[str, Any]       =			content if isinstance(UpperCAmelCase__			,		UpperCAmelCase__						) else (content, {})
    return cls(**UpperCAmelCase__						)
a 			=							{
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -10, 'max': 10}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
a 			=							ClassInstantier(ACTaCLS)
def    a__    (	A       ):
   if activation_string in ACTaFN:
      return ACTaFN[activation_string]
   else:
      raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()       )}'''       )


# For backwards compatibility with: from activations import gelu_python
a 			=							get_activation('gelu_python')
a 			=							get_activation('gelu_new')
a 			=							get_activation('gelu')
a 			=							get_activation('gelu_fast')
a 			=							get_activation('quick_gelu')
a 			=							get_activation('silu')
a 			=							get_activation('mish')






a 			=							get_activation('linear')










from __future__ import annotations

from typing import Any
class 							__snake_case:



 def __init__(						self : str			,		UpperCAmelCase__ : Any			,		UpperCAmelCase__ : Any						):  # noqa: A002

    """simple docstring"""






    lowercase__    :						Optional[int]       =			item
    lowercase__    :						Union[str, Any]       =			next
class 							__snake_case:



 def __init__(						self : List[Any]						):

    """simple docstring"""






    lowercase__    :						Node | None       =			None
    lowercase__    :						Optional[Any]       =			0



 def     UpperCAmelCase     (						self : List[Any]			,		UpperCAmelCase__ : Any						):

    """simple docstring"""






    lowercase__    :						Any       =			Node(UpperCAmelCase__			,		self.head						)
    self.size += 1



 def     UpperCAmelCase     (						self : Dict						):

    """simple docstring"""






    if self.head is None:
       return None
    else:
       lowercase__    :						List[Any]       =			self.head.item
       lowercase__    :						Optional[Any]       =			self.head.next
       self.size -= 1
       return item



 def     UpperCAmelCase     (						self : Union[str, Any]						):

    """simple docstring"""






    return self.head is None



 def __str__(						self : List[str]						):

    """simple docstring"""






    if self.is_empty():
       return ""
    else:
       lowercase__    :						Optional[Any]       =			self.head
       lowercase__    :						str       =			""""""
       lowercase__    :						list[str]       =			[]
       while iterate:
          item_list.append(str(iterate.item						)						)
          lowercase__    :						List[str]       =			iterate.next

       lowercase__    :						List[str]       =			""" --> """.join(UpperCAmelCase__						)

       return item_str







 def __len__(						self : List[Any]						):

    """simple docstring"""






    return self.size


