
"""simple docstring"""






import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


UpperCamelCase_						   =   logging.get_logger(__name__)





class 							snake_case       (     nn.Module		):







   def __init__( self)   ->List[str]:
      super().__init__()
      if version.parse(torch.__version__) < version.parse("1.12.0"):
         raise ImportError(
             F'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
             "PytorchGELUTanh. Please upgrade torch.")







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return nn.functional.gelu(__UpperCAmelCase       ,							approximate="tanh")






class 							snake_case       (     nn.Module		):







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.044_715 * torch.pow(__UpperCAmelCase       ,							3.0))))






class 							snake_case       (     nn.Module		):







   def __init__( self       ,							__UpperCAmelCase = False)   ->Optional[int]:
      super().__init__()
      if use_gelu_python:
         a_						   =     self._gelu_python
      else:
         a_						   =     nn.functional.gelu







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0)))







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return self.act(__UpperCAmelCase)






class 							snake_case       (     nn.Module		):







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)))






class 							snake_case       (     nn.Module		):







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return input * torch.sigmoid(1.702 * input)






class 							snake_case       (     nn.Module		):







   def __init__( self       ,							__UpperCAmelCase       ,							__UpperCAmelCase)   ->int:
      if min > max:
         raise ValueError(F'''min should be < max (got min: {min}, max: {max})''')

      super().__init__()
      a_						   =     min
      a_						   =     max







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return torch.clip(gelu(__UpperCAmelCase)       ,							self.min       ,							self.max)






class 							snake_case       (     nn.Module		):







   def __init__( self)   ->Union[str, Any]:
      super().__init__()
      a_						   =     math.sqrt(2 / math.pi)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(__UpperCAmelCase       ,							3))))






class 							snake_case       (     nn.Module		):







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return nn.functional.silu(__UpperCAmelCase)






class 							snake_case       (     nn.Module		):







   def __init__( self)   ->Dict:
      super().__init__()
      if version.parse(torch.__version__) < version.parse("1.9.0"):
         a_						   =     self._mish_python
      else:
         a_						   =     nn.functional.mish







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return input * torch.tanh(nn.functional.softplus(__UpperCAmelCase))







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return self.act(__UpperCAmelCase)






class 							snake_case       (     nn.Module		):







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Tensor:
      return input






class 							snake_case       (     nn.Module		):







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							__UpperCAmelCase=0.707_107       ,							__UpperCAmelCase=0.282_095)   ->int:
      a_						   =     (input - mu).div(sigma * math.sqrt(2.0))
      return 0.5 * (1.0 + torch.erf(__UpperCAmelCase))






class 							snake_case       (     nn.Module		):







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->int:
      a_						   =     nn.functional.relu(__UpperCAmelCase)
      a_						   =     torch.square(__UpperCAmelCase)
      return squared






class 							snake_case       (     SCREAMING_SNAKE_CASE_		):







   def __getitem__( self       ,							__UpperCAmelCase)   ->List[str]:
      a_						   =     super().__getitem__(__UpperCAmelCase)
      a_   ,			a_						   =     content if isinstance(__UpperCAmelCase       ,							__UpperCAmelCase) else (content, {})
      return cls(**__UpperCAmelCase)






UpperCamelCase_						   =   {
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -10, 'max': 10}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
UpperCamelCase_						   =   ClassInstantier(ACTaCLS)
def    UpperCamelCase						(					UpperCAmelCase     )						->Dict:

   """simple docstring"""



   if activation_string in ACTaFN:
      return ACTaFN[activation_string]
   else:
      raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()     )}'''     )


# For backwards compatibility with: from activations import gelu_python
UpperCamelCase_						   =   get_activation('gelu_python')
UpperCamelCase_						   =   get_activation('gelu_new')
UpperCamelCase_						   =   get_activation('gelu')
UpperCamelCase_						   =   get_activation('gelu_fast')
UpperCamelCase_						   =   get_activation('quick_gelu')
UpperCamelCase_						   =   get_activation('silu')
UpperCamelCase_						   =   get_activation('mish')



UpperCamelCase_						   =   get_activation('linear')

"""simple docstring"""






import copy

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging


UpperCamelCase_						   =   logging.get_logger(__name__)





class 							snake_case       (     SCREAMING_SNAKE_CASE_		):
   a_					:		Tuple     					=     """hybrid-clip"""
   a_					:		Optional[Any]     					=     True







   def __init__( self       ,							__UpperCAmelCase=5_12       ,							**__UpperCAmelCase)   ->Dict:
      super().__init__(**__UpperCAmelCase)

      if "text_config" not in kwargs:
         raise ValueError("`text_config` can not be `None`.")

      if "vision_config" not in kwargs:
         raise ValueError("`vision_config` can not be `None`.")

      a_						   =     kwargs.pop("text_config")
      a_						   =     kwargs.pop("vision_config")

      a_						   =     text_config.pop("model_type")
      a_						   =     vision_config.pop("model_type")

      from transformers import AutoConfig

      a_						   =     AutoConfig.for_model(__UpperCAmelCase       ,							**__UpperCAmelCase)

      if vision_model_type == "clip":
         a_						   =     AutoConfig.for_model(__UpperCAmelCase       ,							**__UpperCAmelCase).vision_config
      elif vision_model_type == "clip_vision_model":
         from transformers import CLIPVisionConfig

         a_						   =     CLIPVisionConfig(**__UpperCAmelCase)
      else:
         a_						   =     AutoConfig.for_model(__UpperCAmelCase       ,							**__UpperCAmelCase)

      a_						   =     projection_dim
      a_						   =     1.0







   @classmethod
   def    UpperCAmelCase__	( cls       ,							__UpperCAmelCase       ,							__UpperCAmelCase       ,							**__UpperCAmelCase)   ->Optional[int]:
      return cls(text_config=text_config.to_dict()       ,							vision_config=vision_config.to_dict()       ,							**__UpperCAmelCase)







   def    UpperCAmelCase__	( self)   ->str:
      a_						   =     copy.deepcopy(self.__dict__)
      a_						   =     self.text_config.to_dict()
      a_						   =     self.vision_config.to_dict()
      a_						   =     self.__class__.model_type
      return output