





"""simple docstring"""


import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


a__       :  Tuple       		=					logging.get_logger(__name__)



class   lowercase_					(  nn.Module				):

       def __init__(      self  ):






        '''simple docstring'''

        super().__init__()
        if version.parse(torch.__version__  ) < version.parse("1.12.0"  ):
         raise ImportError(
             F"You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use "
             "PytorchGELUTanh. Please upgrade torch."  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return nn.functional.gelu(__SCREAMING_SNAKE_CASE ,       approximate="tanh"  )



class   lowercase_					(  nn.Module				):

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi  ) * (input + 0.044715 * torch.pow(__SCREAMING_SNAKE_CASE ,       3.0  ))  ))



class   lowercase_					(  nn.Module				):

       def __init__(      self ,       __SCREAMING_SNAKE_CASE = False  ):






        '''simple docstring'''

        super().__init__()
        if use_gelu_python:
         snake_case__     				=					self._gelu_python
        else:
         snake_case__     				=					nn.functional.gelu

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0  )  ))

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return self.act(__SCREAMING_SNAKE_CASE  )



class   lowercase_					(  nn.Module				):

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return 0.5 * input * (1.0 + torch.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input)  ))



class   lowercase_					(  nn.Module				):

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return input * torch.sigmoid(1.702 * input  )



class   lowercase_					(  nn.Module				):

       def __init__(      self ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        if min > max:
         raise ValueError(F"min should be < max (got min: {min}, max: {max})"  )

        super().__init__()
        snake_case__     				=					min
        snake_case__     				=					max

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return torch.clip(gelu(__SCREAMING_SNAKE_CASE  ) ,       self.min ,       self.max  )



class   lowercase_					(  nn.Module				):

       def __init__(      self  ):






        '''simple docstring'''

        super().__init__()
        snake_case__     				=					math.sqrt(2 / math.pi  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044715 * torch.pow(__SCREAMING_SNAKE_CASE ,       3  ))  ))



class   lowercase_					(  nn.Module				):

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return nn.functional.silu(__SCREAMING_SNAKE_CASE  )



class   lowercase_					(  nn.Module				):

       def __init__(      self  ):






        '''simple docstring'''

        super().__init__()
        if version.parse(torch.__version__  ) < version.parse("1.9.0"  ):
         snake_case__     				=					self._mish_python
        else:
         snake_case__     				=					nn.functional.mish

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return input * torch.tanh(nn.functional.softplus(__SCREAMING_SNAKE_CASE  )  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return self.act(__SCREAMING_SNAKE_CASE  )



class   lowercase_					(  nn.Module				):

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        return input



class   lowercase_					(  nn.Module				):

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE=0.707107 ,       __SCREAMING_SNAKE_CASE=0.282095  ):






        '''simple docstring'''

        snake_case__     				=					(input - mu).div(sigma * math.sqrt(2.0  )  )
        return 0.5 * (1.0 + torch.erf(__SCREAMING_SNAKE_CASE  ))



class   lowercase_					(  nn.Module				):

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        snake_case__     				=					nn.functional.relu(__SCREAMING_SNAKE_CASE  )
        snake_case__     				=					torch.square(__SCREAMING_SNAKE_CASE  )
        return squared



class   lowercase_					(  __lowerCAmelCase				):

       def __getitem__(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        snake_case__     				=					super().__getitem__(__SCREAMING_SNAKE_CASE  )
        snake_case__				, snake_case__     				=					content if isinstance(__SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE  ) else (content, {})
        return cls(**__SCREAMING_SNAKE_CASE  )
a__       :  Optional[Any]       		=					{
    """gelu""": GELUActivation,
    """gelu_10""": (ClippedGELUActivation, {"""min""": -1_0, """max""": 1_0}),
    """gelu_fast""": FastGELUActivation,
    """gelu_new""": NewGELUActivation,
    """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
    """gelu_pytorch_tanh""": PytorchGELUTanh,
    """gelu_accurate""": AccurateGELUActivation,
    """laplace""": LaplaceActivation,
    """linear""": LinearActivation,
    """mish""": MishActivation,
    """quick_gelu""": QuickGELUActivation,
    """relu""": nn.ReLU,
    """relu2""": ReLUSquaredActivation,
    """relu6""": nn.ReLUa,
    """sigmoid""": nn.Sigmoid,
    """silu""": SiLUActivation,
    """swish""": SiLUActivation,
    """tanh""": nn.Tanh,
}
a__       :  str       		=					ClassInstantier(ACTaCLS)





def 				a__      (	_UpperCAmelCase						)     ->	List[str]:

 '''simple docstring'''






 if activation_string in ACTaFN:
  return ACTaFN[activation_string]
 else:
  raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()						)}"						)


# For backwards compatibility with: from activations import gelu_python
a__       :  Optional[Any]       		=					get_activation("""gelu_python""")
a__       :  List[str]       		=					get_activation("""gelu_new""")
a__       :  List[Any]       		=					get_activation("""gelu""")
a__       :  List[Any]       		=					get_activation("""gelu_fast""")
a__       :  Union[str, Any]       		=					get_activation("""quick_gelu""")
a__       :  Any       		=					get_activation("""silu""")
a__       :  Optional[int]       		=					get_activation("""mish""")

a__       :  List[Any]       		=					get_activation("""linear""")








"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


a__       :  List[Any]       		=					logging.get_logger(__name__)

a__       :  int       		=					{
    """microsoft/mpnet-base""": """https://huggingface.co/microsoft/mpnet-base/resolve/main/config.json""",
}



class   lowercase_					(  __lowerCAmelCase				):
       snake_case__					:				int									=  """mpnet"""

       def __init__(      self ,       __SCREAMING_SNAKE_CASE=3_0527 ,       __SCREAMING_SNAKE_CASE=768 ,       __SCREAMING_SNAKE_CASE=12 ,       __SCREAMING_SNAKE_CASE=12 ,       __SCREAMING_SNAKE_CASE=3072 ,       __SCREAMING_SNAKE_CASE="gelu" ,       __SCREAMING_SNAKE_CASE=0.1 ,       __SCREAMING_SNAKE_CASE=0.1 ,       __SCREAMING_SNAKE_CASE=512 ,       __SCREAMING_SNAKE_CASE=0.02 ,       __SCREAMING_SNAKE_CASE=1e-12 ,       __SCREAMING_SNAKE_CASE=32 ,       __SCREAMING_SNAKE_CASE=1 ,       __SCREAMING_SNAKE_CASE=0 ,       __SCREAMING_SNAKE_CASE=2 ,       **__SCREAMING_SNAKE_CASE ,       ):






        '''simple docstring'''

        super().__init__(pad_token_id=__SCREAMING_SNAKE_CASE ,       bos_token_id=__SCREAMING_SNAKE_CASE ,       eos_token_id=__SCREAMING_SNAKE_CASE ,       **__SCREAMING_SNAKE_CASE  )

        snake_case__     				=					vocab_size
        snake_case__     				=					hidden_size
        snake_case__     				=					num_hidden_layers
        snake_case__     				=					num_attention_heads
        snake_case__     				=					hidden_act
        snake_case__     				=					intermediate_size
        snake_case__     				=					hidden_dropout_prob
        snake_case__     				=					attention_probs_dropout_prob
        snake_case__     				=					max_position_embeddings
        snake_case__     				=					initializer_range
        snake_case__     				=					layer_norm_eps
        snake_case__     				=					relative_attention_num_buckets

