
"""simple docstring"""





import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


_a		       =							logging.get_logger(__name__)

# General docstring
_a		       =							'PoolFormerConfig'

# Base docstring
_a		       =							'sail/poolformer_s12'
_a		       =							[1, 512, 7, 7]

# Image classification docstring
_a		       =							'sail/poolformer_s12'
_a		       =							'tabby, tabby cat'

_a		       =							[
    'sail/poolformer_s12',
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]







def 	__a     (		__lowerCamelCase,		__lowerCamelCase = 0.0,		__lowerCamelCase = False  ):
   if drop_prob == 0.0 or not training:
      return input
   UpperCAmelCase_					:					Optional[Any]			     =    1 - drop_prob
   UpperCAmelCase_					:					Union[str, Any]			     =    (input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
   UpperCAmelCase_					:					Union[str, Any]			     =    keep_prob + torch.rand(__lowerCamelCase,		dtype=input.dtype,		device=input.device  )
   random_tensor.floor_()  # binarize
   UpperCAmelCase_					:					Optional[Any]			     =    input.div(__lowerCamelCase  ) * random_tensor
   return output


class A_	(nn.Module      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_ = None  ):

     """simple docstring"""
     super().__init__()
     UpperCAmelCase_					:					str			     =    drop_prob







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     return drop_path(lowercase_				,   self.drop_prob				,   self.training  )







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     return "p={}".format(self.drop_prob  )




class A_	(nn.Module      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_				,   lowercase_				,   lowercase_				,   lowercase_				,   lowercase_				,   lowercase_=None  ):

     """simple docstring"""
     super().__init__()
     UpperCAmelCase_					:					Union[str, Any]			     =    patch_size if isinstance(lowercase_				,   collections.abc.Iterable  ) else (patch_size, patch_size)
     UpperCAmelCase_					:					Dict			     =    stride if isinstance(lowercase_				,   collections.abc.Iterable  ) else (stride, stride)
     UpperCAmelCase_					:					Optional[int]			     =    padding if isinstance(lowercase_				,   collections.abc.Iterable  ) else (padding, padding)

     UpperCAmelCase_					:					Optional[Any]			     =    nn.Convad(lowercase_				,   lowercase_				,   kernel_size=lowercase_				,   stride=lowercase_				,   padding=lowercase_  )
     UpperCAmelCase_					:					Optional[int]			     =    norm_layer(lowercase_  ) if norm_layer else nn.Identity()







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     UpperCAmelCase_					:					List[Any]			     =    self.projection(lowercase_  )
     UpperCAmelCase_					:					List[Any]			     =    self.norm(lowercase_  )
     return embeddings




class A_	(nn.GroupNorm      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_				,   **lowercase_  ):

     """simple docstring"""
     super().__init__(1				,   lowercase_				,   **lowercase_  )




class A_	(nn.Module      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_  ):

     """simple docstring"""
     super().__init__()
     UpperCAmelCase_					:					Optional[Any]			     =    nn.AvgPoolad(lowercase_				,   stride=1				,   padding=pool_size // 2				,   count_include_pad=lowercase_  )







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     return self.pool(lowercase_  ) - hidden_states




class A_	(nn.Module      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_				,   lowercase_				,   lowercase_				,   lowercase_  ):

     """simple docstring"""
     super().__init__()
     UpperCAmelCase_					:					Union[str, Any]			     =    nn.Convad(lowercase_				,   lowercase_				,   1  )
     UpperCAmelCase_					:					Any			     =    nn.Convad(lowercase_				,   lowercase_				,   1  )
     UpperCAmelCase_					:					Union[str, Any]			     =    PoolFormerDropPath(lowercase_  )
     if isinstance(config.hidden_act				,   lowercase_  ):
        UpperCAmelCase_					:					List[str]			     =    ACTaFN[config.hidden_act]
     else:
        UpperCAmelCase_					:					str			     =    config.hidden_act







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     UpperCAmelCase_					:					Tuple			     =    self.conva(lowercase_  )
     UpperCAmelCase_					:					List[Any]			     =    self.act_fn(lowercase_  )
     UpperCAmelCase_					:					Union[str, Any]			     =    self.drop(lowercase_  )
     UpperCAmelCase_					:					Dict			     =    self.conva(lowercase_  )
     UpperCAmelCase_					:					Tuple			     =    self.drop(lowercase_  )

     return hidden_states




class A_	(nn.Module      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_				,   lowercase_				,   lowercase_				,   lowercase_				,   lowercase_				,   lowercase_  ):

     """simple docstring"""
     super().__init__()
     UpperCAmelCase_					:					str			     =    PoolFormerPooling(lowercase_  )
     UpperCAmelCase_					:					Dict			     =    PoolFormerOutput(lowercase_				,   lowercase_				,   lowercase_				,   lowercase_  )
     UpperCAmelCase_					:					List[str]			     =    PoolFormerGroupNorm(lowercase_  )
     UpperCAmelCase_					:					Optional[int]			     =    PoolFormerGroupNorm(lowercase_  )

     # Useful for training neural nets
     UpperCAmelCase_					:					Optional[Any]			     =    PoolFormerDropPath(lowercase_  ) if drop_path > 0.0 else nn.Identity()
     UpperCAmelCase_					:					str			     =    config.use_layer_scale
     if config.use_layer_scale:
        UpperCAmelCase_					:					Union[str, Any]			     =    nn.Parameter(
            config.layer_scale_init_value * torch.ones((lowercase_)  )				,   requires_grad=lowercase_  )
        UpperCAmelCase_					:					Union[str, Any]			     =    nn.Parameter(
            config.layer_scale_init_value * torch.ones((lowercase_)  )				,   requires_grad=lowercase_  )







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     if self.use_layer_scale:
        UpperCAmelCase_					:					Union[str, Any]			     =    self.pooling(self.before_norm(lowercase_  )  )
        UpperCAmelCase_					:					Tuple			     =    self.layer_scale_a.unsqueeze(-1  ).unsqueeze(-1  ) * pooling_output
        # First residual connection
        UpperCAmelCase_					:					Optional[int]			     =    hidden_states + self.drop_path(lowercase_  )
        UpperCAmelCase_					:					List[str]			     =    ()

        UpperCAmelCase_					:					Union[str, Any]			     =    self.output(self.after_norm(lowercase_  )  )
        UpperCAmelCase_					:					Optional[Any]			     =    self.layer_scale_a.unsqueeze(-1  ).unsqueeze(-1  ) * layer_output
        # Second residual connection
        UpperCAmelCase_					:					List[Any]			     =    hidden_states + self.drop_path(lowercase_  )

        UpperCAmelCase_					:					List[Any]			     =    (output,) + outputs
        return outputs

     else:
        UpperCAmelCase_					:					Optional[int]			     =    self.drop_path(self.pooling(self.before_norm(lowercase_  )  )  )
        # First residual connection
        UpperCAmelCase_					:					Optional[int]			     =    pooling_output + hidden_states
        UpperCAmelCase_					:					Union[str, Any]			     =    ()

        # Second residual connection inside the PoolFormerOutput block
        UpperCAmelCase_					:					Optional[Any]			     =    self.drop_path(self.output(self.after_norm(lowercase_  )  )  )
        UpperCAmelCase_					:					Optional[int]			     =    hidden_states + layer_output

        UpperCAmelCase_					:					Tuple			     =    (output,) + outputs
        return outputs




class A_	(nn.Module      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_  ):

     """simple docstring"""
     super().__init__()
     UpperCAmelCase_					:					Optional[int]			     =    config
     # stochastic depth decay rule
     UpperCAmelCase_					:					Union[str, Any]			     =    [x.item() for x in torch.linspace(0				,   config.drop_path_rate				,   sum(config.depths  )  )]

     # patch embeddings
     UpperCAmelCase_					:					List[str]			     =    []
     for i in range(config.num_encoder_blocks  ):
        embeddings.append(
            PoolFormerEmbeddings(
                patch_size=config.patch_sizes[i]				,   stride=config.strides[i]				,   padding=config.padding[i]				,   num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]				,   hidden_size=config.hidden_sizes[i]				,   )  )
     UpperCAmelCase_					:					int			     =    nn.ModuleList(lowercase_  )

     # Transformer blocks
     UpperCAmelCase_					:					List[str]			     =    []
     UpperCAmelCase_					:					Dict			     =    0
     for i in range(config.num_encoder_blocks  ):
        # each block consists of layers
        UpperCAmelCase_					:					str			     =    []
        if i != 0:
           cur += config.depths[i - 1]
        for j in range(config.depths[i]  ):
           layers.append(
               PoolFormerLayer(
                   lowercase_				,   num_channels=config.hidden_sizes[i]				,   pool_size=config.pool_size				,   hidden_size=config.hidden_sizes[i]				,   intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio  )				,   drop_path=dpr[cur + j]				,   )  )
        blocks.append(nn.ModuleList(lowercase_  )  )

     UpperCAmelCase_					:					int			     =    nn.ModuleList(lowercase_  )







  def        UpperCamelCase__					(		self				,   lowercase_				,   lowercase_=False				,   lowercase_=True  ):

     """simple docstring"""
     UpperCAmelCase_					:					int			     =    () if output_hidden_states else None

     UpperCAmelCase_					:					int			     =    pixel_values
     for idx, layers in enumerate(zip(self.patch_embeddings				,   self.block  )  ):
        UpperCAmelCase_     ,   UpperCAmelCase_					:					Tuple			     =    layers
        # Get patch embeddings from hidden_states
        UpperCAmelCase_					:					List[str]			     =    embedding_layer(lowercase_  )
        # Send the embeddings through the blocks
        for _, blk in enumerate(lowercase_  ):
           UpperCAmelCase_					:					Tuple			     =    blk(lowercase_  )
           UpperCAmelCase_					:					int			     =    layer_outputs[0]

        if output_hidden_states:
           UpperCAmelCase_					:					List[Any]			     =    all_hidden_states + (hidden_states,)

     if not return_dict:
        return tuple(v for v in [hidden_states, all_hidden_states] if v is not None  )

     return BaseModelOutputWithNoAttention(last_hidden_state=lowercase_				,   hidden_states=lowercase_  )




class A_	(lowercase__      ):




  '''simple docstring'''
  SCREAMING_SNAKE_CASE__			:						str       =		PoolFormerConfig
  SCREAMING_SNAKE_CASE__			:						Optional[Any]       =		"""poolformer"""
  SCREAMING_SNAKE_CASE__			:						Optional[Any]       =		"""pixel_values"""
  SCREAMING_SNAKE_CASE__			:						Union[str, Any]       =		True







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     if isinstance(lowercase_				,   (nn.Linear, nn.Convad)  ):
        module.weight.data.normal_(mean=0.0				,   std=self.config.initializer_range  )
        if module.bias is not None:
           module.bias.data.zero_()
     elif isinstance(lowercase_				,   nn.LayerNorm  ):
        module.bias.data.zero_()
        module.weight.data.fill_(1.0  )







  def        UpperCamelCase__					(		self				,   lowercase_				,   lowercase_=False  ):

     """simple docstring"""
     if isinstance(lowercase_				,   lowercase_  ):
        UpperCAmelCase_					:					List[Any]			     =    value




_a		       =							R'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use\n    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

_a		       =							R'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`PoolFormerImageProcessor.__call__`] for details.\n'


@add_start_docstrings(
    """The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top."""				,lowercase__				,)
class A_	(lowercase__      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_  ):

     """simple docstring"""
     super().__init__(lowercase_  )
     UpperCAmelCase_					:					Tuple			     =    config

     UpperCAmelCase_					:					Dict			     =    PoolFormerEncoder(lowercase_  )

     # Initialize weights and apply final processing
     self.post_init()







  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     return self.embeddings.patch_embeddings







  @add_start_docstrings_to_model_forward(lowercase_  )
  @add_code_sample_docstrings(
      checkpoint=_CHECKPOINT_FOR_DOC				,   output_type=lowercase_				,   config_class=_CONFIG_FOR_DOC				,   modality="vision"				,   expected_output=_EXPECTED_OUTPUT_SHAPE				,   )
  def        UpperCamelCase__					(		self				,   lowercase_ = None				,   lowercase_ = None				,   lowercase_ = None				,   ):

     """simple docstring"""
     UpperCAmelCase_					:					str			     =    (
         output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
     )
     UpperCAmelCase_					:					int			     =    return_dict if return_dict is not None else self.config.use_return_dict

     if pixel_values is None:
        raise ValueError("You have to specify pixel_values"  )

     UpperCAmelCase_					:					Optional[Any]			     =    self.encoder(
         lowercase_				,   output_hidden_states=lowercase_				,   return_dict=lowercase_				,   )
     UpperCAmelCase_					:					Union[str, Any]			     =    encoder_outputs[0]

     if not return_dict:
        return (sequence_output, None) + encoder_outputs[1:]

     return BaseModelOutputWithNoAttention(
         last_hidden_state=lowercase_				,   hidden_states=encoder_outputs.hidden_states				,   )




class A_	(nn.Module      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_  ):

     """simple docstring"""
     super().__init__()
     UpperCAmelCase_					:					List[str]			     =    nn.Linear(config.hidden_size				,   config.hidden_size  )







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     UpperCAmelCase_					:					Union[str, Any]			     =    self.dense(lowercase_  )
     return output




@add_start_docstrings(
    """
    PoolFormer Model transformer with an image classification head on top
    """				,lowercase__				,)
class A_	(lowercase__      ):




  '''simple docstring'''







  def __init__(		self				,   lowercase_  ):

     """simple docstring"""
     super().__init__(lowercase_  )
     UpperCAmelCase_					:					Tuple			     =    config.num_labels
     UpperCAmelCase_					:					List[Any]			     =    PoolFormerModel(lowercase_  )

     # Final norm
     UpperCAmelCase_					:					Union[str, Any]			     =    PoolFormerGroupNorm(config.hidden_sizes[-1]  )
     # Classifier head
     UpperCAmelCase_					:					Optional[Any]			     =    (
         nn.Linear(config.hidden_sizes[-1]				,   config.num_labels  ) if config.num_labels > 0 else nn.Identity()
     )

     # Initialize weights and apply final processing
     self.post_init()







  @add_start_docstrings_to_model_forward(lowercase_  )
  @add_code_sample_docstrings(
      checkpoint=_IMAGE_CLASS_CHECKPOINT				,   output_type=lowercase_				,   config_class=_CONFIG_FOR_DOC				,   expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT				,   )
  def        UpperCamelCase__					(		self				,   lowercase_ = None				,   lowercase_ = None				,   lowercase_ = None				,   lowercase_ = None				,   ):

     """simple docstring"""
     UpperCAmelCase_					:					Optional[Any]			     =    return_dict if return_dict is not None else self.config.use_return_dict

     UpperCAmelCase_					:					Optional[int]			     =    self.poolformer(
         lowercase_				,   output_hidden_states=lowercase_				,   return_dict=lowercase_				,   )

     UpperCAmelCase_					:					Union[str, Any]			     =    outputs[0]

     UpperCAmelCase_					:					Union[str, Any]			     =    self.classifier(self.norm(lowercase_  ).mean([-2, -1]  )  )

     UpperCAmelCase_					:					str			     =    None
     if labels is not None:
        if self.config.problem_type is None:
           if self.num_labels == 1:
              UpperCAmelCase_					:					Dict			     =    "regression"
           elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
              UpperCAmelCase_					:					Optional[Any]			     =    "single_label_classification"
           else:
              UpperCAmelCase_					:					List[str]			     =    "multi_label_classification"

        if self.config.problem_type == "regression":
           UpperCAmelCase_					:					List[Any]			     =    MSELoss()
           if self.num_labels == 1:
              UpperCAmelCase_					:					Tuple			     =    loss_fct(logits.squeeze()				,   labels.squeeze()  )
           else:
              UpperCAmelCase_					:					Optional[int]			     =    loss_fct(lowercase_				,   lowercase_  )
        elif self.config.problem_type == "single_label_classification":
           UpperCAmelCase_					:					Optional[Any]			     =    CrossEntropyLoss()
           UpperCAmelCase_					:					Optional[Any]			     =    loss_fct(logits.view(-1				,   self.num_labels  )				,   labels.view(-1  )  )
        elif self.config.problem_type == "multi_label_classification":
           UpperCAmelCase_					:					Tuple			     =    BCEWithLogitsLoss()
           UpperCAmelCase_					:					str			     =    loss_fct(lowercase_				,   lowercase_  )

     if not return_dict:
        UpperCAmelCase_					:					Optional[Any]			     =    (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

     return ImageClassifierOutputWithNoAttention(loss=lowercase_				,   logits=lowercase_				,   hidden_states=outputs.hidden_states  )




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


if is_sentencepiece_available():
   from ..ta.tokenization_ta import TaTokenizer
else:
   from ...utils.dummy_sentencepiece_objects import TaTokenizer

__lowerCamelCase							=       TaTokenizer

if is_tokenizers_available():
   from ..ta.tokenization_ta_fast import TaTokenizerFast
else:
   from ...utils.dummy_tokenizers_objects import TaTokenizerFast

__lowerCamelCase							=       TaTokenizerFast

__lowerCamelCase							=       {"""configuration_mt5""": ["""MT5Config""", """MT5OnnxConfig"""]}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase							=       [
       """MT5EncoderModel""",
       """MT5ForConditionalGeneration""",
       """MT5ForQuestionAnswering""",
       """MT5Model""",
       """MT5PreTrainedModel""",
       """MT5Stack""",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase							=       ["""TFMT5EncoderModel""", """TFMT5ForConditionalGeneration""", """TFMT5Model"""]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase							=       ["""FlaxMT5EncoderModel""", """FlaxMT5ForConditionalGeneration""", """FlaxMT5Model"""]


if TYPE_CHECKING:
   from .configuration_mta import MTaConfig, MTaOnnxConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mta import (
          MTaEncoderModel,
          MTaForConditionalGeneration,
          MTaForQuestionAnswering,
          MTaModel,
          MTaPreTrainedModel,
          MTaStack,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_mta import TFMTaEncoderModel, TFMTaForConditionalGeneration, TFMTaModel

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_mta import FlaxMTaEncoderModel, FlaxMTaForConditionalGeneration, FlaxMTaModel

else:
   import sys

   __lowerCamelCase							=       _LazyModule(
       __name__,
       globals()["""__file__"""],
       _import_structure,
       extra_objects={"""MT5Tokenizer""": MTaTokenizer, """MT5TokenizerFast""": MTaTokenizerFast},
       module_spec=__spec__,
   )





