'''simple docstring'''





from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin



@dataclass
class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    A     :    torch.FloatTensor






class   UpperCamelCase					(       a_    ,     a_       ):







    """simple docstring"""





    @register_to_config
    def __init__(					self     :						Optional[int]							,							UpperCAmelCase_     :						int = 1_6							,							UpperCAmelCase_     :						int = 8_8							,							UpperCAmelCase_     :						Optional[int] = None							,							UpperCAmelCase_     :						Optional[int] = None							,							UpperCAmelCase_     :						int = 1							,							UpperCAmelCase_     :						float = 0.0							,							UpperCAmelCase_     :						int = 3_2							,							UpperCAmelCase_     :						Optional[int] = None							,							UpperCAmelCase_     :						bool = False							,							UpperCAmelCase_     :						Optional[int] = None							,							UpperCAmelCase_     :						Optional[int] = None							,							UpperCAmelCase_     :						Optional[int] = None							,							UpperCAmelCase_     :						str = "geglu"							,							UpperCAmelCase_     :						Optional[int] = None							,							UpperCAmelCase_     :						bool = False							,							UpperCAmelCase_     :						bool = False							,							UpperCAmelCase_     :						bool = False							,							UpperCAmelCase_     :						str = "layer_norm"							,							UpperCAmelCase_     :						bool = True							,							):


         """simple docstring"""







         super().__init__()
         a					:							Any							= use_linear_projection
         a					:							Tuple							= num_attention_heads
         a					:							int							= attention_head_dim
         a					:							str							= num_attention_heads * attention_head_dim

         # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
         # Define whether input is continuous or discrete depending on configuration
         a					:							List[str]							= (in_channels is not None) and (patch_size is None)
         a					:							Dict							= num_vector_embeds is not None
         a					:							List[str]							= in_channels is not None and patch_size is not None

         if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
              a					:							Tuple							= (
                  f"""The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"""
                  ' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'
                  ' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'
                  ' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'
                  ' would be very nice if you could open a Pull request for the `transformer/config.json` file'
              )
              deprecate('norm_type!=num_embeds_ada_norm'							,							'1.0.0'							,							UpperCAmelCase_							,							standard_warn=UpperCAmelCase_)
              a					:							List[Any]							= 'ada_norm'

         if self.is_input_continuous and self.is_input_vectorized:
              raise ValueError(
                  f"""Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"""
                  ' sure that either `in_channels` or `num_vector_embeds` is None.')
         elif self.is_input_vectorized and self.is_input_patches:
              raise ValueError(
                  f"""Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"""
                  ' sure that either `num_vector_embeds` or `num_patches` is None.')
         elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
              raise ValueError(
                  f"""Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"""
                  f""" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.""")

         # 2. Define input layers
         if self.is_input_continuous:
              a					:							List[str]							= in_channels

              a					:							Optional[Any]							= torch.nn.GroupNorm(num_groups=UpperCAmelCase_							,							num_channels=UpperCAmelCase_							,							eps=1e-6							,							affine=UpperCAmelCase_)
              if use_linear_projection:
                   a					:							str							= nn.Linear(UpperCAmelCase_							,							UpperCAmelCase_)
              else:
                   a					:							List[str]							= nn.Convad(UpperCAmelCase_							,							UpperCAmelCase_							,							kernel_size=1							,							stride=1							,							padding=0)
         elif self.is_input_vectorized:
              assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
              assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

              a					:							Tuple							= sample_size
              a					:							Optional[int]							= sample_size
              a					:							Union[str, Any]							= num_vector_embeds
              a					:							Union[str, Any]							= self.height * self.width

              a					:							Tuple							= ImagePositionalEmbeddings(
                  num_embed=UpperCAmelCase_							,							embed_dim=UpperCAmelCase_							,							height=self.height							,							width=self.width)
         elif self.is_input_patches:
              assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

              a					:							str							= sample_size
              a					:							Union[str, Any]							= sample_size

              a					:							Dict							= patch_size
              a					:							List[str]							= PatchEmbed(
                  height=UpperCAmelCase_							,							width=UpperCAmelCase_							,							patch_size=UpperCAmelCase_							,							in_channels=UpperCAmelCase_							,							embed_dim=UpperCAmelCase_							,							)

         # 3. Define transformers blocks
         a					:							Dict							= nn.ModuleList(
             [
                 BasicTransformerBlock(
                     UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							dropout=UpperCAmelCase_							,							cross_attention_dim=UpperCAmelCase_							,							activation_fn=UpperCAmelCase_							,							num_embeds_ada_norm=UpperCAmelCase_							,							attention_bias=UpperCAmelCase_							,							only_cross_attention=UpperCAmelCase_							,							upcast_attention=UpperCAmelCase_							,							norm_type=UpperCAmelCase_							,							norm_elementwise_affine=UpperCAmelCase_							,							)
                 for d in range(UpperCAmelCase_)
             ])

         # 4. Define output layers
         a					:							int							= in_channels if out_channels is None else out_channels
         if self.is_input_continuous:
              # TODO: should use out_channels for continuous projections
              if use_linear_projection:
                   a					:							int							= nn.Linear(UpperCAmelCase_							,							UpperCAmelCase_)
              else:
                   a					:							Union[str, Any]							= nn.Convad(UpperCAmelCase_							,							UpperCAmelCase_							,							kernel_size=1							,							stride=1							,							padding=0)
         elif self.is_input_vectorized:
              a					:							Any							= nn.LayerNorm(UpperCAmelCase_)
              a					:							Union[str, Any]							= nn.Linear(UpperCAmelCase_							,							self.num_vector_embeds - 1)
         elif self.is_input_patches:
              a					:							str							= nn.LayerNorm(UpperCAmelCase_							,							elementwise_affine=UpperCAmelCase_							,							eps=1e-6)
              a					:							Dict							= nn.Linear(UpperCAmelCase_							,							2 * inner_dim)
              a					:							List[Any]							= nn.Linear(UpperCAmelCase_							,							patch_size * patch_size * self.out_channels)







    def    SCREAMING_SNAKE_CASE_     (					self     :						str							,							UpperCAmelCase_     :						torch.Tensor							,							UpperCAmelCase_     :						Optional[torch.Tensor] = None							,							UpperCAmelCase_     :						Optional[torch.LongTensor] = None							,							UpperCAmelCase_     :						Optional[torch.LongTensor] = None							,							UpperCAmelCase_     :						Dict[str, Any] = None							,							UpperCAmelCase_     :						Optional[torch.Tensor] = None							,							UpperCAmelCase_     :						Optional[torch.Tensor] = None							,							UpperCAmelCase_     :						bool = True							,							):


         """simple docstring"""







         if attention_mask is not None and attention_mask.ndim == 2:
              # assume that mask is expressed as:
              #   (1 = keep,      0 = discard)
              # convert mask into a bias that can be added to attention scores:
              #       (keep = +0,     discard = -10000.0)
              a					:							Union[str, Any]							= (1 - attention_mask.to(hidden_states.dtype)) * -1_00_00.0
              a					:							str							= attention_mask.unsqueeze(1)

         # convert encoder_attention_mask to a bias the same way we do for attention_mask
         if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
              a					:							Dict							= (1 - encoder_attention_mask.to(hidden_states.dtype)) * -1_00_00.0
              a					:							List[Any]							= encoder_attention_mask.unsqueeze(1)

         # 1. Input
         if self.is_input_continuous:
              a							,			a							,			a							,			a					:							Optional[int]							= hidden_states.shape
              a					:							Dict							= hidden_states

              a					:							Union[str, Any]							= self.norm(UpperCAmelCase_)
              if not self.use_linear_projection:
                   a					:							Dict							= self.proj_in(UpperCAmelCase_)
                   a					:							Tuple							= hidden_states.shape[1]
                   a					:							Tuple							= hidden_states.permute(0							,							2							,							3							,							1).reshape(UpperCAmelCase_							,							height * width							,							UpperCAmelCase_)
              else:
                   a					:							str							= hidden_states.shape[1]
                   a					:							Optional[Any]							= hidden_states.permute(0							,							2							,							3							,							1).reshape(UpperCAmelCase_							,							height * width							,							UpperCAmelCase_)
                   a					:							List[str]							= self.proj_in(UpperCAmelCase_)
         elif self.is_input_vectorized:
              a					:							Union[str, Any]							= self.latent_image_embedding(UpperCAmelCase_)
         elif self.is_input_patches:
              a					:							List[str]							= self.pos_embed(UpperCAmelCase_)

         # 2. Blocks
         for block in self.transformer_blocks:
              a					:							Tuple							= block(
                  UpperCAmelCase_							,							attention_mask=UpperCAmelCase_							,							encoder_hidden_states=UpperCAmelCase_							,							encoder_attention_mask=UpperCAmelCase_							,							timestep=UpperCAmelCase_							,							cross_attention_kwargs=UpperCAmelCase_							,							class_labels=UpperCAmelCase_							,							)

         # 3. Output
         if self.is_input_continuous:
              if not self.use_linear_projection:
                   a					:							int							= hidden_states.reshape(UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_).permute(0							,							3							,							1							,							2).contiguous()
                   a					:							List[Any]							= self.proj_out(UpperCAmelCase_)
              else:
                   a					:							Any							= self.proj_out(UpperCAmelCase_)
                   a					:							str							= hidden_states.reshape(UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_).permute(0							,							3							,							1							,							2).contiguous()

              a					:							List[Any]							= hidden_states + residual
         elif self.is_input_vectorized:
              a					:							Any							= self.norm_out(UpperCAmelCase_)
              a					:							int							= self.out(UpperCAmelCase_)
              # (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
              a					:							List[Any]							= logits.permute(0							,							2							,							1)

              # log(p(x_0))
              a					:							Dict							= F.log_softmax(logits.double()							,							dim=1).float()
         elif self.is_input_patches:
              # TODO: cleanup!
              a					:							Dict							= self.transformer_blocks[0].norma.emb(
                  UpperCAmelCase_							,							UpperCAmelCase_							,							hidden_dtype=hidden_states.dtype)
              a							,			a					:							Optional[Any]							= self.proj_out_a(F.silu(UpperCAmelCase_)).chunk(2							,							dim=1)
              a					:							str							= self.norm_out(UpperCAmelCase_) * (1 + scale[:, None]) + shift[:, None]
              a					:							Tuple							= self.proj_out_a(UpperCAmelCase_)

              # unpatchify
              a					:							Optional[int]							= int(hidden_states.shape[1] ** 0.5)
              a					:							str							= hidden_states.reshape(
                  shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels))
              a					:							List[str]							= torch.einsum('nhwpqc->nchpwq'							,							UpperCAmelCase_)
              a					:							List[Any]							= hidden_states.reshape(
                  shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size))

         if not return_dict:
              return (output,)

         return TransformeraDModelOutput(sample=UpperCAmelCase_)


'''simple docstring'''





from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


UpperCamelCase       :     Any			=    logging.get_logger(__name__)

UpperCamelCase       :     Tuple			=    {
    """google/long-t5-local-base""": """https://huggingface.co/google/long-t5-local-base/blob/main/config.json""",
    """google/long-t5-local-large""": """https://huggingface.co/google/long-t5-local-large/blob/main/config.json""",
    """google/long-t5-tglobal-base""": """https://huggingface.co/google/long-t5-tglobal-base/blob/main/config.json""",
    """google/long-t5-tglobal-large""": """https://huggingface.co/google/long-t5-tglobal-large/blob/main/config.json""",
}



class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    A     :    Union[str, Any]				=			"longt5"
    A     :    List[str]				=			["past_key_values"]
    A     :    Dict				=			{"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}





    def __init__(					self     :						int							,							UpperCAmelCase_     :						Union[str, Any]=3_2_1_2_8							,							UpperCAmelCase_     :						List[Any]=5_1_2							,							UpperCAmelCase_     :						List[Any]=6_4							,							UpperCAmelCase_     :						List[str]=2_0_4_8							,							UpperCAmelCase_     :						List[str]=6							,							UpperCAmelCase_     :						Tuple=None							,							UpperCAmelCase_     :						Tuple=8							,							UpperCAmelCase_     :						str=1_2_7							,							UpperCAmelCase_     :						Dict=1_6							,							UpperCAmelCase_     :						Optional[int]=3_2							,							UpperCAmelCase_     :						Union[str, Any]=1_2_8							,							UpperCAmelCase_     :						str=0.1							,							UpperCAmelCase_     :						Optional[int]=1e-6							,							UpperCAmelCase_     :						Tuple=1.0							,							UpperCAmelCase_     :						str="relu"							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						Dict="local"							,							UpperCAmelCase_     :						Optional[int]=True							,							UpperCAmelCase_     :						Dict=0							,							UpperCAmelCase_     :						Union[str, Any]=1							,							**UpperCAmelCase_     :						Any							,							):


         """simple docstring"""







         a					:							Optional[int]							= vocab_size
         a					:							Tuple							= d_model
         a					:							str							= d_kv
         a					:							str							= d_ff
         a					:							Union[str, Any]							= num_layers
         # default = symmetry
         a					:							Optional[Any]							= num_decoder_layers if num_decoder_layers is not None else self.num_layers
         a					:							List[Any]							= num_heads
         a					:							List[str]							= local_radius
         a					:							Optional[Any]							= global_block_size
         a					:							Optional[int]							= relative_attention_num_buckets
         a					:							List[Any]							= relative_attention_max_distance
         a					:							int							= dropout_rate
         a					:							List[str]							= layer_norm_epsilon
         a					:							Optional[Any]							= initializer_factor
         a					:							List[Any]							= feed_forward_proj
         a					:							Dict							= encoder_attention_type
         a					:							int							= use_cache

         a					:							Optional[Any]							= self.feed_forward_proj.split('-')
         a					:							List[Any]							= act_info[-1]
         a					:							str							= act_info[0] == 'gated'

         if len(UpperCAmelCase_) > 1 and act_info[0] != "gated" or len(UpperCAmelCase_) > 2:
              raise ValueError(
                  f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
                  'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
                  '\'gated-gelu\' or \'relu\'')

         # for backwards compatibility
         if feed_forward_proj == "gated-gelu":
              a					:							Any							= 'gelu_new'

         super().__init__(
             pad_token_id=UpperCAmelCase_							,							eos_token_id=UpperCAmelCase_							,							is_encoder_decoder=UpperCAmelCase_							,							**UpperCAmelCase_							,							)






class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    @property
    def    SCREAMING_SNAKE_CASE_     (					self     :						Union[str, Any]):


         """simple docstring"""







         a					:							List[Any]							= {
             'input_ids': {0: 'batch', 1: 'encoder_sequence'},
             'attention_mask': {0: 'batch', 1: 'encoder_sequence'},
         }
         if self.use_past:
              a					:							List[str]							= 'past_encoder_sequence + sequence'
              a					:							str							= {0: 'batch'}
              a					:							List[Any]							= {0: 'batch', 1: 'past_decoder_sequence + sequence'}
         else:
              a					:							Optional[Any]							= {0: 'batch', 1: 'decoder_sequence'}
              a					:							str							= {0: 'batch', 1: 'decoder_sequence'}

         if self.use_past:
              self.fill_with_past_key_values_(UpperCAmelCase_							,							direction='inputs')

         return common_inputs







    @property
    def    SCREAMING_SNAKE_CASE_     (					self     :						Union[str, Any]):


         """simple docstring"""







         return 1_3

