




from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings






@maybe_allow_in_graph
class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			Optional[Any]    ,    snake_case__    :			Any    ,    snake_case__    :			Union[str, Any]    ,    snake_case__    :			Dict    ,    snake_case__    :			int=0.0    ,    snake_case__    :			Optional[Any] = None    ,    snake_case__    :			str = "geglu"    ,    snake_case__    :			Tuple = None    ,    snake_case__    :			Tuple = False    ,    snake_case__    :			List[str] = False    ,    snake_case__    :			Any = False    ,    snake_case__    :			Union[str, Any] = False    ,    snake_case__    :			Optional[int] = True    ,    snake_case__    :			Tuple = "layer_norm"    ,    snake_case__    :			Any = False    ,    ):




									"""simple docstring"""


									super().__init__()
									_UpperCAmelCase													=     only_cross_attention

									_UpperCAmelCase													=     (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
									_UpperCAmelCase													=     (num_embeds_ada_norm is not None) and norm_type == "ada_norm"

									if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
															raise ValueError(
															    F"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
															    F""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""      )

									# Define 3 blocks. Each block has its own normalization layer.
									# 1. Self-Attn
									if self.use_ada_layer_norm:
															_UpperCAmelCase													=     AdaLayerNorm(snake_case__    ,    snake_case__      )
									elif self.use_ada_layer_norm_zero:
															_UpperCAmelCase													=     AdaLayerNormZero(snake_case__    ,    snake_case__      )
									else:
															_UpperCAmelCase													=     nn.LayerNorm(snake_case__    ,    elementwise_affine=snake_case__      )
									_UpperCAmelCase													=     Attention(
									    query_dim=snake_case__    ,    heads=snake_case__    ,    dim_head=snake_case__    ,    dropout=snake_case__    ,    bias=snake_case__    ,    cross_attention_dim=cross_attention_dim if only_cross_attention else None    ,    upcast_attention=snake_case__    ,    )

									# 2. Cross-Attn
									if cross_attention_dim is not None or double_self_attention:
															# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
															# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
															# the second cross attention block.
															_UpperCAmelCase													=     (
															    AdaLayerNorm(snake_case__    ,    snake_case__      )
															    if self.use_ada_layer_norm
															    else nn.LayerNorm(snake_case__    ,    elementwise_affine=snake_case__      )
															)
															_UpperCAmelCase													=     Attention(
															    query_dim=snake_case__    ,    cross_attention_dim=cross_attention_dim if not double_self_attention else None    ,    heads=snake_case__    ,    dim_head=snake_case__    ,    dropout=snake_case__    ,    bias=snake_case__    ,    upcast_attention=snake_case__    ,    )  # is self-attn if encoder_hidden_states is none
									else:
															_UpperCAmelCase													=     None
															_UpperCAmelCase													=     None

									# 3. Feed-forward
									_UpperCAmelCase													=     nn.LayerNorm(snake_case__    ,    elementwise_affine=snake_case__      )
									_UpperCAmelCase													=     FeedForward(snake_case__    ,    dropout=snake_case__    ,    activation_fn=snake_case__    ,    final_dropout=snake_case__      )

									# let chunk size default to None
									_UpperCAmelCase													=     None
									_UpperCAmelCase													=     0







			def UpperCamelCase			(       self    :			Optional[Any]    ,    snake_case__    :			List[Any]    ,    snake_case__    :			List[str]      ):




									"""simple docstring"""


									_UpperCAmelCase													=     chunk_size
									_UpperCAmelCase													=     dim







			def UpperCamelCase			(       self    :			Dict    ,    snake_case__    :			Tuple    ,    snake_case__    :			str = None    ,    snake_case__    :			str = None    ,    snake_case__    :			List[str] = None    ,    snake_case__    :			Dict = None    ,    snake_case__    :			Tuple = None    ,    snake_case__    :			Optional[int] = None    ,    ):




									"""simple docstring"""


									if self.use_ada_layer_norm:
															_UpperCAmelCase													=     self.norma(snake_case__    ,    snake_case__      )
									elif self.use_ada_layer_norm_zero:
															_UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase													=     self.norma(
															    snake_case__    ,    snake_case__    ,    snake_case__    ,    hidden_dtype=hidden_states.dtype      )
									else:
															_UpperCAmelCase													=     self.norma(snake_case__      )

									_UpperCAmelCase													=     cross_attention_kwargs if cross_attention_kwargs is not None else {}

									_UpperCAmelCase													=     self.attna(
									    snake_case__    ,    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    ,    attention_mask=snake_case__    ,    **snake_case__    ,    )
									if self.use_ada_layer_norm_zero:
															_UpperCAmelCase													=     gate_msa.unsqueeze(1      ) * attn_output
									_UpperCAmelCase													=     attn_output + hidden_states

									# 2. Cross-Attention
									if self.attna is not None:
															_UpperCAmelCase													=     (
															    self.norma(snake_case__    ,    snake_case__      ) if self.use_ada_layer_norm else self.norma(snake_case__      )
															)

															_UpperCAmelCase													=     self.attna(
															    snake_case__    ,    encoder_hidden_states=snake_case__    ,    attention_mask=snake_case__    ,    **snake_case__    ,    )
															_UpperCAmelCase													=     attn_output + hidden_states

									# 3. Feed-forward
									_UpperCAmelCase													=     self.norma(snake_case__      )

									if self.use_ada_layer_norm_zero:
															_UpperCAmelCase													=     norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

									if self._chunk_size is not None:
															# "feed_forward_chunk_size" can be used to save memory
															if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																					raise ValueError(
																					    F"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""      )

															_UpperCAmelCase													=     norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
															_UpperCAmelCase													=     torch.cat(
															    [self.ff(snake_case__      ) for hid_slice in norm_hidden_states.chunk(snake_case__    ,    dim=self._chunk_dim      )]    ,    dim=self._chunk_dim    ,    )
									else:
															_UpperCAmelCase													=     self.ff(snake_case__      )

									if self.use_ada_layer_norm_zero:
															_UpperCAmelCase													=     gate_mlp.unsqueeze(1      ) * ff_output

									_UpperCAmelCase													=     ff_output + hidden_states

									return hidden_states






class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			Tuple    ,    snake_case__    :			List[str]    ,    snake_case__    :			Any = None    ,    snake_case__    :			Dict = 4    ,    snake_case__    :			Tuple = 0.0    ,    snake_case__    :			int = "geglu"    ,    snake_case__    :			Optional[int] = False    ,    ):




									"""simple docstring"""


									super().__init__()
									_UpperCAmelCase													=     int(dim * mult      )
									_UpperCAmelCase													=     dim_out if dim_out is not None else dim

									if activation_fn == "gelu":
															_UpperCAmelCase													=     GELU(snake_case__    ,    snake_case__      )
									if activation_fn == "gelu-approximate":
															_UpperCAmelCase													=     GELU(snake_case__    ,    snake_case__    ,    approximate="tanh"      )
									elif activation_fn == "geglu":
															_UpperCAmelCase													=     GEGLU(snake_case__    ,    snake_case__      )
									elif activation_fn == "geglu-approximate":
															_UpperCAmelCase													=     ApproximateGELU(snake_case__    ,    snake_case__      )

									_UpperCAmelCase													=     nn.ModuleList([]      )
									# project in
									self.net.append(snake_case__      )
									# project dropout
									self.net.append(nn.Dropout(snake_case__      )      )
									# project out
									self.net.append(nn.Linear(snake_case__    ,    snake_case__      )      )
									# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
									if final_dropout:
															self.net.append(nn.Dropout(snake_case__      )      )







			def UpperCamelCase			(       self    :			List[str]    ,    snake_case__    :			Tuple      ):




									"""simple docstring"""


									for module in self.net:
															_UpperCAmelCase													=     module(snake_case__      )
									return hidden_states






class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			Any    ,    snake_case__    :			List[str]    ,    snake_case__    :			Any    ,    snake_case__    :			Union[str, Any] = "none"      ):




									"""simple docstring"""


									super().__init__()
									_UpperCAmelCase													=     nn.Linear(snake_case__    ,    snake_case__      )
									_UpperCAmelCase													=     approximate







			def UpperCamelCase			(       self    :			Optional[int]    ,    snake_case__    :			List[Any]      ):




									"""simple docstring"""


									if gate.device.type != "mps":
															return F.gelu(snake_case__    ,    approximate=self.approximate      )
									# mps: gelu is not implemented for float16
									return F.gelu(gate.to(dtype=torch.floataa      )    ,    approximate=self.approximate      ).to(dtype=gate.dtype      )







			def UpperCamelCase			(       self    :			int    ,    snake_case__    :			Any      ):




									"""simple docstring"""


									_UpperCAmelCase													=     self.proj(snake_case__      )
									_UpperCAmelCase													=     self.gelu(snake_case__      )
									return hidden_states






class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			Optional[int]    ,    snake_case__    :			int    ,    snake_case__    :			str      ):




									"""simple docstring"""


									super().__init__()
									_UpperCAmelCase													=     nn.Linear(snake_case__    ,    dim_out * 2      )







			def UpperCamelCase			(       self    :			Tuple    ,    snake_case__    :			Optional[Any]      ):




									"""simple docstring"""


									if gate.device.type != "mps":
															return F.gelu(snake_case__      )
									# mps: gelu is not implemented for float16
									return F.gelu(gate.to(dtype=torch.floataa      )      ).to(dtype=gate.dtype      )







			def UpperCamelCase			(       self    :			Dict    ,    snake_case__    :			Dict      ):




									"""simple docstring"""


									_UpperCAmelCase							, _UpperCAmelCase													=     self.proj(snake_case__      ).chunk(2    ,    dim=-1      )
									return hidden_states * self.gelu(snake_case__      )






class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			Optional[Any]    ,    snake_case__    :			Dict    ,    snake_case__    :			Union[str, Any]      ):




									"""simple docstring"""


									super().__init__()
									_UpperCAmelCase													=     nn.Linear(snake_case__    ,    snake_case__      )







			def UpperCamelCase			(       self    :			Tuple    ,    snake_case__    :			Dict      ):




									"""simple docstring"""


									_UpperCAmelCase													=     self.proj(snake_case__      )
									return x * torch.sigmoid(1.702 * x      )






class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			List[str]    ,    snake_case__    :			Dict    ,    snake_case__    :			List[Any]      ):




									"""simple docstring"""


									super().__init__()
									_UpperCAmelCase													=     nn.Embedding(snake_case__    ,    snake_case__      )
									_UpperCAmelCase													=     nn.SiLU()
									_UpperCAmelCase													=     nn.Linear(snake_case__    ,    embedding_dim * 2      )
									_UpperCAmelCase													=     nn.LayerNorm(snake_case__    ,    elementwise_affine=snake_case__      )







			def UpperCamelCase			(       self    :			Any    ,    snake_case__    :			Optional[int]    ,    snake_case__    :			int      ):




									"""simple docstring"""


									_UpperCAmelCase													=     self.linear(self.silu(self.emb(snake_case__      )      )      )
									_UpperCAmelCase							, _UpperCAmelCase													=     torch.chunk(snake_case__    ,    2      )
									_UpperCAmelCase													=     self.norm(snake_case__      ) * (1 + scale) + shift
									return x






class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			Any    ,    snake_case__    :			Dict    ,    snake_case__    :			str      ):




									"""simple docstring"""


									super().__init__()

									_UpperCAmelCase													=     CombinedTimestepLabelEmbeddings(snake_case__    ,    snake_case__      )

									_UpperCAmelCase													=     nn.SiLU()
									_UpperCAmelCase													=     nn.Linear(snake_case__    ,    6 * embedding_dim    ,    bias=snake_case__      )
									_UpperCAmelCase													=     nn.LayerNorm(snake_case__    ,    elementwise_affine=snake_case__    ,    eps=1e-6      )







			def UpperCamelCase			(       self    :			Optional[int]    ,    snake_case__    :			Optional[int]    ,    snake_case__    :			Optional[Any]    ,    snake_case__    :			Optional[int]    ,    snake_case__    :			Any=None      ):




									"""simple docstring"""


									_UpperCAmelCase													=     self.linear(self.silu(self.emb(snake_case__    ,    snake_case__    ,    hidden_dtype=snake_case__      )      )      )
									_UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase													=     emb.chunk(6    ,    dim=1      )
									_UpperCAmelCase													=     self.norm(snake_case__      ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
									return x, gate_msa, shift_mlp, scale_mlp, gate_mlp






class     __lowerCAmelCase (       nn.Module ):







			def __init__(       self    :			Tuple    ,    snake_case__    :			Optional[int]    ,    snake_case__    :			int    ,    snake_case__    :			Optional[int]    ,    snake_case__    :			Optional[int] = None    ,    snake_case__    :			Dict = 1e-5      ):




									"""simple docstring"""


									super().__init__()
									_UpperCAmelCase													=     num_groups
									_UpperCAmelCase													=     eps

									if act_fn is None:
															_UpperCAmelCase													=     None
									else:
															_UpperCAmelCase													=     get_activation(snake_case__      )

									_UpperCAmelCase													=     nn.Linear(snake_case__    ,    out_dim * 2      )







			def UpperCamelCase			(       self    :			Optional[int]    ,    snake_case__    :			Optional[Any]    ,    snake_case__    :			Optional[Any]      ):




									"""simple docstring"""


									if self.act:
															_UpperCAmelCase													=     self.act(snake_case__      )
									_UpperCAmelCase													=     self.linear(snake_case__      )
									_UpperCAmelCase													=     emb[:, :, None, None]
									_UpperCAmelCase							, _UpperCAmelCase													=     emb.chunk(2    ,    dim=1      )

									_UpperCAmelCase													=     F.group_norm(snake_case__    ,    self.num_groups    ,    eps=self.eps      )
									_UpperCAmelCase													=     x * (1 + scale) + shift
									return x











"""simple docstring"""





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
       from ...onnx.config import PatchingSpec
       from ...tokenization_utils_base import PreTrainedTokenizerBase


SCREAMING_SNAKE_CASE__					=   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__					=   {
    "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json",
    "allenai/longformer-large-4096": "https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json",
    "allenai/longformer-large-4096-finetuned-triviaqa": (
        "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json"
    ),
    "allenai/longformer-base-4096-extra.pos.embd.only": (
        "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json"
    ),
    "allenai/longformer-large-4096-extra.pos.embd.only": (
        "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json"
    ),
}

class 		lowercase					(					_UpperCAmelCase					):
      _SCREAMING_SNAKE_CASE					=      'longformer'





      def __init__(       self      ,						lowercase = 512      ,						lowercase = 2      ,						lowercase = 1      ,						lowercase = 0      ,						lowercase = 2      ,						lowercase = 30_522      ,						lowercase = 768      ,						lowercase = 12      ,						lowercase = 12      ,						lowercase = 3_072      ,						lowercase = "gelu"      ,						lowercase = 0.1      ,						lowercase = 0.1      ,						lowercase = 512      ,						lowercase = 2      ,						lowercase = 0.02      ,						lowercase = 1e-12      ,						lowercase = False      ,						**lowercase      ,						)		->     Optional[int]:
            super().__init__(pad_token_id=lowercase      ,						**lowercase )

            lowerCAmelCase  						=       attention_window
            lowerCAmelCase  						=       sep_token_id
            lowerCAmelCase  						=       bos_token_id
            lowerCAmelCase  						=       eos_token_id
            lowerCAmelCase  						=       vocab_size
            lowerCAmelCase  						=       hidden_size
            lowerCAmelCase  						=       num_hidden_layers
            lowerCAmelCase  						=       num_attention_heads
            lowerCAmelCase  						=       hidden_act
            lowerCAmelCase  						=       intermediate_size
            lowerCAmelCase  						=       hidden_dropout_prob
            lowerCAmelCase  						=       attention_probs_dropout_prob
            lowerCAmelCase  						=       max_position_embeddings
            lowerCAmelCase  						=       type_vocab_size
            lowerCAmelCase  						=       initializer_range
            lowerCAmelCase  						=       layer_norm_eps
            lowerCAmelCase  						=       onnx_export







class 		lowercase					(					_UpperCAmelCase					):





      def __init__(       self      ,						lowercase      ,						lowercase = "default"      ,						lowercase = None )		->     Tuple:
            super().__init__(lowercase      ,						lowercase      ,						lowercase )
            lowerCAmelCase  						=       True





      @property
      def     _snake_case			(       self )		->     Mapping[str, Mapping[int, str]]:
            if self.task == "multiple-choice":
                  lowerCAmelCase  						=       {0: """batch""", 1: """choice""", 2: """sequence"""}
            else:
                  lowerCAmelCase  						=       {0: """batch""", 1: """sequence"""}
            return OrderedDict(
                [
                    ("""input_ids""", dynamic_axis),
                    ("""attention_mask""", dynamic_axis),
                    ("""global_attention_mask""", dynamic_axis),
                ] )





      @property
      def     _snake_case			(       self )		->     Mapping[str, Mapping[int, str]]:
            lowerCAmelCase  						=       super().outputs
            if self.task == "default":
                  lowerCAmelCase  						=       {0: """batch"""}
            return outputs





      @property
      def     _snake_case			(       self )		->     float:
            return 1e-4





      @property
      def     _snake_case			(       self )		->     int:
            # needs to be >= 14 to support tril operator
            return max(super().default_onnx_opset      ,						14 )





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = -1      ,						lowercase = -1      ,						lowercase = False      ,						lowercase = None      ,						)		->     Mapping[str, Any]:
            lowerCAmelCase  						=       super().generate_dummy_inputs(
                preprocessor=lowercase      ,						batch_size=lowercase      ,						seq_length=lowercase      ,						is_pair=lowercase      ,						framework=lowercase )
            import torch

            # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
            # makes the export fail randomly
            lowerCAmelCase  						=       torch.zeros_like(inputs["""input_ids"""] )
            # make every second token global
            lowerCAmelCase  						=       1

            return inputs






