"""simple docstring"""







from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin

@dataclass
class 						UpperCamelCase     (       lowercase		):
				UpperCAmelCase							: torch.FloatTensor







class 						UpperCamelCase     (       lowercase ,      lowercase		):



				@register_to_config
				def __init__(self							: Union[str, Any]      ,    _A							: int = 16      ,    _A							: int = 88      ,    _A							: Optional[int] = None      ,    _A							: Optional[int] = None      ,    _A							: int = 1      ,    _A							: float = 0.0      ,    _A							: int = 32      ,    _A							: Optional[int] = None      ,    _A							: bool = False      ,    _A							: Optional[int] = None      ,    _A							: Optional[int] = None      ,    _A							: Optional[int] = None      ,    _A							: str = "geglu"      ,    _A							: Optional[int] = None      ,    _A							: bool = False      ,    _A							: bool = False      ,    _A							: bool = False      ,    _A							: str = "layer_norm"      ,    _A							: bool = True      ,    )       ->      Optional[Any]:
					super().__init__()
					__snake_case     :				int 						=					use_linear_projection
					__snake_case     :				Tuple 						=					num_attention_heads
					__snake_case     :				List[Any] 						=					attention_head_dim
					__snake_case     :				List[Any] 						=					num_attention_heads * attention_head_dim

					# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
					# Define whether input is continuous or discrete depending on configuration
					__snake_case     :				List[str] 						=					(in_channels is not None) and (patch_size is None)
					__snake_case     :				int 						=					num_vector_embeds is not None
					__snake_case     :				int 						=					in_channels is not None and patch_size is not None

					if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
						__snake_case     :				Tuple 						=					(
						    f"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"
						    ' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'
						    ' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'
						    ' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'
						    ' would be very nice if you could open a Pull request for the `transformer/config.json` file'
						)
						deprecate('norm_type!=num_embeds_ada_norm'      ,    '1.0.0'      ,    _A      ,    standard_warn=_A)
						__snake_case     :				str 						=					'ada_norm'

					if self.is_input_continuous and self.is_input_vectorized:
						raise ValueError(
						    f"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"
						    ' sure that either `in_channels` or `num_vector_embeds` is None.')
					elif self.is_input_vectorized and self.is_input_patches:
						raise ValueError(
						    f"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"
						    ' sure that either `num_vector_embeds` or `num_patches` is None.')
					elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
						raise ValueError(
						    f"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"
						    f" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.")

					# 2. Define input layers
					if self.is_input_continuous:
						__snake_case     :				int 						=					in_channels

						__snake_case     :				Union[str, Any] 						=					torch.nn.GroupNorm(num_groups=_A      ,    num_channels=_A      ,    eps=1E-6      ,    affine=_A)
						if use_linear_projection:
							__snake_case     :				str 						=					nn.Linear(_A      ,    _A)
						else:
							__snake_case     :				str 						=					nn.Convad(_A      ,    _A      ,    kernel_size=1      ,    stride=1      ,    padding=0)
					elif self.is_input_vectorized:
						assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
						assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

						__snake_case     :				Any 						=					sample_size
						__snake_case     :				Union[str, Any] 						=					sample_size
						__snake_case     :				Dict 						=					num_vector_embeds
						__snake_case     :				Union[str, Any] 						=					self.height * self.width

						__snake_case     :				List[Any] 						=					ImagePositionalEmbeddings(
						    num_embed=_A      ,    embed_dim=_A      ,    height=self.height      ,    width=self.width)
					elif self.is_input_patches:
						assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

						__snake_case     :				Optional[int] 						=					sample_size
						__snake_case     :				Any 						=					sample_size

						__snake_case     :				Dict 						=					patch_size
						__snake_case     :				Optional[Any] 						=					PatchEmbed(
						    height=_A      ,    width=_A      ,    patch_size=_A      ,    in_channels=_A      ,    embed_dim=_A      ,    )

					# 3. Define transformers blocks
					__snake_case     :				Union[str, Any] 						=					nn.ModuleList(
					    [
					        BasicTransformerBlock(
					            _A      ,    _A      ,    _A      ,    dropout=_A      ,    cross_attention_dim=_A      ,    activation_fn=_A      ,    num_embeds_ada_norm=_A      ,    attention_bias=_A      ,    only_cross_attention=_A      ,    upcast_attention=_A      ,    norm_type=_A      ,    norm_elementwise_affine=_A      ,    )
					        for d in range(_A)
					    ])

					# 4. Define output layers
					__snake_case     :				str 						=					in_channels if out_channels is None else out_channels
					if self.is_input_continuous:
						# TODO: should use out_channels for continuous projections
						if use_linear_projection:
							__snake_case     :				int 						=					nn.Linear(_A      ,    _A)
						else:
							__snake_case     :				str 						=					nn.Convad(_A      ,    _A      ,    kernel_size=1      ,    stride=1      ,    padding=0)
					elif self.is_input_vectorized:
						__snake_case     :				Dict 						=					nn.LayerNorm(_A)
						__snake_case     :				int 						=					nn.Linear(_A      ,    self.num_vector_embeds - 1)
					elif self.is_input_patches:
						__snake_case     :				Optional[Any] 						=					nn.LayerNorm(_A      ,    elementwise_affine=_A      ,    eps=1E-6)
						__snake_case     :				Optional[int] 						=					nn.Linear(_A      ,    2 * inner_dim)
						__snake_case     :				List[str] 						=					nn.Linear(_A      ,    patch_size * patch_size * self.out_channels)






				def    _lowercase	(self							: str      ,    _A							: torch.Tensor      ,    _A							: Optional[torch.Tensor] = None      ,    _A							: Optional[torch.LongTensor] = None      ,    _A							: Optional[torch.LongTensor] = None      ,    _A							: Dict[str, Any] = None      ,    _A							: Optional[torch.Tensor] = None      ,    _A							: Optional[torch.Tensor] = None      ,    _A							: bool = True      ,    )       ->      List[Any]:
					# ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
					#   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
					#   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
					# expects mask of shape:
					#   [batch, key_tokens]
					# adds singleton query_tokens dimension:
					#   [batch,                    1, key_tokens]
					# this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
					#   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
					#   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
					if attention_mask is not None and attention_mask.ndim == 2:
						# assume that mask is expressed as:
						#   (1 = keep,      0 = discard)
						# convert mask into a bias that can be added to attention scores:
						#       (keep = +0,     discard = -10000.0)
						__snake_case     :				List[Any] 						=					(1 - attention_mask.to(hidden_states.dtype)) * -10_000.0
						__snake_case     :				Dict 						=					attention_mask.unsqueeze(1)

					# convert encoder_attention_mask to a bias the same way we do for attention_mask
					if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
						__snake_case     :				int 						=					(1 - encoder_attention_mask.to(hidden_states.dtype)) * -10_000.0
						__snake_case     :				Optional[Any] 						=					encoder_attention_mask.unsqueeze(1)

					# 1. Input
					if self.is_input_continuous:
						__snake_case						,							__snake_case						,							__snake_case						,							__snake_case     :				int 						=					hidden_states.shape
						__snake_case     :				List[str] 						=					hidden_states

						__snake_case     :				str 						=					self.norm(_A)
						if not self.use_linear_projection:
							__snake_case     :				Optional[int] 						=					self.proj_in(_A)
							__snake_case     :				int 						=					hidden_states.shape[1]
							__snake_case     :				Any 						=					hidden_states.permute(0      ,    2      ,    3      ,    1).reshape(_A      ,    height * width      ,    _A)
						else:
							__snake_case     :				Optional[int] 						=					hidden_states.shape[1]
							__snake_case     :				Dict 						=					hidden_states.permute(0      ,    2      ,    3      ,    1).reshape(_A      ,    height * width      ,    _A)
							__snake_case     :				Optional[Any] 						=					self.proj_in(_A)
					elif self.is_input_vectorized:
						__snake_case     :				List[Any] 						=					self.latent_image_embedding(_A)
					elif self.is_input_patches:
						__snake_case     :				Optional[int] 						=					self.pos_embed(_A)

					# 2. Blocks
					for block in self.transformer_blocks:
						__snake_case     :				Tuple 						=					block(
						    _A      ,    attention_mask=_A      ,    encoder_hidden_states=_A      ,    encoder_attention_mask=_A      ,    timestep=_A      ,    cross_attention_kwargs=_A      ,    class_labels=_A      ,    )

					# 3. Output
					if self.is_input_continuous:
						if not self.use_linear_projection:
							__snake_case     :				str 						=					hidden_states.reshape(_A      ,    _A      ,    _A      ,    _A).permute(0      ,    3      ,    1      ,    2).contiguous()
							__snake_case     :				List[Any] 						=					self.proj_out(_A)
						else:
							__snake_case     :				Optional[Any] 						=					self.proj_out(_A)
							__snake_case     :				Union[str, Any] 						=					hidden_states.reshape(_A      ,    _A      ,    _A      ,    _A).permute(0      ,    3      ,    1      ,    2).contiguous()

						__snake_case     :				int 						=					hidden_states + residual
					elif self.is_input_vectorized:
						__snake_case     :				str 						=					self.norm_out(_A)
						__snake_case     :				Optional[Any] 						=					self.out(_A)
						# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
						__snake_case     :				Tuple 						=					logits.permute(0      ,    2      ,    1)

						# log(p(x_0))
						__snake_case     :				Tuple 						=					F.log_softmax(logits.double()      ,    dim=1).float()
					elif self.is_input_patches:
						# TODO: cleanup!
						__snake_case     :				int 						=					self.transformer_blocks[0].norma.emb(
						    _A      ,    _A      ,    hidden_dtype=hidden_states.dtype)
						__snake_case						,							__snake_case     :				Tuple 						=					self.proj_out_a(F.silu(_A)).chunk(2      ,    dim=1)
						__snake_case     :				List[Any] 						=					self.norm_out(_A) * (1 + scale[:, None]) + shift[:, None]
						__snake_case     :				Any 						=					self.proj_out_a(_A)

						# unpatchify
						__snake_case     :				Dict 						=					int(hidden_states.shape[1] ** 0.5)
						__snake_case     :				Dict 						=					hidden_states.reshape(
						    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels))
						__snake_case     :				Union[str, Any] 						=					torch.einsum('nhwpqc->nchpwq'      ,    _A)
						__snake_case     :				List[Any] 						=					hidden_states.reshape(
						    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size))

					if not return_dict:
						return (output,)

					return TransformeraDModelOutput(sample=_A)






"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_a	:	Optional[Any]=					{"configuration_gptj": ["GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP", "GPTJConfig", "GPTJOnnxConfig"]}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a	:	Dict=					[
	    "GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "GPTJForCausalLM",
	    "GPTJForQuestionAnswering",
	    "GPTJForSequenceClassification",
	    "GPTJModel",
	    "GPTJPreTrainedModel",
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a	:	Optional[Any]=					[
	    "TFGPTJForCausalLM",
	    "TFGPTJForQuestionAnswering",
	    "TFGPTJForSequenceClassification",
	    "TFGPTJModel",
	    "TFGPTJPreTrainedModel",
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a	:	List[str]=					[
	    "FlaxGPTJForCausalLM",
	    "FlaxGPTJModel",
	    "FlaxGPTJPreTrainedModel",
	]


if TYPE_CHECKING:
	from .configuration_gptj import GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTJConfig, GPTJOnnxConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_gptj import (
		    GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST,
		    GPTJForCausalLM,
		    GPTJForQuestionAnswering,
		    GPTJForSequenceClassification,
		    GPTJModel,
		    GPTJPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_gptj import (
		    TFGPTJForCausalLM,
		    TFGPTJForQuestionAnswering,
		    TFGPTJForSequenceClassification,
		    TFGPTJModel,
		    TFGPTJPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel, FlaxGPTJPreTrainedModel

else:
	import sys

	_a	:	Any=					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





