import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation
def 					UpperCamelCase__		(							A__     ,					A__     ,					A__ = False     ,					A__ = 1     ,					A__ = 1     ,					A__ = 1_0000     ,					)  ->   str:
				assert len(timesteps.shape       ) == 1, "Timesteps should be a 1d-array"

				snake_case__   :      Optional[Any]   				= embedding_dim // 2
				snake_case__   :      Optional[Any]   				= -math.log(A__       ) * torch.arange(
				    start=0     ,					end=A__     ,					dtype=torch.floataa     ,					device=timesteps.device       )
				snake_case__   :      str   				= exponent / (half_dim - downscale_freq_shift)

				snake_case__   :      int   				= torch.exp(A__       )
				snake_case__   :      Optional[Any]   				= timesteps[:, None].float() * emb[None, :]

				# scale embeddings
				snake_case__   :      Optional[Any]   				= scale * emb

				# concat sine and cosine embeddings
				snake_case__   :      str   				= torch.cat([torch.sin(A__       ), torch.cos(A__       )]     ,					dim=-1       )

				# flip sine and cosine embeddings
				if flip_sin_to_cos:
								snake_case__   :      int   				= torch.cat([emb[:, half_dim:], emb[:, :half_dim]]     ,					dim=-1       )

				# zero pad
				if embedding_dim % 2 == 1:
								snake_case__   :      int   				= torch.nn.functional.pad(A__     ,					(0, 1, 0, 0)       )
				return emb
def 					UpperCamelCase__		(							A__     ,					A__     ,					A__=False     ,					A__=0       )  ->   Optional[int]:
				snake_case__   :      str   				= np.arange(A__     ,					dtype=np.floataa       )
				snake_case__   :      List[str]   				= np.arange(A__     ,					dtype=np.floataa       )
				snake_case__   :      str   				= np.meshgrid(A__     ,					A__       )  # here w goes first
				snake_case__   :      Optional[int]   				= np.stack(A__     ,					axis=0       )

				snake_case__   :      Optional[int]   				= grid.reshape([2, 1, grid_size, grid_size]       )
				snake_case__   :      Optional[Any]   				= get_ad_sincos_pos_embed_from_grid(A__     ,					A__       )
				if cls_token and extra_tokens > 0:
								snake_case__   :      Tuple   				= np.concatenate([np.zeros([extra_tokens, embed_dim]       ), pos_embed]     ,					axis=0       )
				return pos_embed
def 					UpperCamelCase__		(							A__     ,					A__       )  ->   List[str]:
				if embed_dim % 2 != 0:
								raise ValueError('embed_dim must be divisible by 2'       )

				# use half of dimensions to encode grid_h
				snake_case__   :      List[Any]   				= get_ad_sincos_pos_embed_from_grid(embed_dim // 2     ,					grid[0]       )  # (H*W, D/2)
				snake_case__   :      Optional[int]   				= get_ad_sincos_pos_embed_from_grid(embed_dim // 2     ,					grid[1]       )  # (H*W, D/2)

				snake_case__   :      Optional[Any]   				= np.concatenate([emb_h, emb_w]     ,					axis=1       )  # (H*W, D)
				return emb
def 					UpperCamelCase__		(							A__     ,					A__       )  ->   Optional[int]:
				if embed_dim % 2 != 0:
								raise ValueError('embed_dim must be divisible by 2'       )

				snake_case__   :      List[str]   				= np.arange(embed_dim // 2     ,					dtype=np.floataa       )
				omega /= embed_dim / 2.0
				snake_case__   :      str   				= 1.0 / 1_0000**omega  # (D/2,)

				snake_case__   :      Optional[int]   				= pos.reshape(-1       )  # (M,)
				snake_case__   :      int   				= np.einsum('m,d->md'     ,					A__     ,					A__       )  # (M, D/2), outer product

				snake_case__   :      Any   				= np.sin(A__       )  # (M, D/2)
				snake_case__   :      Any   				= np.cos(A__       )  # (M, D/2)

				snake_case__   :      Optional[Any]   				= np.concatenate([emb_sin, emb_cos]     ,					axis=1       )  # (M, D)
				return emb





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase=224       ,			__UpperCamelCase=224       ,			__UpperCamelCase=16       ,			__UpperCamelCase=3       ,			__UpperCamelCase=768       ,			__UpperCamelCase=False       ,			__UpperCamelCase=True       ,			__UpperCamelCase=True       ,			)	->    Union[str, Any]:



									'''simple docstring'''







									super().__init__()

									snake_case__   :      Tuple   				= (height // patch_size) * (width // patch_size)
									snake_case__   :      str   				= flatten
									snake_case__   :      List[Any]   				= layer_norm

									snake_case__   :      Tuple   				= nn.Convad(
									    __UpperCamelCase       ,			__UpperCamelCase       ,			kernel_size=(patch_size, patch_size)       ,			stride=__UpperCamelCase       ,			bias=__UpperCamelCase						)
									if layer_norm:
													snake_case__   :      List[str]   				= nn.LayerNorm(__UpperCamelCase       ,			elementwise_affine=__UpperCamelCase       ,			eps=1E-6						)
									else:
													snake_case__   :      str   				= None

									snake_case__   :      Optional[Any]   				= get_ad_sincos_pos_embed(__UpperCamelCase       ,			int(num_patches**0.5						)						)
									self.register_buffer('pos_embed'       ,			torch.from_numpy(__UpperCamelCase						).float().unsqueeze(0						)       ,			persistent=__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase						)	->    int:



									'''simple docstring'''







									snake_case__   :      Tuple   				= self.proj(__UpperCamelCase						)
									if self.flatten:
													snake_case__   :      Any   				= latent.flatten(2						).transpose(1       ,			2						)  # BCHW -> BNC
									if self.layer_norm:
													snake_case__   :      int   				= self.norm(__UpperCamelCase						)
									return latent + self.pos_embed





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase = "silu"       ,			__UpperCamelCase = None       ,			__UpperCamelCase = None       ,			__UpperCamelCase=None       ,			)	->    int:



									'''simple docstring'''







									super().__init__()

									snake_case__   :      str   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)

									if cond_proj_dim is not None:
													snake_case__   :      Any   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase       ,			bias=__UpperCamelCase						)
									else:
													snake_case__   :      Optional[Any]   				= None

									snake_case__   :      List[Any]   				= get_activation(__UpperCamelCase						)

									if out_dim is not None:
													snake_case__   :      List[Any]   				= out_dim
									else:
													snake_case__   :      str   				= time_embed_dim
									snake_case__   :      List[str]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)

									if post_act_fn is None:
													snake_case__   :      Optional[int]   				= None
									else:
													snake_case__   :      Optional[int]   				= get_activation(__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase=None						)	->    Any:



									'''simple docstring'''







									if condition is not None:
													snake_case__   :      Any   				= sample + self.cond_proj(__UpperCamelCase						)
									snake_case__   :      Union[str, Any]   				= self.linear_a(__UpperCamelCase						)

									if self.act is not None:
													snake_case__   :      Union[str, Any]   				= self.act(__UpperCamelCase						)

									snake_case__   :      List[Any]   				= self.linear_a(__UpperCamelCase						)

									if self.post_act is not None:
													snake_case__   :      Tuple   				= self.post_act(__UpperCamelCase						)
									return sample





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)	->    str:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      List[str]   				= num_channels
									snake_case__   :      List[Any]   				= flip_sin_to_cos
									snake_case__   :      Any   				= downscale_freq_shift




					def      __a     (  self       ,			__UpperCamelCase						)	->    int:



									'''simple docstring'''







									snake_case__   :      List[Any]   				= get_timestep_embedding(
									    __UpperCamelCase       ,			self.num_channels       ,			flip_sin_to_cos=self.flip_sin_to_cos       ,			downscale_freq_shift=self.downscale_freq_shift       ,			)
									return t_emb





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase = 256       ,			__UpperCamelCase = 1.0       ,			__UpperCamelCase=True       ,			__UpperCamelCase=True       ,			__UpperCamelCase=False						)	->    str:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      Any   				= nn.Parameter(torch.randn(__UpperCamelCase						) * scale       ,			requires_grad=__UpperCamelCase						)
									snake_case__   :      Optional[int]   				= log
									snake_case__   :      List[str]   				= flip_sin_to_cos

									if set_W_to_weight:
													# to delete later
													snake_case__   :      Dict   				= nn.Parameter(torch.randn(__UpperCamelCase						) * scale       ,			requires_grad=__UpperCamelCase						)

													snake_case__   :      Any   				= self.W




					def      __a     (  self       ,			__UpperCamelCase						)	->    Optional[int]:



									'''simple docstring'''







									if self.log:
													snake_case__   :      Union[str, Any]   				= torch.log(__UpperCamelCase						)

									snake_case__   :      Optional[int]   				= x[:, None] * self.weight[None, :] * 2 * np.pi

									if self.flip_sin_to_cos:
													snake_case__   :      List[Any]   				= torch.cat([torch.cos(__UpperCamelCase						), torch.sin(__UpperCamelCase						)]       ,			dim=-1						)
									else:
													snake_case__   :      int   				= torch.cat([torch.sin(__UpperCamelCase						), torch.cos(__UpperCamelCase						)]       ,			dim=-1						)
									return out





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			)	->    str:



									'''simple docstring'''







									super().__init__()

									snake_case__   :      Dict   				= height
									snake_case__   :      Any   				= width
									snake_case__   :      int   				= num_embed
									snake_case__   :      Any   				= embed_dim

									snake_case__   :      List[Any]   				= nn.Embedding(self.num_embed       ,			__UpperCamelCase						)
									snake_case__   :      Dict   				= nn.Embedding(self.height       ,			__UpperCamelCase						)
									snake_case__   :      List[Any]   				= nn.Embedding(self.width       ,			__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase						)	->    str:



									'''simple docstring'''







									snake_case__   :      int   				= self.emb(__UpperCamelCase						)

									snake_case__   :      Dict   				= self.height_emb(torch.arange(self.height       ,			device=index.device						).view(1       ,			self.height						)						)

									# 1 x H x D -> 1 x H x 1 x D
									snake_case__   :      Any   				= height_emb.unsqueeze(2						)

									snake_case__   :      int   				= self.width_emb(torch.arange(self.width       ,			device=index.device						).view(1       ,			self.width						)						)

									# 1 x W x D -> 1 x 1 x W x D
									snake_case__   :      Optional[int]   				= width_emb.unsqueeze(1						)

									snake_case__   :      Any   				= height_emb + width_emb

									# 1 x H x W x D -> 1 x L xD
									snake_case__   :      Union[str, Any]   				= pos_emb.view(1       ,			self.height * self.width       ,			-1						)

									snake_case__   :      Dict   				= emb + pos_emb[:, : emb.shape[1], :]

									return emb





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)	->    Optional[int]:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      Union[str, Any]   				= dropout_prob > 0
									snake_case__   :      Tuple   				= nn.Embedding(num_classes + use_cfg_embedding       ,			__UpperCamelCase						)
									snake_case__   :      Tuple   				= num_classes
									snake_case__   :      Tuple   				= dropout_prob




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase=None						)	->    Any:



									'''simple docstring'''







									if force_drop_ids is None:
													snake_case__   :      List[Any]   				= torch.rand(labels.shape[0]       ,			device=labels.device						) < self.dropout_prob
									else:
													snake_case__   :      List[Any]   				= torch.tensor(force_drop_ids == 1						)
									snake_case__   :      Dict   				= torch.where(__UpperCamelCase       ,			self.num_classes       ,			__UpperCamelCase						)
									return labels




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase=None						)	->    Union[str, Any]:



									'''simple docstring'''







									snake_case__   :      Any   				= self.dropout_prob > 0
									if (self.training and use_dropout) or (force_drop_ids is not None):
													snake_case__   :      Any   				= self.token_drop(__UpperCamelCase       ,			__UpperCamelCase						)
									snake_case__   :      str   				= self.embedding_table(__UpperCamelCase						)
									return embeddings





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase = 1024       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 10       ,			)	->    int:



									'''simple docstring'''







									super().__init__()

									snake_case__   :      Optional[int]   				= num_image_text_embeds
									snake_case__   :      str   				= nn.Linear(__UpperCamelCase       ,			self.num_image_text_embeds * cross_attention_dim						)
									snake_case__   :      List[str]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase						)	->    Dict:



									'''simple docstring'''







									snake_case__   :      Optional[Any]   				= text_embeds.shape[0]

									# image
									snake_case__   :      Optional[Any]   				= self.image_embeds(__UpperCamelCase						)
									snake_case__   :      str   				= image_text_embeds.reshape(__UpperCamelCase       ,			self.num_image_text_embeds       ,			-1						)

									# text
									snake_case__   :      int   				= self.text_proj(__UpperCamelCase						)

									return torch.cat([image_text_embeds, text_embeds]       ,			dim=1						)





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 32       ,			)	->    List[Any]:



									'''simple docstring'''







									super().__init__()

									snake_case__   :      Optional[int]   				= num_image_text_embeds
									snake_case__   :      Dict   				= nn.Linear(__UpperCamelCase       ,			self.num_image_text_embeds * cross_attention_dim						)
									snake_case__   :      List[Any]   				= nn.LayerNorm(__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase						)	->    Optional[Any]:



									'''simple docstring'''







									snake_case__   :      Optional[Any]   				= image_embeds.shape[0]

									# image
									snake_case__   :      Any   				= self.image_embeds(__UpperCamelCase						)
									snake_case__   :      int   				= image_embeds.reshape(__UpperCamelCase       ,			self.num_image_text_embeds       ,			-1						)
									snake_case__   :      Tuple   				= self.norm(__UpperCamelCase						)
									return image_embeds





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase=0.1						)	->    Any:



									'''simple docstring'''







									super().__init__()

									snake_case__   :      int   				= Timesteps(num_channels=256       ,			flip_sin_to_cos=__UpperCamelCase       ,			downscale_freq_shift=1						)
									snake_case__   :      List[Any]   				= TimestepEmbedding(in_channels=256       ,			time_embed_dim=__UpperCamelCase						)
									snake_case__   :      Optional[Any]   				= LabelEmbedding(__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase=None						)	->    Dict:



									'''simple docstring'''







									snake_case__   :      List[str]   				= self.time_proj(__UpperCamelCase						)
									snake_case__   :      Tuple   				= self.timestep_embedder(timesteps_proj.to(dtype=__UpperCamelCase						)						)  # (N, D)

									snake_case__   :      Any   				= self.class_embedder(__UpperCamelCase						)  # (N, D)

									snake_case__   :      Optional[int]   				= timesteps_emb + class_labels  # (N, D)

									return conditioning





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase = 64						)	->    Optional[int]:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      Optional[int]   				= nn.LayerNorm(__UpperCamelCase						)
									snake_case__   :      Tuple   				= AttentionPooling(__UpperCamelCase       ,			__UpperCamelCase						)
									snake_case__   :      Union[str, Any]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)
									snake_case__   :      int   				= nn.LayerNorm(__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase						)	->    Optional[int]:



									'''simple docstring'''







									snake_case__   :      Any   				= self.norma(__UpperCamelCase						)
									snake_case__   :      Union[str, Any]   				= self.pool(__UpperCamelCase						)
									snake_case__   :      str   				= self.proj(__UpperCamelCase						)
									snake_case__   :      Any   				= self.norma(__UpperCamelCase						)
									return hidden_states





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 1536						)	->    Any:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      Optional[int]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)
									snake_case__   :      List[str]   				= nn.LayerNorm(__UpperCamelCase						)
									snake_case__   :      List[Any]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase						)	->    Dict:



									'''simple docstring'''







									snake_case__   :      Any   				= self.text_proj(__UpperCamelCase						)
									snake_case__   :      Optional[int]   				= self.text_norm(__UpperCamelCase						)

									# image
									snake_case__   :      List[Any]   				= self.image_proj(__UpperCamelCase						)

									return time_image_embeds + time_text_embeds





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 1536						)	->    Optional[Any]:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      Optional[Any]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)
									snake_case__   :      Dict   				= nn.LayerNorm(__UpperCamelCase						)




					def      __a     (  self       ,			__UpperCamelCase						)	->    Optional[int]:



									'''simple docstring'''







									snake_case__   :      Optional[int]   				= self.image_proj(__UpperCamelCase						)
									snake_case__   :      Dict   				= self.image_norm(__UpperCamelCase						)
									return time_image_embeds





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase = 768       ,			__UpperCamelCase = 1536						)	->    str:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      List[str]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase						)
									snake_case__   :      str   				= nn.LayerNorm(__UpperCamelCase						)
									snake_case__   :      Union[str, Any]   				= nn.Sequential(
									    nn.Convad(3       ,			16       ,			3       ,			padding=1						)       ,			nn.SiLU()       ,			nn.Convad(16       ,			16       ,			3       ,			padding=1						)       ,			nn.SiLU()       ,			nn.Convad(16       ,			32       ,			3       ,			padding=1       ,			stride=2						)       ,			nn.SiLU()       ,			nn.Convad(32       ,			32       ,			3       ,			padding=1						)       ,			nn.SiLU()       ,			nn.Convad(32       ,			96       ,			3       ,			padding=1       ,			stride=2						)       ,			nn.SiLU()       ,			nn.Convad(96       ,			96       ,			3       ,			padding=1						)       ,			nn.SiLU()       ,			nn.Convad(96       ,			256       ,			3       ,			padding=1       ,			stride=2						)       ,			nn.SiLU()       ,			nn.Convad(256       ,			4       ,			3       ,			padding=1						)       ,			)




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase						)	->    Any:



									'''simple docstring'''







									snake_case__   :      Any   				= self.image_proj(__UpperCamelCase						)
									snake_case__   :      Union[str, Any]   				= self.image_norm(__UpperCamelCase						)
									snake_case__   :      List[str]   				= self.input_hint_block(__UpperCamelCase						)
									return time_image_embeds, hint





class 					__snake_case    (     nn.Module      ):




					def __init__(  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase=None						)	->    Optional[Any]:



									'''simple docstring'''







									super().__init__()
									snake_case__   :      List[str]   				= dtype
									snake_case__   :      Any   				= nn.Parameter(torch.randn(1       ,			__UpperCamelCase						) / embed_dim**0.5						)
									snake_case__   :      List[str]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase       ,			dtype=self.dtype						)
									snake_case__   :      Optional[int]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase       ,			dtype=self.dtype						)
									snake_case__   :      Optional[Any]   				= nn.Linear(__UpperCamelCase       ,			__UpperCamelCase       ,			dtype=self.dtype						)
									snake_case__   :      Union[str, Any]   				= num_heads
									snake_case__   :      Any   				= embed_dim // self.num_heads




					def      __a     (  self       ,			__UpperCamelCase						)	->    Optional[int]:



									'''simple docstring'''







									snake_case__				,    snake_case__				,    snake_case__   :      List[Any]   				= x.size()

									def shape(__UpperCamelCase						):
													# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
													snake_case__   :      List[Any]   				= x.view(__UpperCamelCase       ,			-1       ,			self.num_heads       ,			self.dim_per_head						)
													# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
													snake_case__   :      str   				= x.transpose(1       ,			2						)
													# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
													snake_case__   :      Tuple   				= x.reshape(bs * self.num_heads       ,			-1       ,			self.dim_per_head						)
													# (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
													snake_case__   :      Dict   				= x.transpose(1       ,			2						)
													return x

									snake_case__   :      Tuple   				= x.mean(dim=1       ,			keepdim=__UpperCamelCase						) + self.positional_embedding.to(x.dtype						)
									snake_case__   :      Any   				= torch.cat([class_token, x]       ,			dim=1						)  # (bs, length+1, width)

									# (bs*n_heads, class_token_length, dim_per_head)
									snake_case__   :      Union[str, Any]   				= shape(self.q_proj(__UpperCamelCase						)						)
									# (bs*n_heads, length+class_token_length, dim_per_head)
									snake_case__   :      List[str]   				= shape(self.k_proj(__UpperCamelCase						)						)
									snake_case__   :      List[str]   				= shape(self.v_proj(__UpperCamelCase						)						)

									# (bs*n_heads, class_token_length, length+class_token_length):
									snake_case__   :      Dict   				= 1 / math.sqrt(math.sqrt(self.dim_per_head						)						)
									snake_case__   :      Union[str, Any]   				= torch.einsum('bct,bcs->bts'       ,			q * scale       ,			k * scale						)  # More stable with f16 than dividing afterwards
									snake_case__   :      str   				= torch.softmax(weight.float()       ,			dim=-1						).type(weight.dtype						)

									# (bs*n_heads, dim_per_head, class_token_length)
									snake_case__   :      Any   				= torch.einsum('bts,bcs->bct'       ,			__UpperCamelCase       ,			__UpperCamelCase						)

									# (bs, length+1, width)
									snake_case__   :      Union[str, Any]   				= a.reshape(__UpperCamelCase       ,			-1       ,			1						).transpose(1       ,			2						)

									return a[:, 0, :]  # cls_token



from ...utils import is_transformers_available
from .pipeline_latent_diffusion_superresolution import LDMSuperResolutionPipeline


if is_transformers_available():
				from .pipeline_latent_diffusion import LDMBertModel, LDMTextToImagePipeline


