






"""simple docstring"""


import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation




def 					a			(  UpperCamelCase_				:					torch.Tensor  ,						UpperCamelCase_				:					int  ,						UpperCamelCase_				:					bool = False  ,						UpperCamelCase_				:					float = 1  ,						UpperCamelCase_				:					float = 1  ,						UpperCamelCase_				:					int = 1_0_0_0_0  ,						)	->		int:
 assert len(timesteps.shape		) == 1, "Timesteps should be a 1d-array"

 __A						:							Union[str, Any]      =						embedding_dim // 2
 __A						:							Optional[Any]      =						-math.log(UpperCamelCase_		) * torch.arange(
     start=0  ,						end=UpperCamelCase_  ,						dtype=torch.floataa  ,						device=timesteps.device		)
 __A						:							Union[str, Any]      =						exponent / (half_dim - downscale_freq_shift)

 __A						:							Optional[int]      =						torch.exp(UpperCamelCase_		)
 __A						:							str      =						timesteps[:, None].float() * emb[None, :]

 # scale embeddings
 __A						:							Union[str, Any]      =						scale * emb

 # concat sine and cosine embeddings
 __A						:							str      =						torch.cat([torch.sin(UpperCamelCase_		), torch.cos(UpperCamelCase_		)]  ,						dim=-1		)

 # flip sine and cosine embeddings
 if flip_sin_to_cos:
  __A						:							List[Any]      =						torch.cat([emb[:, half_dim:], emb[:, :half_dim]]  ,						dim=-1		)

 # zero pad
 if embedding_dim % 2 == 1:
  __A						:							Union[str, Any]      =						torch.nn.functional.pad(UpperCamelCase_  ,						(0, 1, 0, 0)		)
 return emb




def 					a			(  UpperCamelCase_				:					Optional[Any]  ,						UpperCamelCase_				:					Optional[int]  ,						UpperCamelCase_				:					int=False  ,						UpperCamelCase_				:					str=0		)	->		str:
 __A						:							Any      =						np.arange(UpperCamelCase_  ,						dtype=np.floataa		)
 __A						:							List[str]      =						np.arange(UpperCamelCase_  ,						dtype=np.floataa		)
 __A						:							Union[str, Any]      =						np.meshgrid(UpperCamelCase_  ,						UpperCamelCase_		)  # here w goes first
 __A						:							int      =						np.stack(UpperCamelCase_  ,						axis=0		)

 __A						:							Tuple      =						grid.reshape([2, 1, grid_size, grid_size]		)
 __A						:							List[str]      =						get_ad_sincos_pos_embed_from_grid(UpperCamelCase_  ,						UpperCamelCase_		)
 if cls_token and extra_tokens > 0:
  __A						:							List[Any]      =						np.concatenate([np.zeros([extra_tokens, embed_dim]		), pos_embed]  ,						axis=0		)
 return pos_embed




def 					a			(  UpperCamelCase_				:					List[str]  ,						UpperCamelCase_				:					str		)	->		List[str]:

 if embed_dim % 2 != 0:
  raise ValueError('embed_dim must be divisible by 2'		)

 # use half of dimensions to encode grid_h
 __A						:							Dict      =						get_ad_sincos_pos_embed_from_grid(embed_dim // 2  ,						grid[0]		)  # (H*W, D/2)
 __A						:							Optional[int]      =						get_ad_sincos_pos_embed_from_grid(embed_dim // 2  ,						grid[1]		)  # (H*W, D/2)

 __A						:							Optional[int]      =						np.concatenate([emb_h, emb_w]  ,						axis=1		)  # (H*W, D)
 return emb




def 					a			(  UpperCamelCase_				:					Optional[int]  ,						UpperCamelCase_				:					Tuple		)	->		Optional[int]:

 if embed_dim % 2 != 0:
  raise ValueError('embed_dim must be divisible by 2'		)

 __A						:							List[Any]      =						np.arange(embed_dim // 2  ,						dtype=np.floataa		)
 omega /= embed_dim / 2.0
 __A						:							Any      =						1.0 / 1_0_0_0_0**omega  # (D/2,)

 __A						:							Dict      =						pos.reshape(-1		)  # (M,)
 __A						:							List[str]      =						np.einsum('m,d->md'  ,						UpperCamelCase_  ,						UpperCamelCase_		)  # (M, D/2), outer product

 __A						:							str      =						np.sin(UpperCamelCase_		)  # (M, D/2)
 __A						:							List[Any]      =						np.cos(UpperCamelCase_		)  # (M, D/2)

 __A						:							Dict      =						np.concatenate([emb_sin, emb_cos]  ,						axis=1		)  # (M, D)
 return emb






class __a ( nn.Module					):



  def __init__(self	:		Optional[Any]					,      lowercase_	:		Union[str, Any]=224					,      lowercase_	:		List[Any]=224					,      lowercase_	:		Optional[int]=16					,      lowercase_	:		Dict=3					,      lowercase_	:		List[str]=768					,      lowercase_	:		Dict=False					,      lowercase_	:		List[str]=True					,      lowercase_	:		Optional[Any]=True					,      )		->				str:
   super().__init__()

   __A						:							Dict      =						(height // patch_size) * (width // patch_size)
   __A						:							int      =						flatten
   __A						:							Tuple      =						layer_norm

   __A						:							Tuple      =						nn.Convad(
       SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_					,      kernel_size=(patch_size, patch_size)					,      stride=SCREAMING_SNAKE_CASE_					,      bias=SCREAMING_SNAKE_CASE_ )
   if layer_norm:
    __A						:							Any      =						nn.LayerNorm(SCREAMING_SNAKE_CASE_					,      elementwise_affine=SCREAMING_SNAKE_CASE_					,      eps=1E-6 )
   else:
    __A						:							List[str]      =						None

   __A						:							Optional[int]      =						get_ad_sincos_pos_embed(SCREAMING_SNAKE_CASE_					,      int(num_patches**0.5 ) )
   self.register_buffer('pos_embed'					,      torch.from_numpy(SCREAMING_SNAKE_CASE_ ).float().unsqueeze(0 )					,      persistent=SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		List[str]					,      lowercase_	:		Optional[int] )		->				int:
   __A						:							Union[str, Any]      =						self.proj(SCREAMING_SNAKE_CASE_ )
   if self.flatten:
    __A						:							Optional[Any]      =						latent.flatten(2 ).transpose(1					,      2 )  # BCHW -> BNC
   if self.layer_norm:
    __A						:							Dict      =						self.norm(SCREAMING_SNAKE_CASE_ )
   return latent + self.pos_embed






class __a ( nn.Module					):



  def __init__(self	:		Any					,      lowercase_	:		Optional[Any]					,      lowercase_	:		List[str]					,      lowercase_	:		int = "silu"					,      lowercase_	:		Any = None					,      lowercase_	:		Any = None					,      lowercase_	:		Dict=None					,      )		->				Optional[Any]:
   super().__init__()

   __A						:							List[Any]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )

   if cond_proj_dim is not None:
    __A						:							Dict      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_					,      bias=SCREAMING_SNAKE_CASE_ )
   else:
    __A						:							int      =						None

   __A						:							int      =						get_activation(SCREAMING_SNAKE_CASE_ )

   if out_dim is not None:
    __A						:							str      =						out_dim
   else:
    __A						:							Optional[Any]      =						time_embed_dim
   __A						:							str      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )

   if post_act_fn is None:
    __A						:							List[Any]      =						None
   else:
    __A						:							str      =						get_activation(SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		Optional[Any]					,      lowercase_	:		Any					,      lowercase_	:		Dict=None )		->				str:






   if condition is not None:
    __A						:							str      =						sample + self.cond_proj(SCREAMING_SNAKE_CASE_ )
   __A						:							List[Any]      =						self.linear_a(SCREAMING_SNAKE_CASE_ )

   if self.act is not None:
    __A						:							str      =						self.act(SCREAMING_SNAKE_CASE_ )

   __A						:							int      =						self.linear_a(SCREAMING_SNAKE_CASE_ )

   if self.post_act is not None:
    __A						:							Tuple      =						self.post_act(SCREAMING_SNAKE_CASE_ )
   return sample






class __a ( nn.Module					):



  def __init__(self	:		str					,      lowercase_	:		Union[str, Any]					,      lowercase_	:		List[Any]					,      lowercase_	:		Optional[Any] )		->				List[Any]:
   super().__init__()
   __A						:							str      =						num_channels
   __A						:							int      =						flip_sin_to_cos
   __A						:							Any      =						downscale_freq_shift



  def     A_					(self	:		Any					,      lowercase_	:		Tuple )		->				str:
   __A						:							List[Any]      =						get_timestep_embedding(
       SCREAMING_SNAKE_CASE_					,      self.num_channels					,      flip_sin_to_cos=self.flip_sin_to_cos					,      downscale_freq_shift=self.downscale_freq_shift					,      )
   return t_emb






class __a ( nn.Module					):



  def __init__(self	:		str					,      lowercase_	:		Tuple = 256					,      lowercase_	:		Any = 1.0					,      lowercase_	:		Dict=True					,      lowercase_	:		List[Any]=True					,      lowercase_	:		str=False )		->				Optional[Any]:
   super().__init__()
   __A						:							List[str]      =						nn.Parameter(torch.randn(SCREAMING_SNAKE_CASE_ ) * scale					,      requires_grad=SCREAMING_SNAKE_CASE_ )
   __A						:							Optional[Any]      =						log
   __A						:							List[str]      =						flip_sin_to_cos

   if set_W_to_weight:
    # to delete later
    __A						:							Optional[int]      =						nn.Parameter(torch.randn(SCREAMING_SNAKE_CASE_ ) * scale					,      requires_grad=SCREAMING_SNAKE_CASE_ )

    __A						:							int      =						self.W



  def     A_					(self	:		List[Any]					,      lowercase_	:		int )		->				str:






   if self.log:
    __A						:							Optional[Any]      =						torch.log(SCREAMING_SNAKE_CASE_ )

   __A						:							List[Any]      =						x[:, None] * self.weight[None, :] * 2 * np.pi

   if self.flip_sin_to_cos:
    __A						:							Tuple      =						torch.cat([torch.cos(SCREAMING_SNAKE_CASE_ ), torch.sin(SCREAMING_SNAKE_CASE_ )]					,      dim=-1 )
   else:
    __A						:							Optional[Any]      =						torch.cat([torch.sin(SCREAMING_SNAKE_CASE_ ), torch.cos(SCREAMING_SNAKE_CASE_ )]					,      dim=-1 )
   return out






class __a ( nn.Module					):



  def __init__(self	:		Dict					,      lowercase_	:		List[str]					,      lowercase_	:		Union[str, Any]					,      lowercase_	:		int					,      lowercase_	:		str					,      )		->				Dict:
   super().__init__()

   __A						:							int      =						height
   __A						:							Union[str, Any]      =						width
   __A						:							Union[str, Any]      =						num_embed
   __A						:							Tuple      =						embed_dim

   __A						:							int      =						nn.Embedding(self.num_embed					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Optional[Any]      =						nn.Embedding(self.height					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Dict      =						nn.Embedding(self.width					,      SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		int					,      lowercase_	:		int )		->				Dict:
   __A						:							int      =						self.emb(SCREAMING_SNAKE_CASE_ )

   __A						:							List[str]      =						self.height_emb(torch.arange(self.height					,      device=index.device ).view(1					,      self.height ) )

   # 1 x H x D -> 1 x H x 1 x D
   __A						:							int      =						height_emb.unsqueeze(2 )

   __A						:							int      =						self.width_emb(torch.arange(self.width					,      device=index.device ).view(1					,      self.width ) )

   # 1 x W x D -> 1 x 1 x W x D
   __A						:							Any      =						width_emb.unsqueeze(1 )

   __A						:							str      =						height_emb + width_emb

   # 1 x H x W x D -> 1 x L xD
   __A						:							Any      =						pos_emb.view(1					,      self.height * self.width					,      -1 )

   __A						:							str      =						emb + pos_emb[:, : emb.shape[1], :]

   return emb






class __a ( nn.Module					):



  def __init__(self	:		Dict					,      lowercase_	:		Optional[Any]					,      lowercase_	:		List[str]					,      lowercase_	:		Dict )		->				Tuple:
   super().__init__()
   __A						:							List[str]      =						dropout_prob > 0
   __A						:							str      =						nn.Embedding(num_classes + use_cfg_embedding					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Any      =						num_classes
   __A						:							Union[str, Any]      =						dropout_prob



  def     A_					(self	:		Any					,      lowercase_	:		Optional[Any]					,      lowercase_	:		Any=None )		->				Union[str, Any]:






   if force_drop_ids is None:
    __A						:							List[str]      =						torch.rand(labels.shape[0]					,      device=labels.device ) < self.dropout_prob
   else:
    __A						:							Dict      =						torch.tensor(force_drop_ids == 1 )
   __A						:							Any      =						torch.where(SCREAMING_SNAKE_CASE_					,      self.num_classes					,      SCREAMING_SNAKE_CASE_ )
   return labels



  def     A_					(self	:		Optional[int]					,      lowercase_	:		List[Any]					,      lowercase_	:		Dict=None )		->				int:
   __A						:							Union[str, Any]      =						self.dropout_prob > 0
   if (self.training and use_dropout) or (force_drop_ids is not None):
    __A						:							Optional[Any]      =						self.token_drop(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Optional[Any]      =						self.embedding_table(SCREAMING_SNAKE_CASE_ )
   return embeddings






class __a ( nn.Module					):



  def __init__(self	:		Any					,      lowercase_	:		Any = 1_024					,      lowercase_	:		Tuple = 768					,      lowercase_	:		Union[str, Any] = 768					,      lowercase_	:		Tuple = 10					,      )		->				List[Any]:
   super().__init__()

   __A						:							List[Any]      =						num_image_text_embeds
   __A						:							Optional[int]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      self.num_image_text_embeds * cross_attention_dim )
   __A						:							List[Any]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		Dict					,      lowercase_	:		Any					,      lowercase_	:		int )		->				Dict:
   __A						:							Any      =						text_embeds.shape[0]

   # image
   __A						:							Optional[int]      =						self.image_embeds(SCREAMING_SNAKE_CASE_ )
   __A						:							Optional[int]      =						image_text_embeds.reshape(SCREAMING_SNAKE_CASE_					,      self.num_image_text_embeds					,      -1 )

   # text
   __A						:							Tuple      =						self.text_proj(SCREAMING_SNAKE_CASE_ )

   return torch.cat([image_text_embeds, text_embeds]					,      dim=1 )






class __a ( nn.Module					):



  def __init__(self	:		List[Any]					,      lowercase_	:		Union[str, Any] = 768					,      lowercase_	:		Optional[Any] = 768					,      lowercase_	:		int = 32					,      )		->				Optional[int]:
   super().__init__()

   __A						:							Optional[int]      =						num_image_text_embeds
   __A						:							str      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      self.num_image_text_embeds * cross_attention_dim )
   __A						:							Optional[Any]      =						nn.LayerNorm(SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		Any					,      lowercase_	:		List[str] )		->				str:
   __A						:							List[Any]      =						image_embeds.shape[0]

   # image
   __A						:							List[str]      =						self.image_embeds(SCREAMING_SNAKE_CASE_ )
   __A						:							Tuple      =						image_embeds.reshape(SCREAMING_SNAKE_CASE_					,      self.num_image_text_embeds					,      -1 )
   __A						:							Union[str, Any]      =						self.norm(SCREAMING_SNAKE_CASE_ )
   return image_embeds






class __a ( nn.Module					):



  def __init__(self	:		int					,      lowercase_	:		Optional[Any]					,      lowercase_	:		str					,      lowercase_	:		Any=0.1 )		->				Any:
   super().__init__()

   __A						:							int      =						Timesteps(num_channels=256					,      flip_sin_to_cos=SCREAMING_SNAKE_CASE_					,      downscale_freq_shift=1 )
   __A						:							Optional[Any]      =						TimestepEmbedding(in_channels=256					,      time_embed_dim=SCREAMING_SNAKE_CASE_ )
   __A						:							Optional[Any]      =						LabelEmbedding(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		Union[str, Any]					,      lowercase_	:		List[Any]					,      lowercase_	:		List[str]					,      lowercase_	:		List[str]=None )		->				int:
   __A						:							int      =						self.time_proj(SCREAMING_SNAKE_CASE_ )
   __A						:							List[str]      =						self.timestep_embedder(timesteps_proj.to(dtype=SCREAMING_SNAKE_CASE_ ) )  # (N, D)

   __A						:							Dict      =						self.class_embedder(SCREAMING_SNAKE_CASE_ )  # (N, D)

   __A						:							Dict      =						timesteps_emb + class_labels  # (N, D)

   return conditioning






class __a ( nn.Module					):



  def __init__(self	:		Optional[int]					,      lowercase_	:		Optional[Any]					,      lowercase_	:		Optional[Any]					,      lowercase_	:		Optional[Any] = 64 )		->				Optional[int]:
   super().__init__()
   __A						:							Optional[int]      =						nn.LayerNorm(SCREAMING_SNAKE_CASE_ )
   __A						:							Optional[Any]      =						AttentionPooling(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Union[str, Any]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Tuple      =						nn.LayerNorm(SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		int					,      lowercase_	:		Union[str, Any] )		->				Optional[Any]:
   __A						:							Dict      =						self.norma(SCREAMING_SNAKE_CASE_ )
   __A						:							Tuple      =						self.pool(SCREAMING_SNAKE_CASE_ )
   __A						:							Dict      =						self.proj(SCREAMING_SNAKE_CASE_ )
   __A						:							int      =						self.norma(SCREAMING_SNAKE_CASE_ )
   return hidden_states






class __a ( nn.Module					):



  def __init__(self	:		Dict					,      lowercase_	:		Optional[int] = 768					,      lowercase_	:		Union[str, Any] = 768					,      lowercase_	:		int = 1_536 )		->				str:
   super().__init__()
   __A						:							Union[str, Any]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Tuple      =						nn.LayerNorm(SCREAMING_SNAKE_CASE_ )
   __A						:							Any      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		Optional[int]					,      lowercase_	:		Optional[Any]					,      lowercase_	:		str )		->				str:
   __A						:							Tuple      =						self.text_proj(SCREAMING_SNAKE_CASE_ )
   __A						:							Dict      =						self.text_norm(SCREAMING_SNAKE_CASE_ )

   # image
   __A						:							str      =						self.image_proj(SCREAMING_SNAKE_CASE_ )

   return time_image_embeds + time_text_embeds






class __a ( nn.Module					):



  def __init__(self	:		List[Any]					,      lowercase_	:		Union[str, Any] = 768					,      lowercase_	:		List[str] = 1_536 )		->				Optional[Any]:
   super().__init__()
   __A						:							List[str]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )
   __A						:							int      =						nn.LayerNorm(SCREAMING_SNAKE_CASE_ )



  def     A_					(self	:		str					,      lowercase_	:		Dict )		->				List[Any]:
   __A						:							Any      =						self.image_proj(SCREAMING_SNAKE_CASE_ )
   __A						:							str      =						self.image_norm(SCREAMING_SNAKE_CASE_ )
   return time_image_embeds






class __a ( nn.Module					):



  def __init__(self	:		Any					,      lowercase_	:		Union[str, Any] = 768					,      lowercase_	:		str = 1_536 )		->				List[str]:
   super().__init__()
   __A						:							Optional[int]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )
   __A						:							Optional[Any]      =						nn.LayerNorm(SCREAMING_SNAKE_CASE_ )
   __A						:							List[str]      =						nn.Sequential(
       nn.Convad(3					,      16					,      3					,      padding=1 )					,      nn.SiLU()					,      nn.Convad(16					,      16					,      3					,      padding=1 )					,      nn.SiLU()					,      nn.Convad(16					,      32					,      3					,      padding=1					,      stride=2 )					,      nn.SiLU()					,      nn.Convad(32					,      32					,      3					,      padding=1 )					,      nn.SiLU()					,      nn.Convad(32					,      96					,      3					,      padding=1					,      stride=2 )					,      nn.SiLU()					,      nn.Convad(96					,      96					,      3					,      padding=1 )					,      nn.SiLU()					,      nn.Convad(96					,      256					,      3					,      padding=1					,      stride=2 )					,      nn.SiLU()					,      nn.Convad(256					,      4					,      3					,      padding=1 )					,      )



  def     A_					(self	:		List[Any]					,      lowercase_	:		Dict					,      lowercase_	:		Tuple )		->				List[str]:
   __A						:							List[str]      =						self.image_proj(SCREAMING_SNAKE_CASE_ )
   __A						:							List[Any]      =						self.image_norm(SCREAMING_SNAKE_CASE_ )
   __A						:							List[Any]      =						self.input_hint_block(SCREAMING_SNAKE_CASE_ )
   return time_image_embeds, hint






class __a ( nn.Module					):



  def __init__(self	:		str					,      lowercase_	:		List[str]					,      lowercase_	:		Tuple					,      lowercase_	:		Tuple=None )		->				Union[str, Any]:
   super().__init__()
   __A						:							Optional[Any]      =						dtype
   __A						:							str      =						nn.Parameter(torch.randn(1					,      SCREAMING_SNAKE_CASE_ ) / embed_dim**0.5 )
   __A						:							str      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_					,      dtype=self.dtype )
   __A						:							Optional[int]      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_					,      dtype=self.dtype )
   __A						:							Any      =						nn.Linear(SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_					,      dtype=self.dtype )
   __A						:							Tuple      =						num_heads
   __A						:							Optional[int]      =						embed_dim // self.num_heads



  def     A_					(self	:		Optional[int]					,      lowercase_	:		Union[str, Any] )		->				Any:
   __A     ,       __A     ,       __A						:							int      =						x.size()

   def shape(lowercase_	:		Optional[Any] ):
    # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
    __A						:							Any      =						x.view(SCREAMING_SNAKE_CASE_					,      -1					,      self.num_heads					,      self.dim_per_head )
    # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
    __A						:							str      =						x.transpose(1					,      2 )
    # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
    __A						:							str      =						x.reshape(bs * self.num_heads					,      -1					,      self.dim_per_head )
    # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
    __A						:							Any      =						x.transpose(1					,      2 )
    return x

   __A						:							Any      =						x.mean(dim=1					,      keepdim=SCREAMING_SNAKE_CASE_ ) + self.positional_embedding.to(x.dtype )
   __A						:							List[Any]      =						torch.cat([class_token, x]					,      dim=1 )  # (bs, length+1, width)

   # (bs*n_heads, class_token_length, dim_per_head)
   __A						:							Any      =						shape(self.q_proj(SCREAMING_SNAKE_CASE_ ) )
   # (bs*n_heads, length+class_token_length, dim_per_head)
   __A						:							Optional[int]      =						shape(self.k_proj(SCREAMING_SNAKE_CASE_ ) )
   __A						:							Tuple      =						shape(self.v_proj(SCREAMING_SNAKE_CASE_ ) )

   # (bs*n_heads, class_token_length, length+class_token_length):
   __A						:							str      =						1 / math.sqrt(math.sqrt(self.dim_per_head ) )
   __A						:							Dict      =						torch.einsum('bct,bcs->bts'					,      q * scale					,      k * scale )  # More stable with f16 than dividing afterwards
   __A						:							str      =						torch.softmax(weight.float()					,      dim=-1 ).type(weight.dtype )

   # (bs*n_heads, dim_per_head, class_token_length)
   __A						:							str      =						torch.einsum('bts,bcs->bct'					,      SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_ )

   # (bs, length+1, width)
   __A						:							Any      =						a.reshape(SCREAMING_SNAKE_CASE_					,      -1					,      1 ).transpose(1					,      2 )

   return a[:, 0, :]  # cls_token







import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionDualGuidedPipeline
from diffusers.utils.testing_utils import load_image, nightly, require_torch_gpu, torch_device


lowercase__				:				List[str]						=      False






@nightly
@require_torch_gpu
class 							SCREAMING_SNAKE_CASE__     (				unittest.TestCase     ):

							"""simple docstring"""






							def      A__		(	self      )-> List[Any]:





													'''simple docstring'''






													super().tearDown()
													gc.collect()
													torch.cuda.empty_cache()



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      VersatileDiffusionDualGuidedPipeline.from_pretrained('''shi-labs/versatile-diffusion'''      )
													# remove text_unet
													pipe.remove_unused_weights()
													pipe.to(SCREAMING_SNAKE_CASE_      )
													pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE_      )

													__UpperCamelCase  	=      load_image(
													    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'''      )

													__UpperCamelCase  	=      torch.manual_seed(0      )
													__UpperCamelCase  	=      pipe(
													    prompt='''first prompt'''  ,						image=SCREAMING_SNAKE_CASE_  ,						text_to_image_strength=0.7_5  ,						generator=SCREAMING_SNAKE_CASE_  ,						guidance_scale=7.5  ,						num_inference_steps=2  ,						output_type='''numpy'''  ,						).images

													with tempfile.TemporaryDirectory() as tmpdirname:
																			pipe.save_pretrained(SCREAMING_SNAKE_CASE_      )
																			__UpperCamelCase  	=      VersatileDiffusionDualGuidedPipeline.from_pretrained(SCREAMING_SNAKE_CASE_      )

													pipe.to(SCREAMING_SNAKE_CASE_      )
													pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE_      )

													__UpperCamelCase  	=      generator.manual_seed(0      )
													__UpperCamelCase  	=      pipe(
													    prompt='''first prompt'''  ,						image=SCREAMING_SNAKE_CASE_  ,						text_to_image_strength=0.7_5  ,						generator=SCREAMING_SNAKE_CASE_  ,						guidance_scale=7.5  ,						num_inference_steps=2  ,						output_type='''numpy'''  ,						).images

													assert np.abs(image - new_image      ).sum() < 1E-5, "Models don't have the same forward pass"



							def      A__		(	self      )-> int:





													'''simple docstring'''






													__UpperCamelCase  	=      VersatileDiffusionDualGuidedPipeline.from_pretrained('''shi-labs/versatile-diffusion'''      )
													pipe.remove_unused_weights()
													pipe.to(SCREAMING_SNAKE_CASE_      )
													pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE_      )

													__UpperCamelCase  	=      '''cyberpunk 2077'''
													__UpperCamelCase  	=      load_image(
													    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'''      )
													__UpperCamelCase  	=      torch.manual_seed(0      )
													__UpperCamelCase  	=      pipe(
													    prompt=SCREAMING_SNAKE_CASE_  ,						image=SCREAMING_SNAKE_CASE_  ,						text_to_image_strength=0.7_5  ,						generator=SCREAMING_SNAKE_CASE_  ,						guidance_scale=7.5  ,						num_inference_steps=50  ,						output_type='''numpy'''  ,						).images

													__UpperCamelCase  	=      image[0, 253:256, 253:256, -1]

													assert image.shape == (1, 512, 512, 3)
													__UpperCamelCase  	=      np.array([0.0_7_8_7, 0.0_8_4_9, 0.0_8_2_6, 0.0_8_1_2, 0.0_8_0_7, 0.0_7_9_5, 0.0_8_1_8, 0.0_7_9_8, 0.0_7_7_9]      )

													assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2




