





import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation

def      a    (   _lowercase :  torch.Tensor			,    _lowercase :  int			,    _lowercase :  bool = False			,    _lowercase :  float = 1			,    _lowercase :  float = 1			,    _lowercase :  int = 1_0_0_0_0			,    )  ->Any:
		assert len(timesteps.shape							) == 1, "Timesteps should be a 1d-array"

		UpperCamelCase__								=						embedding_dim // 2
		UpperCamelCase__								=						-math.log(_lowercase							) * torch.arange(
		    start=0			,    end=_lowercase			,    dtype=torch.floataa			,    device=timesteps.device							)
		UpperCamelCase__								=						exponent / (half_dim - downscale_freq_shift)

		UpperCamelCase__								=						torch.exp(_lowercase							)
		UpperCamelCase__								=						timesteps[:, None].float() * emb[None, :]

		# scale embeddings
		UpperCamelCase__								=						scale * emb

		# concat sine and cosine embeddings
		UpperCamelCase__								=						torch.cat([torch.sin(_lowercase							), torch.cos(_lowercase							)]			,    dim=-1							)

		# flip sine and cosine embeddings
		if flip_sin_to_cos:
				UpperCamelCase__								=						torch.cat([emb[:, half_dim:], emb[:, :half_dim]]			,    dim=-1							)

		# zero pad
		if embedding_dim % 2 == 1:
				UpperCamelCase__								=						torch.nn.functional.pad(_lowercase			,    (0, 1, 0, 0)							)
		return emb

def      a    (   _lowercase :  Tuple			,    _lowercase :  Optional[int]			,    _lowercase :  Union[str, Any]=False			,    _lowercase :  Dict=0							)  ->Optional[Any]:
		UpperCamelCase__								=						np.arange(_lowercase			,    dtype=np.floataa							)
		UpperCamelCase__								=						np.arange(_lowercase			,    dtype=np.floataa							)
		UpperCamelCase__								=						np.meshgrid(_lowercase			,    _lowercase							)  # here w goes first
		UpperCamelCase__								=						np.stack(_lowercase			,    axis=0							)

		UpperCamelCase__								=						grid.reshape([2, 1, grid_size, grid_size]							)
		UpperCamelCase__								=						get_ad_sincos_pos_embed_from_grid(_lowercase			,    _lowercase							)
		if cls_token and extra_tokens > 0:
				UpperCamelCase__								=						np.concatenate([np.zeros([extra_tokens, embed_dim]							), pos_embed]			,    axis=0							)
		return pos_embed

def      a    (   _lowercase :  Tuple			,    _lowercase :  Any							)  ->Any:
		if embed_dim % 2 != 0:
				raise ValueError("""embed_dim must be divisible by 2"""							)

		# use half of dimensions to encode grid_h
		UpperCamelCase__								=						get_ad_sincos_pos_embed_from_grid(embed_dim // 2			,    grid[0]							)  # (H*W, D/2)
		UpperCamelCase__								=						get_ad_sincos_pos_embed_from_grid(embed_dim // 2			,    grid[1]							)  # (H*W, D/2)

		UpperCamelCase__								=						np.concatenate([emb_h, emb_w]			,    axis=1							)  # (H*W, D)
		return emb

def      a    (   _lowercase :  Any			,    _lowercase :  Optional[Any]							)  ->Optional[Any]:
		if embed_dim % 2 != 0:
				raise ValueError("""embed_dim must be divisible by 2"""							)

		UpperCamelCase__								=						np.arange(embed_dim // 2			,    dtype=np.floataa							)
		omega /= embed_dim / 2.0
		UpperCamelCase__								=						1.0 / 1_0_0_0_0**omega  # (D/2,)

		UpperCamelCase__								=						pos.reshape(-1							)  # (M,)
		UpperCamelCase__								=						np.einsum("""m,d->md"""			,    _lowercase			,    _lowercase							)  # (M, D/2), outer product

		UpperCamelCase__								=						np.sin(_lowercase							)  # (M, D/2)
		UpperCamelCase__								=						np.cos(_lowercase							)  # (M, D/2)

		UpperCamelCase__								=						np.concatenate([emb_sin, emb_cos]			,    axis=1							)  # (M, D)
		return emb




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase=224     ,   __lowerCAmelCase=224     ,   __lowerCAmelCase=16     ,   __lowerCAmelCase=3     ,   __lowerCAmelCase=768     ,   __lowerCAmelCase=False     ,   __lowerCAmelCase=True     ,   __lowerCAmelCase=True     ,   )    ->	List[str]:
							super().__init__()

							UpperCamelCase__								=						(height // patch_size) * (width // patch_size)
							UpperCamelCase__								=						flatten
							UpperCamelCase__								=						layer_norm

							UpperCamelCase__								=						nn.Convad(
							    __lowerCAmelCase     ,   __lowerCAmelCase     ,   kernel_size=(patch_size, patch_size)     ,   stride=__lowerCAmelCase     ,   bias=__lowerCAmelCase							)
							if layer_norm:
									UpperCamelCase__								=						nn.LayerNorm(__lowerCAmelCase     ,   elementwise_affine=__lowerCAmelCase     ,   eps=1E-6							)
							else:
									UpperCamelCase__								=						None

							UpperCamelCase__								=						get_ad_sincos_pos_embed(__lowerCAmelCase     ,   int(num_patches**0.5							)							)
							self.register_buffer("""pos_embed"""     ,   torch.from_numpy(__lowerCAmelCase							).float().unsqueeze(0							)     ,   persistent=__lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Dict:
							UpperCamelCase__								=						self.proj(__lowerCAmelCase							)
							if self.flatten:
									UpperCamelCase__								=						latent.flatten(2							).transpose(1     ,   2							)  # BCHW -> BNC
							if self.layer_norm:
									UpperCamelCase__								=						self.norm(__lowerCAmelCase							)
							return latent + self.pos_embed




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase = "silu"     ,   __lowerCAmelCase = None     ,   __lowerCAmelCase = None     ,   __lowerCAmelCase=None     ,   )    ->	Optional[Any]:
							super().__init__()

							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)

							if cond_proj_dim is not None:
									UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase     ,   bias=__lowerCAmelCase							)
							else:
									UpperCamelCase__								=						None

							UpperCamelCase__								=						get_activation(__lowerCAmelCase							)

							if out_dim is not None:
									UpperCamelCase__								=						out_dim
							else:
									UpperCamelCase__								=						time_embed_dim
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)

							if post_act_fn is None:
									UpperCamelCase__								=						None
							else:
									UpperCamelCase__								=						get_activation(__lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase=None							)    ->	str:
							if condition is not None:
									UpperCamelCase__								=						sample + self.cond_proj(__lowerCAmelCase							)
							UpperCamelCase__								=						self.linear_a(__lowerCAmelCase							)

							if self.act is not None:
									UpperCamelCase__								=						self.act(__lowerCAmelCase							)

							UpperCamelCase__								=						self.linear_a(__lowerCAmelCase							)

							if self.post_act is not None:
									UpperCamelCase__								=						self.post_act(__lowerCAmelCase							)
							return sample




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Union[str, Any]:
							super().__init__()
							UpperCamelCase__								=						num_channels
							UpperCamelCase__								=						flip_sin_to_cos
							UpperCamelCase__								=						downscale_freq_shift




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Any:
							UpperCamelCase__								=						get_timestep_embedding(
							    __lowerCAmelCase     ,   self.num_channels     ,   flip_sin_to_cos=self.flip_sin_to_cos     ,   downscale_freq_shift=self.downscale_freq_shift     ,   )
							return t_emb




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase = 256     ,   __lowerCAmelCase = 1.0     ,   __lowerCAmelCase=True     ,   __lowerCAmelCase=True     ,   __lowerCAmelCase=False							)    ->	Any:
							super().__init__()
							UpperCamelCase__								=						nn.Parameter(torch.randn(__lowerCAmelCase							) * scale     ,   requires_grad=__lowerCAmelCase							)
							UpperCamelCase__								=						log
							UpperCamelCase__								=						flip_sin_to_cos

							if set_W_to_weight:
									# to delete later
									UpperCamelCase__								=						nn.Parameter(torch.randn(__lowerCAmelCase							) * scale     ,   requires_grad=__lowerCAmelCase							)

									UpperCamelCase__								=						self.W




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Any:
							if self.log:
									UpperCamelCase__								=						torch.log(__lowerCAmelCase							)

							UpperCamelCase__								=						x[:, None] * self.weight[None, :] * 2 * np.pi

							if self.flip_sin_to_cos:
									UpperCamelCase__								=						torch.cat([torch.cos(__lowerCAmelCase							), torch.sin(__lowerCAmelCase							)]     ,   dim=-1							)
							else:
									UpperCamelCase__								=						torch.cat([torch.sin(__lowerCAmelCase							), torch.cos(__lowerCAmelCase							)]     ,   dim=-1							)
							return out




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   )    ->	Optional[Any]:
							super().__init__()

							UpperCamelCase__								=						height
							UpperCamelCase__								=						width
							UpperCamelCase__								=						num_embed
							UpperCamelCase__								=						embed_dim

							UpperCamelCase__								=						nn.Embedding(self.num_embed     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						nn.Embedding(self.height     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						nn.Embedding(self.width     ,   __lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	str:
							UpperCamelCase__								=						self.emb(__lowerCAmelCase							)

							UpperCamelCase__								=						self.height_emb(torch.arange(self.height     ,   device=index.device							).view(1     ,   self.height							)							)

							# 1 x H x D -> 1 x H x 1 x D
							UpperCamelCase__								=						height_emb.unsqueeze(2							)

							UpperCamelCase__								=						self.width_emb(torch.arange(self.width     ,   device=index.device							).view(1     ,   self.width							)							)

							# 1 x W x D -> 1 x 1 x W x D
							UpperCamelCase__								=						width_emb.unsqueeze(1							)

							UpperCamelCase__								=						height_emb + width_emb

							# 1 x H x W x D -> 1 x L xD
							UpperCamelCase__								=						pos_emb.view(1     ,   self.height * self.width     ,   -1							)

							UpperCamelCase__								=						emb + pos_emb[:, : emb.shape[1], :]

							return emb




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	List[Any]:
							super().__init__()
							UpperCamelCase__								=						dropout_prob > 0
							UpperCamelCase__								=						nn.Embedding(num_classes + use_cfg_embedding     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						num_classes
							UpperCamelCase__								=						dropout_prob




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase=None							)    ->	Any:
							if force_drop_ids is None:
									UpperCamelCase__								=						torch.rand(labels.shape[0]     ,   device=labels.device							) < self.dropout_prob
							else:
									UpperCamelCase__								=						torch.tensor(force_drop_ids == 1							)
							UpperCamelCase__								=						torch.where(__lowerCAmelCase     ,   self.num_classes     ,   __lowerCAmelCase							)
							return labels




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase=None							)    ->	Union[str, Any]:
							UpperCamelCase__								=						self.dropout_prob > 0
							if (self.training and use_dropout) or (force_drop_ids is not None):
									UpperCamelCase__								=						self.token_drop(__lowerCAmelCase     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						self.embedding_table(__lowerCAmelCase							)
							return embeddings




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase = 1_024     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 10     ,   )    ->	Union[str, Any]:
							super().__init__()

							UpperCamelCase__								=						num_image_text_embeds
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   self.num_image_text_embeds * cross_attention_dim							)
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Any:
							UpperCamelCase__								=						text_embeds.shape[0]

							# image
							UpperCamelCase__								=						self.image_embeds(__lowerCAmelCase							)
							UpperCamelCase__								=						image_text_embeds.reshape(__lowerCAmelCase     ,   self.num_image_text_embeds     ,   -1							)

							# text
							UpperCamelCase__								=						self.text_proj(__lowerCAmelCase							)

							return torch.cat([image_text_embeds, text_embeds]     ,   dim=1							)




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 32     ,   )    ->	Union[str, Any]:
							super().__init__()

							UpperCamelCase__								=						num_image_text_embeds
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   self.num_image_text_embeds * cross_attention_dim							)
							UpperCamelCase__								=						nn.LayerNorm(__lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Any:
							UpperCamelCase__								=						image_embeds.shape[0]

							# image
							UpperCamelCase__								=						self.image_embeds(__lowerCAmelCase							)
							UpperCamelCase__								=						image_embeds.reshape(__lowerCAmelCase     ,   self.num_image_text_embeds     ,   -1							)
							UpperCamelCase__								=						self.norm(__lowerCAmelCase							)
							return image_embeds




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase=0.1							)    ->	str:
							super().__init__()

							UpperCamelCase__								=						Timesteps(num_channels=256     ,   flip_sin_to_cos=__lowerCAmelCase     ,   downscale_freq_shift=1							)
							UpperCamelCase__								=						TimestepEmbedding(in_channels=256     ,   time_embed_dim=__lowerCAmelCase							)
							UpperCamelCase__								=						LabelEmbedding(__lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase=None							)    ->	List[Any]:
							UpperCamelCase__								=						self.time_proj(__lowerCAmelCase							)
							UpperCamelCase__								=						self.timestep_embedder(timesteps_proj.to(dtype=__lowerCAmelCase							)							)  # (N, D)

							UpperCamelCase__								=						self.class_embedder(__lowerCAmelCase							)  # (N, D)

							UpperCamelCase__								=						timesteps_emb + class_labels  # (N, D)

							return conditioning




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase = 64							)    ->	Dict:
							super().__init__()
							UpperCamelCase__								=						nn.LayerNorm(__lowerCAmelCase							)
							UpperCamelCase__								=						AttentionPooling(__lowerCAmelCase     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						nn.LayerNorm(__lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	List[str]:
							UpperCamelCase__								=						self.norma(__lowerCAmelCase							)
							UpperCamelCase__								=						self.pool(__lowerCAmelCase							)
							UpperCamelCase__								=						self.proj(__lowerCAmelCase							)
							UpperCamelCase__								=						self.norma(__lowerCAmelCase							)
							return hidden_states




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 1_536							)    ->	Tuple:
							super().__init__()
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						nn.LayerNorm(__lowerCAmelCase							)
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Union[str, Any]:
							# text
							UpperCamelCase__								=						self.text_proj(__lowerCAmelCase							)
							UpperCamelCase__								=						self.text_norm(__lowerCAmelCase							)

							# image
							UpperCamelCase__								=						self.image_proj(__lowerCAmelCase							)

							return time_image_embeds + time_text_embeds




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 1_536							)    ->	Dict:
							super().__init__()
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						nn.LayerNorm(__lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Dict:
							# image
							UpperCamelCase__								=						self.image_proj(__lowerCAmelCase							)
							UpperCamelCase__								=						self.image_norm(__lowerCAmelCase							)
							return time_image_embeds




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase = 768     ,   __lowerCAmelCase = 1_536							)    ->	Union[str, Any]:
							super().__init__()
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						nn.LayerNorm(__lowerCAmelCase							)
							UpperCamelCase__								=						nn.Sequential(
							    nn.Convad(3     ,   16     ,   3     ,   padding=1							)     ,   nn.SiLU()     ,   nn.Convad(16     ,   16     ,   3     ,   padding=1							)     ,   nn.SiLU()     ,   nn.Convad(16     ,   32     ,   3     ,   padding=1     ,   stride=2							)     ,   nn.SiLU()     ,   nn.Convad(32     ,   32     ,   3     ,   padding=1							)     ,   nn.SiLU()     ,   nn.Convad(32     ,   96     ,   3     ,   padding=1     ,   stride=2							)     ,   nn.SiLU()     ,   nn.Convad(96     ,   96     ,   3     ,   padding=1							)     ,   nn.SiLU()     ,   nn.Convad(96     ,   256     ,   3     ,   padding=1     ,   stride=2							)     ,   nn.SiLU()     ,   nn.Convad(256     ,   4     ,   3     ,   padding=1							)     ,   )




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Optional[Any]:
							# image
							UpperCamelCase__								=						self.image_proj(__lowerCAmelCase							)
							UpperCamelCase__								=						self.image_norm(__lowerCAmelCase							)
							UpperCamelCase__								=						self.input_hint_block(__lowerCAmelCase							)
							return time_image_embeds, hint




class 	_UpperCAmelCase						( nn.Module       ):




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase=None							)    ->	List[str]:
							super().__init__()
							UpperCamelCase__								=						dtype
							UpperCamelCase__								=						nn.Parameter(torch.randn(1     ,   __lowerCAmelCase							) / embed_dim**0.5							)
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase     ,   dtype=self.dtype							)
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase     ,   dtype=self.dtype							)
							UpperCamelCase__								=						nn.Linear(__lowerCAmelCase     ,   __lowerCAmelCase     ,   dtype=self.dtype							)
							UpperCamelCase__								=						num_heads
							UpperCamelCase__								=						embed_dim // self.num_heads




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Dict:
							UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__								=						x.size()

							def shape(__lowerCAmelCase							):
									# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
									UpperCamelCase__								=						x.view(__lowerCAmelCase     ,   -1     ,   self.num_heads     ,   self.dim_per_head							)
									# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
									UpperCamelCase__								=						x.transpose(1     ,   2							)
									# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
									UpperCamelCase__								=						x.reshape(bs * self.num_heads     ,   -1     ,   self.dim_per_head							)
									# (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
									UpperCamelCase__								=						x.transpose(1     ,   2							)
									return x

							UpperCamelCase__								=						x.mean(dim=1     ,   keepdim=__lowerCAmelCase							) + self.positional_embedding.to(x.dtype							)
							UpperCamelCase__								=						torch.cat([class_token, x]     ,   dim=1							)  # (bs, length+1, width)

							# (bs*n_heads, class_token_length, dim_per_head)
							UpperCamelCase__								=						shape(self.q_proj(__lowerCAmelCase							)							)
							# (bs*n_heads, length+class_token_length, dim_per_head)
							UpperCamelCase__								=						shape(self.k_proj(__lowerCAmelCase							)							)
							UpperCamelCase__								=						shape(self.v_proj(__lowerCAmelCase							)							)

							# (bs*n_heads, class_token_length, length+class_token_length):
							UpperCamelCase__								=						1 / math.sqrt(math.sqrt(self.dim_per_head							)							)
							UpperCamelCase__								=						torch.einsum("""bct,bcs->bts"""     ,   q * scale     ,   k * scale							)  # More stable with f16 than dividing afterwards
							UpperCamelCase__								=						torch.softmax(weight.float()     ,   dim=-1							).type(weight.dtype							)

							# (bs*n_heads, dim_per_head, class_token_length)
							UpperCamelCase__								=						torch.einsum("""bts,bcs->bct"""     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)

							# (bs, length+1, width)
							UpperCamelCase__								=						a.reshape(__lowerCAmelCase     ,   -1     ,   1							).transpose(1     ,   2							)

							return a[:, 0, :]  # cls_token












import unittest

import torch

from diffusers import UNetaDModel
from diffusers.utils import floats_tensor, slow, torch_device

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin




class 	_UpperCAmelCase						( _lowercase			,							_lowercase			,							unittest.TestCase       ):
					__A        = UNetaDModel
					__A        = "sample"




					@property
					def 						snake_case_		(    self							)    ->	Any:
							UpperCamelCase__								=						4
							UpperCamelCase__								=						14
							UpperCamelCase__								=						16

							UpperCamelCase__								=						floats_tensor((batch_size, num_features, seq_len)							).to(__lowerCAmelCase							)
							UpperCamelCase__								=						torch.tensor([10] * batch_size							).to(__lowerCAmelCase							)

							return {"sample": noise, "timestep": time_step}




					@property
					def 						snake_case_		(    self							)    ->	Optional[int]:
							return (4, 14, 16)




					@property
					def 						snake_case_		(    self							)    ->	Any:
							return (4, 14, 16)




					def 						snake_case_		(    self							)    ->	Any:
							pass




					def 						snake_case_		(    self							)    ->	Tuple:
							pass




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Any:
							super().test_determinism()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	int:
							super().test_outputs_equivalence()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Optional[Any]:
							super().test_from_save_pretrained()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Union[str, Any]:
							super().test_from_save_pretrained_variant()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Optional[int]:
							super().test_model_from_pretrained()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Dict:
							super().test_output()




					def 						snake_case_		(    self							)    ->	int:
							UpperCamelCase__								=						{
							    """block_out_channels""": (32, 64, 128, 256),
							    """in_channels""": 14,
							    """out_channels""": 14,
							    """time_embedding_type""": """positional""",
							    """use_timestep_embedding""": True,
							    """flip_sin_to_cos""": False,
							    """freq_shift""": 1.0,
							    """out_block_type""": """OutConv1DBlock""",
							    """mid_block_type""": """MidResTemporalBlock1D""",
							    """down_block_types""": ("""DownResnetBlock1D""", """DownResnetBlock1D""", """DownResnetBlock1D""", """DownResnetBlock1D"""),
							    """up_block_types""": ("""UpResnetBlock1D""", """UpResnetBlock1D""", """UpResnetBlock1D"""),
							    """act_fn""": """mish""",
							}
							UpperCamelCase__								=						self.dummy_input
							return init_dict, inputs_dict




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	List[Any]:
							UpperCamelCase__	,      UpperCamelCase__								=						UNetaDModel.from_pretrained(
							    """bglick13/hopper-medium-v2-value-function-hor32"""     ,   output_loading_info=__lowerCAmelCase     ,   subfolder="""unet"""							)
							self.assertIsNotNone(__lowerCAmelCase							)
							self.assertEqual(len(loading_info["""missing_keys"""]							)     ,   0							)

							model.to(__lowerCAmelCase							)
							UpperCamelCase__								=						model(**self.dummy_input							)

							assert image is not None, "Make sure output is not None"




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Optional[Any]:
							UpperCamelCase__								=						UNetaDModel.from_pretrained("""bglick13/hopper-medium-v2-value-function-hor32"""     ,   subfolder="""unet"""							)
							torch.manual_seed(0							)
							if torch.cuda.is_available():
									torch.cuda.manual_seed_all(0							)

							UpperCamelCase__								=						model.config.in_channels
							UpperCamelCase__								=						16
							UpperCamelCase__								=						torch.randn((1, seq_len, num_features)							).permute(
							    0     ,   2     ,   1							)  # match original, we can update values and remove
							UpperCamelCase__								=						torch.full((num_features,)     ,   0							)

							with torch.no_grad():
									UpperCamelCase__								=						model(__lowerCAmelCase     ,   __lowerCAmelCase							).sample.permute(0     ,   2     ,   1							)

							UpperCamelCase__								=						output[0, -3:, -3:].flatten()
							# fmt: off
							UpperCamelCase__								=						torch.tensor([-2.1_3_7_1_7_2, 1.1_4_2_6_0_1_6, 0.3_6_8_8_6_8_7, -0.7_6_6_9_2_2, 0.7_3_0_3_1_4_6, 0.1_1_0_3_8_8_6_4, -0.4_7_6_0_6_3_3, 0.1_3_2_7_0_1_7_2, 0.0_2_5_9_1_3_4_8]							)
							# fmt: on
							self.assertTrue(torch.allclose(__lowerCAmelCase     ,   __lowerCAmelCase     ,   rtol=1E-3							)							)




					def 						snake_case_		(    self							)    ->	int:
							# Not implemented yet for this UNet
							pass




					@slow
					def 						snake_case_		(    self							)    ->	Tuple:
							UpperCamelCase__								=						"""harmonai/maestro-150k"""
							UpperCamelCase__								=						UNetaDModel.from_pretrained(__lowerCAmelCase     ,   subfolder="""unet"""							)
							model.to(__lowerCAmelCase							)

							UpperCamelCase__								=						65_536
							UpperCamelCase__								=						torch.sin(torch.arange(__lowerCAmelCase							)[None, None, :].repeat(1     ,   2     ,   1							)							).to(__lowerCAmelCase							)
							UpperCamelCase__								=						torch.tensor([1]							).to(__lowerCAmelCase							)

							with torch.no_grad():
									UpperCamelCase__								=						model(__lowerCAmelCase     ,   __lowerCAmelCase							).sample

							UpperCamelCase__								=						output.abs().sum()
							UpperCamelCase__								=						output.abs().max()

							assert (output_sum - 2_2_4.0_8_9_6).abs() < 0.5
							assert (output_max - 0.0_6_0_7).abs() < 4E-4




class 	_UpperCAmelCase						( _lowercase			,							_lowercase			,							unittest.TestCase       ):
					__A        = UNetaDModel
					__A        = "sample"




					@property
					def 						snake_case_		(    self							)    ->	Any:
							UpperCamelCase__								=						4
							UpperCamelCase__								=						14
							UpperCamelCase__								=						16

							UpperCamelCase__								=						floats_tensor((batch_size, num_features, seq_len)							).to(__lowerCAmelCase							)
							UpperCamelCase__								=						torch.tensor([10] * batch_size							).to(__lowerCAmelCase							)

							return {"sample": noise, "timestep": time_step}




					@property
					def 						snake_case_		(    self							)    ->	Union[str, Any]:
							return (4, 14, 16)




					@property
					def 						snake_case_		(    self							)    ->	List[Any]:
							return (4, 14, 1)




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Dict:
							super().test_determinism()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Optional[int]:
							super().test_outputs_equivalence()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Optional[int]:
							super().test_from_save_pretrained()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Optional[Any]:
							super().test_from_save_pretrained_variant()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Optional[Any]:
							super().test_model_from_pretrained()




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	List[str]:
							# UNetRL is a value-function is different output shape
							UpperCamelCase__	,      UpperCamelCase__								=						self.prepare_init_args_and_inputs_for_common()
							UpperCamelCase__								=						self.model_class(**__lowerCAmelCase							)
							model.to(__lowerCAmelCase							)
							model.eval()

							with torch.no_grad():
									UpperCamelCase__								=						model(**__lowerCAmelCase							)

									if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							):
											UpperCamelCase__								=						output.sample

							self.assertIsNotNone(__lowerCAmelCase							)
							UpperCamelCase__								=						torch.Size((inputs_dict["""sample"""].shape[0], 1)							)
							self.assertEqual(output.shape     ,   __lowerCAmelCase     ,   """Input and output shapes do not match"""							)




					def 						snake_case_		(    self							)    ->	Optional[int]:
							pass




					def 						snake_case_		(    self							)    ->	Union[str, Any]:
							pass




					def 						snake_case_		(    self							)    ->	Dict:
							UpperCamelCase__								=						{
							    """in_channels""": 14,
							    """out_channels""": 14,
							    """down_block_types""": ["""DownResnetBlock1D""", """DownResnetBlock1D""", """DownResnetBlock1D""", """DownResnetBlock1D"""],
							    """up_block_types""": [],
							    """out_block_type""": """ValueFunction""",
							    """mid_block_type""": """ValueFunctionMidBlock1D""",
							    """block_out_channels""": [32, 64, 128, 256],
							    """layers_per_block""": 1,
							    """downsample_each_block""": True,
							    """use_timestep_embedding""": True,
							    """freq_shift""": 1.0,
							    """flip_sin_to_cos""": False,
							    """time_embedding_type""": """positional""",
							    """act_fn""": """mish""",
							}
							UpperCamelCase__								=						self.dummy_input
							return init_dict, inputs_dict




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	Tuple:
							UpperCamelCase__	,      UpperCamelCase__								=						UNetaDModel.from_pretrained(
							    """bglick13/hopper-medium-v2-value-function-hor32"""     ,   output_loading_info=__lowerCAmelCase     ,   subfolder="""value_function"""							)
							self.assertIsNotNone(__lowerCAmelCase							)
							self.assertEqual(len(vf_loading_info["""missing_keys"""]							)     ,   0							)

							value_function.to(__lowerCAmelCase							)
							UpperCamelCase__								=						value_function(**self.dummy_input							)

							assert image is not None, "Make sure output is not None"




					@unittest.skipIf(torch_device == """mps"""     ,   """mish op not supported in MPS"""							)
					def 						snake_case_		(    self							)    ->	List[Any]:
							UpperCamelCase__	,      UpperCamelCase__								=						UNetaDModel.from_pretrained(
							    """bglick13/hopper-medium-v2-value-function-hor32"""     ,   output_loading_info=__lowerCAmelCase     ,   subfolder="""value_function"""							)
							torch.manual_seed(0							)
							if torch.cuda.is_available():
									torch.cuda.manual_seed_all(0							)

							UpperCamelCase__								=						value_function.config.in_channels
							UpperCamelCase__								=						14
							UpperCamelCase__								=						torch.randn((1, seq_len, num_features)							).permute(
							    0     ,   2     ,   1							)  # match original, we can update values and remove
							UpperCamelCase__								=						torch.full((num_features,)     ,   0							)

							with torch.no_grad():
									UpperCamelCase__								=						value_function(__lowerCAmelCase     ,   __lowerCAmelCase							).sample

							# fmt: off
							UpperCamelCase__								=						torch.tensor([1_6_5.2_5] * seq_len							)
							# fmt: on
							self.assertTrue(torch.allclose(__lowerCAmelCase     ,   __lowerCAmelCase     ,   rtol=1E-3							)							)




					def 						snake_case_		(    self							)    ->	Union[str, Any]:
							# Not implemented yet for this UNet
							pass





