

import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation
def   _lowerCAmelCase(						__A ,							__A ,							__A = False ,							__A = 1 ,							__A = 1 ,							__A = 10000 ,							):
  assert len(timesteps.shape      ) == 1, "Timesteps should be a 1d-array"

  UpperCAmelCase          =	embedding_dim // 2
  UpperCAmelCase          =	-math.log(__A      ) * torch.arange(
      start=0 ,							end=__A ,							dtype=torch.floataa ,							device=timesteps.device      )
  UpperCAmelCase          =	exponent / (half_dim - downscale_freq_shift)

  UpperCAmelCase          =	torch.exp(__A      )
  UpperCAmelCase          =	timesteps[:, None].float() * emb[None, :]

  # scale embeddings
  UpperCAmelCase          =	scale * emb

  # concat sine and cosine embeddings
  UpperCAmelCase          =	torch.cat([torch.sin(__A      ), torch.cos(__A      )] ,							dim=-1      )

  # flip sine and cosine embeddings
  if flip_sin_to_cos:
    UpperCAmelCase          =	torch.cat([emb[:, half_dim:], emb[:, :half_dim]] ,							dim=-1      )

  # zero pad
  if embedding_dim % 2 == 1:
    UpperCAmelCase          =	torch.nn.functional.pad(__A ,							(0, 1, 0, 0)      )
  return emb
def   _lowerCAmelCase(						__A ,							__A ,							__A=False ,							__A=0      ):
  UpperCAmelCase          =	np.arange(__A ,							dtype=np.floataa      )
  UpperCAmelCase          =	np.arange(__A ,							dtype=np.floataa      )
  UpperCAmelCase          =	np.meshgrid(__A ,							__A      )  # here w goes first
  UpperCAmelCase          =	np.stack(__A ,							axis=0      )

  UpperCAmelCase          =	grid.reshape([2, 1, grid_size, grid_size]      )
  UpperCAmelCase          =	get_ad_sincos_pos_embed_from_grid(__A ,							__A      )
  if cls_token and extra_tokens > 0:
    UpperCAmelCase          =	np.concatenate([np.zeros([extra_tokens, embed_dim]      ), pos_embed] ,							axis=0      )
  return pos_embed
def   _lowerCAmelCase(						__A ,							__A      ):
  if embed_dim % 2 != 0:
    raise ValueError("embed_dim must be divisible by 2"      )

  # use half of dimensions to encode grid_h
  UpperCAmelCase          =	get_ad_sincos_pos_embed_from_grid(embed_dim // 2 ,							grid[0]      )  # (H*W, D/2)
  UpperCAmelCase          =	get_ad_sincos_pos_embed_from_grid(embed_dim // 2 ,							grid[1]      )  # (H*W, D/2)

  UpperCAmelCase          =	np.concatenate([emb_h, emb_w] ,							axis=1      )  # (H*W, D)
  return emb
def   _lowerCAmelCase(						__A ,							__A      ):
  if embed_dim % 2 != 0:
    raise ValueError("embed_dim must be divisible by 2"      )

  UpperCAmelCase          =	np.arange(embed_dim // 2 ,							dtype=np.floataa      )
  omega /= embed_dim / 2.0
  UpperCAmelCase          =	1.0 / 10000**omega  # (D/2,)

  UpperCAmelCase          =	pos.reshape(-1      )  # (M,)
  UpperCAmelCase          =	np.einsum("m,d->md" ,							__A ,							__A      )  # (M, D/2), outer product

  UpperCAmelCase          =	np.sin(__A      )  # (M, D/2)
  UpperCAmelCase          =	np.cos(__A      )  # (M, D/2)

  UpperCAmelCase          =	np.concatenate([emb_sin, emb_cos] ,							axis=1      )  # (M, D)
  return emb






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Optional[Any]      ,    lowerCAmelCase__		:			Optional[int]=2_2_4      ,    lowerCAmelCase__		:			Tuple=2_2_4      ,    lowerCAmelCase__		:			Optional[Any]=1_6      ,    lowerCAmelCase__		:			int=3      ,    lowerCAmelCase__		:			Optional[Any]=7_6_8      ,    lowerCAmelCase__		:			str=False      ,    lowerCAmelCase__		:			Optional[Any]=True      ,    lowerCAmelCase__		:			Tuple=True      ,    )     ->   int:
   super().__init__()

   UpperCAmelCase          =	(height // patch_size) * (width // patch_size)
   UpperCAmelCase          =	flatten
   UpperCAmelCase          =	layer_norm

   UpperCAmelCase          =	nn.Convad(
       lowerCAmelCase__      ,    lowerCAmelCase__      ,    kernel_size=(patch_size, patch_size)      ,    stride=lowerCAmelCase__      ,    bias=lowerCAmelCase__     )
   if layer_norm:
     UpperCAmelCase          =	nn.LayerNorm(lowerCAmelCase__      ,    elementwise_affine=lowerCAmelCase__      ,    eps=1e-6     )
   else:
     UpperCAmelCase          =	None

   UpperCAmelCase          =	get_ad_sincos_pos_embed(lowerCAmelCase__      ,    int(num_patches**0.5     )     )
   self.register_buffer("pos_embed"      ,    torch.from_numpy(lowerCAmelCase__     ).float().unsqueeze(0     )      ,    persistent=lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Optional[int]      ,    lowerCAmelCase__		:			Tuple     )     ->   Dict:
   UpperCAmelCase          =	self.proj(lowerCAmelCase__     )
   if self.flatten:
     UpperCAmelCase          =	latent.flatten(2     ).transpose(1      ,    2     )  # BCHW -> BNC
   if self.layer_norm:
     UpperCAmelCase          =	self.norm(lowerCAmelCase__     )
   return latent + self.pos_embed






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Tuple      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			str = "silu"      ,    lowerCAmelCase__		:			int = None      ,    lowerCAmelCase__		:			Optional[str] = None      ,    lowerCAmelCase__		:			Tuple=None      ,    )     ->   Optional[int]:
   super().__init__()

   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )

   if cond_proj_dim is not None:
     UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__      ,    bias=lowerCAmelCase__     )
   else:
     UpperCAmelCase          =	None

   UpperCAmelCase          =	get_activation(lowerCAmelCase__     )

   if out_dim is not None:
     UpperCAmelCase          =	out_dim
   else:
     UpperCAmelCase          =	time_embed_dim
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )

   if post_act_fn is None:
     UpperCAmelCase          =	None
   else:
     UpperCAmelCase          =	get_activation(lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Optional[int]      ,    lowerCAmelCase__		:			Any      ,    lowerCAmelCase__		:			Tuple=None     )     ->   List[Any]:
   if condition is not None:
     UpperCAmelCase          =	sample + self.cond_proj(lowerCAmelCase__     )
   UpperCAmelCase          =	self.linear_a(lowerCAmelCase__     )

   if self.act is not None:
     UpperCAmelCase          =	self.act(lowerCAmelCase__     )

   UpperCAmelCase          =	self.linear_a(lowerCAmelCase__     )

   if self.post_act is not None:
     UpperCAmelCase          =	self.post_act(lowerCAmelCase__     )
   return sample






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Optional[Any]      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			bool      ,    lowerCAmelCase__		:			float     )     ->   Dict:
   super().__init__()
   UpperCAmelCase          =	num_channels
   UpperCAmelCase          =	flip_sin_to_cos
   UpperCAmelCase          =	downscale_freq_shift


 def _UpperCamelCase						(    self		:			Dict      ,    lowerCAmelCase__		:			List[Any]     )     ->   Tuple:
   UpperCAmelCase          =	get_timestep_embedding(
       lowerCAmelCase__      ,    self.num_channels      ,    flip_sin_to_cos=self.flip_sin_to_cos      ,    downscale_freq_shift=self.downscale_freq_shift      ,    )
   return t_emb






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Dict      ,    lowerCAmelCase__		:			int = 2_5_6      ,    lowerCAmelCase__		:			float = 1.0      ,    lowerCAmelCase__		:			Optional[Any]=True      ,    lowerCAmelCase__		:			Any=True      ,    lowerCAmelCase__		:			Dict=False     )     ->   int:
   super().__init__()
   UpperCAmelCase          =	nn.Parameter(torch.randn(lowerCAmelCase__     ) * scale      ,    requires_grad=lowerCAmelCase__     )
   UpperCAmelCase          =	log
   UpperCAmelCase          =	flip_sin_to_cos

   if set_W_to_weight:
     # to delete later
     UpperCAmelCase          =	nn.Parameter(torch.randn(lowerCAmelCase__     ) * scale      ,    requires_grad=lowerCAmelCase__     )

     UpperCAmelCase          =	self.W


 def _UpperCamelCase						(    self		:			Tuple      ,    lowerCAmelCase__		:			Dict     )     ->   Optional[int]:
   if self.log:
     UpperCAmelCase          =	torch.log(lowerCAmelCase__     )

   UpperCAmelCase          =	x[:, None] * self.weight[None, :] * 2 * np.pi

   if self.flip_sin_to_cos:
     UpperCAmelCase          =	torch.cat([torch.cos(lowerCAmelCase__     ), torch.sin(lowerCAmelCase__     )]      ,    dim=-1     )
   else:
     UpperCAmelCase          =	torch.cat([torch.sin(lowerCAmelCase__     ), torch.cos(lowerCAmelCase__     )]      ,    dim=-1     )
   return out






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Optional[Any]      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			int      ,    )     ->   Optional[int]:
   super().__init__()

   UpperCAmelCase          =	height
   UpperCAmelCase          =	width
   UpperCAmelCase          =	num_embed
   UpperCAmelCase          =	embed_dim

   UpperCAmelCase          =	nn.Embedding(self.num_embed      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	nn.Embedding(self.height      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	nn.Embedding(self.width      ,    lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			str     )     ->   List[Any]:
   UpperCAmelCase          =	self.emb(lowerCAmelCase__     )

   UpperCAmelCase          =	self.height_emb(torch.arange(self.height      ,    device=index.device     ).view(1      ,    self.height     )     )

   # 1 x H x D -> 1 x H x 1 x D
   UpperCAmelCase          =	height_emb.unsqueeze(2     )

   UpperCAmelCase          =	self.width_emb(torch.arange(self.width      ,    device=index.device     ).view(1      ,    self.width     )     )

   # 1 x W x D -> 1 x 1 x W x D
   UpperCAmelCase          =	width_emb.unsqueeze(1     )

   UpperCAmelCase          =	height_emb + width_emb

   # 1 x H x W x D -> 1 x L xD
   UpperCAmelCase          =	pos_emb.view(1      ,    self.height * self.width      ,    -1     )

   UpperCAmelCase          =	emb + pos_emb[:, : emb.shape[1], :]

   return emb






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Any      ,    lowerCAmelCase__		:			Any      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			Optional[Any]     )     ->   int:
   super().__init__()
   UpperCAmelCase          =	dropout_prob > 0
   UpperCAmelCase          =	nn.Embedding(num_classes + use_cfg_embedding      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	num_classes
   UpperCAmelCase          =	dropout_prob


 def _UpperCamelCase						(    self		:			int      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			Optional[int]=None     )     ->   Union[str, Any]:
   if force_drop_ids is None:
     UpperCAmelCase          =	torch.rand(labels.shape[0]      ,    device=labels.device     ) < self.dropout_prob
   else:
     UpperCAmelCase          =	torch.tensor(force_drop_ids == 1     )
   UpperCAmelCase          =	torch.where(lowerCAmelCase__      ,    self.num_classes      ,    lowerCAmelCase__     )
   return labels


 def _UpperCamelCase						(    self		:			Tuple      ,    lowerCAmelCase__		:			torch.LongTensor      ,    lowerCAmelCase__		:			Dict=None     )     ->   List[str]:
   UpperCAmelCase          =	self.dropout_prob > 0
   if (self.training and use_dropout) or (force_drop_ids is not None):
     UpperCAmelCase          =	self.token_drop(lowerCAmelCase__      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	self.embedding_table(lowerCAmelCase__     )
   return embeddings






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			List[Any]      ,    lowerCAmelCase__		:			int = 1_0_2_4      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 1_0      ,    )     ->   Union[str, Any]:
   super().__init__()

   UpperCAmelCase          =	num_image_text_embeds
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    self.num_image_text_embeds * cross_attention_dim     )
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			List[Any]      ,    lowerCAmelCase__		:			torch.FloatTensor      ,    lowerCAmelCase__		:			torch.FloatTensor     )     ->   List[Any]:
   UpperCAmelCase          =	text_embeds.shape[0]

   # image
   UpperCAmelCase          =	self.image_embeds(lowerCAmelCase__     )
   UpperCAmelCase          =	image_text_embeds.reshape(lowerCAmelCase__      ,    self.num_image_text_embeds      ,    -1     )

   # text
   UpperCAmelCase          =	self.text_proj(lowerCAmelCase__     )

   return torch.cat([image_text_embeds, text_embeds]      ,    dim=1     )






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 3_2      ,    )     ->   Dict:
   super().__init__()

   UpperCAmelCase          =	num_image_text_embeds
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    self.num_image_text_embeds * cross_attention_dim     )
   UpperCAmelCase          =	nn.LayerNorm(lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			int      ,    lowerCAmelCase__		:			torch.FloatTensor     )     ->   str:
   UpperCAmelCase          =	image_embeds.shape[0]

   # image
   UpperCAmelCase          =	self.image_embeds(lowerCAmelCase__     )
   UpperCAmelCase          =	image_embeds.reshape(lowerCAmelCase__      ,    self.num_image_text_embeds      ,    -1     )
   UpperCAmelCase          =	self.norm(lowerCAmelCase__     )
   return image_embeds






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Tuple      ,    lowerCAmelCase__		:			Any      ,    lowerCAmelCase__		:			List[str]      ,    lowerCAmelCase__		:			Optional[int]=0.1     )     ->   List[Any]:
   super().__init__()

   UpperCAmelCase          =	Timesteps(num_channels=2_5_6      ,    flip_sin_to_cos=lowerCAmelCase__      ,    downscale_freq_shift=1     )
   UpperCAmelCase          =	TimestepEmbedding(in_channels=2_5_6      ,    time_embed_dim=lowerCAmelCase__     )
   UpperCAmelCase          =	LabelEmbedding(lowerCAmelCase__      ,    lowerCAmelCase__      ,    lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Optional[int]      ,    lowerCAmelCase__		:			Dict      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			Union[str, Any]=None     )     ->   Tuple:
   UpperCAmelCase          =	self.time_proj(lowerCAmelCase__     )
   UpperCAmelCase          =	self.timestep_embedder(timesteps_proj.to(dtype=lowerCAmelCase__     )     )  # (N, D)

   UpperCAmelCase          =	self.class_embedder(lowerCAmelCase__     )  # (N, D)

   UpperCAmelCase          =	timesteps_emb + class_labels  # (N, D)

   return conditioning






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Dict      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			int      ,    lowerCAmelCase__		:			int = 6_4     )     ->   int:
   super().__init__()
   UpperCAmelCase          =	nn.LayerNorm(lowerCAmelCase__     )
   UpperCAmelCase          =	AttentionPooling(lowerCAmelCase__      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	nn.LayerNorm(lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Tuple      ,    lowerCAmelCase__		:			Dict     )     ->   str:
   UpperCAmelCase          =	self.norma(lowerCAmelCase__     )
   UpperCAmelCase          =	self.pool(lowerCAmelCase__     )
   UpperCAmelCase          =	self.proj(lowerCAmelCase__     )
   UpperCAmelCase          =	self.norma(lowerCAmelCase__     )
   return hidden_states






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			str      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 1_5_3_6     )     ->   Optional[Any]:
   super().__init__()
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	nn.LayerNorm(lowerCAmelCase__     )
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Dict      ,    lowerCAmelCase__		:			torch.FloatTensor      ,    lowerCAmelCase__		:			torch.FloatTensor     )     ->   List[str]:
   # text
   UpperCAmelCase          =	self.text_proj(lowerCAmelCase__     )
   UpperCAmelCase          =	self.text_norm(lowerCAmelCase__     )

   # image
   UpperCAmelCase          =	self.image_proj(lowerCAmelCase__     )

   return time_image_embeds + time_text_embeds






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 1_5_3_6     )     ->   Optional[Any]:
   super().__init__()
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	nn.LayerNorm(lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			int      ,    lowerCAmelCase__		:			torch.FloatTensor     )     ->   Tuple:
   # image
   UpperCAmelCase          =	self.image_proj(lowerCAmelCase__     )
   UpperCAmelCase          =	self.image_norm(lowerCAmelCase__     )
   return time_image_embeds






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			int      ,    lowerCAmelCase__		:			int = 7_6_8      ,    lowerCAmelCase__		:			int = 1_5_3_6     )     ->   Any:
   super().__init__()
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__     )
   UpperCAmelCase          =	nn.LayerNorm(lowerCAmelCase__     )
   UpperCAmelCase          =	nn.Sequential(
       nn.Convad(3      ,    1_6      ,    3      ,    padding=1     )      ,    nn.SiLU()      ,    nn.Convad(1_6      ,    1_6      ,    3      ,    padding=1     )      ,    nn.SiLU()      ,    nn.Convad(1_6      ,    3_2      ,    3      ,    padding=1      ,    stride=2     )      ,    nn.SiLU()      ,    nn.Convad(3_2      ,    3_2      ,    3      ,    padding=1     )      ,    nn.SiLU()      ,    nn.Convad(3_2      ,    9_6      ,    3      ,    padding=1      ,    stride=2     )      ,    nn.SiLU()      ,    nn.Convad(9_6      ,    9_6      ,    3      ,    padding=1     )      ,    nn.SiLU()      ,    nn.Convad(9_6      ,    2_5_6      ,    3      ,    padding=1      ,    stride=2     )      ,    nn.SiLU()      ,    nn.Convad(2_5_6      ,    4      ,    3      ,    padding=1     )      ,    )


 def _UpperCamelCase						(    self		:			Dict      ,    lowerCAmelCase__		:			torch.FloatTensor      ,    lowerCAmelCase__		:			torch.FloatTensor     )     ->   str:
   # image
   UpperCAmelCase          =	self.image_proj(lowerCAmelCase__     )
   UpperCAmelCase          =	self.image_norm(lowerCAmelCase__     )
   UpperCAmelCase          =	self.input_hint_block(lowerCAmelCase__     )
   return time_image_embeds, hint






class     __magic_name__     (							nn.Module     ):


 def __init__(    self		:			int      ,    lowerCAmelCase__		:			Tuple      ,    lowerCAmelCase__		:			Dict      ,    lowerCAmelCase__		:			List[Any]=None     )     ->   Any:
   super().__init__()
   UpperCAmelCase          =	dtype
   UpperCAmelCase          =	nn.Parameter(torch.randn(1      ,    lowerCAmelCase__     ) / embed_dim**0.5     )
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__      ,    dtype=self.dtype     )
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__      ,    dtype=self.dtype     )
   UpperCAmelCase          =	nn.Linear(lowerCAmelCase__      ,    lowerCAmelCase__      ,    dtype=self.dtype     )
   UpperCAmelCase          =	num_heads
   UpperCAmelCase          =	embed_dim // self.num_heads


 def _UpperCamelCase						(    self		:			Optional[int]      ,    lowerCAmelCase__		:			int     )     ->   Any:
   UpperCAmelCase   ,						UpperCAmelCase   ,						UpperCAmelCase          =	x.size()

   def shape(lowerCAmelCase__		:			Optional[Any]     ):
     # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
     UpperCAmelCase          =	x.view(lowerCAmelCase__      ,    -1      ,    self.num_heads      ,    self.dim_per_head     )
     # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
     UpperCAmelCase          =	x.transpose(1      ,    2     )
     # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
     UpperCAmelCase          =	x.reshape(bs * self.num_heads      ,    -1      ,    self.dim_per_head     )
     # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
     UpperCAmelCase          =	x.transpose(1      ,    2     )
     return x

   UpperCAmelCase          =	x.mean(dim=1      ,    keepdim=lowerCAmelCase__     ) + self.positional_embedding.to(x.dtype     )
   UpperCAmelCase          =	torch.cat([class_token, x]      ,    dim=1     )  # (bs, length+1, width)

   # (bs*n_heads, class_token_length, dim_per_head)
   UpperCAmelCase          =	shape(self.q_proj(lowerCAmelCase__     )     )
   # (bs*n_heads, length+class_token_length, dim_per_head)
   UpperCAmelCase          =	shape(self.k_proj(lowerCAmelCase__     )     )
   UpperCAmelCase          =	shape(self.v_proj(lowerCAmelCase__     )     )

   # (bs*n_heads, class_token_length, length+class_token_length):
   UpperCAmelCase          =	1 / math.sqrt(math.sqrt(self.dim_per_head     )     )
   UpperCAmelCase          =	torch.einsum("bct,bcs->bts"      ,    q * scale      ,    k * scale     )  # More stable with f16 than dividing afterwards
   UpperCAmelCase          =	torch.softmax(weight.float()      ,    dim=-1     ).type(weight.dtype     )

   # (bs*n_heads, dim_per_head, class_token_length)
   UpperCAmelCase          =	torch.einsum("bts,bcs->bct"      ,    lowerCAmelCase__      ,    lowerCAmelCase__     )

   # (bs, length+1, width)
   UpperCAmelCase          =	a.reshape(lowerCAmelCase__      ,    -1      ,    1     ).transpose(1      ,    2     )

   return a[:, 0, :]  # cls_token






import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available
def   _lowerCAmelCase(						__A ,							__A ,							__A ,							__A=None      ):
  UpperCAmelCase          =	None
  # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
  if str(weight.dtype      ) == "torch.bfloat16":
    # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
    UpperCAmelCase          =	weight.view(torch.intaa      )
    UpperCAmelCase          =	"bfloat16"
  UpperCAmelCase          =	weight.cpu().numpy()
  UpperCAmelCase          =	os.path.join(__A ,							F"{weight_name}.dat"      )
  if index is not None:
    if dtype is None:
      UpperCAmelCase          =	str(array.dtype      )
    UpperCAmelCase          =	{"dtype": dtype, "shape": list(array.shape      )}
  if array.ndim == 0:
    UpperCAmelCase          =	array[None]
  UpperCAmelCase          =	np.memmap(__A ,							dtype=array.dtype ,							mode="w+" ,							shape=array.shape      )
  UpperCAmelCase          =	array[:]
  file_array.flush()
  return index
def   _lowerCAmelCase(						__A ,							__A      ):
  UpperCAmelCase          =	tuple(weight_info["shape"]      )
  if shape == ():
    # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
    UpperCAmelCase          =	(1,)

  UpperCAmelCase          =	weight_info["dtype"]
  if dtype == "bfloat16":
    # NumPy does not support bfloat16 so this was saved as a int16
    UpperCAmelCase          =	"int16"

  UpperCAmelCase          =	np.memmap(__A ,							dtype=__A ,							shape=__A ,							mode="r"      )

  if len(weight_info["shape"]      ) == 0:
    UpperCAmelCase          =	weight[0]
  UpperCAmelCase          =	torch.tensor(__A      )
  if weight_info["dtype"] == "bfloat16":
    UpperCAmelCase          =	weight.view(torch.bfloataa      )

  return weight
def   _lowerCAmelCase(						__A ,							__A      ):
  if index is None or len(__A      ) == 0:
    # Nothing to save
    return

  UpperCAmelCase          =	os.path.join(__A ,							"index.json"      )
  if os.path.isfile(__A      ):
    with open(__A ,							"r" ,							encoding="utf-8"      ) as f:
      UpperCAmelCase          =	json.load(__A      )
  else:
    UpperCAmelCase          =	{}
  current_index.update(__A      )

  with open(__A ,							"w" ,							encoding="utf-8"      ) as f:
    json.dump(__A ,							__A ,							indent=2      )
def   _lowerCAmelCase(						__A ,							__A      ):
  os.makedirs(__A ,							exist_ok=__A      )
  UpperCAmelCase          =	{}
  for name, parameter in state_dict.items():
    UpperCAmelCase          =	offload_weight(__A ,							__A ,							__A ,							index=__A      )

  # Update index
  save_offload_index(__A ,							__A      )






class     __magic_name__     (							_snake_case     ):


 def __init__(    self		:			Tuple      ,    lowerCAmelCase__		:			Mapping      ,    lowerCAmelCase__		:			str     )     ->   Tuple:
   UpperCAmelCase          =	dataset
   UpperCAmelCase          =	prefix


 def __getitem__(    self		:			Dict      ,    lowerCAmelCase__		:			Any     )     ->   Optional[int]:
   return self.dataset[f"{self.prefix}{key}"]


 def __iter__(    self		:			Optional[int]     )     ->   Any:
   return iter([key for key in self.dataset if key.startswith(self.prefix     )]     )


 def __len__(    self		:			Union[str, Any]     )     ->   int:
   return len(self.dataset     )






class     __magic_name__     (							_snake_case     ):


 def __init__(    self		:			Optional[int]      ,    lowerCAmelCase__		:			Dict[str, torch.Tensor] = None      ,    lowerCAmelCase__		:			Optional[Union[str, os.PathLike]] = None      ,    lowerCAmelCase__		:			Mapping = None      ,    lowerCAmelCase__		:			Dict=None      ,    )     ->   Dict:
   if state_dict is None and save_folder is None:
     raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights."     )

   UpperCAmelCase          =	{} if state_dict is None else state_dict
   UpperCAmelCase          =	save_folder
   if index is None and save_folder is not None:
     with open(os.path.join(lowerCAmelCase__      ,    "index.json"     )     ) as f:
       UpperCAmelCase          =	json.load(lowerCAmelCase__     )
   UpperCAmelCase          =	{} if index is None else index
   UpperCAmelCase          =	list(self.state_dict.keys()     )
   self.all_keys.extend([key for key in self.index if key not in self.all_keys]     )
   UpperCAmelCase          =	device


 def __getitem__(    self		:			str      ,    lowerCAmelCase__		:			str     )     ->   Dict:
   # State dict gets priority
   if key in self.state_dict:
     return self.state_dict[key]
   UpperCAmelCase          =	self.index[key]
   if weight_info.get("safetensors_file"     ) is not None:
     if not is_safetensors_available():
       raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`."     )

     from safetensors import safe_open

     UpperCAmelCase          =	"cpu" if self.device is None else self.device
     with safe_open(weight_info["safetensors_file"]      ,    framework="pt"      ,    device=lowerCAmelCase__     ) as f:
       UpperCAmelCase          =	f.get_tensor(weight_info.get("weight_name"      ,    lowerCAmelCase__     )     )

     if "dtype" in weight_info:
       return tensor.to(getattr(lowerCAmelCase__      ,    weight_info["dtype"]     )     )
     else:
       return tensor

   UpperCAmelCase          =	os.path.join(self.save_folder      ,    f"{key}.dat"     )
   return load_offloaded_weight(lowerCAmelCase__      ,    lowerCAmelCase__     )


 def __iter__(    self		:			str     )     ->   Dict:
   return iter(self.all_keys     )


 def __len__(    self		:			Optional[Any]     )     ->   List[Any]:
   return len(self.all_keys     )

def   _lowerCAmelCase(						__A ,							__A      ):
  UpperCAmelCase          =	{}
  for module_name in submodule_names:
    # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
    # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
    result.update(
        {
            key: param
            for key, param in state_dict.items()
            if key == module_name or key.startswith(module_name + "."      )
        }      )
  return result



