




'''simple docstring'''

import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation

require_version('''tokenizers>=0.13.3''')

if is_sentencepiece_available():
		from .tokenization_llama import LlamaTokenizer
else:
		a__         =     None

a__         =     logging.get_logger(__name__)
a__         =     {'''vocab_file''': '''tokenizer.model''', '''tokenizer_file''': '''tokenizer.json'''}

a__    , a__         =     '''[INST]''', '''[/INST]'''
a__    , a__         =     '''<<SYS>>\n''', '''\n<</SYS>>\n\n'''

# fmt: off
a__         =     '''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don\'t know the answer to a question, please don\'t share false information.'''





class __magic_name__(						__lowerCAmelCase      ):
						UpperCAmelCase_		:					str    =  VOCAB_FILES_NAMES
						UpperCAmelCase_		:					List[Any]    =  LlamaTokenizer
						UpperCAmelCase_		:					Any    =  """left"""
						UpperCAmelCase_		:					Optional[int]    =  ["""input_ids""", """attention_mask"""]






						def __init__(   self						:						Any					,   __UpperCamelCase						:						Tuple=None					,   __UpperCamelCase						:						Any=None					,   __UpperCamelCase						:						List[str]=False					,   __UpperCamelCase						:						List[Any]="<unk>"					,   __UpperCamelCase						:						Optional[int]="<s>"					,   __UpperCamelCase						:						Tuple="</s>"					,   __UpperCamelCase						:						Any=True					,   __UpperCamelCase						:						Any=False					,   **__UpperCamelCase						:						Tuple					,   ):



									'''simple docstring'''






									super().__init__(
									    vocab_file=__UpperCamelCase					,   tokenizer_file=__UpperCamelCase					,   clean_up_tokenization_spaces=__UpperCamelCase					,   unk_token=__UpperCamelCase					,   bos_token=__UpperCamelCase					,   eos_token=__UpperCamelCase					,   **__UpperCamelCase					,   )
									snake_case__          =						add_bos_token
									snake_case__          =						add_eos_token
									self.update_post_processor()

									snake_case__          =						vocab_file
									snake_case__          =						False if not self.vocab_file else True






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.bos_token
									snake_case__          =						self.bos_token_id

									snake_case__          =						self.eos_token
									snake_case__          =						self.eos_token_id

									snake_case__          =						f"""{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}"""
									snake_case__          =						f"""{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}"""

									snake_case__          =						[]
									if self.add_bos_token:
												special_tokens.append((bos, bos_token_id)       )
									if self.add_eos_token:
												special_tokens.append((eos, eos_token_id)       )
									snake_case__          =						processors.TemplateProcessing(
									    single=__UpperCamelCase					,   pair=__UpperCamelCase					,   special_tokens=__UpperCamelCase       )






						@property
						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									return self._add_eos_token






						@property
						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return self._add_bos_token






						@add_eos_token.setter
						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						Any       ):



									'''simple docstring'''






									snake_case__          =						value
									self.update_post_processor()






						@add_bos_token.setter
						def       __lowerCAmelCase(   self						:						int					,   __UpperCamelCase						:						Any       ):



									'''simple docstring'''






									snake_case__          =						value
									self.update_post_processor()






						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Optional[str] = None       ):



									'''simple docstring'''






									if not self.can_save_slow_tokenizer:
												raise ValueError(
												    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
												    """tokenizer."""       )

									if not os.path.isdir(__UpperCamelCase       ):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
												return
									snake_case__          =						os.path.join(
									    __UpperCamelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

									if os.path.abspath(self.vocab_file       ) != os.path.abspath(__UpperCamelCase       ):
												copyfile(self.vocab_file					,   __UpperCamelCase       )

									return (out_vocab_file,)







						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						"Conversation"       ):



									'''simple docstring'''






									snake_case__          =						list(conversation.iter_texts()       )
									if not all([is_user for is_user, msg in dialogue[::2]]       ) or not all(
									    [not is_user for is_user, msg in dialogue[1::2]]       ):
												raise ValueError(
												    """The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)"""       )

									snake_case__          =						[]
									if len(conversation.past_user_inputs       ) > 0:
												if not conversation.past_user_inputs[0].startswith(__UpperCamelCase       ) or E_SYS not in conversation.past_user_inputs[0]:
															snake_case__          =						(
															    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
															)
									elif not dialogue[0][1].startswith(__UpperCamelCase       ) or E_SYS not in dialogue[0][1]:
												snake_case__          =						(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

									dialog_tokens += sum(
									    [
									        [self.bos_token_id]
									        + self.encode(
									            f"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """					,   add_special_tokens=__UpperCamelCase       )
									        + [self.eos_token_id]
									        for prompt, answer in zip(dialogue[::2]					,   dialogue[1::2]       )
									    ]					,   []					,   )
									if not (dialogue[-1][0]):
												raise ValueError(f"""Last message must be from user, got {dialogue[-1]["role"]}"""       )
									dialog_tokens += [self.bos_token_id] + self.encode(
									    f"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""					,   add_special_tokens=__UpperCamelCase       )
									return dialog_tokens
import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation






def  __lowerCamelCase						(		lowerCAmelCase__      ,    lowerCAmelCase__      ,    lowerCAmelCase__ = False      ,    lowerCAmelCase__ = 1      ,    lowerCAmelCase__ = 1      ,    lowerCAmelCase__ = 1_0_0_0_0      ,    ):
       assert len(timesteps.shape      ) == 1, "Timesteps should be a 1d-array"

       lowerCAmelCase__	      =   embedding_dim // 2
       lowerCAmelCase__	      =   -math.log(lowerCAmelCase__      ) * torch.arange(
           start=0      ,    end=lowerCAmelCase__      ,    dtype=torch.floataa      ,    device=timesteps.device      )
       lowerCAmelCase__	      =   exponent / (half_dim - downscale_freq_shift)

       lowerCAmelCase__	      =   torch.exp(lowerCAmelCase__      )
       lowerCAmelCase__	      =   timesteps[:, None].float() * emb[None, :]

       # scale embeddings
       lowerCAmelCase__	      =   scale * emb

       # concat sine and cosine embeddings
       lowerCAmelCase__	      =   torch.cat([torch.sin(lowerCAmelCase__      ), torch.cos(lowerCAmelCase__      )]      ,    dim=-1      )

       # flip sine and cosine embeddings
       if flip_sin_to_cos:
              lowerCAmelCase__	      =   torch.cat([emb[:, half_dim:], emb[:, :half_dim]]      ,    dim=-1      )

       # zero pad
       if embedding_dim % 2 == 1:
              lowerCAmelCase__	      =   torch.nn.functional.pad(lowerCAmelCase__      ,    (0, 1, 0, 0)      )
       return emb






def  __lowerCamelCase						(		lowerCAmelCase__      ,    lowerCAmelCase__      ,    lowerCAmelCase__=False      ,    lowerCAmelCase__=0      ):
       lowerCAmelCase__	      =   np.arange(lowerCAmelCase__      ,    dtype=np.floataa      )
       lowerCAmelCase__	      =   np.arange(lowerCAmelCase__      ,    dtype=np.floataa      )
       lowerCAmelCase__	      =   np.meshgrid(lowerCAmelCase__      ,    lowerCAmelCase__      )  # here w goes first
       lowerCAmelCase__	      =   np.stack(lowerCAmelCase__      ,    axis=0      )

       lowerCAmelCase__	      =   grid.reshape([2, 1, grid_size, grid_size]      )
       lowerCAmelCase__	      =   get_ad_sincos_pos_embed_from_grid(lowerCAmelCase__      ,    lowerCAmelCase__      )
       if cls_token and extra_tokens > 0:
              lowerCAmelCase__	      =   np.concatenate([np.zeros([extra_tokens, embed_dim]      ), pos_embed]      ,    axis=0      )
       return pos_embed






def  __lowerCamelCase						(		lowerCAmelCase__      ,    lowerCAmelCase__      ):
       if embed_dim % 2 != 0:
              raise ValueError('embed_dim must be divisible by 2'      )

       # use half of dimensions to encode grid_h
       lowerCAmelCase__	      =   get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,    grid[0]      )  # (H*W, D/2)
       lowerCAmelCase__	      =   get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,    grid[1]      )  # (H*W, D/2)

       lowerCAmelCase__	      =   np.concatenate([emb_h, emb_w]      ,    axis=1      )  # (H*W, D)
       return emb






def  __lowerCamelCase						(		lowerCAmelCase__      ,    lowerCAmelCase__      ):
       if embed_dim % 2 != 0:
              raise ValueError('embed_dim must be divisible by 2'      )

       lowerCAmelCase__	      =   np.arange(embed_dim // 2      ,    dtype=np.floataa      )
       omega /= embed_dim / 2.0
       lowerCAmelCase__	      =   1.0 / 1_0_0_0_0**omega  # (D/2,)

       lowerCAmelCase__	      =   pos.reshape(-1      )  # (M,)
       lowerCAmelCase__	      =   np.einsum('m,d->md'      ,    lowerCAmelCase__      ,    lowerCAmelCase__      )  # (M, D/2), outer product

       lowerCAmelCase__	      =   np.sin(lowerCAmelCase__      )  # (M, D/2)
       lowerCAmelCase__	      =   np.cos(lowerCAmelCase__      )  # (M, D/2)

       lowerCAmelCase__	      =   np.concatenate([emb_sin, emb_cos]      ,    axis=1      )  # (M, D)
       return emb
class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    Optional[int]     ,       lowercase__   :    Any=224     ,       lowercase__   :    Any=224     ,       lowercase__   :    int=16     ,       lowercase__   :    List[Any]=3     ,       lowercase__   :    Dict=768     ,       lowercase__   :    Tuple=False     ,       lowercase__   :    Any=True     ,       lowercase__   :    List[str]=True     ,       ):







        '''simple docstring'''



        super().__init__()

        lowerCAmelCase__	      =   (height // patch_size) * (width // patch_size)
        lowerCAmelCase__	      =   flatten
        lowerCAmelCase__	      =   layer_norm

        lowerCAmelCase__	      =   nn.Convad(
            lowercase__     ,       lowercase__     ,       kernel_size=(patch_size, patch_size)     ,       stride=lowercase__     ,       bias=lowercase__)
        if layer_norm:
               lowerCAmelCase__	      =   nn.LayerNorm(lowercase__     ,       elementwise_affine=lowercase__     ,       eps=1e-6)
        else:
               lowerCAmelCase__	      =   None

        lowerCAmelCase__	      =   get_ad_sincos_pos_embed(lowercase__     ,       int(num_patches**0.5))
        self.register_buffer('pos_embed'     ,       torch.from_numpy(lowercase__).float().unsqueeze(0)     ,       persistent=lowercase__)




 def 						__snake_case		(     self   :    Union[str, Any]     ,       lowercase__   :    List[str]):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.proj(lowercase__)
        if self.flatten:
               lowerCAmelCase__	      =   latent.flatten(2).transpose(1     ,       2)  # BCHW -> BNC
        if self.layer_norm:
               lowerCAmelCase__	      =   self.norm(lowercase__)
        return latent + self.pos_embed






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    str     ,       lowercase__   :    int     ,       lowercase__   :    int     ,       lowercase__   :    str = "silu"     ,       lowercase__   :    int = None     ,       lowercase__   :    Optional[str] = None     ,       lowercase__   :    List[str]=None     ,       ):







        '''simple docstring'''



        super().__init__()

        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)

        if cond_proj_dim is not None:
               lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__     ,       bias=lowercase__)
        else:
               lowerCAmelCase__	      =   None

        lowerCAmelCase__	      =   get_activation(lowercase__)

        if out_dim is not None:
               lowerCAmelCase__	      =   out_dim
        else:
               lowerCAmelCase__	      =   time_embed_dim
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)

        if post_act_fn is None:
               lowerCAmelCase__	      =   None
        else:
               lowerCAmelCase__	      =   get_activation(lowercase__)




 def 						__snake_case		(     self   :    Union[str, Any]     ,       lowercase__   :    Tuple     ,       lowercase__   :    List[Any]=None):







        '''simple docstring'''



        if condition is not None:
               lowerCAmelCase__	      =   sample + self.cond_proj(lowercase__)
        lowerCAmelCase__	      =   self.linear_a(lowercase__)

        if self.act is not None:
               lowerCAmelCase__	      =   self.act(lowercase__)

        lowerCAmelCase__	      =   self.linear_a(lowercase__)

        if self.post_act is not None:
               lowerCAmelCase__	      =   self.post_act(lowercase__)
        return sample






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    Optional[int]     ,       lowercase__   :    int     ,       lowercase__   :    bool     ,       lowercase__   :    float):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   num_channels
        lowerCAmelCase__	      =   flip_sin_to_cos
        lowerCAmelCase__	      =   downscale_freq_shift




 def 						__snake_case		(     self   :    int     ,       lowercase__   :    Union[str, Any]):







        '''simple docstring'''



        lowerCAmelCase__	      =   get_timestep_embedding(
            lowercase__     ,       self.num_channels     ,       flip_sin_to_cos=self.flip_sin_to_cos     ,       downscale_freq_shift=self.downscale_freq_shift     ,       )
        return t_emb






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    List[str]     ,       lowercase__   :    int = 256     ,       lowercase__   :    float = 1.0     ,       lowercase__   :    str=True     ,       lowercase__   :    List[str]=True     ,       lowercase__   :    Any=False):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   nn.Parameter(torch.randn(lowercase__) * scale     ,       requires_grad=lowercase__)
        lowerCAmelCase__	      =   log
        lowerCAmelCase__	      =   flip_sin_to_cos

        if set_W_to_weight:
               # to delete later
               lowerCAmelCase__	      =   nn.Parameter(torch.randn(lowercase__) * scale     ,       requires_grad=lowercase__)

               lowerCAmelCase__	      =   self.W




 def 						__snake_case		(     self   :    Dict     ,       lowercase__   :    List[str]):







        '''simple docstring'''



        if self.log:
               lowerCAmelCase__	      =   torch.log(lowercase__)

        lowerCAmelCase__	      =   x[:, None] * self.weight[None, :] * 2 * np.pi

        if self.flip_sin_to_cos:
               lowerCAmelCase__	      =   torch.cat([torch.cos(lowercase__), torch.sin(lowercase__)]     ,       dim=-1)
        else:
               lowerCAmelCase__	      =   torch.cat([torch.sin(lowercase__), torch.cos(lowercase__)]     ,       dim=-1)
        return out






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    List[str]     ,       lowercase__   :    int     ,       lowercase__   :    int     ,       lowercase__   :    int     ,       lowercase__   :    int     ,       ):







        '''simple docstring'''



        super().__init__()

        lowerCAmelCase__	      =   height
        lowerCAmelCase__	      =   width
        lowerCAmelCase__	      =   num_embed
        lowerCAmelCase__	      =   embed_dim

        lowerCAmelCase__	      =   nn.Embedding(self.num_embed     ,       lowercase__)
        lowerCAmelCase__	      =   nn.Embedding(self.height     ,       lowercase__)
        lowerCAmelCase__	      =   nn.Embedding(self.width     ,       lowercase__)




 def 						__snake_case		(     self   :    Dict     ,       lowercase__   :    Any):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.emb(lowercase__)

        lowerCAmelCase__	      =   self.height_emb(torch.arange(self.height     ,       device=index.device).view(1     ,       self.height))

        # 1 x H x D -> 1 x H x 1 x D
        lowerCAmelCase__	      =   height_emb.unsqueeze(2)

        lowerCAmelCase__	      =   self.width_emb(torch.arange(self.width     ,       device=index.device).view(1     ,       self.width))

        # 1 x W x D -> 1 x 1 x W x D
        lowerCAmelCase__	      =   width_emb.unsqueeze(1)

        lowerCAmelCase__	      =   height_emb + width_emb

        # 1 x H x W x D -> 1 x L xD
        lowerCAmelCase__	      =   pos_emb.view(1     ,       self.height * self.width     ,       -1)

        lowerCAmelCase__	      =   emb + pos_emb[:, : emb.shape[1], :]

        return emb






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    Any     ,       lowercase__   :    int     ,       lowercase__   :    Dict     ,       lowercase__   :    Any):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   dropout_prob > 0
        lowerCAmelCase__	      =   nn.Embedding(num_classes + use_cfg_embedding     ,       lowercase__)
        lowerCAmelCase__	      =   num_classes
        lowerCAmelCase__	      =   dropout_prob




 def 						__snake_case		(     self   :    Any     ,       lowercase__   :    List[Any]     ,       lowercase__   :    List[Any]=None):







        '''simple docstring'''



        if force_drop_ids is None:
               lowerCAmelCase__	      =   torch.rand(labels.shape[0]     ,       device=labels.device) < self.dropout_prob
        else:
               lowerCAmelCase__	      =   torch.tensor(force_drop_ids == 1)
        lowerCAmelCase__	      =   torch.where(lowercase__     ,       self.num_classes     ,       lowercase__)
        return labels




 def 						__snake_case		(     self   :    Union[str, Any]     ,       lowercase__   :    torch.LongTensor     ,       lowercase__   :    str=None):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.dropout_prob > 0
        if (self.training and use_dropout) or (force_drop_ids is not None):
               lowerCAmelCase__	      =   self.token_drop(lowercase__     ,       lowercase__)
        lowerCAmelCase__	      =   self.embedding_table(lowercase__)
        return embeddings






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    Optional[Any]     ,       lowercase__   :    int = 1_024     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 10     ,       ):







        '''simple docstring'''



        super().__init__()

        lowerCAmelCase__	      =   num_image_text_embeds
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       self.num_image_text_embeds * cross_attention_dim)
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)




 def 						__snake_case		(     self   :    List[Any]     ,       lowercase__   :    torch.FloatTensor     ,       lowercase__   :    torch.FloatTensor):







        '''simple docstring'''



        lowerCAmelCase__	      =   text_embeds.shape[0]

        # image
        lowerCAmelCase__	      =   self.image_embeds(lowercase__)
        lowerCAmelCase__	      =   image_text_embeds.reshape(lowercase__     ,       self.num_image_text_embeds     ,       -1)

        # text
        lowerCAmelCase__	      =   self.text_proj(lowercase__)

        return torch.cat([image_text_embeds, text_embeds]     ,       dim=1)






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    str     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 32     ,       ):







        '''simple docstring'''



        super().__init__()

        lowerCAmelCase__	      =   num_image_text_embeds
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       self.num_image_text_embeds * cross_attention_dim)
        lowerCAmelCase__	      =   nn.LayerNorm(lowercase__)




 def 						__snake_case		(     self   :    Union[str, Any]     ,       lowercase__   :    torch.FloatTensor):







        '''simple docstring'''



        lowerCAmelCase__	      =   image_embeds.shape[0]

        # image
        lowerCAmelCase__	      =   self.image_embeds(lowercase__)
        lowerCAmelCase__	      =   image_embeds.reshape(lowercase__     ,       self.num_image_text_embeds     ,       -1)
        lowerCAmelCase__	      =   self.norm(lowercase__)
        return image_embeds






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    str     ,       lowercase__   :    Optional[int]     ,       lowercase__   :    Dict     ,       lowercase__   :    Tuple=0.1):







        '''simple docstring'''



        super().__init__()

        lowerCAmelCase__	      =   Timesteps(num_channels=256     ,       flip_sin_to_cos=lowercase__     ,       downscale_freq_shift=1)
        lowerCAmelCase__	      =   TimestepEmbedding(in_channels=256     ,       time_embed_dim=lowercase__)
        lowerCAmelCase__	      =   LabelEmbedding(lowercase__     ,       lowercase__     ,       lowercase__)




 def 						__snake_case		(     self   :    List[Any]     ,       lowercase__   :    List[str]     ,       lowercase__   :    Dict     ,       lowercase__   :    Dict=None):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.time_proj(lowercase__)
        lowerCAmelCase__	      =   self.timestep_embedder(timesteps_proj.to(dtype=lowercase__))  # (N, D)

        lowerCAmelCase__	      =   self.class_embedder(lowercase__)  # (N, D)

        lowerCAmelCase__	      =   timesteps_emb + class_labels  # (N, D)

        return conditioning






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    Optional[Any]     ,       lowercase__   :    int     ,       lowercase__   :    int     ,       lowercase__   :    int = 64):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   nn.LayerNorm(lowercase__)
        lowerCAmelCase__	      =   AttentionPooling(lowercase__     ,       lowercase__)
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)
        lowerCAmelCase__	      =   nn.LayerNorm(lowercase__)




 def 						__snake_case		(     self   :    Optional[int]     ,       lowercase__   :    Any):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.norma(lowercase__)
        lowerCAmelCase__	      =   self.pool(lowercase__)
        lowerCAmelCase__	      =   self.proj(lowercase__)
        lowerCAmelCase__	      =   self.norma(lowercase__)
        return hidden_states






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    List[str]     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 1_536):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)
        lowerCAmelCase__	      =   nn.LayerNorm(lowercase__)
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)




 def 						__snake_case		(     self   :    Union[str, Any]     ,       lowercase__   :    torch.FloatTensor     ,       lowercase__   :    torch.FloatTensor):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.text_proj(lowercase__)
        lowerCAmelCase__	      =   self.text_norm(lowercase__)

        # image
        lowerCAmelCase__	      =   self.image_proj(lowercase__)

        return time_image_embeds + time_text_embeds






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    List[Any]     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 1_536):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)
        lowerCAmelCase__	      =   nn.LayerNorm(lowercase__)




 def 						__snake_case		(     self   :    Union[str, Any]     ,       lowercase__   :    torch.FloatTensor):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.image_proj(lowercase__)
        lowerCAmelCase__	      =   self.image_norm(lowercase__)
        return time_image_embeds






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    Optional[int]     ,       lowercase__   :    int = 768     ,       lowercase__   :    int = 1_536):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__)
        lowerCAmelCase__	      =   nn.LayerNorm(lowercase__)
        lowerCAmelCase__	      =   nn.Sequential(
            nn.Convad(3     ,       16     ,       3     ,       padding=1)     ,       nn.SiLU()     ,       nn.Convad(16     ,       16     ,       3     ,       padding=1)     ,       nn.SiLU()     ,       nn.Convad(16     ,       32     ,       3     ,       padding=1     ,       stride=2)     ,       nn.SiLU()     ,       nn.Convad(32     ,       32     ,       3     ,       padding=1)     ,       nn.SiLU()     ,       nn.Convad(32     ,       96     ,       3     ,       padding=1     ,       stride=2)     ,       nn.SiLU()     ,       nn.Convad(96     ,       96     ,       3     ,       padding=1)     ,       nn.SiLU()     ,       nn.Convad(96     ,       256     ,       3     ,       padding=1     ,       stride=2)     ,       nn.SiLU()     ,       nn.Convad(256     ,       4     ,       3     ,       padding=1)     ,       )




 def 						__snake_case		(     self   :    List[Any]     ,       lowercase__   :    torch.FloatTensor     ,       lowercase__   :    torch.FloatTensor):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.image_proj(lowercase__)
        lowerCAmelCase__	      =   self.image_norm(lowercase__)
        lowerCAmelCase__	      =   self.input_hint_block(lowercase__)
        return time_image_embeds, hint






class 							a_					(		nn.Module		):






 '''simple docstring'''






 def __init__(     self   :    Union[str, Any]     ,       lowercase__   :    Optional[Any]     ,       lowercase__   :    Optional[int]     ,       lowercase__   :    Optional[int]=None):







        '''simple docstring'''



        super().__init__()
        lowerCAmelCase__	      =   dtype
        lowerCAmelCase__	      =   nn.Parameter(torch.randn(1     ,       lowercase__) / embed_dim**0.5)
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__     ,       dtype=self.dtype)
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__     ,       dtype=self.dtype)
        lowerCAmelCase__	      =   nn.Linear(lowercase__     ,       lowercase__     ,       dtype=self.dtype)
        lowerCAmelCase__	      =   num_heads
        lowerCAmelCase__	      =   embed_dim // self.num_heads




 def 						__snake_case		(     self   :    List[Any]     ,       lowercase__   :    List[Any]):







        '''simple docstring'''



        lowerCAmelCase__      ,   lowerCAmelCase__      ,   lowerCAmelCase__	      =   x.size()

        def shape(lowercase__   :    Dict):
               # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
               lowerCAmelCase__	      =   x.view(lowercase__     ,       -1     ,       self.num_heads     ,       self.dim_per_head)
               # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
               lowerCAmelCase__	      =   x.transpose(1     ,       2)
               # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
               lowerCAmelCase__	      =   x.reshape(bs * self.num_heads     ,       -1     ,       self.dim_per_head)
               # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
               lowerCAmelCase__	      =   x.transpose(1     ,       2)
               return x

        lowerCAmelCase__	      =   x.mean(dim=1     ,       keepdim=lowercase__) + self.positional_embedding.to(x.dtype)
        lowerCAmelCase__	      =   torch.cat([class_token, x]     ,       dim=1)  # (bs, length+1, width)

        # (bs*n_heads, class_token_length, dim_per_head)
        lowerCAmelCase__	      =   shape(self.q_proj(lowercase__))
        # (bs*n_heads, length+class_token_length, dim_per_head)
        lowerCAmelCase__	      =   shape(self.k_proj(lowercase__))
        lowerCAmelCase__	      =   shape(self.v_proj(lowercase__))

        # (bs*n_heads, class_token_length, length+class_token_length):
        lowerCAmelCase__	      =   1 / math.sqrt(math.sqrt(self.dim_per_head))
        lowerCAmelCase__	      =   torch.einsum('bct,bcs->bts'     ,       q * scale     ,       k * scale)  # More stable with f16 than dividing afterwards
        lowerCAmelCase__	      =   torch.softmax(weight.float()     ,       dim=-1).type(weight.dtype)

        # (bs*n_heads, dim_per_head, class_token_length)
        lowerCAmelCase__	      =   torch.einsum('bts,bcs->bct'     ,       lowercase__     ,       lowercase__)

        # (bs, length+1, width)
        lowerCAmelCase__	      =   a.reshape(lowercase__     ,       -1     ,       1).transpose(1     ,       2)

        return a[:, 0, :]  # cls_token
