



import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation





def 				A      (	_UpperCAmelCase							:  torch.Tensor				,  _UpperCAmelCase							:  int				,  _UpperCAmelCase							:  bool = False				,  _UpperCAmelCase							:  float = 1				,  _UpperCAmelCase							:  float = 1				,  _UpperCAmelCase							:  int = 10_000				,  )							->					Tuple:







			'''simple docstring'''
			assert len(timesteps.shape       ) == 1, "Timesteps should be a 1d-array"

			_UpperCAmelCase									=  embedding_dim // 2
			_UpperCAmelCase									=  -math.log(_UpperCAmelCase       ) * torch.arange(
			    start=0				,  end=_UpperCAmelCase				,  dtype=torch.floataa				,  device=timesteps.device       )
			_UpperCAmelCase									=  exponent / (half_dim - downscale_freq_shift)

			_UpperCAmelCase									=  torch.exp(_UpperCAmelCase       )
			_UpperCAmelCase									=  timesteps[:, None].float() * emb[None, :]

			# scale embeddings
			_UpperCAmelCase									=  scale * emb

			# concat sine and cosine embeddings
			_UpperCAmelCase									=  torch.cat([torch.sin(_UpperCAmelCase       ), torch.cos(_UpperCAmelCase       )]				,  dim=-1       )

			# flip sine and cosine embeddings
			if flip_sin_to_cos:
						_UpperCAmelCase									=  torch.cat([emb[:, half_dim:], emb[:, :half_dim]]				,  dim=-1       )

			# zero pad
			if embedding_dim % 2 == 1:
						_UpperCAmelCase									=  torch.nn.functional.pad(_UpperCAmelCase				,  (0, 1, 0, 0)       )
			return emb





def 				A      (	_UpperCAmelCase							:  Any				,  _UpperCAmelCase							:  List[Any]				,  _UpperCAmelCase							:  Dict=False				,  _UpperCAmelCase							:  int=0       )							->					Any:







			'''simple docstring'''
			_UpperCAmelCase									=  np.arange(_UpperCAmelCase				,  dtype=np.floataa       )
			_UpperCAmelCase									=  np.arange(_UpperCAmelCase				,  dtype=np.floataa       )
			_UpperCAmelCase									=  np.meshgrid(_UpperCAmelCase				,  _UpperCAmelCase       )  # here w goes first
			_UpperCAmelCase									=  np.stack(_UpperCAmelCase				,  axis=0       )

			_UpperCAmelCase									=  grid.reshape([2, 1, grid_size, grid_size]       )
			_UpperCAmelCase									=  get_ad_sincos_pos_embed_from_grid(_UpperCAmelCase				,  _UpperCAmelCase       )
			if cls_token and extra_tokens > 0:
						_UpperCAmelCase									=  np.concatenate([np.zeros([extra_tokens, embed_dim]       ), pos_embed]				,  axis=0       )
			return pos_embed





def 				A      (	_UpperCAmelCase							:  List[str]				,  _UpperCAmelCase							:  Union[str, Any]       )							->					Tuple:







			'''simple docstring'''
			if embed_dim % 2 != 0:
						raise ValueError('embed_dim must be divisible by 2'       )

			# use half of dimensions to encode grid_h
			_UpperCAmelCase									=  get_ad_sincos_pos_embed_from_grid(embed_dim // 2				,  grid[0]       )  # (H*W, D/2)
			_UpperCAmelCase									=  get_ad_sincos_pos_embed_from_grid(embed_dim // 2				,  grid[1]       )  # (H*W, D/2)

			_UpperCAmelCase									=  np.concatenate([emb_h, emb_w]				,  axis=1       )  # (H*W, D)
			return emb





def 				A      (	_UpperCAmelCase							:  Optional[Any]				,  _UpperCAmelCase							:  Optional[int]       )							->					int:







			'''simple docstring'''
			if embed_dim % 2 != 0:
						raise ValueError('embed_dim must be divisible by 2'       )

			_UpperCAmelCase									=  np.arange(embed_dim // 2				,  dtype=np.floataa       )
			omega /= embed_dim / 2.0
			_UpperCAmelCase									=  1.0 / 10_000**omega  # (D/2,)

			_UpperCAmelCase									=  pos.reshape(-1       )  # (M,)
			_UpperCAmelCase									=  np.einsum('m,d->md'				,  _UpperCAmelCase				,  _UpperCAmelCase       )  # (M, D/2), outer product

			_UpperCAmelCase									=  np.sin(_UpperCAmelCase       )  # (M, D/2)
			_UpperCAmelCase									=  np.cos(_UpperCAmelCase       )  # (M, D/2)

			_UpperCAmelCase									=  np.concatenate([emb_sin, emb_cos]				,  axis=1       )  # (M, D)
			return emb







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    List[Any]					,						A  :    Dict=2_24					,						A  :    Union[str, Any]=2_24					,						A  :    List[str]=16					,						A  :    int=3					,						A  :    Optional[Any]=7_68					,						A  :    List[str]=False					,						A  :    Optional[int]=True					,						A  :    List[Any]=True					,						)  ->       List[str]:



						"""simple docstring"""






						super().__init__()

						_UpperCAmelCase									=  (height // patch_size) * (width // patch_size)
						_UpperCAmelCase									=  flatten
						_UpperCAmelCase									=  layer_norm

						_UpperCAmelCase									=  nn.Convad(
						    A					,						A					,						kernel_size=(patch_size, patch_size)					,						stride=A					,						bias=A)
						if layer_norm:
									_UpperCAmelCase									=  nn.LayerNorm(A					,						elementwise_affine=A					,						eps=1E-6)
						else:
									_UpperCAmelCase									=  None

						_UpperCAmelCase									=  get_ad_sincos_pos_embed(A					,						int(num_patches**0.5))
						self.register_buffer('pos_embed'					,						torch.from_numpy(A).float().unsqueeze(0)					,						persistent=A)



			def        _lowerCamelCase			(		self  :    Dict					,						A  :    int)  ->       str:



						"""simple docstring"""






						_UpperCAmelCase									=  self.proj(A)
						if self.flatten:
									_UpperCAmelCase									=  latent.flatten(2).transpose(1					,						2)  # BCHW -> BNC
						if self.layer_norm:
									_UpperCAmelCase									=  self.norm(A)
						return latent + self.pos_embed







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    List[Any]					,						A  :    int					,						A  :    int					,						A  :    str = "silu"					,						A  :    int = None					,						A  :    Optional[str] = None					,						A  :    Optional[Any]=None					,						)  ->       Tuple:



						"""simple docstring"""






						super().__init__()

						_UpperCAmelCase									=  nn.Linear(A					,						A)

						if cond_proj_dim is not None:
									_UpperCAmelCase									=  nn.Linear(A					,						A					,						bias=A)
						else:
									_UpperCAmelCase									=  None

						_UpperCAmelCase									=  get_activation(A)

						if out_dim is not None:
									_UpperCAmelCase									=  out_dim
						else:
									_UpperCAmelCase									=  time_embed_dim
						_UpperCAmelCase									=  nn.Linear(A					,						A)

						if post_act_fn is None:
									_UpperCAmelCase									=  None
						else:
									_UpperCAmelCase									=  get_activation(A)



			def        _lowerCamelCase			(		self  :    Union[str, Any]					,						A  :    Union[str, Any]					,						A  :    Dict=None)  ->       List[Any]:



						"""simple docstring"""






						if condition is not None:
									_UpperCAmelCase									=  sample + self.cond_proj(A)
						_UpperCAmelCase									=  self.linear_a(A)

						if self.act is not None:
									_UpperCAmelCase									=  self.act(A)

						_UpperCAmelCase									=  self.linear_a(A)

						if self.post_act is not None:
									_UpperCAmelCase									=  self.post_act(A)
						return sample







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Union[str, Any]					,						A  :    int					,						A  :    bool					,						A  :    float)  ->       List[Any]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  num_channels
						_UpperCAmelCase									=  flip_sin_to_cos
						_UpperCAmelCase									=  downscale_freq_shift



			def        _lowerCamelCase			(		self  :    Optional[Any]					,						A  :    Optional[Any])  ->       Tuple:



						"""simple docstring"""






						_UpperCAmelCase									=  get_timestep_embedding(
						    A					,						self.num_channels					,						flip_sin_to_cos=self.flip_sin_to_cos					,						downscale_freq_shift=self.downscale_freq_shift					,						)
						return t_emb







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Any					,						A  :    int = 2_56					,						A  :    float = 1.0					,						A  :    Union[str, Any]=True					,						A  :    List[str]=True					,						A  :    Union[str, Any]=False)  ->       Optional[int]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  nn.Parameter(torch.randn(A) * scale					,						requires_grad=A)
						_UpperCAmelCase									=  log
						_UpperCAmelCase									=  flip_sin_to_cos

						if set_W_to_weight:
									# to delete later
									_UpperCAmelCase									=  nn.Parameter(torch.randn(A) * scale					,						requires_grad=A)

									_UpperCAmelCase									=  self.W



			def        _lowerCamelCase			(		self  :    Optional[Any]					,						A  :    int)  ->       List[str]:



						"""simple docstring"""






						if self.log:
									_UpperCAmelCase									=  torch.log(A)

						_UpperCAmelCase									=  x[:, None] * self.weight[None, :] * 2 * np.pi

						if self.flip_sin_to_cos:
									_UpperCAmelCase									=  torch.cat([torch.cos(A), torch.sin(A)]					,						dim=-1)
						else:
									_UpperCAmelCase									=  torch.cat([torch.sin(A), torch.cos(A)]					,						dim=-1)
						return out







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    List[str]					,						A  :    int					,						A  :    int					,						A  :    int					,						A  :    int					,						)  ->       Dict:



						"""simple docstring"""






						super().__init__()

						_UpperCAmelCase									=  height
						_UpperCAmelCase									=  width
						_UpperCAmelCase									=  num_embed
						_UpperCAmelCase									=  embed_dim

						_UpperCAmelCase									=  nn.Embedding(self.num_embed					,						A)
						_UpperCAmelCase									=  nn.Embedding(self.height					,						A)
						_UpperCAmelCase									=  nn.Embedding(self.width					,						A)



			def        _lowerCamelCase			(		self  :    Any					,						A  :    Any)  ->       Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase									=  self.emb(A)

						_UpperCAmelCase									=  self.height_emb(torch.arange(self.height					,						device=index.device).view(1					,						self.height))

						# 1 x H x D -> 1 x H x 1 x D
						_UpperCAmelCase									=  height_emb.unsqueeze(2)

						_UpperCAmelCase									=  self.width_emb(torch.arange(self.width					,						device=index.device).view(1					,						self.width))

						# 1 x W x D -> 1 x 1 x W x D
						_UpperCAmelCase									=  width_emb.unsqueeze(1)

						_UpperCAmelCase									=  height_emb + width_emb

						# 1 x H x W x D -> 1 x L xD
						_UpperCAmelCase									=  pos_emb.view(1					,						self.height * self.width					,						-1)

						_UpperCAmelCase									=  emb + pos_emb[:, : emb.shape[1], :]

						return emb







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Union[str, Any]					,						A  :    str					,						A  :    List[str]					,						A  :    Dict)  ->       Union[str, Any]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  dropout_prob > 0
						_UpperCAmelCase									=  nn.Embedding(num_classes + use_cfg_embedding					,						A)
						_UpperCAmelCase									=  num_classes
						_UpperCAmelCase									=  dropout_prob



			def        _lowerCamelCase			(		self  :    Dict					,						A  :    Optional[int]					,						A  :    List[Any]=None)  ->       Optional[Any]:



						"""simple docstring"""






						if force_drop_ids is None:
									_UpperCAmelCase									=  torch.rand(labels.shape[0]					,						device=labels.device) < self.dropout_prob
						else:
									_UpperCAmelCase									=  torch.tensor(force_drop_ids == 1)
						_UpperCAmelCase									=  torch.where(A					,						self.num_classes					,						A)
						return labels



			def        _lowerCamelCase			(		self  :    int					,						A  :    torch.LongTensor					,						A  :    Union[str, Any]=None)  ->       Dict:



						"""simple docstring"""






						_UpperCAmelCase									=  self.dropout_prob > 0
						if (self.training and use_dropout) or (force_drop_ids is not None):
									_UpperCAmelCase									=  self.token_drop(A					,						A)
						_UpperCAmelCase									=  self.embedding_table(A)
						return embeddings







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Tuple					,						A  :    int = 10_24					,						A  :    int = 7_68					,						A  :    int = 7_68					,						A  :    int = 10					,						)  ->       Tuple:



						"""simple docstring"""






						super().__init__()

						_UpperCAmelCase									=  num_image_text_embeds
						_UpperCAmelCase									=  nn.Linear(A					,						self.num_image_text_embeds * cross_attention_dim)
						_UpperCAmelCase									=  nn.Linear(A					,						A)



			def        _lowerCamelCase			(		self  :    Union[str, Any]					,						A  :    torch.FloatTensor					,						A  :    torch.FloatTensor)  ->       List[str]:



						"""simple docstring"""






						_UpperCAmelCase									=  text_embeds.shape[0]

						# image
						_UpperCAmelCase									=  self.image_embeds(A)
						_UpperCAmelCase									=  image_text_embeds.reshape(A					,						self.num_image_text_embeds					,						-1)

						# text
						_UpperCAmelCase									=  self.text_proj(A)

						return torch.cat([image_text_embeds, text_embeds]					,						dim=1)







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Optional[int]					,						A  :    int = 7_68					,						A  :    int = 7_68					,						A  :    int = 32					,						)  ->       Any:



						"""simple docstring"""






						super().__init__()

						_UpperCAmelCase									=  num_image_text_embeds
						_UpperCAmelCase									=  nn.Linear(A					,						self.num_image_text_embeds * cross_attention_dim)
						_UpperCAmelCase									=  nn.LayerNorm(A)



			def        _lowerCamelCase			(		self  :    str					,						A  :    torch.FloatTensor)  ->       str:



						"""simple docstring"""






						_UpperCAmelCase									=  image_embeds.shape[0]

						# image
						_UpperCAmelCase									=  self.image_embeds(A)
						_UpperCAmelCase									=  image_embeds.reshape(A					,						self.num_image_text_embeds					,						-1)
						_UpperCAmelCase									=  self.norm(A)
						return image_embeds







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    List[Any]					,						A  :    str					,						A  :    Optional[int]					,						A  :    Optional[int]=0.1)  ->       Tuple:



						"""simple docstring"""






						super().__init__()

						_UpperCAmelCase									=  Timesteps(num_channels=2_56					,						flip_sin_to_cos=A					,						downscale_freq_shift=1)
						_UpperCAmelCase									=  TimestepEmbedding(in_channels=2_56					,						time_embed_dim=A)
						_UpperCAmelCase									=  LabelEmbedding(A					,						A					,						A)



			def        _lowerCamelCase			(		self  :    Tuple					,						A  :    Optional[Any]					,						A  :    Any					,						A  :    List[Any]=None)  ->       Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase									=  self.time_proj(A)
						_UpperCAmelCase									=  self.timestep_embedder(timesteps_proj.to(dtype=A))  # (N, D)

						_UpperCAmelCase									=  self.class_embedder(A)  # (N, D)

						_UpperCAmelCase									=  timesteps_emb + class_labels  # (N, D)

						return conditioning







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Optional[int]					,						A  :    int					,						A  :    int					,						A  :    int = 64)  ->       Optional[Any]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  nn.LayerNorm(A)
						_UpperCAmelCase									=  AttentionPooling(A					,						A)
						_UpperCAmelCase									=  nn.Linear(A					,						A)
						_UpperCAmelCase									=  nn.LayerNorm(A)



			def        _lowerCamelCase			(		self  :    Optional[int]					,						A  :    List[Any])  ->       List[str]:



						"""simple docstring"""






						_UpperCAmelCase									=  self.norma(A)
						_UpperCAmelCase									=  self.pool(A)
						_UpperCAmelCase									=  self.proj(A)
						_UpperCAmelCase									=  self.norma(A)
						return hidden_states







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    List[str]					,						A  :    int = 7_68					,						A  :    int = 7_68					,						A  :    int = 15_36)  ->       List[Any]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  nn.Linear(A					,						A)
						_UpperCAmelCase									=  nn.LayerNorm(A)
						_UpperCAmelCase									=  nn.Linear(A					,						A)



			def        _lowerCamelCase			(		self  :    List[Any]					,						A  :    torch.FloatTensor					,						A  :    torch.FloatTensor)  ->       Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase									=  self.text_proj(A)
						_UpperCAmelCase									=  self.text_norm(A)

						# image
						_UpperCAmelCase									=  self.image_proj(A)

						return time_image_embeds + time_text_embeds







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    str					,						A  :    int = 7_68					,						A  :    int = 15_36)  ->       Any:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  nn.Linear(A					,						A)
						_UpperCAmelCase									=  nn.LayerNorm(A)



			def        _lowerCamelCase			(		self  :    str					,						A  :    torch.FloatTensor)  ->       Any:



						"""simple docstring"""






						_UpperCAmelCase									=  self.image_proj(A)
						_UpperCAmelCase									=  self.image_norm(A)
						return time_image_embeds







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Optional[Any]					,						A  :    int = 7_68					,						A  :    int = 15_36)  ->       str:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  nn.Linear(A					,						A)
						_UpperCAmelCase									=  nn.LayerNorm(A)
						_UpperCAmelCase									=  nn.Sequential(
						    nn.Convad(3					,						16					,						3					,						padding=1)					,						nn.SiLU()					,						nn.Convad(16					,						16					,						3					,						padding=1)					,						nn.SiLU()					,						nn.Convad(16					,						32					,						3					,						padding=1					,						stride=2)					,						nn.SiLU()					,						nn.Convad(32					,						32					,						3					,						padding=1)					,						nn.SiLU()					,						nn.Convad(32					,						96					,						3					,						padding=1					,						stride=2)					,						nn.SiLU()					,						nn.Convad(96					,						96					,						3					,						padding=1)					,						nn.SiLU()					,						nn.Convad(96					,						2_56					,						3					,						padding=1					,						stride=2)					,						nn.SiLU()					,						nn.Convad(2_56					,						4					,						3					,						padding=1)					,						)



			def        _lowerCamelCase			(		self  :    List[str]					,						A  :    torch.FloatTensor					,						A  :    torch.FloatTensor)  ->       int:



						"""simple docstring"""






						_UpperCAmelCase									=  self.image_proj(A)
						_UpperCAmelCase									=  self.image_norm(A)
						_UpperCAmelCase									=  self.input_hint_block(A)
						return time_image_embeds, hint







class      __lowerCAmelCase       (     nn.Module		):



			def __init__(		self  :    Tuple					,						A  :    List[Any]					,						A  :    Any					,						A  :    str=None)  ->       Optional[Any]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase									=  dtype
						_UpperCAmelCase									=  nn.Parameter(torch.randn(1					,						A) / embed_dim**0.5)
						_UpperCAmelCase									=  nn.Linear(A					,						A					,						dtype=self.dtype)
						_UpperCAmelCase									=  nn.Linear(A					,						A					,						dtype=self.dtype)
						_UpperCAmelCase									=  nn.Linear(A					,						A					,						dtype=self.dtype)
						_UpperCAmelCase									=  num_heads
						_UpperCAmelCase									=  embed_dim // self.num_heads



			def        _lowerCamelCase			(		self  :    Optional[int]					,						A  :    Dict)  ->       Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase			,	_UpperCAmelCase			,	_UpperCAmelCase									=  x.size()

						def shape(A  :    int):
									# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
									_UpperCAmelCase									=  x.view(A					,						-1					,						self.num_heads					,						self.dim_per_head)
									# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
									_UpperCAmelCase									=  x.transpose(1					,						2)
									# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
									_UpperCAmelCase									=  x.reshape(bs * self.num_heads					,						-1					,						self.dim_per_head)
									# (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
									_UpperCAmelCase									=  x.transpose(1					,						2)
									return x

						_UpperCAmelCase									=  x.mean(dim=1					,						keepdim=A) + self.positional_embedding.to(x.dtype)
						_UpperCAmelCase									=  torch.cat([class_token, x]					,						dim=1)  # (bs, length+1, width)

						# (bs*n_heads, class_token_length, dim_per_head)
						_UpperCAmelCase									=  shape(self.q_proj(A))
						# (bs*n_heads, length+class_token_length, dim_per_head)
						_UpperCAmelCase									=  shape(self.k_proj(A))
						_UpperCAmelCase									=  shape(self.v_proj(A))

						# (bs*n_heads, class_token_length, length+class_token_length):
						_UpperCAmelCase									=  1 / math.sqrt(math.sqrt(self.dim_per_head))
						_UpperCAmelCase									=  torch.einsum('bct,bcs->bts'					,						q * scale					,						k * scale)  # More stable with f16 than dividing afterwards
						_UpperCAmelCase									=  torch.softmax(weight.float()					,						dim=-1).type(weight.dtype)

						# (bs*n_heads, dim_per_head, class_token_length)
						_UpperCAmelCase									=  torch.einsum('bts,bcs->bct'					,						A					,						A)

						# (bs, length+1, width)
						_UpperCAmelCase									=  a.reshape(A					,						-1					,						1).transpose(1					,						2)

						return a[:, 0, :]  # cls_token





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__  =			logging.get_logger(__name__)

UpperCAmelCase__  =			{
    "dandelin/vilt-b32-mlm": "https://huggingface.co/dandelin/vilt-b32-mlm/blob/main/config.json"
}







class      __lowerCAmelCase       (     A		):
			UpperCamelCase     =	'''vilt'''



			def __init__(		self  :    List[Any]					,						A  :    Dict=3_05_22					,						A  :    Union[str, Any]=2					,						A  :    Tuple=2					,						A  :    List[Any]=40					,						A  :    Optional[int]=7_68					,						A  :    Any=12					,						A  :    Optional[int]=12					,						A  :    Optional[Any]=30_72					,						A  :    Optional[Any]="gelu"					,						A  :    str=0.0					,						A  :    Any=0.0					,						A  :    Union[str, Any]=0.0_2					,						A  :    Union[str, Any]=1E-12					,						A  :    Optional[Any]=3_84					,						A  :    Dict=32					,						A  :    int=3					,						A  :    int=True					,						A  :    Union[str, Any]=-1					,						A  :    int=False					,						A  :    int=-1					,						**A  :    Tuple					,						)  ->       Dict:



						"""simple docstring"""






						super().__init__(tie_word_embeddings=A					,						**A)

						_UpperCAmelCase									=  vocab_size
						_UpperCAmelCase									=  type_vocab_size
						_UpperCAmelCase									=  modality_type_vocab_size
						_UpperCAmelCase									=  max_position_embeddings

						_UpperCAmelCase									=  hidden_size
						_UpperCAmelCase									=  num_hidden_layers
						_UpperCAmelCase									=  num_attention_heads
						_UpperCAmelCase									=  intermediate_size
						_UpperCAmelCase									=  hidden_act
						_UpperCAmelCase									=  hidden_dropout_prob
						_UpperCAmelCase									=  attention_probs_dropout_prob
						_UpperCAmelCase									=  initializer_range
						_UpperCAmelCase									=  layer_norm_eps

						_UpperCAmelCase									=  image_size
						_UpperCAmelCase									=  patch_size
						_UpperCAmelCase									=  num_channels
						_UpperCAmelCase									=  qkv_bias
						_UpperCAmelCase									=  max_image_length
						_UpperCAmelCase									=  num_images
