

import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp




def        _lowercase  (					a_					:       List[str]       ,a_					:       Dict       ,a_					:       str       ,a_					:       str       ,a_					:       int = 4_0_9_6					)					->						Any:




					'''simple docstring'''







					__magic_name__,						__magic_name__,						__magic_name__				=     key.shape[-3:]
					__magic_name__				=     value.shape[-1]
					__magic_name__				=     min(a_       ,a_					)
					__magic_name__				=     query / jnp.sqrt(a_					)

					@functools.partial(jax.checkpoint       ,prevent_cse=a_					)
					def summarize_chunk(a_					:       Tuple       ,a_					:       Union[str, Any]       ,a_					:       Union[str, Any]					):
										__magic_name__				=     jnp.einsum('...qhd,...khd->...qhk'       ,a_       ,a_       ,precision=a_					)

										__magic_name__				=     jnp.max(a_       ,axis=-1       ,keepdims=a_					)
										__magic_name__				=     jax.lax.stop_gradient(a_					)
										__magic_name__				=     jnp.exp(attn_weights - max_score					)

										__magic_name__				=     jnp.einsum('...vhf,...qhv->...qhf'       ,a_       ,a_       ,precision=a_					)
										__magic_name__				=     jnp.einsum('...qhk->...qh'       ,a_					)

										return (exp_values, exp_weights.sum(axis=-1					), max_score)

					def chunk_scanner(a_					:       Optional[Any]					):
										# julienne key array
										__magic_name__				=     jax.lax.dynamic_slice(
										    operand=a_       ,start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]       ,slice_sizes=list(key.shape[:-3]					) + [key_chunk_size, num_heads, k_features]       ,)

										# julienne value array
										__magic_name__				=     jax.lax.dynamic_slice(
										    operand=a_       ,start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]       ,slice_sizes=list(value.shape[:-3]					) + [key_chunk_size, num_heads, v_features]       ,)

										return summarize_chunk(a_       ,a_       ,a_					)

					__magic_name__,						__magic_name__,						__magic_name__				=     jax.lax.map(f=a_       ,xs=jnp.arange(0       ,a_       ,a_					)					)

					__magic_name__				=     jnp.max(a_       ,axis=0       ,keepdims=a_					)
					__magic_name__				=     jnp.exp(chunk_max - global_max					)

					chunk_values *= jnp.expand_dims(a_       ,axis=-1					)
					chunk_weights *= max_diffs

					__magic_name__				=     chunk_values.sum(axis=0					)
					__magic_name__				=     jnp.expand_dims(a_       ,-1					).sum(axis=0					)

					return all_values / all_weights




def        _lowercase  (					a_					:       str       ,a_					:       Union[str, Any]       ,a_					:       Dict       ,a_					:       Union[str, Any]=jax.lax.Precision.HIGHEST       ,a_					:       int = 1_0_2_4       ,a_					:       int = 4_0_9_6					)					->						Optional[int]:




					'''simple docstring'''







					__magic_name__,						__magic_name__,						__magic_name__				=     query.shape[-3:]

					def chunk_scanner(a_					:       Optional[Any]       ,a_					:       Optional[int]					):
										# julienne query array
										__magic_name__				=     jax.lax.dynamic_slice(
										    operand=a_       ,start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]       ,slice_sizes=list(query.shape[:-3]					) + [min(a_       ,a_					), num_heads, q_features]       ,)

										return (
										    chunk_idx + query_chunk_size,  # unused ignore it
										    _query_chunk_attention(
										        query=a_       ,key=a_       ,value=a_       ,precision=a_       ,key_chunk_size=a_					),
										)

					__magic_name__,						__magic_name__				=     jax.lax.scan(
					    f=a_       ,init=0       ,xs=a_       ,length=math.ceil(num_q / query_chunk_size					)					)

					return jnp.concatenate(a_       ,axis=-3					)  # fuse the chunked result back

class 					__UpperCamelCase					(       nn.Module      ):
	_lowercase	: int
	_lowercase	: int												=	8
	_lowercase	: int												=	6_4
	_lowercase	: float												=	0.0
	_lowercase	: bool												=	False
	_lowercase	: jnp.dtype												=	jnp.floataa







	def 						_SCREAMING_SNAKE_CASE				( self:			int   ):

						'''simple docstring'''





						__magic_name__				=     self.dim_head * self.heads
						__magic_name__				=     self.dim_head**-0.5

						# Weights were exported with old names {to_q, to_k, to_v, to_out}
						__magic_name__				=     nn.Dense(__UpperCamelCase	,			use_bias=__UpperCamelCase	,			dtype=self.dtype	,			name='to_q'   )
						__magic_name__				=     nn.Dense(__UpperCamelCase	,			use_bias=__UpperCamelCase	,			dtype=self.dtype	,			name='to_k'   )
						__magic_name__				=     nn.Dense(__UpperCamelCase	,			use_bias=__UpperCamelCase	,			dtype=self.dtype	,			name='to_v'   )

						__magic_name__				=     nn.Dense(self.query_dim	,			dtype=self.dtype	,			name='to_out_0'   )
						__magic_name__				=     nn.Dropout(rate=self.dropout   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Dict	,			__UpperCamelCase:			Dict   ):

						'''simple docstring'''





						__magic_name__,						__magic_name__,						__magic_name__				=     tensor.shape
						__magic_name__				=     self.heads
						__magic_name__				=     tensor.reshape(__UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase	,			dim // head_size   )
						__magic_name__				=     jnp.transpose(__UpperCamelCase	,			(0, 2, 1, 3)   )
						__magic_name__				=     tensor.reshape(batch_size * head_size	,			__UpperCamelCase	,			dim // head_size   )
						return tensor







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			Any   ):

						'''simple docstring'''





						__magic_name__,						__magic_name__,						__magic_name__				=     tensor.shape
						__magic_name__				=     self.heads
						__magic_name__				=     tensor.reshape(batch_size // head_size	,			__UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase   )
						__magic_name__				=     jnp.transpose(__UpperCamelCase	,			(0, 2, 1, 3)   )
						__magic_name__				=     tensor.reshape(batch_size // head_size	,			__UpperCamelCase	,			dim * head_size   )
						return tensor







	def __call__( self:			Optional[Any]	,			__UpperCamelCase:			int	,			__UpperCamelCase:			Optional[Any]=None	,			__UpperCamelCase:			Union[str, Any]=True   ):

						'''simple docstring'''





						__magic_name__				=     hidden_states if context is None else context

						__magic_name__				=     self.query(__UpperCamelCase   )
						__magic_name__				=     self.key(__UpperCamelCase   )
						__magic_name__				=     self.value(__UpperCamelCase   )

						__magic_name__				=     self.reshape_heads_to_batch_dim(__UpperCamelCase   )
						__magic_name__				=     self.reshape_heads_to_batch_dim(__UpperCamelCase   )
						__magic_name__				=     self.reshape_heads_to_batch_dim(__UpperCamelCase   )

						if self.use_memory_efficient_attention:
											__magic_name__				=     query_states.transpose(1	,			0	,			2   )
											__magic_name__				=     key_states.transpose(1	,			0	,			2   )
											__magic_name__				=     value_states.transpose(1	,			0	,			2   )

											# this if statement create a chunk size for each layer of the unet
											# the chunk size is equal to the query_length dimension of the deepest layer of the unet

											__magic_name__				=     query_states.shape[-3]
											if flatten_latent_dim % 64 == 0:
																__magic_name__				=     int(flatten_latent_dim / 64   )
											elif flatten_latent_dim % 16 == 0:
																__magic_name__				=     int(flatten_latent_dim / 16   )
											elif flatten_latent_dim % 4 == 0:
																__magic_name__				=     int(flatten_latent_dim / 4   )
											else:
																__magic_name__				=     int(__UpperCamelCase   )

											__magic_name__				=     jax_memory_efficient_attention(
											    __UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase	,			query_chunk_size=__UpperCamelCase	,			key_chunk_size=40_96 * 4   )

											__magic_name__				=     hidden_states.transpose(1	,			0	,			2   )
						else:
											# compute attentions
											__magic_name__				=     jnp.einsum('b i d, b j d->b i j'	,			__UpperCamelCase	,			__UpperCamelCase   )
											__magic_name__				=     attention_scores * self.scale
											__magic_name__				=     nn.softmax(__UpperCamelCase	,			axis=2   )

											# attend to values
											__magic_name__				=     jnp.einsum('b i j, b j d -> b i d'	,			__UpperCamelCase	,			__UpperCamelCase   )

						__magic_name__				=     self.reshape_batch_dim_to_heads(__UpperCamelCase   )
						__magic_name__				=     self.proj_attn(__UpperCamelCase   )
						return self.dropout_layer(__UpperCamelCase	,			deterministic=__UpperCamelCase   )




class 					__UpperCamelCase					(       nn.Module      ):
	_lowercase	: int
	_lowercase	: int
	_lowercase	: int
	_lowercase	: float												=	0.0
	_lowercase	: bool												=	False
	_lowercase	: jnp.dtype												=	jnp.floataa
	_lowercase	: bool												=	False







	def 						_SCREAMING_SNAKE_CASE				( self:			int   ):

						'''simple docstring'''





						__magic_name__				=     FlaxAttention(
						    self.dim	,			self.n_heads	,			self.d_head	,			self.dropout	,			self.use_memory_efficient_attention	,			dtype=self.dtype   )
						# cross attention
						__magic_name__				=     FlaxAttention(
						    self.dim	,			self.n_heads	,			self.d_head	,			self.dropout	,			self.use_memory_efficient_attention	,			dtype=self.dtype   )
						__magic_name__				=     FlaxFeedForward(dim=self.dim	,			dropout=self.dropout	,			dtype=self.dtype   )
						__magic_name__				=     nn.LayerNorm(epsilon=1E-5	,			dtype=self.dtype   )
						__magic_name__				=     nn.LayerNorm(epsilon=1E-5	,			dtype=self.dtype   )
						__magic_name__				=     nn.LayerNorm(epsilon=1E-5	,			dtype=self.dtype   )
						__magic_name__				=     nn.Dropout(rate=self.dropout   )







	def __call__( self:			int	,			__UpperCamelCase:			str	,			__UpperCamelCase:			List[Any]	,			__UpperCamelCase:			Any=True   ):

						'''simple docstring'''





						__magic_name__				=     hidden_states
						if self.only_cross_attention:
											__magic_name__				=     self.attna(self.norma(__UpperCamelCase   )	,			__UpperCamelCase	,			deterministic=__UpperCamelCase   )
						else:
											__magic_name__				=     self.attna(self.norma(__UpperCamelCase   )	,			deterministic=__UpperCamelCase   )
						__magic_name__				=     hidden_states + residual

						# cross attention
						__magic_name__				=     hidden_states
						__magic_name__				=     self.attna(self.norma(__UpperCamelCase   )	,			__UpperCamelCase	,			deterministic=__UpperCamelCase   )
						__magic_name__				=     hidden_states + residual

						# feed forward
						__magic_name__				=     hidden_states
						__magic_name__				=     self.ff(self.norma(__UpperCamelCase   )	,			deterministic=__UpperCamelCase   )
						__magic_name__				=     hidden_states + residual

						return self.dropout_layer(__UpperCamelCase	,			deterministic=__UpperCamelCase   )




class 					__UpperCamelCase					(       nn.Module      ):
	_lowercase	: int
	_lowercase	: int
	_lowercase	: int
	_lowercase	: int												=	1
	_lowercase	: float												=	0.0
	_lowercase	: bool												=	False
	_lowercase	: bool												=	False
	_lowercase	: jnp.dtype												=	jnp.floataa
	_lowercase	: bool												=	False







	def 						_SCREAMING_SNAKE_CASE				( self:			str   ):

						'''simple docstring'''





						__magic_name__				=     nn.GroupNorm(num_groups=32	,			epsilon=1E-5   )

						__magic_name__				=     self.n_heads * self.d_head
						if self.use_linear_projection:
											__magic_name__				=     nn.Dense(__UpperCamelCase	,			dtype=self.dtype   )
						else:
											__magic_name__				=     nn.Conv(
											    __UpperCamelCase	,			kernel_size=(1, 1)	,			strides=(1, 1)	,			padding='VALID'	,			dtype=self.dtype	,			)

						__magic_name__				=     [
						    FlaxBasicTransformerBlock(
						        __UpperCamelCase	,			self.n_heads	,			self.d_head	,			dropout=self.dropout	,			only_cross_attention=self.only_cross_attention	,			dtype=self.dtype	,			use_memory_efficient_attention=self.use_memory_efficient_attention	,			)
						    for _ in range(self.depth   )
						]

						if self.use_linear_projection:
											__magic_name__				=     nn.Dense(__UpperCamelCase	,			dtype=self.dtype   )
						else:
											__magic_name__				=     nn.Conv(
											    __UpperCamelCase	,			kernel_size=(1, 1)	,			strides=(1, 1)	,			padding='VALID'	,			dtype=self.dtype	,			)

						__magic_name__				=     nn.Dropout(rate=self.dropout   )







	def __call__( self:			Dict	,			__UpperCamelCase:			int	,			__UpperCamelCase:			Dict	,			__UpperCamelCase:			List[str]=True   ):

						'''simple docstring'''





						__magic_name__,						__magic_name__,						__magic_name__,						__magic_name__				=     hidden_states.shape
						__magic_name__				=     hidden_states
						__magic_name__				=     self.norm(__UpperCamelCase   )
						if self.use_linear_projection:
											__magic_name__				=     hidden_states.reshape(__UpperCamelCase	,			height * width	,			__UpperCamelCase   )
											__magic_name__				=     self.proj_in(__UpperCamelCase   )
						else:
											__magic_name__				=     self.proj_in(__UpperCamelCase   )
											__magic_name__				=     hidden_states.reshape(__UpperCamelCase	,			height * width	,			__UpperCamelCase   )

						for transformer_block in self.transformer_blocks:
											__magic_name__				=     transformer_block(__UpperCamelCase	,			__UpperCamelCase	,			deterministic=__UpperCamelCase   )

						if self.use_linear_projection:
											__magic_name__				=     self.proj_out(__UpperCamelCase   )
											__magic_name__				=     hidden_states.reshape(__UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase   )
						else:
											__magic_name__				=     hidden_states.reshape(__UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase   )
											__magic_name__				=     self.proj_out(__UpperCamelCase   )

						__magic_name__				=     hidden_states + residual
						return self.dropout_layer(__UpperCamelCase	,			deterministic=__UpperCamelCase   )




class 					__UpperCamelCase					(       nn.Module      ):
	_lowercase	: int
	_lowercase	: float												=	0.0
	_lowercase	: jnp.dtype												=	jnp.floataa







	def 						_SCREAMING_SNAKE_CASE				( self:			Any   ):

						'''simple docstring'''





						__magic_name__				=     FlaxGEGLU(self.dim	,			self.dropout	,			self.dtype   )
						__magic_name__				=     nn.Dense(self.dim	,			dtype=self.dtype   )







	def __call__( self:			Dict	,			__UpperCamelCase:			List[Any]	,			__UpperCamelCase:			int=True   ):

						'''simple docstring'''





						__magic_name__				=     self.net_a(__UpperCamelCase	,			deterministic=__UpperCamelCase   )
						__magic_name__				=     self.net_a(__UpperCamelCase   )
						return hidden_states




class 					__UpperCamelCase					(       nn.Module      ):
	_lowercase	: int
	_lowercase	: float												=	0.0
	_lowercase	: jnp.dtype												=	jnp.floataa







	def 						_SCREAMING_SNAKE_CASE				( self:			Dict   ):

						'''simple docstring'''





						__magic_name__				=     self.dim * 4
						__magic_name__				=     nn.Dense(inner_dim * 2	,			dtype=self.dtype   )
						__magic_name__				=     nn.Dropout(rate=self.dropout   )







	def __call__( self:			Any	,			__UpperCamelCase:			List[Any]	,			__UpperCamelCase:			Optional[int]=True   ):

						'''simple docstring'''





						__magic_name__				=     self.proj(__UpperCamelCase   )
						__magic_name__,						__magic_name__				=     jnp.split(__UpperCamelCase	,			2	,			axis=2   )
						return self.dropout_layer(hidden_linear * nn.gelu(__UpperCamelCase   )	,			deterministic=__UpperCamelCase   )







from hashlib import mda, shaaaa

A__							=				(shaaaa, mda)

class 					__UpperCamelCase					:







	def __init__( self:			Optional[int]	,			__UpperCamelCase:			int = 8   ):

						'''simple docstring'''





						__magic_name__				=     0b0
						__magic_name__				=     size







	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						__magic_name__				=     self.hash_(__UpperCamelCase   )
						self.bitarray |= h







	def 						_SCREAMING_SNAKE_CASE				( self:			List[str]	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						__magic_name__				=     self.hash_(__UpperCamelCase   )
						return (h & self.bitarray) == h







	def __contains__( self:			Optional[int]	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						return self.exists(__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			int   ):

						'''simple docstring'''





						__magic_name__				=     bin(__UpperCamelCase   )[2:]
						return res.zfill(self.size   )







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			str   ):

						'''simple docstring'''





						return self.format_bin(self.bitarray   )







	def 						_SCREAMING_SNAKE_CASE				( self:			str	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						__magic_name__				=     0b0
						for func in HASH_FUNCTIONS:
											__magic_name__				=     (
											    int.from_bytes(func(value.encode()   ).digest()	,			'little'   ) % self.size
											)
											res |= 2**position
						return res







	def 						_SCREAMING_SNAKE_CASE				( self:			Dict	,			__UpperCamelCase:			str   ):

						'''simple docstring'''





						return self.format_bin(self.hash_(__UpperCamelCase   )   )







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[int]   ):

						'''simple docstring'''





						__magic_name__				=     bin(self.bitarray   ).count('1'   )
						return (n_ones / self.size) ** len(__UpperCamelCase   )




