






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp


def __A       ( __lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase = 4_096      )->    Union[str, Any]:





						"""simple docstring"""



						_UpperCAmelCase   ,					_UpperCAmelCase   ,					_UpperCAmelCase							   =     key.shape[-3:]
						_UpperCAmelCase							   =     value.shape[-1]
						_UpperCAmelCase							   =     min(__lowerCAmelCase					,						__lowerCAmelCase      )
						_UpperCAmelCase							   =     query / jnp.sqrt(__lowerCAmelCase      )

						@functools.partial(jax.checkpoint					,						prevent_cse=__lowerCAmelCase      )
						def summarize_chunk(__lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase      ):
												_UpperCAmelCase							   =     jnp.einsum('...qhd,...khd->...qhk'					,						__lowerCAmelCase					,						__lowerCAmelCase					,						precision=__lowerCAmelCase      )

												_UpperCAmelCase							   =     jnp.max(__lowerCAmelCase					,						axis=-1					,						keepdims=__lowerCAmelCase      )
												_UpperCAmelCase							   =     jax.lax.stop_gradient(__lowerCAmelCase      )
												_UpperCAmelCase							   =     jnp.exp(attn_weights - max_score      )

												_UpperCAmelCase							   =     jnp.einsum('...vhf,...qhv->...qhf'					,						__lowerCAmelCase					,						__lowerCAmelCase					,						precision=__lowerCAmelCase      )
												_UpperCAmelCase							   =     jnp.einsum('...qhk->...qh'					,						__lowerCAmelCase      )

												return (exp_values, exp_weights.sum(axis=-1      ), max_score)

						def chunk_scanner(__lowerCAmelCase      ):
												# julienne key array
												_UpperCAmelCase							   =     jax.lax.dynamic_slice(
												    operand=__lowerCAmelCase					,						start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]					,						slice_sizes=list(key.shape[:-3]      ) + [key_chunk_size, num_heads, k_features]					,						)

												# julienne value array
												_UpperCAmelCase							   =     jax.lax.dynamic_slice(
												    operand=__lowerCAmelCase					,						start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]					,						slice_sizes=list(value.shape[:-3]      ) + [key_chunk_size, num_heads, v_features]					,						)

												return summarize_chunk(__lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase      )

						_UpperCAmelCase   ,					_UpperCAmelCase   ,					_UpperCAmelCase							   =     jax.lax.map(f=__lowerCAmelCase					,						xs=jnp.arange(0					,						__lowerCAmelCase					,						__lowerCAmelCase      )      )

						_UpperCAmelCase							   =     jnp.max(__lowerCAmelCase					,						axis=0					,						keepdims=__lowerCAmelCase      )
						_UpperCAmelCase							   =     jnp.exp(chunk_max - global_max      )

						chunk_values *= jnp.expand_dims(__lowerCAmelCase					,						axis=-1      )
						chunk_weights *= max_diffs

						_UpperCAmelCase							   =     chunk_values.sum(axis=0      )
						_UpperCAmelCase							   =     jnp.expand_dims(__lowerCAmelCase					,						-1      ).sum(axis=0      )

						return all_values / all_weights


def __A       ( __lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase=jax.lax.Precision.HIGHEST					,						__lowerCAmelCase = 1_024					,						__lowerCAmelCase = 4_096      )->    int:





						"""simple docstring"""



						_UpperCAmelCase   ,					_UpperCAmelCase   ,					_UpperCAmelCase							   =     query.shape[-3:]

						def chunk_scanner(__lowerCAmelCase					,						__lowerCAmelCase      ):
												# julienne query array
												_UpperCAmelCase							   =     jax.lax.dynamic_slice(
												    operand=__lowerCAmelCase					,						start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]					,						slice_sizes=list(query.shape[:-3]      ) + [min(__lowerCAmelCase					,						__lowerCAmelCase      ), num_heads, q_features]					,						)

												return (
												    chunk_idx + query_chunk_size,  # unused ignore it
												    _query_chunk_attention(
												        query=__lowerCAmelCase					,						key=__lowerCAmelCase					,						value=__lowerCAmelCase					,						precision=__lowerCAmelCase					,						key_chunk_size=__lowerCAmelCase      ),
												)

						_UpperCAmelCase   ,					_UpperCAmelCase							   =     jax.lax.scan(
						    f=__lowerCAmelCase					,						init=0					,						xs=__lowerCAmelCase					,						length=math.ceil(num_q / query_chunk_size      )      )

						return jnp.concatenate(__lowerCAmelCase					,						axis=-3      )  # fuse the chunked result back





class __lowerCamelCase					( nn.Module):





	"""simple docstring"""






	UpperCamelCase__								= 42
	UpperCamelCase__								= 8
	UpperCamelCase__								= 64
	UpperCamelCase__								= 0.0
	UpperCamelCase__								= False
	UpperCamelCase__								= jnp.floataa

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.dim_head * self.heads
							_UpperCAmelCase							   =     self.dim_head**-0.5

							# Weights were exported with old names {to_q, to_k, to_v, to_out}
							_UpperCAmelCase							   =     nn.Dense(UpperCAmelCase			,       use_bias=UpperCAmelCase			,       dtype=self.dtype			,       name='to_q'       )
							_UpperCAmelCase							   =     nn.Dense(UpperCAmelCase			,       use_bias=UpperCAmelCase			,       dtype=self.dtype			,       name='to_k'       )
							_UpperCAmelCase							   =     nn.Dense(UpperCAmelCase			,       use_bias=UpperCAmelCase			,       dtype=self.dtype			,       name='to_v'       )

							_UpperCAmelCase							   =     nn.Dense(self.query_dim			,       dtype=self.dtype			,       name='to_out_0'       )
							_UpperCAmelCase							   =     nn.Dropout(rate=self.dropout       )

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase   ,					_UpperCAmelCase   ,					_UpperCAmelCase							   =     tensor.shape
							_UpperCAmelCase							   =     self.heads
							_UpperCAmelCase							   =     tensor.reshape(UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase			,       dim // head_size       )
							_UpperCAmelCase							   =     jnp.transpose(UpperCAmelCase			,       (0, 2, 1, 3)       )
							_UpperCAmelCase							   =     tensor.reshape(batch_size * head_size			,       UpperCAmelCase			,       dim // head_size       )
							return tensor

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase   ,					_UpperCAmelCase   ,					_UpperCAmelCase							   =     tensor.shape
							_UpperCAmelCase							   =     self.heads
							_UpperCAmelCase							   =     tensor.reshape(batch_size // head_size			,       UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase       )
							_UpperCAmelCase							   =     jnp.transpose(UpperCAmelCase			,       (0, 2, 1, 3)       )
							_UpperCAmelCase							   =     tensor.reshape(batch_size // head_size			,       UpperCAmelCase			,       dim * head_size       )
							return tensor

	def __call__(				self			,       UpperCAmelCase			,       UpperCAmelCase=None			,       UpperCAmelCase=True       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     hidden_states if context is None else context

							_UpperCAmelCase							   =     self.query(UpperCAmelCase       )
							_UpperCAmelCase							   =     self.key(UpperCAmelCase       )
							_UpperCAmelCase							   =     self.value(UpperCAmelCase       )

							_UpperCAmelCase							   =     self.reshape_heads_to_batch_dim(UpperCAmelCase       )
							_UpperCAmelCase							   =     self.reshape_heads_to_batch_dim(UpperCAmelCase       )
							_UpperCAmelCase							   =     self.reshape_heads_to_batch_dim(UpperCAmelCase       )

							if self.use_memory_efficient_attention:
													_UpperCAmelCase							   =     query_states.transpose(1			,       0			,       2       )
													_UpperCAmelCase							   =     key_states.transpose(1			,       0			,       2       )
													_UpperCAmelCase							   =     value_states.transpose(1			,       0			,       2       )

													# this if statement create a chunk size for each layer of the unet
													# the chunk size is equal to the query_length dimension of the deepest layer of the unet

													_UpperCAmelCase							   =     query_states.shape[-3]
													if flatten_latent_dim % 64 == 0:
																			_UpperCAmelCase							   =     int(flatten_latent_dim / 64       )
													elif flatten_latent_dim % 16 == 0:
																			_UpperCAmelCase							   =     int(flatten_latent_dim / 16       )
													elif flatten_latent_dim % 4 == 0:
																			_UpperCAmelCase							   =     int(flatten_latent_dim / 4       )
													else:
																			_UpperCAmelCase							   =     int(UpperCAmelCase       )

													_UpperCAmelCase							   =     jax_memory_efficient_attention(
													    UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase			,       query_chunk_size=UpperCAmelCase			,       key_chunk_size=4096 * 4       )

													_UpperCAmelCase							   =     hidden_states.transpose(1			,       0			,       2       )
							else:
													# compute attentions
													_UpperCAmelCase							   =     jnp.einsum('b i d, b j d->b i j'			,       UpperCAmelCase			,       UpperCAmelCase       )
													_UpperCAmelCase							   =     attention_scores * self.scale
													_UpperCAmelCase							   =     nn.softmax(UpperCAmelCase			,       axis=2       )

													# attend to values
													_UpperCAmelCase							   =     jnp.einsum('b i j, b j d -> b i d'			,       UpperCAmelCase			,       UpperCAmelCase       )

							_UpperCAmelCase							   =     self.reshape_batch_dim_to_heads(UpperCAmelCase       )
							_UpperCAmelCase							   =     self.proj_attn(UpperCAmelCase       )
							return self.dropout_layer(UpperCAmelCase			,       deterministic=UpperCAmelCase       )





class __lowerCamelCase					( nn.Module):





	"""simple docstring"""






	UpperCamelCase__								= 42
	UpperCamelCase__								= 42
	UpperCamelCase__								= 42
	UpperCamelCase__								= 0.0
	UpperCamelCase__								= False
	UpperCamelCase__								= jnp.floataa
	UpperCamelCase__								= False

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     FlaxAttention(
							    self.dim			,       self.n_heads			,       self.d_head			,       self.dropout			,       self.use_memory_efficient_attention			,       dtype=self.dtype       )
							# cross attention
							_UpperCAmelCase							   =     FlaxAttention(
							    self.dim			,       self.n_heads			,       self.d_head			,       self.dropout			,       self.use_memory_efficient_attention			,       dtype=self.dtype       )
							_UpperCAmelCase							   =     FlaxFeedForward(dim=self.dim			,       dropout=self.dropout			,       dtype=self.dtype       )
							_UpperCAmelCase							   =     nn.LayerNorm(epsilon=1e-5			,       dtype=self.dtype       )
							_UpperCAmelCase							   =     nn.LayerNorm(epsilon=1e-5			,       dtype=self.dtype       )
							_UpperCAmelCase							   =     nn.LayerNorm(epsilon=1e-5			,       dtype=self.dtype       )
							_UpperCAmelCase							   =     nn.Dropout(rate=self.dropout       )

	def __call__(				self			,       UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase=True       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     hidden_states
							if self.only_cross_attention:
													_UpperCAmelCase							   =     self.attna(self.norma(UpperCAmelCase       )			,       UpperCAmelCase			,       deterministic=UpperCAmelCase       )
							else:
													_UpperCAmelCase							   =     self.attna(self.norma(UpperCAmelCase       )			,       deterministic=UpperCAmelCase       )
							_UpperCAmelCase							   =     hidden_states + residual

							# cross attention
							_UpperCAmelCase							   =     hidden_states
							_UpperCAmelCase							   =     self.attna(self.norma(UpperCAmelCase       )			,       UpperCAmelCase			,       deterministic=UpperCAmelCase       )
							_UpperCAmelCase							   =     hidden_states + residual

							# feed forward
							_UpperCAmelCase							   =     hidden_states
							_UpperCAmelCase							   =     self.ff(self.norma(UpperCAmelCase       )			,       deterministic=UpperCAmelCase       )
							_UpperCAmelCase							   =     hidden_states + residual

							return self.dropout_layer(UpperCAmelCase			,       deterministic=UpperCAmelCase       )





class __lowerCamelCase					( nn.Module):





	"""simple docstring"""






	UpperCamelCase__								= 42
	UpperCamelCase__								= 42
	UpperCamelCase__								= 42
	UpperCamelCase__								= 1
	UpperCamelCase__								= 0.0
	UpperCamelCase__								= False
	UpperCamelCase__								= False
	UpperCamelCase__								= jnp.floataa
	UpperCamelCase__								= False

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     nn.GroupNorm(num_groups=32			,       epsilon=1e-5       )

							_UpperCAmelCase							   =     self.n_heads * self.d_head
							if self.use_linear_projection:
													_UpperCAmelCase							   =     nn.Dense(UpperCAmelCase			,       dtype=self.dtype       )
							else:
													_UpperCAmelCase							   =     nn.Conv(
													    UpperCAmelCase			,       kernel_size=(1, 1)			,       strides=(1, 1)			,       padding='VALID'			,       dtype=self.dtype			,       )

							_UpperCAmelCase							   =     [
							    FlaxBasicTransformerBlock(
							        UpperCAmelCase			,       self.n_heads			,       self.d_head			,       dropout=self.dropout			,       only_cross_attention=self.only_cross_attention			,       dtype=self.dtype			,       use_memory_efficient_attention=self.use_memory_efficient_attention			,       )
							    for _ in range(self.depth       )
							]

							if self.use_linear_projection:
													_UpperCAmelCase							   =     nn.Dense(UpperCAmelCase			,       dtype=self.dtype       )
							else:
													_UpperCAmelCase							   =     nn.Conv(
													    UpperCAmelCase			,       kernel_size=(1, 1)			,       strides=(1, 1)			,       padding='VALID'			,       dtype=self.dtype			,       )

							_UpperCAmelCase							   =     nn.Dropout(rate=self.dropout       )

	def __call__(				self			,       UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase=True       ):
							"""simple docstring"""







							_UpperCAmelCase   ,					_UpperCAmelCase   ,					_UpperCAmelCase   ,					_UpperCAmelCase							   =     hidden_states.shape
							_UpperCAmelCase							   =     hidden_states
							_UpperCAmelCase							   =     self.norm(UpperCAmelCase       )
							if self.use_linear_projection:
													_UpperCAmelCase							   =     hidden_states.reshape(UpperCAmelCase			,       height * width			,       UpperCAmelCase       )
													_UpperCAmelCase							   =     self.proj_in(UpperCAmelCase       )
							else:
													_UpperCAmelCase							   =     self.proj_in(UpperCAmelCase       )
													_UpperCAmelCase							   =     hidden_states.reshape(UpperCAmelCase			,       height * width			,       UpperCAmelCase       )

							for transformer_block in self.transformer_blocks:
													_UpperCAmelCase							   =     transformer_block(UpperCAmelCase			,       UpperCAmelCase			,       deterministic=UpperCAmelCase       )

							if self.use_linear_projection:
													_UpperCAmelCase							   =     self.proj_out(UpperCAmelCase       )
													_UpperCAmelCase							   =     hidden_states.reshape(UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase       )
							else:
													_UpperCAmelCase							   =     hidden_states.reshape(UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase			,       UpperCAmelCase       )
													_UpperCAmelCase							   =     self.proj_out(UpperCAmelCase       )

							_UpperCAmelCase							   =     hidden_states + residual
							return self.dropout_layer(UpperCAmelCase			,       deterministic=UpperCAmelCase       )





class __lowerCamelCase					( nn.Module):





	"""simple docstring"""






	UpperCamelCase__								= 42
	UpperCamelCase__								= 0.0
	UpperCamelCase__								= jnp.floataa

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     FlaxGEGLU(self.dim			,       self.dropout			,       self.dtype       )
							_UpperCAmelCase							   =     nn.Dense(self.dim			,       dtype=self.dtype       )

	def __call__(				self			,       UpperCAmelCase			,       UpperCAmelCase=True       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.net_a(UpperCAmelCase			,       deterministic=UpperCAmelCase       )
							_UpperCAmelCase							   =     self.net_a(UpperCAmelCase       )
							return hidden_states





class __lowerCamelCase					( nn.Module):





	"""simple docstring"""






	UpperCamelCase__								= 42
	UpperCamelCase__								= 0.0
	UpperCamelCase__								= jnp.floataa

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.dim * 4
							_UpperCAmelCase							   =     nn.Dense(inner_dim * 2			,       dtype=self.dtype       )
							_UpperCAmelCase							   =     nn.Dropout(rate=self.dropout       )

	def __call__(				self			,       UpperCAmelCase			,       UpperCAmelCase=True       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.proj(UpperCAmelCase       )
							_UpperCAmelCase   ,					_UpperCAmelCase							   =     jnp.split(UpperCAmelCase			,       2			,       axis=2       )
							return self.dropout_layer(hidden_linear * nn.gelu(UpperCAmelCase       )			,       deterministic=UpperCAmelCase       )








from __future__ import annotations

import re


def __A       ( __lowerCAmelCase      )->    list[str]:





						"""simple docstring"""



						def alphanum_key(__lowerCAmelCase      ):
												return [int(__lowerCAmelCase      ) if s.isdigit() else s.lower() for s in re.split('([0-9]+)'					,						__lowerCAmelCase      )]

						return sorted(__lowerCAmelCase					,						key=__lowerCAmelCase      )


if __name__ == "__main__":
		import doctest

		doctest.testmod()
