






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp





def 						A_					(						lowercase_	,  lowercase_	,  lowercase_	,  lowercase_	,  lowercase_ = 4_0_9_6			)				->Any:







 """simple docstring"""

 SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		key.shape[-3:]
 SCREAMING_SNAKE_CASE				    =		value.shape[-1]
 SCREAMING_SNAKE_CASE				    =		min(lowercase_	,  lowercase_			)
 SCREAMING_SNAKE_CASE				    =		query / jnp.sqrt(lowercase_			)

 @functools.partial(jax.checkpoint	,  prevent_cse=lowercase_			)
 def summarize_chunk(lowercase_	,  lowercase_	,  lowercase_			):
  SCREAMING_SNAKE_CASE				    =		jnp.einsum('...qhd,...khd->...qhk'	,  lowercase_	,  lowercase_	,  precision=lowercase_			)

  SCREAMING_SNAKE_CASE				    =		jnp.max(lowercase_	,  axis=-1	,  keepdims=lowercase_			)
  SCREAMING_SNAKE_CASE				    =		jax.lax.stop_gradient(lowercase_			)
  SCREAMING_SNAKE_CASE				    =		jnp.exp(attn_weights - max_score			)

  SCREAMING_SNAKE_CASE				    =		jnp.einsum('...vhf,...qhv->...qhf'	,  lowercase_	,  lowercase_	,  precision=lowercase_			)
  SCREAMING_SNAKE_CASE				    =		jnp.einsum('...qhk->...qh'	,  lowercase_			)

  return (exp_values, exp_weights.sum(axis=-1			), max_score)

 def chunk_scanner(lowercase_			):
  # julienne key array
  SCREAMING_SNAKE_CASE				    =		jax.lax.dynamic_slice(
      operand=lowercase_	,  start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]	,  slice_sizes=list(key.shape[:-3]			) + [key_chunk_size, num_heads, k_features]	,  )

  # julienne value array
  SCREAMING_SNAKE_CASE				    =		jax.lax.dynamic_slice(
      operand=lowercase_	,  start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]	,  slice_sizes=list(value.shape[:-3]			) + [key_chunk_size, num_heads, v_features]	,  )

  return summarize_chunk(lowercase_	,  lowercase_	,  lowercase_			)

 SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		jax.lax.map(f=lowercase_	,  xs=jnp.arange(0	,  lowercase_	,  lowercase_			)			)

 SCREAMING_SNAKE_CASE				    =		jnp.max(lowercase_	,  axis=0	,  keepdims=lowercase_			)
 SCREAMING_SNAKE_CASE				    =		jnp.exp(chunk_max - global_max			)

 chunk_values *= jnp.expand_dims(lowercase_	,  axis=-1			)
 chunk_weights *= max_diffs

 SCREAMING_SNAKE_CASE				    =		chunk_values.sum(axis=0			)
 SCREAMING_SNAKE_CASE				    =		jnp.expand_dims(lowercase_	,  -1			).sum(axis=0			)

 return all_values / all_weights





def 						A_					(						lowercase_	,  lowercase_	,  lowercase_	,  lowercase_=jax.lax.Precision.HIGHEST	,  lowercase_ = 1_0_2_4	,  lowercase_ = 4_0_9_6			)				->List[Any]:







 """simple docstring"""

 SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		query.shape[-3:]

 def chunk_scanner(lowercase_	,  lowercase_			):
  # julienne query array
  SCREAMING_SNAKE_CASE				    =		jax.lax.dynamic_slice(
      operand=lowercase_	,  start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]	,  slice_sizes=list(query.shape[:-3]			) + [min(lowercase_	,  lowercase_			), num_heads, q_features]	,  )

  return (
      chunk_idx + query_chunk_size,  # unused ignore it
      _query_chunk_attention(
          query=lowercase_	,  key=lowercase_	,  value=lowercase_	,  precision=lowercase_	,  key_chunk_size=lowercase_			),
  )

 SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		jax.lax.scan(
     f=lowercase_	,  init=0	,  xs=lowercase_	,  length=math.ceil(num_q / query_chunk_size			)			)

 return jnp.concatenate(lowercase_	,  axis=-3			)  # fuse the chunked result back
class 	a_(		nn.Module	):



     """simple docstring"""



     __snake_case		:				int
     __snake_case		:				int  					=8
     __snake_case		:				int  					=64
     __snake_case		:				float  					=0.0
     __snake_case		:				bool  					=False
     __snake_case		:				jnp.dtype  					=jnp.floataa





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	Optional[int]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.dim_head * self.heads
      SCREAMING_SNAKE_CASE				    =		self.dim_head**-0.5

      # Weights were exported with old names {to_q, to_k, to_v, to_out}
      SCREAMING_SNAKE_CASE				    =		nn.Dense(lowerCAmelCase__  ,  use_bias=lowerCAmelCase__  ,  dtype=self.dtype  ,  name='to_q')
      SCREAMING_SNAKE_CASE				    =		nn.Dense(lowerCAmelCase__  ,  use_bias=lowerCAmelCase__  ,  dtype=self.dtype  ,  name='to_k')
      SCREAMING_SNAKE_CASE				    =		nn.Dense(lowerCAmelCase__  ,  use_bias=lowerCAmelCase__  ,  dtype=self.dtype  ,  name='to_v')

      SCREAMING_SNAKE_CASE				    =		nn.Dense(self.query_dim  ,  dtype=self.dtype  ,  name='to_out_0')
      SCREAMING_SNAKE_CASE				    =		nn.Dropout(rate=self.dropout)





     def     __UpperCamelCase						(		self					:	int  ,  lowerCAmelCase__					:	List[Any])							->	int:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		tensor.shape
      SCREAMING_SNAKE_CASE				    =		self.heads
      SCREAMING_SNAKE_CASE				    =		tensor.reshape(lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__  ,  dim // head_size)
      SCREAMING_SNAKE_CASE				    =		jnp.transpose(lowerCAmelCase__  ,  (0, 2, 1, 3))
      SCREAMING_SNAKE_CASE				    =		tensor.reshape(batch_size * head_size  ,  lowerCAmelCase__  ,  dim // head_size)
      return tensor





     def     __UpperCamelCase						(		self					:	List[str]  ,  lowerCAmelCase__					:	Dict)							->	Dict:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		tensor.shape
      SCREAMING_SNAKE_CASE				    =		self.heads
      SCREAMING_SNAKE_CASE				    =		tensor.reshape(batch_size // head_size  ,  lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		jnp.transpose(lowerCAmelCase__  ,  (0, 2, 1, 3))
      SCREAMING_SNAKE_CASE				    =		tensor.reshape(batch_size // head_size  ,  lowerCAmelCase__  ,  dim * head_size)
      return tensor





     def __call__(		self					:	int  ,  lowerCAmelCase__					:	str  ,  lowerCAmelCase__					:	List[str]=None  ,  lowerCAmelCase__					:	Dict=True)							->	int:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		hidden_states if context is None else context

      SCREAMING_SNAKE_CASE				    =		self.query(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		self.key(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		self.value(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		self.reshape_heads_to_batch_dim(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		self.reshape_heads_to_batch_dim(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		self.reshape_heads_to_batch_dim(lowerCAmelCase__)

      if self.use_memory_efficient_attention:
       SCREAMING_SNAKE_CASE				    =		query_states.transpose(1  ,  0  ,  2)
       SCREAMING_SNAKE_CASE				    =		key_states.transpose(1  ,  0  ,  2)
       SCREAMING_SNAKE_CASE				    =		value_states.transpose(1  ,  0  ,  2)

       # this if statement create a chunk size for each layer of the unet
       # the chunk size is equal to the query_length dimension of the deepest layer of the unet

       SCREAMING_SNAKE_CASE				    =		query_states.shape[-3]
       if flatten_latent_dim % 6_4 == 0:
        SCREAMING_SNAKE_CASE				    =		int(flatten_latent_dim / 6_4)
       elif flatten_latent_dim % 1_6 == 0:
        SCREAMING_SNAKE_CASE				    =		int(flatten_latent_dim / 1_6)
       elif flatten_latent_dim % 4 == 0:
        SCREAMING_SNAKE_CASE				    =		int(flatten_latent_dim / 4)
       else:
        SCREAMING_SNAKE_CASE				    =		int(lowerCAmelCase__)

       SCREAMING_SNAKE_CASE				    =		jax_memory_efficient_attention(
           lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__  ,  query_chunk_size=lowerCAmelCase__  ,  key_chunk_size=4_0_9_6 * 4)

       SCREAMING_SNAKE_CASE				    =		hidden_states.transpose(1  ,  0  ,  2)
      else:
       # compute attentions
       SCREAMING_SNAKE_CASE				    =		jnp.einsum('b i d, b j d->b i j'  ,  lowerCAmelCase__  ,  lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		attention_scores * self.scale
       SCREAMING_SNAKE_CASE				    =		nn.softmax(lowerCAmelCase__  ,  axis=2)

       # attend to values
       SCREAMING_SNAKE_CASE				    =		jnp.einsum('b i j, b j d -> b i d'  ,  lowerCAmelCase__  ,  lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		self.reshape_batch_dim_to_heads(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		self.proj_attn(lowerCAmelCase__)
      return self.dropout_layer(lowerCAmelCase__  ,  deterministic=lowerCAmelCase__)

class 	a_(		nn.Module	):



     """simple docstring"""



     __snake_case		:				int
     __snake_case		:				int
     __snake_case		:				int
     __snake_case		:				float  					=0.0
     __snake_case		:				bool  					=False
     __snake_case		:				jnp.dtype  					=jnp.floataa
     __snake_case		:				bool  					=False





     def     __UpperCamelCase						(		self					:	Dict)							->	Optional[Any]:

      """simple docstring"""
      # self attention (or cross_attention if only_cross_attention is True)
      SCREAMING_SNAKE_CASE				    =		FlaxAttention(
          self.dim  ,  self.n_heads  ,  self.d_head  ,  self.dropout  ,  self.use_memory_efficient_attention  ,  dtype=self.dtype)
      # cross attention
      SCREAMING_SNAKE_CASE				    =		FlaxAttention(
          self.dim  ,  self.n_heads  ,  self.d_head  ,  self.dropout  ,  self.use_memory_efficient_attention  ,  dtype=self.dtype)
      SCREAMING_SNAKE_CASE				    =		FlaxFeedForward(dim=self.dim  ,  dropout=self.dropout  ,  dtype=self.dtype)
      SCREAMING_SNAKE_CASE				    =		nn.LayerNorm(epsilon=1e-5  ,  dtype=self.dtype)
      SCREAMING_SNAKE_CASE				    =		nn.LayerNorm(epsilon=1e-5  ,  dtype=self.dtype)
      SCREAMING_SNAKE_CASE				    =		nn.LayerNorm(epsilon=1e-5  ,  dtype=self.dtype)
      SCREAMING_SNAKE_CASE				    =		nn.Dropout(rate=self.dropout)





     def __call__(		self					:	List[str]  ,  lowerCAmelCase__					:	Tuple  ,  lowerCAmelCase__					:	str  ,  lowerCAmelCase__					:	Dict=True)							->	int:

      """simple docstring"""
      # self attention
      SCREAMING_SNAKE_CASE				    =		hidden_states
      if self.only_cross_attention:
       SCREAMING_SNAKE_CASE				    =		self.attna(self.norma(lowerCAmelCase__)  ,  lowerCAmelCase__  ,  deterministic=lowerCAmelCase__)
      else:
       SCREAMING_SNAKE_CASE				    =		self.attna(self.norma(lowerCAmelCase__)  ,  deterministic=lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		hidden_states + residual

      # cross attention
      SCREAMING_SNAKE_CASE				    =		hidden_states
      SCREAMING_SNAKE_CASE				    =		self.attna(self.norma(lowerCAmelCase__)  ,  lowerCAmelCase__  ,  deterministic=lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		hidden_states + residual

      # feed forward
      SCREAMING_SNAKE_CASE				    =		hidden_states
      SCREAMING_SNAKE_CASE				    =		self.ff(self.norma(lowerCAmelCase__)  ,  deterministic=lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		hidden_states + residual

      return self.dropout_layer(lowerCAmelCase__  ,  deterministic=lowerCAmelCase__)

class 	a_(		nn.Module	):



     """simple docstring"""



     __snake_case		:				int
     __snake_case		:				int
     __snake_case		:				int
     __snake_case		:				int  					=1
     __snake_case		:				float  					=0.0
     __snake_case		:				bool  					=False
     __snake_case		:				bool  					=False
     __snake_case		:				jnp.dtype  					=jnp.floataa
     __snake_case		:				bool  					=False





     def     __UpperCamelCase						(		self					:	Optional[int])							->	Optional[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		nn.GroupNorm(num_groups=3_2  ,  epsilon=1e-5)

      SCREAMING_SNAKE_CASE				    =		self.n_heads * self.d_head
      if self.use_linear_projection:
       SCREAMING_SNAKE_CASE				    =		nn.Dense(lowerCAmelCase__  ,  dtype=self.dtype)
      else:
       SCREAMING_SNAKE_CASE				    =		nn.Conv(
           lowerCAmelCase__  ,  kernel_size=(1, 1)  ,  strides=(1, 1)  ,  padding='VALID'  ,  dtype=self.dtype  ,  )

      SCREAMING_SNAKE_CASE				    =		[
          FlaxBasicTransformerBlock(
              lowerCAmelCase__  ,  self.n_heads  ,  self.d_head  ,  dropout=self.dropout  ,  only_cross_attention=self.only_cross_attention  ,  dtype=self.dtype  ,  use_memory_efficient_attention=self.use_memory_efficient_attention  ,  )
          for _ in range(self.depth)
      ]

      if self.use_linear_projection:
       SCREAMING_SNAKE_CASE				    =		nn.Dense(lowerCAmelCase__  ,  dtype=self.dtype)
      else:
       SCREAMING_SNAKE_CASE				    =		nn.Conv(
           lowerCAmelCase__  ,  kernel_size=(1, 1)  ,  strides=(1, 1)  ,  padding='VALID'  ,  dtype=self.dtype  ,  )

      SCREAMING_SNAKE_CASE				    =		nn.Dropout(rate=self.dropout)





     def __call__(		self					:	List[str]  ,  lowerCAmelCase__					:	Dict  ,  lowerCAmelCase__					:	Optional[Any]  ,  lowerCAmelCase__					:	List[str]=True)							->	Dict:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		hidden_states.shape
      SCREAMING_SNAKE_CASE				    =		hidden_states
      SCREAMING_SNAKE_CASE				    =		self.norm(lowerCAmelCase__)
      if self.use_linear_projection:
       SCREAMING_SNAKE_CASE				    =		hidden_states.reshape(lowerCAmelCase__  ,  height * width  ,  lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		self.proj_in(lowerCAmelCase__)
      else:
       SCREAMING_SNAKE_CASE				    =		self.proj_in(lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		hidden_states.reshape(lowerCAmelCase__  ,  height * width  ,  lowerCAmelCase__)

      for transformer_block in self.transformer_blocks:
       SCREAMING_SNAKE_CASE				    =		transformer_block(lowerCAmelCase__  ,  lowerCAmelCase__  ,  deterministic=lowerCAmelCase__)

      if self.use_linear_projection:
       SCREAMING_SNAKE_CASE				    =		self.proj_out(lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		hidden_states.reshape(lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__)
      else:
       SCREAMING_SNAKE_CASE				    =		hidden_states.reshape(lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		self.proj_out(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		hidden_states + residual
      return self.dropout_layer(lowerCAmelCase__  ,  deterministic=lowerCAmelCase__)

class 	a_(		nn.Module	):



     """simple docstring"""



     __snake_case		:				int
     __snake_case		:				float  					=0.0
     __snake_case		:				jnp.dtype  					=jnp.floataa





     def     __UpperCamelCase						(		self					:	Union[str, Any])							->	Any:

      """simple docstring"""
      # The second linear layer needs to be called
      # net_2 for now to match the index of the Sequential layer
      SCREAMING_SNAKE_CASE				    =		FlaxGEGLU(self.dim  ,  self.dropout  ,  self.dtype)
      SCREAMING_SNAKE_CASE				    =		nn.Dense(self.dim  ,  dtype=self.dtype)





     def __call__(		self					:	Union[str, Any]  ,  lowerCAmelCase__					:	List[str]  ,  lowerCAmelCase__					:	Optional[Any]=True)							->	int:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.net_a(lowerCAmelCase__  ,  deterministic=lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		self.net_a(lowerCAmelCase__)
      return hidden_states

class 	a_(		nn.Module	):



     """simple docstring"""



     __snake_case		:				int
     __snake_case		:				float  					=0.0
     __snake_case		:				jnp.dtype  					=jnp.floataa





     def     __UpperCamelCase						(		self					:	int)							->	Dict:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.dim * 4
      SCREAMING_SNAKE_CASE				    =		nn.Dense(inner_dim * 2  ,  dtype=self.dtype)
      SCREAMING_SNAKE_CASE				    =		nn.Dropout(rate=self.dropout)





     def __call__(		self					:	List[str]  ,  lowerCAmelCase__					:	str  ,  lowerCAmelCase__					:	Dict=True)							->	List[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.proj(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		jnp.split(lowerCAmelCase__  ,  2  ,  axis=2)
      return self.dropout_layer(hidden_linear * nn.gelu(lowerCAmelCase__)  ,  deterministic=lowerCAmelCase__)












def 						A_					(						lowercase_	,  lowercase_			)				->List[Any]:







 """simple docstring"""

 SCREAMING_SNAKE_CASE				    =		1
 while b > 0:
  if b & 1:
   res *= a

  a *= a
  b >>= 1

 return res





def 						A_					(						lowercase_	,  lowercase_	,  lowercase_			)				->Any:







 """simple docstring"""

 SCREAMING_SNAKE_CASE				    =		1
 while b > 0:
  if b & 1:
   SCREAMING_SNAKE_CASE				    =		((res % c) * (a % c)) % c

  a *= a
  b >>= 1

 return res




