




'''simple docstring'''





import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_ = 4_096						):
    _a       , _a       , _a				:							Optional[int]							       =      key.shape[-3:]
    _a				:							Tuple							       =      value.shape[-1]
    _a				:							int							       =      min(UpperCAmelCase_   ,	UpperCAmelCase_						)
    _a				:							Tuple							       =      query / jnp.sqrt(UpperCAmelCase_						)

    @functools.partial(jax.checkpoint   ,	prevent_cse=UpperCAmelCase_						)
    def summarize_chunk(UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_						):
        _a				:							Dict							       =      jnp.einsum("""...qhd,...khd->...qhk"""   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	precision=UpperCAmelCase_						)

        _a				:							List[str]							       =      jnp.max(UpperCAmelCase_   ,	axis=-1   ,	keepdims=UpperCAmelCase_						)
        _a				:							Any							       =      jax.lax.stop_gradient(UpperCAmelCase_						)
        _a				:							Tuple							       =      jnp.exp(attn_weights - max_score						)

        _a				:							List[str]							       =      jnp.einsum("""...vhf,...qhv->...qhf"""   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	precision=UpperCAmelCase_						)
        _a				:							List[Any]							       =      jnp.einsum("""...qhk->...qh"""   ,	UpperCAmelCase_						)

        return (exp_values, exp_weights.sum(axis=-1						), max_score)

    def chunk_scanner(UpperCAmelCase_						):
        # julienne key array
        _a				:							Any							       =      jax.lax.dynamic_slice(
            operand=UpperCAmelCase_   ,	start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]   ,	slice_sizes=list(key.shape[:-3]						) + [key_chunk_size, num_heads, k_features]   ,	)

        # julienne value array
        _a				:							Optional[Any]							       =      jax.lax.dynamic_slice(
            operand=UpperCAmelCase_   ,	start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]   ,	slice_sizes=list(value.shape[:-3]						) + [key_chunk_size, num_heads, v_features]   ,	)

        return summarize_chunk(UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_						)

    _a       , _a       , _a				:							Optional[int]							       =      jax.lax.map(f=UpperCAmelCase_   ,	xs=jnp.arange(0   ,	UpperCAmelCase_   ,	UpperCAmelCase_						)						)

    _a				:							Optional[Any]							       =      jnp.max(UpperCAmelCase_   ,	axis=0   ,	keepdims=UpperCAmelCase_						)
    _a				:							List[str]							       =      jnp.exp(chunk_max - global_max						)

    chunk_values *= jnp.expand_dims(UpperCAmelCase_   ,	axis=-1						)
    chunk_weights *= max_diffs

    _a				:							Optional[Any]							       =      chunk_values.sum(axis=0						)
    _a				:							List[str]							       =      jnp.expand_dims(UpperCAmelCase_   ,	-1						).sum(axis=0						)

    return all_values / all_weights





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_=jax.lax.Precision.HIGHEST   ,	UpperCAmelCase_ = 1_024   ,	UpperCAmelCase_ = 4_096						):
    _a       , _a       , _a				:							int							       =      query.shape[-3:]

    def chunk_scanner(UpperCAmelCase_   ,	UpperCAmelCase_						):
        # julienne query array
        _a				:							Any							       =      jax.lax.dynamic_slice(
            operand=UpperCAmelCase_   ,	start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]   ,	slice_sizes=list(query.shape[:-3]						) + [min(UpperCAmelCase_   ,	UpperCAmelCase_						), num_heads, q_features]   ,	)

        return (
            chunk_idx + query_chunk_size,  # unused ignore it
            _query_chunk_attention(
                query=UpperCAmelCase_   ,	key=UpperCAmelCase_   ,	value=UpperCAmelCase_   ,	precision=UpperCAmelCase_   ,	key_chunk_size=UpperCAmelCase_						),
        )

    _a       , _a				:							List[str]							       =      jax.lax.scan(
        f=UpperCAmelCase_   ,	init=0   ,	xs=UpperCAmelCase_   ,	length=math.ceil(num_q / query_chunk_size						)						)

    return jnp.concatenate(UpperCAmelCase_   ,	axis=-3						)  # fuse the chunked result back




class      UpperCamelCase       (	nn.Module       ):
  a     :int
  a     :int          =					8
  a     :int          =					6_4
  a     :float          =					0.0
  a     :bool          =					False
  a     :jnp.dtype          =					jnp.floataa



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							Union[str, Any]							       =      self.dim_head * self.heads
      _a				:							Any							       =      self.dim_head**-0.5

      # Weights were exported with old names {to_q, to_k, to_v, to_out}
      _a				:							Any							       =      nn.Dense(__lowerCAmelCase		,     use_bias=__lowerCAmelCase		,     dtype=self.dtype		,     name="""to_q""" )
      _a				:							Tuple							       =      nn.Dense(__lowerCAmelCase		,     use_bias=__lowerCAmelCase		,     dtype=self.dtype		,     name="""to_k""" )
      _a				:							Dict							       =      nn.Dense(__lowerCAmelCase		,     use_bias=__lowerCAmelCase		,     dtype=self.dtype		,     name="""to_v""" )

      _a				:							Union[str, Any]							       =      nn.Dense(self.query_dim		,     dtype=self.dtype		,     name="""to_out_0""" )
      _a				:							List[str]							       =      nn.Dropout(rate=self.dropout )



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a       , _a       , _a				:							List[str]							       =      tensor.shape
      _a				:							List[str]							       =      self.heads
      _a				:							List[Any]							       =      tensor.reshape(__lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase		,     dim // head_size )
      _a				:							Optional[Any]							       =      jnp.transpose(__lowerCAmelCase		,     (0, 2, 1, 3) )
      _a				:							Optional[Any]							       =      tensor.reshape(batch_size * head_size		,     __lowerCAmelCase		,     dim // head_size )
      return tensor



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a       , _a       , _a				:							str							       =      tensor.shape
      _a				:							Optional[int]							       =      self.heads
      _a				:							int							       =      tensor.reshape(batch_size // head_size		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase )
      _a				:							List[Any]							       =      jnp.transpose(__lowerCAmelCase		,     (0, 2, 1, 3) )
      _a				:							Tuple							       =      tensor.reshape(batch_size // head_size		,     __lowerCAmelCase		,     dim * head_size )
      return tensor



  def __call__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase=None		,     __lowerCAmelCase=True ):







      '''simple docstring'''


      _a				:							Union[str, Any]							       =      hidden_states if context is None else context

      _a				:							Dict							       =      self.query(__lowerCAmelCase )
      _a				:							str							       =      self.key(__lowerCAmelCase )
      _a				:							Dict							       =      self.value(__lowerCAmelCase )

      _a				:							str							       =      self.reshape_heads_to_batch_dim(__lowerCAmelCase )
      _a				:							str							       =      self.reshape_heads_to_batch_dim(__lowerCAmelCase )
      _a				:							Tuple							       =      self.reshape_heads_to_batch_dim(__lowerCAmelCase )

      if self.use_memory_efficient_attention:
          _a				:							List[str]							       =      query_states.transpose(1		,     0		,     2 )
          _a				:							Dict							       =      key_states.transpose(1		,     0		,     2 )
          _a				:							int							       =      value_states.transpose(1		,     0		,     2 )

          # this if statement create a chunk size for each layer of the unet
          # the chunk size is equal to the query_length dimension of the deepest layer of the unet

          _a				:							Dict							       =      query_states.shape[-3]
          if flatten_latent_dim % 64 == 0:
              _a				:							Union[str, Any]							       =      int(flatten_latent_dim / 64 )
          elif flatten_latent_dim % 16 == 0:
              _a				:							Dict							       =      int(flatten_latent_dim / 16 )
          elif flatten_latent_dim % 4 == 0:
              _a				:							List[Any]							       =      int(flatten_latent_dim / 4 )
          else:
              _a				:							Optional[int]							       =      int(__lowerCAmelCase )

          _a				:							List[str]							       =      jax_memory_efficient_attention(
              __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase		,     query_chunk_size=__lowerCAmelCase		,     key_chunk_size=4096 * 4 )

          _a				:							Optional[Any]							       =      hidden_states.transpose(1		,     0		,     2 )
      else:
          # compute attentions
          _a				:							Dict							       =      jnp.einsum("""b i d, b j d->b i j"""		,     __lowerCAmelCase		,     __lowerCAmelCase )
          _a				:							Tuple							       =      attention_scores * self.scale
          _a				:							Union[str, Any]							       =      nn.softmax(__lowerCAmelCase		,     axis=2 )

          # attend to values
          _a				:							Optional[Any]							       =      jnp.einsum("""b i j, b j d -> b i d"""		,     __lowerCAmelCase		,     __lowerCAmelCase )

      _a				:							List[Any]							       =      self.reshape_batch_dim_to_heads(__lowerCAmelCase )
      _a				:							Optional[Any]							       =      self.proj_attn(__lowerCAmelCase )
      return self.dropout_layer(__lowerCAmelCase		,     deterministic=__lowerCAmelCase )






class      UpperCamelCase       (	nn.Module       ):
  a     :int
  a     :int
  a     :int
  a     :float          =					0.0
  a     :bool          =					False
  a     :jnp.dtype          =					jnp.floataa
  a     :bool          =					False



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							Any							       =      FlaxAttention(
          self.dim		,     self.n_heads		,     self.d_head		,     self.dropout		,     self.use_memory_efficient_attention		,     dtype=self.dtype )
      # cross attention
      _a				:							Union[str, Any]							       =      FlaxAttention(
          self.dim		,     self.n_heads		,     self.d_head		,     self.dropout		,     self.use_memory_efficient_attention		,     dtype=self.dtype )
      _a				:							List[str]							       =      FlaxFeedForward(dim=self.dim		,     dropout=self.dropout		,     dtype=self.dtype )
      _a				:							str							       =      nn.LayerNorm(epsilon=1e-5		,     dtype=self.dtype )
      _a				:							Dict							       =      nn.LayerNorm(epsilon=1e-5		,     dtype=self.dtype )
      _a				:							Optional[int]							       =      nn.LayerNorm(epsilon=1e-5		,     dtype=self.dtype )
      _a				:							int							       =      nn.Dropout(rate=self.dropout )



  def __call__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase=True ):







      '''simple docstring'''


      _a				:							Optional[Any]							       =      hidden_states
      if self.only_cross_attention:
          _a				:							Dict							       =      self.attna(self.norma(__lowerCAmelCase )		,     __lowerCAmelCase		,     deterministic=__lowerCAmelCase )
      else:
          _a				:							Optional[Any]							       =      self.attna(self.norma(__lowerCAmelCase )		,     deterministic=__lowerCAmelCase )
      _a				:							int							       =      hidden_states + residual

      # cross attention
      _a				:							int							       =      hidden_states
      _a				:							Any							       =      self.attna(self.norma(__lowerCAmelCase )		,     __lowerCAmelCase		,     deterministic=__lowerCAmelCase )
      _a				:							List[Any]							       =      hidden_states + residual

      # feed forward
      _a				:							List[str]							       =      hidden_states
      _a				:							str							       =      self.ff(self.norma(__lowerCAmelCase )		,     deterministic=__lowerCAmelCase )
      _a				:							Tuple							       =      hidden_states + residual

      return self.dropout_layer(__lowerCAmelCase		,     deterministic=__lowerCAmelCase )






class      UpperCamelCase       (	nn.Module       ):
  a     :int
  a     :int
  a     :int
  a     :int          =					1
  a     :float          =					0.0
  a     :bool          =					False
  a     :bool          =					False
  a     :jnp.dtype          =					jnp.floataa
  a     :bool          =					False



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							Union[str, Any]							       =      nn.GroupNorm(num_groups=32		,     epsilon=1e-5 )

      _a				:							Any							       =      self.n_heads * self.d_head
      if self.use_linear_projection:
          _a				:							Dict							       =      nn.Dense(__lowerCAmelCase		,     dtype=self.dtype )
      else:
          _a				:							List[Any]							       =      nn.Conv(
              __lowerCAmelCase		,     kernel_size=(1, 1)		,     strides=(1, 1)		,     padding="""VALID"""		,     dtype=self.dtype		,     )

      _a				:							Union[str, Any]							       =      [
          FlaxBasicTransformerBlock(
              __lowerCAmelCase		,     self.n_heads		,     self.d_head		,     dropout=self.dropout		,     only_cross_attention=self.only_cross_attention		,     dtype=self.dtype		,     use_memory_efficient_attention=self.use_memory_efficient_attention		,     )
          for _ in range(self.depth )
      ]

      if self.use_linear_projection:
          _a				:							List[str]							       =      nn.Dense(__lowerCAmelCase		,     dtype=self.dtype )
      else:
          _a				:							int							       =      nn.Conv(
              __lowerCAmelCase		,     kernel_size=(1, 1)		,     strides=(1, 1)		,     padding="""VALID"""		,     dtype=self.dtype		,     )

      _a				:							Optional[Any]							       =      nn.Dropout(rate=self.dropout )



  def __call__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase=True ):







      '''simple docstring'''


      _a       , _a       , _a       , _a				:							int							       =      hidden_states.shape
      _a				:							Dict							       =      hidden_states
      _a				:							Optional[int]							       =      self.norm(__lowerCAmelCase )
      if self.use_linear_projection:
          _a				:							Union[str, Any]							       =      hidden_states.reshape(__lowerCAmelCase		,     height * width		,     __lowerCAmelCase )
          _a				:							Any							       =      self.proj_in(__lowerCAmelCase )
      else:
          _a				:							Tuple							       =      self.proj_in(__lowerCAmelCase )
          _a				:							List[Any]							       =      hidden_states.reshape(__lowerCAmelCase		,     height * width		,     __lowerCAmelCase )

      for transformer_block in self.transformer_blocks:
          _a				:							Dict							       =      transformer_block(__lowerCAmelCase		,     __lowerCAmelCase		,     deterministic=__lowerCAmelCase )

      if self.use_linear_projection:
          _a				:							str							       =      self.proj_out(__lowerCAmelCase )
          _a				:							int							       =      hidden_states.reshape(__lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase )
      else:
          _a				:							Union[str, Any]							       =      hidden_states.reshape(__lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase )
          _a				:							Tuple							       =      self.proj_out(__lowerCAmelCase )

      _a				:							Union[str, Any]							       =      hidden_states + residual
      return self.dropout_layer(__lowerCAmelCase		,     deterministic=__lowerCAmelCase )






class      UpperCamelCase       (	nn.Module       ):
  a     :int
  a     :float          =					0.0
  a     :jnp.dtype          =					jnp.floataa



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							List[str]							       =      FlaxGEGLU(self.dim		,     self.dropout		,     self.dtype )
      _a				:							Any							       =      nn.Dense(self.dim		,     dtype=self.dtype )



  def __call__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase=True ):







      '''simple docstring'''


      _a				:							int							       =      self.net_a(__lowerCAmelCase		,     deterministic=__lowerCAmelCase )
      _a				:							Tuple							       =      self.net_a(__lowerCAmelCase )
      return hidden_states






class      UpperCamelCase       (	nn.Module       ):
  a     :int
  a     :float          =					0.0
  a     :jnp.dtype          =					jnp.floataa



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							Dict							       =      self.dim * 4
      _a				:							Optional[int]							       =      nn.Dense(inner_dim * 2		,     dtype=self.dtype )
      _a				:							List[Any]							       =      nn.Dropout(rate=self.dropout )



  def __call__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase=True ):







      '''simple docstring'''


      _a				:							Dict							       =      self.proj(__lowerCAmelCase )
      _a       , _a				:							Union[str, Any]							       =      jnp.split(__lowerCAmelCase		,     2		,     axis=2 )
      return self.dropout_layer(hidden_linear * nn.gelu(__lowerCAmelCase )		,     deterministic=__lowerCAmelCase )




'''simple docstring'''





from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
 if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from ...utils.dummy_torch_and_transformers_objects import (
     AudioLDMPipeline,
 )
else:
 from .pipeline_audioldm import AudioLDMPipeline






