





"""simple docstring"""




from ..utils import DummyObject, requires_backends

class 					_UpperCAmelCase    (  metaclass=UpperCAmelCase__						):



      '''simple docstring'''
      lowercase_ :					List[Any]             =  ["""torch""", """librosa"""]

      def __init__(	self   , *snake_case_   , **snake_case_    ):

        """simple docstring"""

        requires_backends(self   , ['torch', 'librosa']    )

      @classmethod
      def    lowerCamelCase_     (	cls   , *snake_case_   , **snake_case_    ):

        """simple docstring"""

        requires_backends(cls   , ['torch', 'librosa']    )

      @classmethod
      def    lowerCamelCase_     (	cls   , *snake_case_   , **snake_case_    ):

        """simple docstring"""

        requires_backends(cls   , ['torch', 'librosa']    )


class 					_UpperCAmelCase    (  metaclass=UpperCAmelCase__						):



      '''simple docstring'''
      lowercase_ :					List[str]             =  ["""torch""", """librosa"""]

      def __init__(	self   , *snake_case_   , **snake_case_    ):

        """simple docstring"""

        requires_backends(self   , ['torch', 'librosa']    )

      @classmethod
      def    lowerCamelCase_     (	cls   , *snake_case_   , **snake_case_    ):

        """simple docstring"""

        requires_backends(cls   , ['torch', 'librosa']    )

      @classmethod
      def    lowerCamelCase_     (	cls   , *snake_case_   , **snake_case_    ):

        """simple docstring"""

        requires_backends(cls   , ['torch', 'librosa']    )






"""simple docstring"""




import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase = 4096   ):






  """simple docstring"""





  A_		,			A_		,			A_    :			List[Any]									=    key.shape[-3:]
  A_    :			Dict									=    value.shape[-1]
  A_    :			Union[str, Any]									=    min(_UpperCAmelCase					,   _UpperCAmelCase   )
  A_    :			str									=    query / jnp.sqrt(_UpperCAmelCase   )

  @functools.partial(jax.checkpoint					,   prevent_cse=_UpperCAmelCase   )
  def summarize_chunk(_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase   ):
    A_    :			Any									=    jnp.einsum('...qhd,...khd->...qhk'					,   _UpperCAmelCase					,   _UpperCAmelCase					,   precision=_UpperCAmelCase   )

    A_    :			int									=    jnp.max(_UpperCAmelCase					,   axis=-1					,   keepdims=_UpperCAmelCase   )
    A_    :			List[str]									=    jax.lax.stop_gradient(_UpperCAmelCase   )
    A_    :			Dict									=    jnp.exp(attn_weights - max_score   )

    A_    :			int									=    jnp.einsum('...vhf,...qhv->...qhf'					,   _UpperCAmelCase					,   _UpperCAmelCase					,   precision=_UpperCAmelCase   )
    A_    :			Optional[Any]									=    jnp.einsum('...qhk->...qh'					,   _UpperCAmelCase   )

    return (exp_values, exp_weights.sum(axis=-1   ), max_score)

  def chunk_scanner(_UpperCAmelCase   ):
    # julienne key array
    A_    :			Dict									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]					,   slice_sizes=list(key.shape[:-3]   ) + [key_chunk_size, num_heads, k_features]					,   )

    # julienne value array
    A_    :			Optional[Any]									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]					,   slice_sizes=list(value.shape[:-3]   ) + [key_chunk_size, num_heads, v_features]					,   )

    return summarize_chunk(_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase   )

  A_		,			A_		,			A_    :			Dict									=    jax.lax.map(f=_UpperCAmelCase					,   xs=jnp.arange(0					,   _UpperCAmelCase					,   _UpperCAmelCase   )   )

  A_    :			int									=    jnp.max(_UpperCAmelCase					,   axis=0					,   keepdims=_UpperCAmelCase   )
  A_    :			Tuple									=    jnp.exp(chunk_max - global_max   )

  chunk_values *= jnp.expand_dims(_UpperCAmelCase					,   axis=-1   )
  chunk_weights *= max_diffs

  A_    :			List[Any]									=    chunk_values.sum(axis=0   )
  A_    :			Dict									=    jnp.expand_dims(_UpperCAmelCase					,   -1   ).sum(axis=0   )

  return all_values / all_weights





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase=jax.lax.Precision.HIGHEST					,   _UpperCAmelCase = 1024					,   _UpperCAmelCase = 4096   ):






  """simple docstring"""





  A_		,			A_		,			A_    :			Any									=    query.shape[-3:]

  def chunk_scanner(_UpperCAmelCase					,   _UpperCAmelCase   ):
    # julienne query array
    A_    :			Tuple									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]					,   slice_sizes=list(query.shape[:-3]   ) + [min(_UpperCAmelCase					,   _UpperCAmelCase   ), num_heads, q_features]					,   )

    return (
        chunk_idx + query_chunk_size,  # unused ignore it
        _query_chunk_attention(
            query=_UpperCAmelCase					,   key=_UpperCAmelCase					,   value=_UpperCAmelCase					,   precision=_UpperCAmelCase					,   key_chunk_size=_UpperCAmelCase   ),
    )

  A_		,			A_    :			Union[str, Any]									=    jax.lax.scan(
      f=_UpperCAmelCase					,   init=0					,   xs=_UpperCAmelCase					,   length=math.ceil(num_q / query_chunk_size   )   )

  return jnp.concatenate(_UpperCAmelCase					,   axis=-3   )  # fuse the chunked result back

class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int             =  8
      lowercase_ :					int             =  64
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    self.dim_head * self.heads
        A_    :			Tuple									=    self.dim_head**-0.5

        # Weights were exported with old names {to_q, to_k, to_v, to_out}
        A_    :			Any									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_q'    )
        A_    :			int									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_k'    )
        A_    :			Optional[int]									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_v'    )

        A_    :			Union[str, Any]									=    nn.Dense(self.query_dim   , dtype=self.dtype   , name='to_out_0'    )
        A_    :			str									=    nn.Dropout(rate=self.dropout    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_    :			Union[str, Any]									=    tensor.shape
        A_    :			List[str]									=    self.heads
        A_    :			str									=    tensor.reshape(snake_case_   , snake_case_   , snake_case_   , dim // head_size    )
        A_    :			Optional[Any]									=    jnp.transpose(snake_case_   , (0, 2, 1, 3)    )
        A_    :			Any									=    tensor.reshape(batch_size * head_size   , snake_case_   , dim // head_size    )
        return tensor

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_    :			str									=    tensor.shape
        A_    :			Optional[Any]									=    self.heads
        A_    :			List[Any]									=    tensor.reshape(batch_size // head_size   , snake_case_   , snake_case_   , snake_case_    )
        A_    :			List[Any]									=    jnp.transpose(snake_case_   , (0, 2, 1, 3)    )
        A_    :			Any									=    tensor.reshape(batch_size // head_size   , snake_case_   , dim * head_size    )
        return tensor

      def __call__(	self   , snake_case_   , snake_case_=None   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    hidden_states if context is None else context

        A_    :			str									=    self.query(snake_case_    )
        A_    :			Any									=    self.key(snake_case_    )
        A_    :			Union[str, Any]									=    self.value(snake_case_    )

        A_    :			int									=    self.reshape_heads_to_batch_dim(snake_case_    )
        A_    :			List[str]									=    self.reshape_heads_to_batch_dim(snake_case_    )
        A_    :			str									=    self.reshape_heads_to_batch_dim(snake_case_    )

        if self.use_memory_efficient_attention:
          A_    :			str									=    query_states.transpose(1   , 0   , 2    )
          A_    :			Optional[int]									=    key_states.transpose(1   , 0   , 2    )
          A_    :			Optional[Any]									=    value_states.transpose(1   , 0   , 2    )

          # this if statement create a chunk size for each layer of the unet
          # the chunk size is equal to the query_length dimension of the deepest layer of the unet

          A_    :			Union[str, Any]									=    query_states.shape[-3]
          if flatten_latent_dim % 6_4 == 0:
            A_    :			Any									=    int(flatten_latent_dim / 6_4    )
          elif flatten_latent_dim % 1_6 == 0:
            A_    :			Optional[int]									=    int(flatten_latent_dim / 1_6    )
          elif flatten_latent_dim % 4 == 0:
            A_    :			Dict									=    int(flatten_latent_dim / 4    )
          else:
            A_    :			Optional[int]									=    int(snake_case_    )

          A_    :			Any									=    jax_memory_efficient_attention(
              snake_case_   , snake_case_   , snake_case_   , query_chunk_size=snake_case_   , key_chunk_size=4_0_9_6 * 4    )

          A_    :			Any									=    hidden_states.transpose(1   , 0   , 2    )
        else:
          # compute attentions
          A_    :			str									=    jnp.einsum('b i d, b j d->b i j'   , snake_case_   , snake_case_    )
          A_    :			Dict									=    attention_scores * self.scale
          A_    :			List[str]									=    nn.softmax(snake_case_   , axis=2    )

          # attend to values
          A_    :			Optional[Any]									=    jnp.einsum('b i j, b j d -> b i d'   , snake_case_   , snake_case_    )

        A_    :			List[str]									=    self.reshape_batch_dim_to_heads(snake_case_    )
        A_    :			Tuple									=    self.proj_attn(snake_case_    )
        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa
      lowercase_ :					bool             =  False

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    FlaxAttention(
            self.dim   , self.n_heads   , self.d_head   , self.dropout   , self.use_memory_efficient_attention   , dtype=self.dtype    )
        # cross attention
        A_    :			Optional[Any]									=    FlaxAttention(
            self.dim   , self.n_heads   , self.d_head   , self.dropout   , self.use_memory_efficient_attention   , dtype=self.dtype    )
        A_    :			Optional[Any]									=    FlaxFeedForward(dim=self.dim   , dropout=self.dropout   , dtype=self.dtype    )
        A_    :			Any									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			Dict									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			Optional[int]									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			List[str]									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[int]									=    hidden_states
        if self.only_cross_attention:
          A_    :			Union[str, Any]									=    self.attna(self.norma(snake_case_    )   , snake_case_   , deterministic=snake_case_    )
        else:
          A_    :			Tuple									=    self.attna(self.norma(snake_case_    )   , deterministic=snake_case_    )
        A_    :			Optional[Any]									=    hidden_states + residual

        # cross attention
        A_    :			List[str]									=    hidden_states
        A_    :			List[str]									=    self.attna(self.norma(snake_case_    )   , snake_case_   , deterministic=snake_case_    )
        A_    :			Any									=    hidden_states + residual

        # feed forward
        A_    :			Dict									=    hidden_states
        A_    :			str									=    self.ff(self.norma(snake_case_    )   , deterministic=snake_case_    )
        A_    :			Optional[int]									=    hidden_states + residual

        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int             =  1
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa
      lowercase_ :					bool             =  False

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    nn.GroupNorm(num_groups=3_2   , epsilon=1E-5    )

        A_    :			Dict									=    self.n_heads * self.d_head
        if self.use_linear_projection:
          A_    :			Union[str, Any]									=    nn.Dense(snake_case_   , dtype=self.dtype    )
        else:
          A_    :			Any									=    nn.Conv(
              snake_case_   , kernel_size=(1, 1)   , strides=(1, 1)   , padding='VALID'   , dtype=self.dtype   , )

        A_    :			List[str]									=    [
            FlaxBasicTransformerBlock(
                snake_case_   , self.n_heads   , self.d_head   , dropout=self.dropout   , only_cross_attention=self.only_cross_attention   , dtype=self.dtype   , use_memory_efficient_attention=self.use_memory_efficient_attention   , )
            for _ in range(self.depth    )
        ]

        if self.use_linear_projection:
          A_    :			List[Any]									=    nn.Dense(snake_case_   , dtype=self.dtype    )
        else:
          A_    :			Dict									=    nn.Conv(
              snake_case_   , kernel_size=(1, 1)   , strides=(1, 1)   , padding='VALID'   , dtype=self.dtype   , )

        A_    :			List[str]									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_		,			A_		,			A_		,			A_    :			Any									=    hidden_states.shape
        A_    :			Any									=    hidden_states
        A_    :			List[Any]									=    self.norm(snake_case_    )
        if self.use_linear_projection:
          A_    :			str									=    hidden_states.reshape(snake_case_   , height * width   , snake_case_    )
          A_    :			Optional[Any]									=    self.proj_in(snake_case_    )
        else:
          A_    :			List[str]									=    self.proj_in(snake_case_    )
          A_    :			List[Any]									=    hidden_states.reshape(snake_case_   , height * width   , snake_case_    )

        for transformer_block in self.transformer_blocks:
          A_    :			int									=    transformer_block(snake_case_   , snake_case_   , deterministic=snake_case_    )

        if self.use_linear_projection:
          A_    :			Union[str, Any]									=    self.proj_out(snake_case_    )
          A_    :			Any									=    hidden_states.reshape(snake_case_   , snake_case_   , snake_case_   , snake_case_    )
        else:
          A_    :			int									=    hidden_states.reshape(snake_case_   , snake_case_   , snake_case_   , snake_case_    )
          A_    :			Union[str, Any]									=    self.proj_out(snake_case_    )

        A_    :			Union[str, Any]									=    hidden_states + residual
        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    FlaxGEGLU(self.dim   , self.dropout   , self.dtype    )
        A_    :			int									=    nn.Dense(self.dim   , dtype=self.dtype    )

      def __call__(	self   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Dict									=    self.net_a(snake_case_   , deterministic=snake_case_    )
        A_    :			Any									=    self.net_a(snake_case_    )
        return hidden_states


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Optional[int]									=    self.dim * 4
        A_    :			List[str]									=    nn.Dense(inner_dim * 2   , dtype=self.dtype    )
        A_    :			int									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    self.proj(snake_case_    )
        A_		,			A_    :			Any									=    jnp.split(snake_case_   , 2   , axis=2    )
        return self.dropout_layer(hidden_linear * nn.gelu(snake_case_    )   , deterministic=snake_case_    )