


import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp
def 	_lowerCamelCase  (		A_  :	Tuple							,  A_  :	Tuple							,  A_  :	Tuple							,  A_  :	Optional[Any]							,  A_  :	int = 4_0_9_6      )			->			Union[str, Any]:

   '''simple docstring'''






   UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			Union[str, Any] 						=key.shape[-3:]
   UpperCamelCase__ :			List[str] 						=value.shape[-1]
   UpperCamelCase__ :			str 						=min(A_							,  A_      )
   UpperCamelCase__ :			Any 						=query / jnp.sqrt(A_      )

   @functools.partial(jax.checkpoint							,  prevent_cse=A_      )
   def summarize_chunk(A_  :	List[str]							,  A_  :	Any							,  A_  :	Union[str, Any]      ):
      UpperCamelCase__ :			Union[str, Any] 						=jnp.einsum("...qhd,...khd->...qhk"							,  A_							,  A_							,  precision=A_      )

      UpperCamelCase__ :			int 						=jnp.max(A_							,  axis=-1							,  keepdims=A_      )
      UpperCamelCase__ :			Tuple 						=jax.lax.stop_gradient(A_      )
      UpperCamelCase__ :			Tuple 						=jnp.exp(attn_weights - max_score      )

      UpperCamelCase__ :			Optional[Any] 						=jnp.einsum("...vhf,...qhv->...qhf"							,  A_							,  A_							,  precision=A_      )
      UpperCamelCase__ :			int 						=jnp.einsum("...qhk->...qh"							,  A_      )

      return (exp_values, exp_weights.sum(axis=-1      ), max_score)

   def chunk_scanner(A_  :	Any      ):
      # julienne key array
      UpperCamelCase__ :			Optional[int] 						=jax.lax.dynamic_slice(
          operand=A_							,  start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]							,  slice_sizes=list(key.shape[:-3]      ) + [key_chunk_size, num_heads, k_features]							,  )

      # julienne value array
      UpperCamelCase__ :			List[str] 						=jax.lax.dynamic_slice(
          operand=A_							,  start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]							,  slice_sizes=list(value.shape[:-3]      ) + [key_chunk_size, num_heads, v_features]							,  )

      return summarize_chunk(A_							,  A_							,  A_      )

   UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			int 						=jax.lax.map(f=A_							,  xs=jnp.arange(0							,  A_							,  A_      )      )

   UpperCamelCase__ :			Any 						=jnp.max(A_							,  axis=0							,  keepdims=A_      )
   UpperCamelCase__ :			Optional[int] 						=jnp.exp(chunk_max - global_max      )

   chunk_values *= jnp.expand_dims(A_							,  axis=-1      )
   chunk_weights *= max_diffs

   UpperCamelCase__ :			str 						=chunk_values.sum(axis=0      )
   UpperCamelCase__ :			str 						=jnp.expand_dims(A_							,  -1      ).sum(axis=0      )

   return all_values / all_weights
def 	_lowerCamelCase  (		A_  :	str							,  A_  :	Any							,  A_  :	str							,  A_  :	Optional[Any]=jax.lax.Precision.HIGHEST							,  A_  :	int = 1_0_2_4							,  A_  :	int = 4_0_9_6      )			->			Any:

   '''simple docstring'''






   UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			int 						=query.shape[-3:]

   def chunk_scanner(A_  :	Any							,  A_  :	Optional[Any]      ):
      # julienne query array
      UpperCamelCase__ :			List[str] 						=jax.lax.dynamic_slice(
          operand=A_							,  start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]							,  slice_sizes=list(query.shape[:-3]      ) + [min(A_							,  A_      ), num_heads, q_features]							,  )

      return (
          chunk_idx + query_chunk_size,  # unused ignore it
          _query_chunk_attention(
              query=A_							,  key=A_							,  value=A_							,  precision=A_							,  key_chunk_size=A_      ),
      )

   UpperCamelCase__						,	UpperCamelCase__ :			Optional[int] 						=jax.lax.scan(
       f=A_							,  init=0							,  xs=A_							,  length=math.ceil(num_q / query_chunk_size      )      )

   return jnp.concatenate(A_							,  axis=-3      )  # fuse the chunked result back



class 							lowercase__(						nn.Module			):






      '''simple docstring'''







      snake_case__       =					42
      snake_case__       =					8
      snake_case__       =					6_4
      snake_case__       =					0.0
      snake_case__       =					False
      snake_case__       =					jnp.floataa




      def    UpperCAmelCase							(		self)					->						Optional[Any]:







         """simple docstring"""







         UpperCamelCase__ :			str 						=self.dim_head * self.heads
         UpperCamelCase__ :			Dict 						=self.dim_head**-0.5

         # Weights were exported with old names {to_q, to_k, to_v, to_out}
         UpperCamelCase__ :			int 						=nn.Dense(__SCREAMING_SNAKE_CASE   ,			use_bias=__SCREAMING_SNAKE_CASE   ,			dtype=self.dtype   ,			name="to_q")
         UpperCamelCase__ :			List[str] 						=nn.Dense(__SCREAMING_SNAKE_CASE   ,			use_bias=__SCREAMING_SNAKE_CASE   ,			dtype=self.dtype   ,			name="to_k")
         UpperCamelCase__ :			Union[str, Any] 						=nn.Dense(__SCREAMING_SNAKE_CASE   ,			use_bias=__SCREAMING_SNAKE_CASE   ,			dtype=self.dtype   ,			name="to_v")

         UpperCamelCase__ :			str 						=nn.Dense(self.query_dim   ,			dtype=self.dtype   ,			name="to_out_0")
         UpperCamelCase__ :			int 						=nn.Dropout(rate=self.dropout)




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE)					->						Union[str, Any]:







         """simple docstring"""







         UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			Tuple 						=tensor.shape
         UpperCamelCase__ :			str 						=self.heads
         UpperCamelCase__ :			Any 						=tensor.reshape(__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			dim // head_size)
         UpperCamelCase__ :			str 						=jnp.transpose(__SCREAMING_SNAKE_CASE   ,			(0, 2, 1, 3))
         UpperCamelCase__ :			Tuple 						=tensor.reshape(batch_size * head_size   ,			__SCREAMING_SNAKE_CASE   ,			dim // head_size)
         return tensor




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE)					->						Union[str, Any]:







         """simple docstring"""







         UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			Union[str, Any] 						=tensor.shape
         UpperCamelCase__ :			Union[str, Any] 						=self.heads
         UpperCamelCase__ :			Optional[int] 						=tensor.reshape(batch_size // head_size   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			Optional[int] 						=jnp.transpose(__SCREAMING_SNAKE_CASE   ,			(0, 2, 1, 3))
         UpperCamelCase__ :			Any 						=tensor.reshape(batch_size // head_size   ,			__SCREAMING_SNAKE_CASE   ,			dim * head_size)
         return tensor




      def __call__(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE=None   ,			__SCREAMING_SNAKE_CASE=True)					->						int:







         """simple docstring"""







         UpperCamelCase__ :			List[str] 						=hidden_states if context is None else context

         UpperCamelCase__ :			int 						=self.query(__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			Any 						=self.key(__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			List[Any] 						=self.value(__SCREAMING_SNAKE_CASE)

         UpperCamelCase__ :			List[Any] 						=self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			Optional[Any] 						=self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			str 						=self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE)

         if self.use_memory_efficient_attention:
            UpperCamelCase__ :			List[Any] 						=query_states.transpose(1   ,			0   ,			2)
            UpperCamelCase__ :			List[str] 						=key_states.transpose(1   ,			0   ,			2)
            UpperCamelCase__ :			List[Any] 						=value_states.transpose(1   ,			0   ,			2)

            # this if statement create a chunk size for each layer of the unet
            # the chunk size is equal to the query_length dimension of the deepest layer of the unet

            UpperCamelCase__ :			List[str] 						=query_states.shape[-3]
            if flatten_latent_dim % 64 == 0:
               UpperCamelCase__ :			str 						=int(flatten_latent_dim / 64)
            elif flatten_latent_dim % 16 == 0:
               UpperCamelCase__ :			str 						=int(flatten_latent_dim / 16)
            elif flatten_latent_dim % 4 == 0:
               UpperCamelCase__ :			Any 						=int(flatten_latent_dim / 4)
            else:
               UpperCamelCase__ :			str 						=int(__SCREAMING_SNAKE_CASE)

            UpperCamelCase__ :			Any 						=jax_memory_efficient_attention(
                __SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			query_chunk_size=__SCREAMING_SNAKE_CASE   ,			key_chunk_size=40_96 * 4)

            UpperCamelCase__ :			int 						=hidden_states.transpose(1   ,			0   ,			2)
         else:
            # compute attentions
            UpperCamelCase__ :			Dict 						=jnp.einsum("b i d, b j d->b i j"   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)
            UpperCamelCase__ :			Tuple 						=attention_scores * self.scale
            UpperCamelCase__ :			int 						=nn.softmax(__SCREAMING_SNAKE_CASE   ,			axis=2)

            # attend to values
            UpperCamelCase__ :			List[Any] 						=jnp.einsum("b i j, b j d -> b i d"   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)

         UpperCamelCase__ :			Union[str, Any] 						=self.reshape_batch_dim_to_heads(__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			Tuple 						=self.proj_attn(__SCREAMING_SNAKE_CASE)
         return self.dropout_layer(__SCREAMING_SNAKE_CASE   ,			deterministic=__SCREAMING_SNAKE_CASE)




class 							lowercase__(						nn.Module			):






      '''simple docstring'''







      snake_case__       =					42
      snake_case__       =					42
      snake_case__       =					42
      snake_case__       =					0.0
      snake_case__       =					False
      snake_case__       =					jnp.floataa
      snake_case__       =					False




      def    UpperCAmelCase							(		self)					->						Tuple:







         """simple docstring"""







         UpperCamelCase__ :			str 						=FlaxAttention(
             self.dim   ,			self.n_heads   ,			self.d_head   ,			self.dropout   ,			self.use_memory_efficient_attention   ,			dtype=self.dtype)
         # cross attention
         UpperCamelCase__ :			str 						=FlaxAttention(
             self.dim   ,			self.n_heads   ,			self.d_head   ,			self.dropout   ,			self.use_memory_efficient_attention   ,			dtype=self.dtype)
         UpperCamelCase__ :			Optional[int] 						=FlaxFeedForward(dim=self.dim   ,			dropout=self.dropout   ,			dtype=self.dtype)
         UpperCamelCase__ :			Dict 						=nn.LayerNorm(epsilon=1E-5   ,			dtype=self.dtype)
         UpperCamelCase__ :			List[str] 						=nn.LayerNorm(epsilon=1E-5   ,			dtype=self.dtype)
         UpperCamelCase__ :			str 						=nn.LayerNorm(epsilon=1E-5   ,			dtype=self.dtype)
         UpperCamelCase__ :			Any 						=nn.Dropout(rate=self.dropout)




      def __call__(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE=True)					->						Union[str, Any]:







         """simple docstring"""







         UpperCamelCase__ :			Tuple 						=hidden_states
         if self.only_cross_attention:
            UpperCamelCase__ :			List[Any] 						=self.attna(self.norma(__SCREAMING_SNAKE_CASE)   ,			__SCREAMING_SNAKE_CASE   ,			deterministic=__SCREAMING_SNAKE_CASE)
         else:
            UpperCamelCase__ :			Dict 						=self.attna(self.norma(__SCREAMING_SNAKE_CASE)   ,			deterministic=__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			List[Any] 						=hidden_states + residual

         # cross attention
         UpperCamelCase__ :			List[str] 						=hidden_states
         UpperCamelCase__ :			Optional[Any] 						=self.attna(self.norma(__SCREAMING_SNAKE_CASE)   ,			__SCREAMING_SNAKE_CASE   ,			deterministic=__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			Optional[Any] 						=hidden_states + residual

         # feed forward
         UpperCamelCase__ :			Any 						=hidden_states
         UpperCamelCase__ :			Any 						=self.ff(self.norma(__SCREAMING_SNAKE_CASE)   ,			deterministic=__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			int 						=hidden_states + residual

         return self.dropout_layer(__SCREAMING_SNAKE_CASE   ,			deterministic=__SCREAMING_SNAKE_CASE)




class 							lowercase__(						nn.Module			):






      '''simple docstring'''







      snake_case__       =					42
      snake_case__       =					42
      snake_case__       =					42
      snake_case__       =					1
      snake_case__       =					0.0
      snake_case__       =					False
      snake_case__       =					False
      snake_case__       =					jnp.floataa
      snake_case__       =					False




      def    UpperCAmelCase							(		self)					->						Optional[Any]:







         """simple docstring"""







         UpperCamelCase__ :			str 						=nn.GroupNorm(num_groups=32   ,			epsilon=1E-5)

         UpperCamelCase__ :			Optional[Any] 						=self.n_heads * self.d_head
         if self.use_linear_projection:
            UpperCamelCase__ :			Dict 						=nn.Dense(__SCREAMING_SNAKE_CASE   ,			dtype=self.dtype)
         else:
            UpperCamelCase__ :			Union[str, Any] 						=nn.Conv(
                __SCREAMING_SNAKE_CASE   ,			kernel_size=(1, 1)   ,			strides=(1, 1)   ,			padding="VALID"   ,			dtype=self.dtype   ,			)

         UpperCamelCase__ :			List[Any] 						=[
             FlaxBasicTransformerBlock(
                 __SCREAMING_SNAKE_CASE   ,			self.n_heads   ,			self.d_head   ,			dropout=self.dropout   ,			only_cross_attention=self.only_cross_attention   ,			dtype=self.dtype   ,			use_memory_efficient_attention=self.use_memory_efficient_attention   ,			)
             for _ in range(self.depth)
         ]

         if self.use_linear_projection:
            UpperCamelCase__ :			Tuple 						=nn.Dense(__SCREAMING_SNAKE_CASE   ,			dtype=self.dtype)
         else:
            UpperCamelCase__ :			List[Any] 						=nn.Conv(
                __SCREAMING_SNAKE_CASE   ,			kernel_size=(1, 1)   ,			strides=(1, 1)   ,			padding="VALID"   ,			dtype=self.dtype   ,			)

         UpperCamelCase__ :			Dict 						=nn.Dropout(rate=self.dropout)




      def __call__(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE=True)					->						Optional[Any]:







         """simple docstring"""







         UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			str 						=hidden_states.shape
         UpperCamelCase__ :			List[Any] 						=hidden_states
         UpperCamelCase__ :			Union[str, Any] 						=self.norm(__SCREAMING_SNAKE_CASE)
         if self.use_linear_projection:
            UpperCamelCase__ :			str 						=hidden_states.reshape(__SCREAMING_SNAKE_CASE   ,			height * width   ,			__SCREAMING_SNAKE_CASE)
            UpperCamelCase__ :			Optional[int] 						=self.proj_in(__SCREAMING_SNAKE_CASE)
         else:
            UpperCamelCase__ :			List[Any] 						=self.proj_in(__SCREAMING_SNAKE_CASE)
            UpperCamelCase__ :			str 						=hidden_states.reshape(__SCREAMING_SNAKE_CASE   ,			height * width   ,			__SCREAMING_SNAKE_CASE)

         for transformer_block in self.transformer_blocks:
            UpperCamelCase__ :			int 						=transformer_block(__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			deterministic=__SCREAMING_SNAKE_CASE)

         if self.use_linear_projection:
            UpperCamelCase__ :			int 						=self.proj_out(__SCREAMING_SNAKE_CASE)
            UpperCamelCase__ :			Optional[Any] 						=hidden_states.reshape(__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)
         else:
            UpperCamelCase__ :			List[str] 						=hidden_states.reshape(__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)
            UpperCamelCase__ :			str 						=self.proj_out(__SCREAMING_SNAKE_CASE)

         UpperCamelCase__ :			List[str] 						=hidden_states + residual
         return self.dropout_layer(__SCREAMING_SNAKE_CASE   ,			deterministic=__SCREAMING_SNAKE_CASE)




class 							lowercase__(						nn.Module			):






      '''simple docstring'''







      snake_case__       =					42
      snake_case__       =					0.0
      snake_case__       =					jnp.floataa




      def    UpperCAmelCase							(		self)					->						int:







         """simple docstring"""







         UpperCamelCase__ :			Union[str, Any] 						=FlaxGEGLU(self.dim   ,			self.dropout   ,			self.dtype)
         UpperCamelCase__ :			Tuple 						=nn.Dense(self.dim   ,			dtype=self.dtype)




      def __call__(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE=True)					->						Dict:







         """simple docstring"""







         UpperCamelCase__ :			Optional[Any] 						=self.net_a(__SCREAMING_SNAKE_CASE   ,			deterministic=__SCREAMING_SNAKE_CASE)
         UpperCamelCase__ :			List[Any] 						=self.net_a(__SCREAMING_SNAKE_CASE)
         return hidden_states




class 							lowercase__(						nn.Module			):






      '''simple docstring'''







      snake_case__       =					42
      snake_case__       =					0.0
      snake_case__       =					jnp.floataa




      def    UpperCAmelCase							(		self)					->						List[Any]:







         """simple docstring"""







         UpperCamelCase__ :			Optional[Any] 						=self.dim * 4
         UpperCamelCase__ :			Optional[Any] 						=nn.Dense(inner_dim * 2   ,			dtype=self.dtype)
         UpperCamelCase__ :			Tuple 						=nn.Dropout(rate=self.dropout)




      def __call__(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE=True)					->						Tuple:







         """simple docstring"""







         UpperCamelCase__ :			Tuple 						=self.proj(__SCREAMING_SNAKE_CASE)
         UpperCamelCase__						,	UpperCamelCase__ :			List[Any] 						=jnp.split(__SCREAMING_SNAKE_CASE   ,			2   ,			axis=2)
         return self.dropout_layer(hidden_linear * nn.gelu(__SCREAMING_SNAKE_CASE)   ,			deterministic=__SCREAMING_SNAKE_CASE)





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__UpperCAmelCase      			=  {"""configuration_sew_d""": ["""SEW_D_PRETRAINED_CONFIG_ARCHIVE_MAP""", """SEWDConfig"""]}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __UpperCAmelCase      			=  [
         """SEW_D_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """SEWDForCTC""",
         """SEWDForSequenceClassification""",
         """SEWDModel""",
         """SEWDPreTrainedModel""",
     ]

if TYPE_CHECKING:
     from .configuration_sew_d import SEW_D_PRETRAINED_CONFIG_ARCHIVE_MAP, SEWDConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_sew_d import (
              SEW_D_PRETRAINED_MODEL_ARCHIVE_LIST,
              SEWDForCTC,
              SEWDForSequenceClassification,
              SEWDModel,
              SEWDPreTrainedModel,
          )


else:
     import sys

     __UpperCAmelCase      			=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

