
import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp



def 		lowercase_     (  A__					,						A__					,						A__					,						A__					,						A__ = 4096						)				->				List[str]:
    """simple docstring"""

    snake_case	,				snake_case	,				snake_case						=   key.shape[-3:]
    snake_case						=   value.shape[-1]
    snake_case						=   min(A__					,						A__						)
    snake_case						=   query / jnp.sqrt(A__						)

    @functools.partial(jax.checkpoint					,						prevent_cse=A__						)
    def summarize_chunk(A__					,						A__					,						A__						):
        snake_case						=   jnp.einsum("...qhd,...khd->...qhk"					,						A__					,						A__					,						precision=A__						)

        snake_case						=   jnp.max(A__					,						axis=-1					,						keepdims=A__						)
        snake_case						=   jax.lax.stop_gradient(A__						)
        snake_case						=   jnp.exp(attn_weights - max_score						)

        snake_case						=   jnp.einsum("...vhf,...qhv->...qhf"					,						A__					,						A__					,						precision=A__						)
        snake_case						=   jnp.einsum("...qhk->...qh"					,						A__						)

        return (exp_values, exp_weights.sum(axis=-1						), max_score)

    def chunk_scanner(A__						):
        # julienne key array
        snake_case						=   jax.lax.dynamic_slice(
            operand=A__					,						start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]					,						slice_sizes=list(key.shape[:-3]						) + [key_chunk_size, num_heads, k_features]					,						)

        # julienne value array
        snake_case						=   jax.lax.dynamic_slice(
            operand=A__					,						start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]					,						slice_sizes=list(value.shape[:-3]						) + [key_chunk_size, num_heads, v_features]					,						)

        return summarize_chunk(A__					,						A__					,						A__						)

    snake_case	,				snake_case	,				snake_case						=   jax.lax.map(f=A__					,						xs=jnp.arange(0					,						A__					,						A__						)						)

    snake_case						=   jnp.max(A__					,						axis=0					,						keepdims=A__						)
    snake_case						=   jnp.exp(chunk_max - global_max						)

    chunk_values *= jnp.expand_dims(A__					,						axis=-1						)
    chunk_weights *= max_diffs

    snake_case						=   chunk_values.sum(axis=0						)
    snake_case						=   jnp.expand_dims(A__					,						-1						).sum(axis=0						)

    return all_values / all_weights



def 		lowercase_     (  A__					,						A__					,						A__					,						A__=jax.lax.Precision.HIGHEST					,						A__ = 1024					,						A__ = 4096						)				->				Dict:
    """simple docstring"""

    snake_case	,				snake_case	,				snake_case						=   query.shape[-3:]

    def chunk_scanner(A__					,						A__						):
        # julienne query array
        snake_case						=   jax.lax.dynamic_slice(
            operand=A__					,						start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]					,						slice_sizes=list(query.shape[:-3]						) + [min(A__					,						A__						), num_heads, q_features]					,						)

        return (
            chunk_idx + query_chunk_size,  # unused ignore it
            _query_chunk_attention(
                query=A__					,						key=A__					,						value=A__					,						precision=A__					,						key_chunk_size=A__						),
        )

    snake_case	,				snake_case						=   jax.lax.scan(
        f=A__					,						init=0					,						xs=A__					,						length=math.ceil(num_q / query_chunk_size						)						)

    return jnp.concatenate(A__					,						axis=-3						)  # fuse the chunked result back







class 	lowerCamelCase						(    nn.Module    ):
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				int 	=						8
     UpperCAmelCase__							:				int 	=						64
     UpperCAmelCase__							:				float 	=						0.0
     UpperCAmelCase__							:				bool 	=						False
     UpperCAmelCase__							:				jnp.dtype 	=						jnp.floataa





     def 						UpperCAmelCase(self :       Optional[int]  )		->		Optional[int]:
         snake_case						=   self.dim_head * self.heads
         snake_case						=   self.dim_head**-0.5

         # Weights were exported with old names {to_q, to_k, to_v, to_out}
         snake_case						=   nn.Dense(_A							,       use_bias=_A							,       dtype=self.dtype							,       name="to_q"  )
         snake_case						=   nn.Dense(_A							,       use_bias=_A							,       dtype=self.dtype							,       name="to_k"  )
         snake_case						=   nn.Dense(_A							,       use_bias=_A							,       dtype=self.dtype							,       name="to_v"  )

         snake_case						=   nn.Dense(self.query_dim							,       dtype=self.dtype							,       name="to_out_0"  )
         snake_case						=   nn.Dropout(rate=self.dropout  )





     def 						UpperCAmelCase(self :       List[str]							,       _A :       List[Any]  )		->		Any:
         snake_case	,				snake_case	,				snake_case						=   tensor.shape
         snake_case						=   self.heads
         snake_case						=   tensor.reshape(_A							,       _A							,       _A							,       dim // head_size  )
         snake_case						=   jnp.transpose(_A							,       (0, 2, 1, 3)  )
         snake_case						=   tensor.reshape(batch_size * head_size							,       _A							,       dim // head_size  )
         return tensor





     def 						UpperCAmelCase(self :       Tuple							,       _A :       str  )		->		str:
         snake_case	,				snake_case	,				snake_case						=   tensor.shape
         snake_case						=   self.heads
         snake_case						=   tensor.reshape(batch_size // head_size							,       _A							,       _A							,       _A  )
         snake_case						=   jnp.transpose(_A							,       (0, 2, 1, 3)  )
         snake_case						=   tensor.reshape(batch_size // head_size							,       _A							,       dim * head_size  )
         return tensor





     def __call__(self :       Tuple							,       _A :       int							,       _A :       int=None							,       _A :       Tuple=True  )		->		Tuple:
         snake_case						=   hidden_states if context is None else context

         snake_case						=   self.query(_A  )
         snake_case						=   self.key(_A  )
         snake_case						=   self.value(_A  )

         snake_case						=   self.reshape_heads_to_batch_dim(_A  )
         snake_case						=   self.reshape_heads_to_batch_dim(_A  )
         snake_case						=   self.reshape_heads_to_batch_dim(_A  )

         if self.use_memory_efficient_attention:
             snake_case						=   query_states.transpose(1							,       0							,       2  )
             snake_case						=   key_states.transpose(1							,       0							,       2  )
             snake_case						=   value_states.transpose(1							,       0							,       2  )

             # this if statement create a chunk size for each layer of the unet
             # the chunk size is equal to the query_length dimension of the deepest layer of the unet

             snake_case						=   query_states.shape[-3]
             if flatten_latent_dim % 6_4 == 0:
                 snake_case						=   int(flatten_latent_dim / 6_4  )
             elif flatten_latent_dim % 1_6 == 0:
                 snake_case						=   int(flatten_latent_dim / 1_6  )
             elif flatten_latent_dim % 4 == 0:
                 snake_case						=   int(flatten_latent_dim / 4  )
             else:
                 snake_case						=   int(_A  )

             snake_case						=   jax_memory_efficient_attention(
                 _A							,       _A							,       _A							,       query_chunk_size=_A							,       key_chunk_size=4_0_9_6 * 4  )

             snake_case						=   hidden_states.transpose(1							,       0							,       2  )
         else:
             # compute attentions
             snake_case						=   jnp.einsum("b i d, b j d->b i j"							,       _A							,       _A  )
             snake_case						=   attention_scores * self.scale
             snake_case						=   nn.softmax(_A							,       axis=2  )

             # attend to values
             snake_case						=   jnp.einsum("b i j, b j d -> b i d"							,       _A							,       _A  )

         snake_case						=   self.reshape_batch_dim_to_heads(_A  )
         snake_case						=   self.proj_attn(_A  )
         return self.dropout_layer(_A							,       deterministic=_A  )







class 	lowerCamelCase						(    nn.Module    ):
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				float 	=						0.0
     UpperCAmelCase__							:				bool 	=						False
     UpperCAmelCase__							:				jnp.dtype 	=						jnp.floataa
     UpperCAmelCase__							:				bool 	=						False





     def 						UpperCAmelCase(self :       Any  )		->		List[str]:
         # self attention (or cross_attention if only_cross_attention is True)
         snake_case						=   FlaxAttention(
             self.dim							,       self.n_heads							,       self.d_head							,       self.dropout							,       self.use_memory_efficient_attention							,       dtype=self.dtype  )
         # cross attention
         snake_case						=   FlaxAttention(
             self.dim							,       self.n_heads							,       self.d_head							,       self.dropout							,       self.use_memory_efficient_attention							,       dtype=self.dtype  )
         snake_case						=   FlaxFeedForward(dim=self.dim							,       dropout=self.dropout							,       dtype=self.dtype  )
         snake_case						=   nn.LayerNorm(epsilon=1E-5							,       dtype=self.dtype  )
         snake_case						=   nn.LayerNorm(epsilon=1E-5							,       dtype=self.dtype  )
         snake_case						=   nn.LayerNorm(epsilon=1E-5							,       dtype=self.dtype  )
         snake_case						=   nn.Dropout(rate=self.dropout  )





     def __call__(self :       Tuple							,       _A :       Optional[Any]							,       _A :       str							,       _A :       List[str]=True  )		->		Union[str, Any]:
         # self attention
         snake_case						=   hidden_states
         if self.only_cross_attention:
             snake_case						=   self.attna(self.norma(_A  )							,       _A							,       deterministic=_A  )
         else:
             snake_case						=   self.attna(self.norma(_A  )							,       deterministic=_A  )
         snake_case						=   hidden_states + residual

         # cross attention
         snake_case						=   hidden_states
         snake_case						=   self.attna(self.norma(_A  )							,       _A							,       deterministic=_A  )
         snake_case						=   hidden_states + residual

         # feed forward
         snake_case						=   hidden_states
         snake_case						=   self.ff(self.norma(_A  )							,       deterministic=_A  )
         snake_case						=   hidden_states + residual

         return self.dropout_layer(_A							,       deterministic=_A  )







class 	lowerCamelCase						(    nn.Module    ):
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				int 	=						1
     UpperCAmelCase__							:				float 	=						0.0
     UpperCAmelCase__							:				bool 	=						False
     UpperCAmelCase__							:				bool 	=						False
     UpperCAmelCase__							:				jnp.dtype 	=						jnp.floataa
     UpperCAmelCase__							:				bool 	=						False





     def 						UpperCAmelCase(self :       Optional[Any]  )		->		Dict:
         snake_case						=   nn.GroupNorm(num_groups=3_2							,       epsilon=1E-5  )

         snake_case						=   self.n_heads * self.d_head
         if self.use_linear_projection:
             snake_case						=   nn.Dense(_A							,       dtype=self.dtype  )
         else:
             snake_case						=   nn.Conv(
                 _A							,       kernel_size=(1, 1)							,       strides=(1, 1)							,       padding="VALID"							,       dtype=self.dtype							,       )

         snake_case						=   [
             FlaxBasicTransformerBlock(
                 _A							,       self.n_heads							,       self.d_head							,       dropout=self.dropout							,       only_cross_attention=self.only_cross_attention							,       dtype=self.dtype							,       use_memory_efficient_attention=self.use_memory_efficient_attention							,       )
             for _ in range(self.depth  )
         ]

         if self.use_linear_projection:
             snake_case						=   nn.Dense(_A							,       dtype=self.dtype  )
         else:
             snake_case						=   nn.Conv(
                 _A							,       kernel_size=(1, 1)							,       strides=(1, 1)							,       padding="VALID"							,       dtype=self.dtype							,       )

         snake_case						=   nn.Dropout(rate=self.dropout  )





     def __call__(self :       List[str]							,       _A :       Tuple							,       _A :       Optional[int]							,       _A :       Dict=True  )		->		int:
         snake_case	,				snake_case	,				snake_case	,				snake_case						=   hidden_states.shape
         snake_case						=   hidden_states
         snake_case						=   self.norm(_A  )
         if self.use_linear_projection:
             snake_case						=   hidden_states.reshape(_A							,       height * width							,       _A  )
             snake_case						=   self.proj_in(_A  )
         else:
             snake_case						=   self.proj_in(_A  )
             snake_case						=   hidden_states.reshape(_A							,       height * width							,       _A  )

         for transformer_block in self.transformer_blocks:
             snake_case						=   transformer_block(_A							,       _A							,       deterministic=_A  )

         if self.use_linear_projection:
             snake_case						=   self.proj_out(_A  )
             snake_case						=   hidden_states.reshape(_A							,       _A							,       _A							,       _A  )
         else:
             snake_case						=   hidden_states.reshape(_A							,       _A							,       _A							,       _A  )
             snake_case						=   self.proj_out(_A  )

         snake_case						=   hidden_states + residual
         return self.dropout_layer(_A							,       deterministic=_A  )







class 	lowerCamelCase						(    nn.Module    ):
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				float 	=						0.0
     UpperCAmelCase__							:				jnp.dtype 	=						jnp.floataa





     def 						UpperCAmelCase(self :       List[str]  )		->		List[Any]:
         # The second linear layer needs to be called
         # net_2 for now to match the index of the Sequential layer
         snake_case						=   FlaxGEGLU(self.dim							,       self.dropout							,       self.dtype  )
         snake_case						=   nn.Dense(self.dim							,       dtype=self.dtype  )





     def __call__(self :       str							,       _A :       Optional[Any]							,       _A :       Optional[int]=True  )		->		List[str]:
         snake_case						=   self.net_a(_A							,       deterministic=_A  )
         snake_case						=   self.net_a(_A  )
         return hidden_states







class 	lowerCamelCase						(    nn.Module    ):
     UpperCAmelCase__							:				int
     UpperCAmelCase__							:				float 	=						0.0
     UpperCAmelCase__							:				jnp.dtype 	=						jnp.floataa





     def 						UpperCAmelCase(self :       Dict  )		->		Union[str, Any]:
         snake_case						=   self.dim * 4
         snake_case						=   nn.Dense(inner_dim * 2							,       dtype=self.dtype  )
         snake_case						=   nn.Dropout(rate=self.dropout  )





     def __call__(self :       str							,       _A :       Dict							,       _A :       Any=True  )		->		Optional[Any]:
         snake_case						=   self.proj(_A  )
         snake_case	,				snake_case						=   jnp.split(_A							,       2							,       axis=2  )
         return self.dropout_layer(hidden_linear * nn.gelu(_A  )							,       deterministic=_A  )



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_A       =					{
    "configuration_splinter": ["SPLINTER_PRETRAINED_CONFIG_ARCHIVE_MAP", "SplinterConfig"],
    "tokenization_splinter": ["SplinterTokenizer"],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A       =					["SplinterTokenizerFast"]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A       =					[
       "SPLINTER_PRETRAINED_MODEL_ARCHIVE_LIST",
       "SplinterForQuestionAnswering",
       "SplinterForPreTraining",
       "SplinterLayer",
       "SplinterModel",
       "SplinterPreTrainedModel",
   ]


if TYPE_CHECKING:
   from .configuration_splinter import SPLINTER_PRETRAINED_CONFIG_ARCHIVE_MAP, SplinterConfig
   from .tokenization_splinter import SplinterTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_splinter_fast import SplinterTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_splinter import (
          SPLINTER_PRETRAINED_MODEL_ARCHIVE_LIST,
          SplinterForPreTraining,
          SplinterForQuestionAnswering,
          SplinterLayer,
          SplinterModel,
          SplinterPreTrainedModel,
      )


else:
   import sys

   _A       =					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

