






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp



def      UpperCAmelCase						( lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__ = 4096	):


		'''simple docstring'''

		__A ,   __A ,   __A    =							key.shape[-3:]
		__A    =							value.shape[-1]
		__A    =							min(A__					,     A__	)
		__A    =							query / jnp.sqrt(A__	)

		@functools.partial(jax.checkpoint					,     prevent_cse=A__	)
		def summarize_chunk(lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__	):
				__A    =							jnp.einsum("...qhd,...khd->...qhk"					,     A__					,     A__					,     precision=A__	)

				__A    =							jnp.max(A__					,     axis=-1					,     keepdims=A__	)
				__A    =							jax.lax.stop_gradient(A__	)
				__A    =							jnp.exp(attn_weights - max_score	)

				__A    =							jnp.einsum("...vhf,...qhv->...qhf"					,     A__					,     A__					,     precision=A__	)
				__A    =							jnp.einsum("...qhk->...qh"					,     A__	)

				return (exp_values, exp_weights.sum(axis=-1	), max_score)

		def chunk_scanner(lowerCAmelCase__	):
				# julienne key array
				__A    =							jax.lax.dynamic_slice(
				    operand=A__					,     start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]					,     slice_sizes=list(key.shape[:-3]	) + [key_chunk_size, num_heads, k_features]					,     )

				# julienne value array
				__A    =							jax.lax.dynamic_slice(
				    operand=A__					,     start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]					,     slice_sizes=list(value.shape[:-3]	) + [key_chunk_size, num_heads, v_features]					,     )

				return summarize_chunk(A__					,     A__					,     A__	)

		__A ,   __A ,   __A    =							jax.lax.map(f=A__					,     xs=jnp.arange(0					,     A__					,     A__	)	)

		__A    =							jnp.max(A__					,     axis=0					,     keepdims=A__	)
		__A    =							jnp.exp(chunk_max - global_max	)

		chunk_values *= jnp.expand_dims(A__					,     axis=-1	)
		chunk_weights *= max_diffs

		__A    =							chunk_values.sum(axis=0	)
		__A    =							jnp.expand_dims(A__					,     -1	).sum(axis=0	)

		return all_values / all_weights



def      UpperCAmelCase						( lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__=jax.lax.Precision.HIGHEST					,     lowerCAmelCase__ = 1024					,     lowerCAmelCase__ = 4096	):


		'''simple docstring'''

		__A ,   __A ,   __A    =							query.shape[-3:]

		def chunk_scanner(lowerCAmelCase__					,     lowerCAmelCase__	):
				# julienne query array
				__A    =							jax.lax.dynamic_slice(
				    operand=A__					,     start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]					,     slice_sizes=list(query.shape[:-3]	) + [min(A__					,     A__	), num_heads, q_features]					,     )

				return (
				    chunk_idx + query_chunk_size,  # unused ignore it
				    _query_chunk_attention(
				        query=A__					,     key=A__					,     value=A__					,     precision=A__					,     key_chunk_size=A__	),
				)

		__A ,   __A    =							jax.lax.scan(
		    f=A__					,     init=0					,     xs=A__					,     length=math.ceil(num_q / query_chunk_size	)	)

		return jnp.concatenate(A__					,     axis=-3	)  # fuse the chunked result back





class        a__      (	nn.Module		):
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       int 					= 8
		UpperCAmelCase_							:       int 					= 64
		UpperCAmelCase_							:       float 					= 0.0
		UpperCAmelCase_							:       bool 					= False
		UpperCAmelCase_							:       jnp.dtype 					= jnp.floataa






		def 					_lowerCamelCase					(      self						)				->				Optional[int]:
				__A    =							self.dim_head * self.heads
				__A    =							self.dim_head**-0.5

				# Weights were exported with old names {to_q, to_k, to_v, to_out}
				__A    =							nn.Dense(_A							, use_bias=_A							, dtype=self.dtype							, name="to_q"						)
				__A    =							nn.Dense(_A							, use_bias=_A							, dtype=self.dtype							, name="to_k"						)
				__A    =							nn.Dense(_A							, use_bias=_A							, dtype=self.dtype							, name="to_v"						)

				__A    =							nn.Dense(self.query_dim							, dtype=self.dtype							, name="to_out_0"						)
				__A    =							nn.Dropout(rate=self.dropout						)






		def 					_lowerCamelCase					(      self							, lowercase__						)				->				Any:
				__A ,   __A ,   __A    =							tensor.shape
				__A    =							self.heads
				__A    =							tensor.reshape(_A							, _A							, _A							, dim // head_size						)
				__A    =							jnp.transpose(_A							, (0, 2, 1, 3)						)
				__A    =							tensor.reshape(batch_size * head_size							, _A							, dim // head_size						)
				return tensor






		def 					_lowerCamelCase					(      self							, lowercase__						)				->				str:
				__A ,   __A ,   __A    =							tensor.shape
				__A    =							self.heads
				__A    =							tensor.reshape(batch_size // head_size							, _A							, _A							, _A						)
				__A    =							jnp.transpose(_A							, (0, 2, 1, 3)						)
				__A    =							tensor.reshape(batch_size // head_size							, _A							, dim * head_size						)
				return tensor






		def __call__(      self							, lowercase__							, lowercase__=None							, lowercase__=True						)				->				Tuple:
				__A    =							hidden_states if context is None else context

				__A    =							self.query(_A						)
				__A    =							self.key(_A						)
				__A    =							self.value(_A						)

				__A    =							self.reshape_heads_to_batch_dim(_A						)
				__A    =							self.reshape_heads_to_batch_dim(_A						)
				__A    =							self.reshape_heads_to_batch_dim(_A						)

				if self.use_memory_efficient_attention:
						__A    =							query_states.transpose(1							, 0							, 2						)
						__A    =							key_states.transpose(1							, 0							, 2						)
						__A    =							value_states.transpose(1							, 0							, 2						)

						# this if statement create a chunk size for each layer of the unet
						# the chunk size is equal to the query_length dimension of the deepest layer of the unet

						__A    =							query_states.shape[-3]
						if flatten_latent_dim % 64 == 0:
								__A    =							int(flatten_latent_dim / 64						)
						elif flatten_latent_dim % 16 == 0:
								__A    =							int(flatten_latent_dim / 16						)
						elif flatten_latent_dim % 4 == 0:
								__A    =							int(flatten_latent_dim / 4						)
						else:
								__A    =							int(_A						)

						__A    =							jax_memory_efficient_attention(
						    _A							, _A							, _A							, query_chunk_size=_A							, key_chunk_size=4096 * 4						)

						__A    =							hidden_states.transpose(1							, 0							, 2						)
				else:
						# compute attentions
						__A    =							jnp.einsum("b i d, b j d->b i j"							, _A							, _A						)
						__A    =							attention_scores * self.scale
						__A    =							nn.softmax(_A							, axis=2						)

						# attend to values
						__A    =							jnp.einsum("b i j, b j d -> b i d"							, _A							, _A						)

				__A    =							self.reshape_batch_dim_to_heads(_A						)
				__A    =							self.proj_attn(_A						)
				return self.dropout_layer(_A							, deterministic=_A						)





class        a__      (	nn.Module		):
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       float 					= 0.0
		UpperCAmelCase_							:       bool 					= False
		UpperCAmelCase_							:       jnp.dtype 					= jnp.floataa
		UpperCAmelCase_							:       bool 					= False






		def 					_lowerCamelCase					(      self						)				->				List[str]:
				# self attention (or cross_attention if only_cross_attention is True)
				__A    =							FlaxAttention(
				    self.dim							, self.n_heads							, self.d_head							, self.dropout							, self.use_memory_efficient_attention							, dtype=self.dtype						)
				# cross attention
				__A    =							FlaxAttention(
				    self.dim							, self.n_heads							, self.d_head							, self.dropout							, self.use_memory_efficient_attention							, dtype=self.dtype						)
				__A    =							FlaxFeedForward(dim=self.dim							, dropout=self.dropout							, dtype=self.dtype						)
				__A    =							nn.LayerNorm(epsilon=1e-5							, dtype=self.dtype						)
				__A    =							nn.LayerNorm(epsilon=1e-5							, dtype=self.dtype						)
				__A    =							nn.LayerNorm(epsilon=1e-5							, dtype=self.dtype						)
				__A    =							nn.Dropout(rate=self.dropout						)






		def __call__(      self							, lowercase__							, lowercase__							, lowercase__=True						)				->				Union[str, Any]:
				# self attention
				__A    =							hidden_states
				if self.only_cross_attention:
						__A    =							self.attna(self.norma(_A						)							, _A							, deterministic=_A						)
				else:
						__A    =							self.attna(self.norma(_A						)							, deterministic=_A						)
				__A    =							hidden_states + residual

				# cross attention
				__A    =							hidden_states
				__A    =							self.attna(self.norma(_A						)							, _A							, deterministic=_A						)
				__A    =							hidden_states + residual

				# feed forward
				__A    =							hidden_states
				__A    =							self.ff(self.norma(_A						)							, deterministic=_A						)
				__A    =							hidden_states + residual

				return self.dropout_layer(_A							, deterministic=_A						)





class        a__      (	nn.Module		):
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       int 					= 1
		UpperCAmelCase_							:       float 					= 0.0
		UpperCAmelCase_							:       bool 					= False
		UpperCAmelCase_							:       bool 					= False
		UpperCAmelCase_							:       jnp.dtype 					= jnp.floataa
		UpperCAmelCase_							:       bool 					= False






		def 					_lowerCamelCase					(      self						)				->				Dict:
				__A    =							nn.GroupNorm(num_groups=32							, epsilon=1e-5						)

				__A    =							self.n_heads * self.d_head
				if self.use_linear_projection:
						__A    =							nn.Dense(_A							, dtype=self.dtype						)
				else:
						__A    =							nn.Conv(
						    _A							, kernel_size=(1, 1)							, strides=(1, 1)							, padding="VALID"							, dtype=self.dtype							, )

				__A    =							[
				    FlaxBasicTransformerBlock(
				        _A							, self.n_heads							, self.d_head							, dropout=self.dropout							, only_cross_attention=self.only_cross_attention							, dtype=self.dtype							, use_memory_efficient_attention=self.use_memory_efficient_attention							, )
				    for _ in range(self.depth						)
				]

				if self.use_linear_projection:
						__A    =							nn.Dense(_A							, dtype=self.dtype						)
				else:
						__A    =							nn.Conv(
						    _A							, kernel_size=(1, 1)							, strides=(1, 1)							, padding="VALID"							, dtype=self.dtype							, )

				__A    =							nn.Dropout(rate=self.dropout						)






		def __call__(      self							, lowercase__							, lowercase__							, lowercase__=True						)				->				int:
				__A ,   __A ,   __A ,   __A    =							hidden_states.shape
				__A    =							hidden_states
				__A    =							self.norm(_A						)
				if self.use_linear_projection:
						__A    =							hidden_states.reshape(_A							, height * width							, _A						)
						__A    =							self.proj_in(_A						)
				else:
						__A    =							self.proj_in(_A						)
						__A    =							hidden_states.reshape(_A							, height * width							, _A						)

				for transformer_block in self.transformer_blocks:
						__A    =							transformer_block(_A							, _A							, deterministic=_A						)

				if self.use_linear_projection:
						__A    =							self.proj_out(_A						)
						__A    =							hidden_states.reshape(_A							, _A							, _A							, _A						)
				else:
						__A    =							hidden_states.reshape(_A							, _A							, _A							, _A						)
						__A    =							self.proj_out(_A						)

				__A    =							hidden_states + residual
				return self.dropout_layer(_A							, deterministic=_A						)





class        a__      (	nn.Module		):
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       float 					= 0.0
		UpperCAmelCase_							:       jnp.dtype 					= jnp.floataa






		def 					_lowerCamelCase					(      self						)				->				List[Any]:
				# The second linear layer needs to be called
				# net_2 for now to match the index of the Sequential layer
				__A    =							FlaxGEGLU(self.dim							, self.dropout							, self.dtype						)
				__A    =							nn.Dense(self.dim							, dtype=self.dtype						)






		def __call__(      self							, lowercase__							, lowercase__=True						)				->				List[str]:
				__A    =							self.net_a(_A							, deterministic=_A						)
				__A    =							self.net_a(_A						)
				return hidden_states





class        a__      (	nn.Module		):
		UpperCAmelCase_							:       int
		UpperCAmelCase_							:       float 					= 0.0
		UpperCAmelCase_							:       jnp.dtype 					= jnp.floataa






		def 					_lowerCamelCase					(      self						)				->				Union[str, Any]:
				__A    =							self.dim * 4
				__A    =							nn.Dense(inner_dim * 2							, dtype=self.dtype						)
				__A    =							nn.Dropout(rate=self.dropout						)






		def __call__(      self							, lowercase__							, lowercase__=True						)				->				Optional[Any]:
				__A    =							self.proj(_A						)
				__A ,   __A    =							jnp.split(_A							, 2							, axis=2						)
				return self.dropout_layer(hidden_linear * nn.gelu(_A						)							, deterministic=_A						)



import unittest

from transformers import load_tool
from transformers.tools.agent_types import AGENT_TYPE_MAPPING

from .test_tools_common import ToolTesterMixin, output_types







class 	lowerCamelCase						(    unittest.TestCase				,					A_    ):





     def 						UpperCAmelCase(self :       Any  )		->		int:
         snake_case						=   load_tool("translation"  )
         self.tool.setup()
         snake_case						=   load_tool("translation"							,       remote=_A  )





     def 						UpperCAmelCase(self :       Optional[Any]  )		->		Optional[int]:
         snake_case						=   self.tool("Hey, what's up?"							,       src_lang="English"							,       tgt_lang="French"  )
         self.assertEqual(_A							,       "- Hé, comment ça va?"  )





     def 						UpperCAmelCase(self :       str  )		->		int:
         snake_case						=   self.remote_tool("Hey, what's up?"							,       src_lang="English"							,       tgt_lang="French"  )
         self.assertEqual(_A							,       "- Hé, comment ça va?"  )





     def 						UpperCAmelCase(self :       str  )		->		List[str]:
         snake_case						=   self.tool(text="Hey, what's up?"							,       src_lang="English"							,       tgt_lang="French"  )
         self.assertEqual(_A							,       "- Hé, comment ça va?"  )





     def 						UpperCAmelCase(self :       Optional[Any]  )		->		str:
         snake_case						=   self.remote_tool(text="Hey, what's up?"							,       src_lang="English"							,       tgt_lang="French"  )
         self.assertEqual(_A							,       "- Hé, comment ça va?"  )





     def 						UpperCAmelCase(self :       Any  )		->		Tuple:
         snake_case						=   ["Hey, what's up?", "English", "Spanish"]
         snake_case						=   self.tool(*_A  )

         # There is a single output
         if len(self.tool.outputs  ) == 1:
             snake_case						=   [outputs]

         self.assertListEqual(output_types(_A  )							,       self.tool.outputs  )





     def 						UpperCAmelCase(self :       List[str]  )		->		List[Any]:
         snake_case						=   ["Hey, what's up?", "English", "Spanish"]
         snake_case						=   self.tool(*_A  )

         if not isinstance(_A							,       _A  ):
             snake_case						=   [outputs]

         self.assertEqual(len(_A  )							,       len(self.tool.outputs  )  )

         for output, output_type in zip(_A							,       self.tool.outputs  ):
             snake_case						=   AGENT_TYPE_MAPPING[output_type]
             self.assertTrue(isinstance(_A							,       _A  )  )





     def 						UpperCAmelCase(self :       int  )		->		Any:
         snake_case						=   ["Hey, what's up?", "English", "Spanish"]

         snake_case						=   []

         for _input, input_type in zip(_A							,       self.tool.inputs  ):
             if isinstance(_A							,       _A  ):
                 _inputs.append([AGENT_TYPE_MAPPING[_input_type](_input  ) for _input_type in input_type]  )
             else:
                 _inputs.append(AGENT_TYPE_MAPPING[input_type](_input  )  )

        # Should not raise an error
         snake_case						=   self.tool(*_A  )

         if not isinstance(_A							,       _A  ):
             snake_case						=   [outputs]

         self.assertEqual(len(_A  )							,       len(self.tool.outputs  )  )

