
from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


_UpperCAmelCase   :					Tuple    =			logging.get_logger(__name__)


class 							lowercase   (						nn.Module  ):
		__lowercase			:			HybridCLIPConfig
		__lowercase			:			jnp.dtype      	=							jnp.floataa



		def __UpperCamelCase					(					self			)    ->   Optional[int]:
				"""simple docstring"""
				UpperCamelCase            =	self.config.text_config
				UpperCamelCase            =	self.config.vision_config

				UpperCamelCase            =	self.config.projection_dim
				UpperCamelCase            =	text_config.hidden_size
				UpperCamelCase            =	vision_config.hidden_size

				UpperCamelCase            =	FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
				UpperCamelCase            =	FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__							, lowerCAmelCase_			).module_class

				UpperCamelCase            =	text_module(lowerCAmelCase_							, dtype=self.dtype			)
				UpperCamelCase            =	vision_module(lowerCAmelCase_							, dtype=self.dtype			)

				UpperCamelCase            =	nn.Dense(
				    self.projection_dim							, dtype=self.dtype							, kernel_init=jax.nn.initializers.normal(0.02			)							, use_bias=lowerCAmelCase_							, )
				UpperCamelCase            =	nn.Dense(
				    self.projection_dim							, dtype=self.dtype							, kernel_init=jax.nn.initializers.normal(0.02			)							, use_bias=lowerCAmelCase_							, )
				UpperCamelCase            =	self.param('logit_scale'							, jax.nn.initializers.ones							, []			)



		def __call__(					self							, A_=None							, A_=None							, A_=None							, A_=None							, A_=None							, A_ = True							, A_=None							, A_=None							, A_=None							, )    ->   List[str]:
				"""simple docstring"""
				UpperCamelCase            =	return_dict if return_dict is not None else self.config.return_dict

				UpperCamelCase            =	self.vision_model(
				    pixel_values=lowerCAmelCase_							, deterministic=lowerCAmelCase_							, output_attentions=lowerCAmelCase_							, output_hidden_states=lowerCAmelCase_							, return_dict=lowerCAmelCase_							, )

				UpperCamelCase            =	self.text_model(
				    input_ids=lowerCAmelCase_							, attention_mask=lowerCAmelCase_							, token_type_ids=lowerCAmelCase_							, position_ids=lowerCAmelCase_							, deterministic=lowerCAmelCase_							, output_attentions=lowerCAmelCase_							, output_hidden_states=lowerCAmelCase_							, return_dict=lowerCAmelCase_							, )

				UpperCamelCase            =	vision_outputs[1]
				UpperCamelCase            =	self.visual_projection(lowerCAmelCase_			)

				UpperCamelCase            =	text_outputs[1]
				UpperCamelCase            =	self.text_projection(lowerCAmelCase_			)

				# normalized features
				UpperCamelCase            =	image_embeds / jnp.linalg.norm(lowerCAmelCase_							, axis=-1							, keepdims=lowerCAmelCase_			)
				UpperCamelCase            =	text_embeds / jnp.linalg.norm(lowerCAmelCase_							, axis=-1							, keepdims=lowerCAmelCase_			)

				# cosine similarity as logits
				UpperCamelCase            =	jnp.exp(self.logit_scale			)
				UpperCamelCase            =	jnp.matmul(lowerCAmelCase_							, image_embeds.T			) * logit_scale
				UpperCamelCase            =	logits_per_text.T

				if not return_dict:
						return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

				return FlaxCLIPOutput(
				    logits_per_image=lowerCAmelCase_							, logits_per_text=lowerCAmelCase_							, text_embeds=lowerCAmelCase_							, image_embeds=lowerCAmelCase_							, text_model_output=lowerCAmelCase_							, vision_model_output=lowerCAmelCase_							, )






class 							lowercase   (						SCREAMING_SNAKE_CASE__  ):
		__lowercase			:			Any      	=							HybridCLIPConfig
		__lowercase			:			Union[str, Any]      	=							FlaxHybridCLIPModule



		def __init__(					self							, A_							, A_ = None							, A_ = 0							, A_ = jnp.floataa							, **A_							, )    ->   str:
				"""simple docstring"""
				if input_shape is None:
						UpperCamelCase            =	((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

				UpperCamelCase            =	self.module_class(config=lowerCAmelCase_							, dtype=lowerCAmelCase_							, **lowerCAmelCase_			)
				super().__init__(lowerCAmelCase_							, lowerCAmelCase_							, input_shape=lowerCAmelCase_							, seed=lowerCAmelCase_							, dtype=lowerCAmelCase_			)



		def __UpperCamelCase					(					self							, A_							, A_							, A_ = None			)    ->   Optional[Any]:
				"""simple docstring"""
				UpperCamelCase            =	jnp.zeros(input_shape[0]							, dtype='i4'			)
				UpperCamelCase            =	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_			).shape[-1]			)							, input_shape[0]			)
				UpperCamelCase            =	jnp.ones_like(lowerCAmelCase_			)
				UpperCamelCase            =	jnp.ones_like(lowerCAmelCase_			)

				UpperCamelCase            =	jax.random.normal(lowerCAmelCase_							, input_shape[1]			)

				UpperCamelCase            =	jax.random.split(lowerCAmelCase_			)
				UpperCamelCase            =	{'''params''': params_rng, '''dropout''': dropout_rng}

				return self.module.init(lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_			)["params"]



		def __call__(					self							, A_							, A_							, A_=None							, A_=None							, A_=None							, A_ = None							, A_ = None							, A_ = False							, A_ = None							, A_ = None							, A_ = None							, )    ->   Union[str, Any]:
				"""simple docstring"""
				UpperCamelCase            =	output_attentions if output_attentions is not None else self.config.output_attentions
				UpperCamelCase            =	(
				    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
				)
				UpperCamelCase            =	return_dict if return_dict is not None else self.config.return_dict

				if position_ids is None:
						UpperCamelCase            =	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_			).shape[-1]			)							, input_ids.shape			)

				if token_type_ids is None:
						UpperCamelCase            =	jnp.zeros_like(lowerCAmelCase_			)

				if attention_mask is None:
						UpperCamelCase            =	jnp.ones_like(lowerCAmelCase_			)

				# Handle any PRNG if needed
				UpperCamelCase            =	{}
				if dropout_rng is not None:
						UpperCamelCase            =	dropout_rng

				return self.module.apply(
				    {'params': params or self.params}							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, jnp.array(lowerCAmelCase_							, dtype=jnp.floataa			)							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, not train							, lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_							, rngs=lowerCAmelCase_							, )



		def __UpperCamelCase					(					self							, A_							, A_=None							, A_=None							, A_=None							, A_ = None							, A_ = None							, A_=False							, )    ->   Any:
				"""simple docstring"""
				if position_ids is None:
						UpperCamelCase            =	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_			).shape[-1]			)							, input_ids.shape			)

				if token_type_ids is None:
						UpperCamelCase            =	jnp.zeros_like(lowerCAmelCase_			)

				if attention_mask is None:
						UpperCamelCase            =	jnp.ones_like(lowerCAmelCase_			)

				# Handle any PRNG if needed
				UpperCamelCase            =	{}
				if dropout_rng is not None:
						UpperCamelCase            =	dropout_rng

				def _get_features(A_							, A_							, A_							, A_							, A_							, A_			):
						UpperCamelCase            =	module.text_model(
						    input_ids=lowerCAmelCase_							, attention_mask=lowerCAmelCase_							, position_ids=lowerCAmelCase_							, token_type_ids=lowerCAmelCase_							, deterministic=lowerCAmelCase_							, )
						UpperCamelCase            =	text_outputs[1]
						UpperCamelCase            =	module.text_projection(lowerCAmelCase_			)
						return text_features

				return self.module.apply(
				    {'params': params or self.params}							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, jnp.array(lowerCAmelCase_							, dtype='i4'			)							, not train							, method=_get_features							, rngs=lowerCAmelCase_							, )



		def __UpperCamelCase					(					self							, A_							, A_ = None							, A_ = None							, A_=False			)    ->   int:
				"""simple docstring"""
				UpperCamelCase            =	{}
				if dropout_rng is not None:
						UpperCamelCase            =	dropout_rng

				def _get_features(A_							, A_							, A_			):
						UpperCamelCase            =	module.vision_model(pixel_values=lowerCAmelCase_							, deterministic=lowerCAmelCase_			)
						UpperCamelCase            =	vision_outputs[1]  # pooled_output
						UpperCamelCase            =	module.visual_projection(lowerCAmelCase_			)
						return image_features

				return self.module.apply(
				    {'params': params or self.params}							, jnp.array(lowerCAmelCase_							, dtype=jnp.floataa			)							, not train							, method=_get_features							, rngs=lowerCAmelCase_							, )



		@classmethod
		def __UpperCamelCase					(					cls							, A_ = None							, A_ = None							, *A_							, **A_							, )    ->   List[str]:
				"""simple docstring"""
				UpperCamelCase            =	{
				    argument[len('text_'			) :]: value for argument, value in kwargs.items() if argument.startswith('text_'			)
				}

				UpperCamelCase            =	{
				    argument[len('vision_'			) :]: value for argument, value in kwargs.items() if argument.startswith('vision_'			)
				}

				# remove text, vision kwargs from kwargs
				for key in kwargs_text.keys():
						del kwargs["text_" + key]
				for key in kwargs_vision.keys():
						del kwargs["vision_" + key]

				# Load and initialize the text and vision model
				UpperCamelCase            =	kwargs_text.pop('model'							, lowerCAmelCase_			)
				if text_model is None:
						assert (
						    text_model_name_or_path is not None
						), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
						from transformers import FlaxAutoModel

						if "config" not in kwargs_text:
								from transformers import AutoConfig

								UpperCamelCase            =	AutoConfig.from_pretrained(lowerCAmelCase_			)
								UpperCamelCase            =	text_config

						UpperCamelCase            =	FlaxAutoModel.from_pretrained(lowerCAmelCase_							, *lowerCAmelCase_							, **lowerCAmelCase_			)

				UpperCamelCase            =	kwargs_vision.pop('model'							, lowerCAmelCase_			)
				if vision_model is None:
						assert (
						    vision_model_name_or_path is not None
						), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
						from transformers import FlaxAutoModel

						if "config" not in kwargs_vision:
								from transformers import AutoConfig

								UpperCamelCase            =	AutoConfig.from_pretrained(lowerCAmelCase_			)
								UpperCamelCase            =	vision_config

						UpperCamelCase            =	FlaxAutoModel.from_pretrained(lowerCAmelCase_							, *lowerCAmelCase_							, **lowerCAmelCase_			)

				# instantiate config with corresponding kwargs
				UpperCamelCase            =	kwargs.pop('dtype'							, jnp.floataa			)
				UpperCamelCase            =	HybridCLIPConfig.from_text_vision_configs(text_model.config							, vision_model.config							, **lowerCAmelCase_			)

				# init model
				UpperCamelCase            =	cls(lowerCAmelCase_							, *lowerCAmelCase_							, dtype=lowerCAmelCase_							, **lowerCAmelCase_			)

				if vision_config.model_type == "clip":
						UpperCamelCase            =	vision_model.params['''vision_model''']
						UpperCamelCase            =	vision_model.params['''visual_projection''']['''kernel''']
				else:
						UpperCamelCase            =	vision_model.params

				UpperCamelCase            =	text_model.params

				return model


import math as m






def     lowerCamelCase__				(   a						,						a						,						a=0.001 ) ->  Optional[int]:
   return (f(a + h ) - f(a - h )) / (2 * h)






def     lowerCamelCase__				(   a						,						a=0						,						a=1_00						,						a=0.0001						,						a=1E-6						,						a=False ) ->  List[Any]:
   _A:    Optional[Any]						     =			xa  # set the initial guess
   _A:    Dict						     =			[a]
   _A:    Tuple						     =			abs(f(a ) )
   _A:    Tuple						     =			lambda a : calc_derivative(a						,						a						,						h=a )  # noqa: E731  Derivative of f(x)
   for _ in range(a ):
      if fa(a ) == 0:
         raise ValueError('''No converging solution found''' )
      _A:    Any						     =			a - f(a ) / fa(a )  # Calculate the next estimate
      if logsteps:
         steps.append(a )
      if error < maxerror:
         break
   else:
      raise ValueError('''Iteration limit reached, no converging solution found''' )
   if logsteps:
      # If logstep is true, then log intermediate steps
      return a, error, steps
   return a, error


if __name__ == "__main__":
   from matplotlib import pyplot as plt

   UpperCAmelCase__						:	Optional[Any]	     = lambda x: m.tanh(x) ** 2 - m.exp(3 * x)  # noqa: E731
   UpperCAmelCase__     ,UpperCAmelCase__     ,UpperCAmelCase__						:	str	     = newton_raphson(
       f, xa=10, maxiter=1000, step=1E-6, logsteps=True
   )
   plt.plot([abs(f(x)) for x in steps])
   plt.xlabel('step')
   plt.ylabel('error')
   plt.show()
   print(F"""solution = {{{solution:f}}}, error = {{{error:f}}}""")



