
from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


UpperCAmelCase__						:	Tuple	     = logging.get_logger(__name__)






class UpperCAmelCase  (		nn.Module  ):





   '''simple docstring'''
   __UpperCamelCase						:       HybridCLIPConfig
   __UpperCamelCase						:       jnp.dtype												=       jnp.floataa


   def 					__magic_name__							(   self	:				Tuple	):

      """simple docstring"""
      _A:    str						     =			self.config.text_config
      _A:    int						     =			self.config.vision_config

      _A:    Optional[Any]						     =			self.config.projection_dim
      _A:    Optional[int]						     =			text_config.hidden_size
      _A:    Union[str, Any]						     =			vision_config.hidden_size

      _A:    Tuple						     =			FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
      _A:    Union[str, Any]						     =			FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__     ,     lowerCAmelCase_	).module_class

      _A:    str						     =			text_module(lowerCAmelCase_     ,     dtype=self.dtype	)
      _A:    int						     =			vision_module(lowerCAmelCase_     ,     dtype=self.dtype	)

      _A:    Union[str, Any]						     =			nn.Dense(
          self.projection_dim     ,     dtype=self.dtype     ,     kernel_init=jax.nn.initializers.normal(0.02	)     ,     use_bias=lowerCAmelCase_     ,     )
      _A:    List[str]						     =			nn.Dense(
          self.projection_dim     ,     dtype=self.dtype     ,     kernel_init=jax.nn.initializers.normal(0.02	)     ,     use_bias=lowerCAmelCase_     ,     )
      _A:    Tuple						     =			self.param('''logit_scale'''     ,     jax.nn.initializers.ones     ,     []	)


   def __call__(   self	:				Optional[int]     ,     lowerCAmelCase_	:				int=None     ,     lowerCAmelCase_	:				Any=None     ,     lowerCAmelCase_	:				Optional[Any]=None     ,     lowerCAmelCase_	:				Optional[Any]=None     ,     lowerCAmelCase_	:				Tuple=None     ,     lowerCAmelCase_	:				bool = True     ,     lowerCAmelCase_	:				List[str]=None     ,     lowerCAmelCase_	:				List[Any]=None     ,     lowerCAmelCase_	:				int=None     ,     ):

      """simple docstring"""
      _A:    str						     =			return_dict if return_dict is not None else self.config.return_dict

      _A:    str						     =			self.vision_model(
          pixel_values=lowerCAmelCase_     ,     deterministic=lowerCAmelCase_     ,     output_attentions=lowerCAmelCase_     ,     output_hidden_states=lowerCAmelCase_     ,     return_dict=lowerCAmelCase_     ,     )

      _A:    Tuple						     =			self.text_model(
          input_ids=lowerCAmelCase_     ,     attention_mask=lowerCAmelCase_     ,     token_type_ids=lowerCAmelCase_     ,     position_ids=lowerCAmelCase_     ,     deterministic=lowerCAmelCase_     ,     output_attentions=lowerCAmelCase_     ,     output_hidden_states=lowerCAmelCase_     ,     return_dict=lowerCAmelCase_     ,     )

      _A:    Tuple						     =			vision_outputs[1]
      _A:    Union[str, Any]						     =			self.visual_projection(lowerCAmelCase_	)

      _A:    Tuple						     =			text_outputs[1]
      _A:    Optional[Any]						     =			self.text_projection(lowerCAmelCase_	)

      # normalized features
      _A:    str						     =			image_embeds / jnp.linalg.norm(lowerCAmelCase_     ,     axis=-1     ,     keepdims=lowerCAmelCase_	)
      _A:    Dict						     =			text_embeds / jnp.linalg.norm(lowerCAmelCase_     ,     axis=-1     ,     keepdims=lowerCAmelCase_	)

      # cosine similarity as logits
      _A:    str						     =			jnp.exp(self.logit_scale	)
      _A:    int						     =			jnp.matmul(lowerCAmelCase_     ,     image_embeds.T	) * logit_scale
      _A:    Optional[int]						     =			logits_per_text.T

      if not return_dict:
         return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

      return FlaxCLIPOutput(
          logits_per_image=lowerCAmelCase_     ,     logits_per_text=lowerCAmelCase_     ,     text_embeds=lowerCAmelCase_     ,     image_embeds=lowerCAmelCase_     ,     text_model_output=lowerCAmelCase_     ,     vision_model_output=lowerCAmelCase_     ,     )






class UpperCAmelCase  (		SCREAMING_SNAKE_CASE__  ):





   '''simple docstring'''
   __UpperCamelCase						:       Any												=       HybridCLIPConfig
   __UpperCamelCase						:       Union[str, Any]												=       FlaxHybridCLIPModule


   def __init__(   self	:				Tuple     ,     lowerCAmelCase_	:				HybridCLIPConfig     ,     lowerCAmelCase_	:				Optional[Tuple] = None     ,     lowerCAmelCase_	:				int = 0     ,     lowerCAmelCase_	:				jnp.dtype = jnp.floataa     ,     **lowerCAmelCase_	:				Tuple     ,     ):

      """simple docstring"""
      if input_shape is None:
         _A:    Tuple						     =			((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

      _A:    List[Any]						     =			self.module_class(config=lowerCAmelCase_     ,     dtype=lowerCAmelCase_     ,     **lowerCAmelCase_	)
      super().__init__(lowerCAmelCase_     ,     lowerCAmelCase_     ,     input_shape=lowerCAmelCase_     ,     seed=lowerCAmelCase_     ,     dtype=lowerCAmelCase_	)


   def 					__magic_name__							(   self	:				Optional[int]     ,     lowerCAmelCase_	:				jax.random.PRNGKey     ,     lowerCAmelCase_	:				Tuple     ,     lowerCAmelCase_	:				FrozenDict = None	):

      """simple docstring"""
      # init input tensor
      _A:    List[Any]						     =			jnp.zeros(input_shape[0]     ,     dtype='''i4'''	)
      _A:    str						     =			jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_	).shape[-1]	)     ,     input_shape[0]	)
      _A:    Dict						     =			jnp.ones_like(lowerCAmelCase_	)
      _A:    Dict						     =			jnp.ones_like(lowerCAmelCase_	)

      _A:    Dict						     =			jax.random.normal(lowerCAmelCase_     ,     input_shape[1]	)

      _A     ,    _A:    List[Any]						     =			jax.random.split(lowerCAmelCase_	)
      _A:    Tuple						     =			{'''params''': params_rng, '''dropout''': dropout_rng}

      return self.module.init(lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_	)["params"]


   def __call__(   self	:				Optional[int]     ,     lowerCAmelCase_	:				List[str]     ,     lowerCAmelCase_	:				Dict     ,     lowerCAmelCase_	:				List[str]=None     ,     lowerCAmelCase_	:				Any=None     ,     lowerCAmelCase_	:				List[Any]=None     ,     lowerCAmelCase_	:				dict = None     ,     lowerCAmelCase_	:				jax.random.PRNGKey = None     ,     lowerCAmelCase_	:				bool = False     ,     lowerCAmelCase_	:				Optional[bool] = None     ,     lowerCAmelCase_	:				Optional[bool] = None     ,     lowerCAmelCase_	:				Optional[bool] = None     ,     ):

      """simple docstring"""
      _A:    List[str]						     =			output_attentions if output_attentions is not None else self.config.output_attentions
      _A:    Optional[Any]						     =			(
          output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
      )
      _A:    Any						     =			return_dict if return_dict is not None else self.config.return_dict

      if position_ids is None:
         _A:    List[Any]						     =			jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_	).shape[-1]	)     ,     input_ids.shape	)

      if token_type_ids is None:
         _A:    Tuple						     =			jnp.zeros_like(lowerCAmelCase_	)

      if attention_mask is None:
         _A:    Any						     =			jnp.ones_like(lowerCAmelCase_	)

      # Handle any PRNG if needed
      _A:    Dict						     =			{}
      if dropout_rng is not None:
         _A:    Dict						     =			dropout_rng

      return self.module.apply(
          {'''params''': params or self.params}     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     jnp.array(lowerCAmelCase_     ,     dtype=jnp.floataa	)     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     not train     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     lowerCAmelCase_     ,     rngs=lowerCAmelCase_     ,     )


   def 					__magic_name__							(   self	:				Optional[int]     ,     lowerCAmelCase_	:				int     ,     lowerCAmelCase_	:				Union[str, Any]=None     ,     lowerCAmelCase_	:				List[Any]=None     ,     lowerCAmelCase_	:				Union[str, Any]=None     ,     lowerCAmelCase_	:				dict = None     ,     lowerCAmelCase_	:				jax.random.PRNGKey = None     ,     lowerCAmelCase_	:				str=False     ,     ):

      """simple docstring"""
      if position_ids is None:
         _A:    Tuple						     =			jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_	).shape[-1]	)     ,     input_ids.shape	)

      if token_type_ids is None:
         _A:    Union[str, Any]						     =			jnp.zeros_like(lowerCAmelCase_	)

      if attention_mask is None:
         _A:    int						     =			jnp.ones_like(lowerCAmelCase_	)

      # Handle any PRNG if needed
      _A:    str						     =			{}
      if dropout_rng is not None:
         _A:    Optional[int]						     =			dropout_rng

      def _get_features(lowerCAmelCase_	:				str     ,     lowerCAmelCase_	:				List[str]     ,     lowerCAmelCase_	:				List[str]     ,     lowerCAmelCase_	:				int     ,     lowerCAmelCase_	:				str     ,     lowerCAmelCase_	:				Optional[int]	):
         _A:    List[str]						     =			module.text_model(
             input_ids=lowerCAmelCase_     ,     attention_mask=lowerCAmelCase_     ,     position_ids=lowerCAmelCase_     ,     token_type_ids=lowerCAmelCase_     ,     deterministic=lowerCAmelCase_     ,     )
         _A:    int						     =			text_outputs[1]
         _A:    Union[str, Any]						     =			module.text_projection(lowerCAmelCase_	)
         return text_features

      return self.module.apply(
          {'''params''': params or self.params}     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     jnp.array(lowerCAmelCase_     ,     dtype='''i4'''	)     ,     not train     ,     method=_get_features     ,     rngs=lowerCAmelCase_     ,     )


   def 					__magic_name__							(   self	:				List[str]     ,     lowerCAmelCase_	:				Any     ,     lowerCAmelCase_	:				dict = None     ,     lowerCAmelCase_	:				jax.random.PRNGKey = None     ,     lowerCAmelCase_	:				Tuple=False	):

      """simple docstring"""
      _A:    Any						     =			{}
      if dropout_rng is not None:
         _A:    List[str]						     =			dropout_rng

      def _get_features(lowerCAmelCase_	:				List[Any]     ,     lowerCAmelCase_	:				int     ,     lowerCAmelCase_	:				Tuple	):
         _A:    Dict						     =			module.vision_model(pixel_values=lowerCAmelCase_     ,     deterministic=lowerCAmelCase_	)
         _A:    List[Any]						     =			vision_outputs[1]  # pooled_output
         _A:    Any						     =			module.visual_projection(lowerCAmelCase_	)
         return image_features

      return self.module.apply(
          {'''params''': params or self.params}     ,     jnp.array(lowerCAmelCase_     ,     dtype=jnp.floataa	)     ,     not train     ,     method=_get_features     ,     rngs=lowerCAmelCase_     ,     )


   @classmethod
   def 					__magic_name__							(   cls	:				Dict     ,     lowerCAmelCase_	:				str = None     ,     lowerCAmelCase_	:				str = None     ,     *lowerCAmelCase_	:				List[Any]     ,     **lowerCAmelCase_	:				Union[str, Any]     ,     ):

      """simple docstring"""
      _A:    int						     =			{
          argument[len('''text_'''	) :]: value for argument, value in kwargs.items() if argument.startswith('''text_'''	)
      }

      _A:    List[str]						     =			{
          argument[len('''vision_'''	) :]: value for argument, value in kwargs.items() if argument.startswith('''vision_'''	)
      }

      # remove text, vision kwargs from kwargs
      for key in kwargs_text.keys():
         del kwargs["text_" + key]
      for key in kwargs_vision.keys():
         del kwargs["vision_" + key]

      # Load and initialize the text and vision model
      _A:    Optional[Any]						     =			kwargs_text.pop('''model'''     ,     lowerCAmelCase_	)
      if text_model is None:
         assert (
             text_model_name_or_path is not None
         ), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
         from transformers import FlaxAutoModel

         if "config" not in kwargs_text:
            from transformers import AutoConfig

            _A:    Optional[int]						     =			AutoConfig.from_pretrained(lowerCAmelCase_	)
            _A:    Tuple						     =			text_config

         _A:    int						     =			FlaxAutoModel.from_pretrained(lowerCAmelCase_     ,     *lowerCAmelCase_     ,     **lowerCAmelCase_	)

      _A:    Optional[int]						     =			kwargs_vision.pop('''model'''     ,     lowerCAmelCase_	)
      if vision_model is None:
         assert (
             vision_model_name_or_path is not None
         ), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
         from transformers import FlaxAutoModel

         if "config" not in kwargs_vision:
            from transformers import AutoConfig

            _A:    List[Any]						     =			AutoConfig.from_pretrained(lowerCAmelCase_	)
            _A:    Optional[int]						     =			vision_config

         _A:    Any						     =			FlaxAutoModel.from_pretrained(lowerCAmelCase_     ,     *lowerCAmelCase_     ,     **lowerCAmelCase_	)

      # instantiate config with corresponding kwargs
      _A:    Optional[Any]						     =			kwargs.pop('''dtype'''     ,     jnp.floataa	)
      _A:    int						     =			HybridCLIPConfig.from_text_vision_configs(text_model.config     ,     vision_model.config     ,     **lowerCAmelCase_	)

      # init model
      _A:    Any						     =			cls(lowerCAmelCase_     ,     *lowerCAmelCase_     ,     dtype=lowerCAmelCase_     ,     **lowerCAmelCase_	)

      if vision_config.model_type == "clip":
         _A:    str						     =			vision_model.params['''vision_model''']
         _A:    List[Any]						     =			vision_model.params['''visual_projection''']['''kernel''']
      else:
         _A:    Union[str, Any]						     =			vision_model.params

      _A:    Tuple						     =			text_model.params

      return model





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase__						:	List[str]	     = logging.get_logger(__name__)

UpperCAmelCase__						:	str	     = {
    'flaubert/flaubert_small_cased': 'https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/config.json',
    'flaubert/flaubert_base_uncased': 'https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/config.json',
    'flaubert/flaubert_base_cased': 'https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/config.json',
    'flaubert/flaubert_large_cased': 'https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/config.json',
}






class UpperCAmelCase  (		SCREAMING_SNAKE_CASE__  ):





   '''simple docstring'''
   __UpperCamelCase						:       Tuple												=       '''flaubert'''
   __UpperCamelCase						:       str												=       {
       '''hidden_size''': '''emb_dim''',
       '''num_attention_heads''': '''n_heads''',
       '''num_hidden_layers''': '''n_layers''',
       '''n_words''': '''vocab_size''',  # For backward compatibility
   }


   def __init__(   self	:				Union[str, Any]     ,     lowerCAmelCase_	:				Union[str, Any]=False     ,     lowerCAmelCase_	:				List[str]=0.0     ,     lowerCAmelCase_	:				Dict=3_0_1_4_5     ,     lowerCAmelCase_	:				Tuple=2_0_4_8     ,     lowerCAmelCase_	:				List[Any]=1_2     ,     lowerCAmelCase_	:				str=1_6     ,     lowerCAmelCase_	:				Union[str, Any]=0.1     ,     lowerCAmelCase_	:				List[str]=0.1     ,     lowerCAmelCase_	:				str=True     ,     lowerCAmelCase_	:				Optional[int]=False     ,     lowerCAmelCase_	:				Any=False     ,     lowerCAmelCase_	:				int=False     ,     lowerCAmelCase_	:				int=1     ,     lowerCAmelCase_	:				List[str]=True     ,     lowerCAmelCase_	:				Dict=5_1_2     ,     lowerCAmelCase_	:				Any=2_0_4_8**-0.5     ,     lowerCAmelCase_	:				List[str]=1e-12     ,     lowerCAmelCase_	:				Any=0.02     ,     lowerCAmelCase_	:				Optional[Any]=0     ,     lowerCAmelCase_	:				Union[str, Any]=1     ,     lowerCAmelCase_	:				List[Any]=2     ,     lowerCAmelCase_	:				Dict=3     ,     lowerCAmelCase_	:				Optional[Any]=5     ,     lowerCAmelCase_	:				Any=True     ,     lowerCAmelCase_	:				Union[str, Any]="first"     ,     lowerCAmelCase_	:				List[Any]=True     ,     lowerCAmelCase_	:				Tuple=None     ,     lowerCAmelCase_	:				Optional[int]=True     ,     lowerCAmelCase_	:				int=0.1     ,     lowerCAmelCase_	:				Optional[int]=5     ,     lowerCAmelCase_	:				Union[str, Any]=5     ,     lowerCAmelCase_	:				List[Any]=0     ,     lowerCAmelCase_	:				Dict=0     ,     lowerCAmelCase_	:				Union[str, Any]=2     ,     lowerCAmelCase_	:				Optional[int]=0     ,     **lowerCAmelCase_	:				Any     ,     ):

      """simple docstring"""
      _A:    List[str]						     =			pre_norm
      _A:    List[Any]						     =			layerdrop
      _A:    Dict						     =			vocab_size
      _A:    List[Any]						     =			emb_dim
      _A:    Optional[Any]						     =			n_layers
      _A:    Union[str, Any]						     =			n_heads
      _A:    Optional[Any]						     =			dropout
      _A:    Optional[Any]						     =			attention_dropout
      _A:    Dict						     =			gelu_activation
      _A:    Tuple						     =			sinusoidal_embeddings
      _A:    Tuple						     =			causal
      _A:    List[Any]						     =			asm
      _A:    Dict						     =			n_langs
      _A:    Optional[int]						     =			use_lang_emb
      _A:    Any						     =			layer_norm_eps
      _A:    str						     =			bos_index
      _A:    Optional[int]						     =			eos_index
      _A:    int						     =			pad_index
      _A:    Tuple						     =			unk_index
      _A:    int						     =			mask_index
      _A:    int						     =			is_encoder
      _A:    Union[str, Any]						     =			max_position_embeddings
      _A:    List[str]						     =			embed_init_std
      _A:    List[str]						     =			init_std
      _A:    Tuple						     =			summary_type
      _A:    List[str]						     =			summary_use_proj
      _A:    Union[str, Any]						     =			summary_activation
      _A:    Union[str, Any]						     =			summary_proj_to_labels
      _A:    int						     =			summary_first_dropout
      _A:    Optional[Any]						     =			start_n_top
      _A:    Optional[Any]						     =			end_n_top
      _A:    Dict						     =			mask_token_id
      _A:    str						     =			lang_id

      if "n_words" in kwargs:
         _A:    int						     =			kwargs['''n_words''']

      super().__init__(pad_token_id=lowerCAmelCase_     ,     bos_token_id=lowerCAmelCase_     ,     **lowerCAmelCase_	)






class UpperCAmelCase  (		SCREAMING_SNAKE_CASE__  ):





   '''simple docstring'''


   @property
   def 					__magic_name__							(   self	:				Tuple	):

      """simple docstring"""
      if self.task == "multiple-choice":
         _A:    int						     =			{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
      else:
         _A:    Any						     =			{0: '''batch''', 1: '''sequence'''}
      return OrderedDict(
          [
              ('''input_ids''', dynamic_axis),
              ('''attention_mask''', dynamic_axis),
          ]	)



