


from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


SCREAMING_SNAKE_CASE__       			=				logging.get_logger(__name__)







class     __lowerCamelCase     (  nn.Module     ):


							"""simple docstring"""

							lowerCAmelCase__          =				42
							lowerCAmelCase__          =				jnp.floataa




							def 				A__      (  self  )     ->       Union[str, Any]:






								'''simple docstring'''






								lowercase_			 = self.config.text_config
								lowercase_			 = self.config.vision_config

								lowercase_			 = self.config.projection_dim
								lowercase_			 = text_config.hidden_size
								lowercase_			 = vision_config.hidden_size

								lowercase_			 = FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
								lowercase_			 = FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__						,       lowerCAmelCase_  ).module_class

								lowercase_			 = text_module(lowerCAmelCase_						,       dtype=self.dtype  )
								lowercase_			 = vision_module(lowerCAmelCase_						,       dtype=self.dtype  )

								lowercase_			 = nn.Dense(
								    self.projection_dim						,       dtype=self.dtype						,       kernel_init=jax.nn.initializers.normal(0.02  )						,       use_bias=lowerCAmelCase_						,       )
								lowercase_			 = nn.Dense(
								    self.projection_dim						,       dtype=self.dtype						,       kernel_init=jax.nn.initializers.normal(0.02  )						,       use_bias=lowerCAmelCase_						,       )
								lowercase_			 = self.param("logit_scale"						,       jax.nn.initializers.ones						,       []  )




							def __call__(  self						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase = True						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       )     ->       Dict:






								'''simple docstring'''






								lowercase_			 = return_dict if return_dict is not None else self.config.return_dict

								lowercase_			 = self.vision_model(
								    pixel_values=lowerCAmelCase_						,       deterministic=lowerCAmelCase_						,       output_attentions=lowerCAmelCase_						,       output_hidden_states=lowerCAmelCase_						,       return_dict=lowerCAmelCase_						,       )

								lowercase_			 = self.text_model(
								    input_ids=lowerCAmelCase_						,       attention_mask=lowerCAmelCase_						,       token_type_ids=lowerCAmelCase_						,       position_ids=lowerCAmelCase_						,       deterministic=lowerCAmelCase_						,       output_attentions=lowerCAmelCase_						,       output_hidden_states=lowerCAmelCase_						,       return_dict=lowerCAmelCase_						,       )

								lowercase_			 = vision_outputs[1]
								lowercase_			 = self.visual_projection(lowerCAmelCase_  )

								lowercase_			 = text_outputs[1]
								lowercase_			 = self.text_projection(lowerCAmelCase_  )

								# normalized features
								lowercase_			 = image_embeds / jnp.linalg.norm(lowerCAmelCase_						,       axis=-1						,       keepdims=lowerCAmelCase_  )
								lowercase_			 = text_embeds / jnp.linalg.norm(lowerCAmelCase_						,       axis=-1						,       keepdims=lowerCAmelCase_  )

								# cosine similarity as logits
								lowercase_			 = jnp.exp(self.logit_scale  )
								lowercase_			 = jnp.matmul(lowerCAmelCase_						,       image_embeds.T  ) * logit_scale
								lowercase_			 = logits_per_text.T

								if not return_dict:
									return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

								return FlaxCLIPOutput(
								    logits_per_image=lowerCAmelCase_						,       logits_per_text=lowerCAmelCase_						,       text_embeds=lowerCAmelCase_						,       image_embeds=lowerCAmelCase_						,       text_model_output=lowerCAmelCase_						,       vision_model_output=lowerCAmelCase_						,       )







class     __lowerCamelCase     (  __a     ):


							"""simple docstring"""

							lowerCAmelCase__          =				HybridCLIPConfig
							lowerCAmelCase__          =				FlaxHybridCLIPModule




							def __init__(  self						,       UpperCAmelCase						,       UpperCAmelCase = None						,       UpperCAmelCase = 0						,       UpperCAmelCase = jnp.floataa						,       **UpperCAmelCase						,       )     ->       Dict:






								'''simple docstring'''






								if input_shape is None:
									lowercase_			 = ((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

								lowercase_			 = self.module_class(config=lowerCAmelCase_						,       dtype=lowerCAmelCase_						,       **lowerCAmelCase_  )
								super().__init__(lowerCAmelCase_						,       lowerCAmelCase_						,       input_shape=lowerCAmelCase_						,       seed=lowerCAmelCase_						,       dtype=lowerCAmelCase_  )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase = None  )     ->       FrozenDict:






								'''simple docstring'''






								lowercase_			 = jnp.zeros(input_shape[0]						,       dtype="i4"  )
								lowercase_			 = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_  ).shape[-1]  )						,       input_shape[0]  )
								lowercase_			 = jnp.ones_like(lowerCAmelCase_  )
								lowercase_			 = jnp.ones_like(lowerCAmelCase_  )

								lowercase_			 = jax.random.normal(lowerCAmelCase_						,       input_shape[1]  )

								lowercase_ ,	lowercase_			 = jax.random.split(lowerCAmelCase_  )
								lowercase_			 = {"params": params_rng, "dropout": dropout_rng}

								return self.module.init(lowerCAmelCase_						,       lowerCAmelCase_						,       lowerCAmelCase_						,       lowerCAmelCase_						,       lowerCAmelCase_						,       lowerCAmelCase_  )["params"]




							def __call__(  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase = None						,       UpperCAmelCase = None						,       UpperCAmelCase = False						,       UpperCAmelCase = None						,       UpperCAmelCase = None						,       UpperCAmelCase = None						,       )     ->       str:






								'''simple docstring'''






								lowercase_			 = output_attentions if output_attentions is not None else self.config.output_attentions
								lowercase_			 = (
								    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
								)
								lowercase_			 = return_dict if return_dict is not None else self.config.return_dict

								if position_ids is None:
									lowercase_			 = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_  ).shape[-1]  )						,       input_ids.shape  )

								if token_type_ids is None:
									lowercase_			 = jnp.zeros_like(lowerCAmelCase_  )

								if attention_mask is None:
									lowercase_			 = jnp.ones_like(lowerCAmelCase_  )

								# Handle any PRNG if needed
								lowercase_			 = {}
								if dropout_rng is not None:
									lowercase_			 = dropout_rng

								return self.module.apply(
								    {"params": params or self.params}						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       jnp.array(lowerCAmelCase_						,       dtype=jnp.floataa  )						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       not train						,       lowerCAmelCase_						,       lowerCAmelCase_						,       lowerCAmelCase_						,       rngs=lowerCAmelCase_						,       )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase = None						,       UpperCAmelCase = None						,       UpperCAmelCase=False						,       )     ->       Dict:






								'''simple docstring'''






								if position_ids is None:
									lowercase_			 = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(lowerCAmelCase_  ).shape[-1]  )						,       input_ids.shape  )

								if token_type_ids is None:
									lowercase_			 = jnp.zeros_like(lowerCAmelCase_  )

								if attention_mask is None:
									lowercase_			 = jnp.ones_like(lowerCAmelCase_  )

								# Handle any PRNG if needed
								lowercase_			 = {}
								if dropout_rng is not None:
									lowercase_			 = dropout_rng

								def _get_features(UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  ):
									lowercase_			 = module.text_model(
									    input_ids=lowerCAmelCase_						,       attention_mask=lowerCAmelCase_						,       position_ids=lowerCAmelCase_						,       token_type_ids=lowerCAmelCase_						,       deterministic=lowerCAmelCase_						,       )
									lowercase_			 = text_outputs[1]
									lowercase_			 = module.text_projection(lowerCAmelCase_  )
									return text_features

								return self.module.apply(
								    {"params": params or self.params}						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       jnp.array(lowerCAmelCase_						,       dtype="i4"  )						,       not train						,       method=_get_features						,       rngs=lowerCAmelCase_						,       )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase = None						,       UpperCAmelCase = None						,       UpperCAmelCase=False  )     ->       Optional[int]:






								'''simple docstring'''






								lowercase_			 = {}
								if dropout_rng is not None:
									lowercase_			 = dropout_rng

								def _get_features(UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  ):
									lowercase_			 = module.vision_model(pixel_values=lowerCAmelCase_						,       deterministic=lowerCAmelCase_  )
									lowercase_			 = vision_outputs[1]  # pooled_output
									lowercase_			 = module.visual_projection(lowerCAmelCase_  )
									return image_features

								return self.module.apply(
								    {"params": params or self.params}						,       jnp.array(lowerCAmelCase_						,       dtype=jnp.floataa  )						,       not train						,       method=_get_features						,       rngs=lowerCAmelCase_						,       )




							@classmethod
							def 				A__      (  cls						,       UpperCAmelCase = None						,       UpperCAmelCase = None						,       *UpperCAmelCase						,       **UpperCAmelCase						,       )     ->       FlaxPreTrainedModel:






								'''simple docstring'''






								lowercase_			 = {
								    argument[len("text_"  ) :]: value for argument, value in kwargs.items() if argument.startswith("text_"  )
								}

								lowercase_			 = {
								    argument[len("vision_"  ) :]: value for argument, value in kwargs.items() if argument.startswith("vision_"  )
								}

								# remove text, vision kwargs from kwargs
								for key in kwargs_text.keys():
									del kwargs["text_" + key]
								for key in kwargs_vision.keys():
									del kwargs["vision_" + key]

								# Load and initialize the text and vision model
								lowercase_			 = kwargs_text.pop("model"						,       lowerCAmelCase_  )
								if text_model is None:
									assert (
									    text_model_name_or_path is not None
									), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
									from transformers import FlaxAutoModel

									if "config" not in kwargs_text:
										from transformers import AutoConfig

										lowercase_			 = AutoConfig.from_pretrained(lowerCAmelCase_  )
										lowercase_			 = text_config

									lowercase_			 = FlaxAutoModel.from_pretrained(lowerCAmelCase_						,       *lowerCAmelCase_						,       **lowerCAmelCase_  )

								lowercase_			 = kwargs_vision.pop("model"						,       lowerCAmelCase_  )
								if vision_model is None:
									assert (
									    vision_model_name_or_path is not None
									), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
									from transformers import FlaxAutoModel

									if "config" not in kwargs_vision:
										from transformers import AutoConfig

										lowercase_			 = AutoConfig.from_pretrained(lowerCAmelCase_  )
										lowercase_			 = vision_config

									lowercase_			 = FlaxAutoModel.from_pretrained(lowerCAmelCase_						,       *lowerCAmelCase_						,       **lowerCAmelCase_  )

								# instantiate config with corresponding kwargs
								lowercase_			 = kwargs.pop("dtype"						,       jnp.floataa  )
								lowercase_			 = HybridCLIPConfig.from_text_vision_configs(text_model.config						,       vision_model.config						,       **lowerCAmelCase_  )

								# init model
								lowercase_			 = cls(lowerCAmelCase_						,       *lowerCAmelCase_						,       dtype=lowerCAmelCase_						,       **lowerCAmelCase_  )

								if vision_config.model_type == "clip":
									lowercase_			 = vision_model.params["vision_model"]
									lowercase_			 = vision_model.params["visual_projection"]["kernel"]
								else:
									lowercase_			 = vision_model.params

								lowercase_			 = text_model.params

								return model






'''simple docstring'''


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowercase											=      logging.get_logger(__name__)

lowercase											=      {
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 __magic_name__       :			str	=   "gpt2"
 __magic_name__       :			str	=   ["past_key_values"]
 __magic_name__       :			Dict	=   {
     "hidden_size": "n_embd",
     "max_position_embeddings": "n_positions",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }




 def __init__( self				,	lowerCAmelCase_=5_0_2_5_7				,	lowerCAmelCase_=1_0_2_4				,	lowerCAmelCase_=7_6_8				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=None				,	lowerCAmelCase_="gelu_new"				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=1e-5				,	lowerCAmelCase_=0.0_2				,	lowerCAmelCase_="cls_index"				,	lowerCAmelCase_=True				,	lowerCAmelCase_=None				,	lowerCAmelCase_=True				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False				,	**lowerCAmelCase_				,	)			->   Dict:



      """simple docstring"""







      a_									=vocab_size
      a_									=n_positions
      a_									=n_embd
      a_									=n_layer
      a_									=n_head
      a_									=n_inner
      a_									=activation_function
      a_									=resid_pdrop
      a_									=embd_pdrop
      a_									=attn_pdrop
      a_									=layer_norm_epsilon
      a_									=initializer_range
      a_									=summary_type
      a_									=summary_use_proj
      a_									=summary_activation
      a_									=summary_first_dropout
      a_									=summary_proj_to_labels
      a_									=scale_attn_weights
      a_									=use_cache
      a_									=scale_attn_by_inverse_layer_idx
      a_									=reorder_and_upcast_attn

      a_									=bos_token_id
      a_									=eos_token_id

      super().__init__(bos_token_id=lowerCAmelCase_				,	eos_token_id=lowerCAmelCase_				,	**lowerCAmelCase_)







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 def __init__( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = "default"				,	lowerCAmelCase_ = None				,	lowerCAmelCase_ = False				,	)			->   List[str]:



      """simple docstring"""







      super().__init__(lowerCAmelCase_				,	task=lowerCAmelCase_				,	patching_specs=lowerCAmelCase_				,	use_past=lowerCAmelCase_)
      if not getattr(self._config				,	"pad_token_id"				,	lowerCAmelCase_):
           # TODO: how to do that better?
           a_									=0




 @property
 def    lowercase_	( self)			->   Mapping[str, Mapping[int, str]]:



      """simple docstring"""







      a_									=OrderedDict({"input_ids": {0: "batch", 1: "sequence"}})
      if self.use_past:
           self.fill_with_past_key_values_(lowerCAmelCase_				,	direction="inputs")
           a_									={0: "batch", 1: "past_sequence + sequence"}
      else:
           a_									={0: "batch", 1: "sequence"}

      return common_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_layer




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_head




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = False				,	lowerCAmelCase_ = None				,	)			->   Mapping[str, Any]:



      """simple docstring"""







      a_									=super(lowerCAmelCase_				,	self).generate_dummy_inputs(
          lowerCAmelCase_				,	batch_size=lowerCAmelCase_				,	seq_length=lowerCAmelCase_				,	is_pair=lowerCAmelCase_				,	framework=lowerCAmelCase_)

      # We need to order the input in the way they appears in the forward()
      a_									=OrderedDict({"input_ids": common_inputs["input_ids"]})

      # Need to add the past_keys
      if self.use_past:
           if not is_torch_available():
                raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.")
           else:
                import torch

                a_		,    a_									=common_inputs["input_ids"].shape
                # Not using the same length for past_key_values
                a_									=seqlen + 2
                a_									=(
                    batch,
                    self.num_attention_heads,
                    past_key_values_length,
                    self._config.hidden_size // self.num_attention_heads,
                )
                a_									=[
                    (torch.zeros(lowerCAmelCase_), torch.zeros(lowerCAmelCase_)) for _ in range(self.num_layers)
                ]

      a_									=common_inputs["attention_mask"]
      if self.use_past:
           a_									=ordered_inputs["attention_mask"].dtype
           a_									=torch.cat(
               [ordered_inputs["attention_mask"], torch.ones(lowerCAmelCase_				,	lowerCAmelCase_				,	dtype=lowerCAmelCase_)]				,	dim=1)

      return ordered_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return 1_3


