






"""simple docstring"""



from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


A_     =				logging.get_logger(__name__)






class 				__SCREAMING_SNAKE_CASE    (					nn.Module		):
							snake_case_  =				42
							snake_case_  =				jnp.floataa


							def     _UpperCamelCase							(      self   :    Optional[Any]				):
													'''simple docstring'''





													A__    :	Dict			=	self.config.text_config
													A__    :	Tuple			=	self.config.vision_config

													A__    :	Union[str, Any]			=	self.config.projection_dim
													A__    :	str			=	text_config.hidden_size
													A__    :	Any			=	vision_config.hidden_size

													A__    :	Tuple			=	FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
													A__    :	int			=	FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__      ,     snake_case				).module_class

													A__    :	List[Any]			=	text_module(snake_case      ,     dtype=self.dtype				)
													A__    :	List[Any]			=	vision_module(snake_case      ,     dtype=self.dtype				)

													A__    :	List[str]			=	nn.Dense(
													    self.projection_dim      ,     dtype=self.dtype      ,     kernel_init=jax.nn.initializers.normal(0.02				)      ,     use_bias=snake_case      ,     )
													A__    :	Any			=	nn.Dense(
													    self.projection_dim      ,     dtype=self.dtype      ,     kernel_init=jax.nn.initializers.normal(0.02				)      ,     use_bias=snake_case      ,     )
													A__    :	Any			=	self.param("""logit_scale"""      ,     jax.nn.initializers.ones      ,     []				)


							def __call__(      self   :    Optional[int]      ,     snake_case   :    str=None      ,     snake_case   :    Dict=None      ,     snake_case   :    int=None      ,     snake_case   :    Any=None      ,     snake_case   :    List[str]=None      ,     snake_case   :    bool = True      ,     snake_case   :    Dict=None      ,     snake_case   :    List[Any]=None      ,     snake_case   :    Any=None      ,     ):
													'''simple docstring'''





													A__    :	Tuple			=	return_dict if return_dict is not None else self.config.return_dict

													A__    :	Optional[int]			=	self.vision_model(
													    pixel_values=snake_case      ,     deterministic=snake_case      ,     output_attentions=snake_case      ,     output_hidden_states=snake_case      ,     return_dict=snake_case      ,     )

													A__    :	Union[str, Any]			=	self.text_model(
													    input_ids=snake_case      ,     attention_mask=snake_case      ,     token_type_ids=snake_case      ,     position_ids=snake_case      ,     deterministic=snake_case      ,     output_attentions=snake_case      ,     output_hidden_states=snake_case      ,     return_dict=snake_case      ,     )

													A__    :	int			=	vision_outputs[1]
													A__    :	Dict			=	self.visual_projection(snake_case				)

													A__    :	Optional[Any]			=	text_outputs[1]
													A__    :	Optional[Any]			=	self.text_projection(snake_case				)

													# normalized features
													A__    :	str			=	image_embeds / jnp.linalg.norm(snake_case      ,     axis=-1      ,     keepdims=snake_case				)
													A__    :	Dict			=	text_embeds / jnp.linalg.norm(snake_case      ,     axis=-1      ,     keepdims=snake_case				)

													# cosine similarity as logits
													A__    :	List[Any]			=	jnp.exp(self.logit_scale				)
													A__    :	Optional[int]			=	jnp.matmul(snake_case      ,     image_embeds.T				) * logit_scale
													A__    :	Optional[int]			=	logits_per_text.T

													if not return_dict:
																			return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

													return FlaxCLIPOutput(
													    logits_per_image=snake_case      ,     logits_per_text=snake_case      ,     text_embeds=snake_case      ,     image_embeds=snake_case      ,     text_model_output=snake_case      ,     vision_model_output=snake_case      ,     )






class 				__SCREAMING_SNAKE_CASE    (					_a		):
							snake_case_  =				HybridCLIPConfig
							snake_case_  =				FlaxHybridCLIPModule


							def __init__(      self   :    Tuple      ,     snake_case   :    HybridCLIPConfig      ,     snake_case   :    Optional[Tuple] = None      ,     snake_case   :    int = 0      ,     snake_case   :    jnp.dtype = jnp.floataa      ,     **snake_case   :    Tuple      ,     ):
													'''simple docstring'''





													if input_shape is None:
																			A__    :	int			=	((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

													A__    :	Any			=	self.module_class(config=snake_case      ,     dtype=snake_case      ,     **snake_case				)
													super().__init__(snake_case      ,     snake_case      ,     input_shape=snake_case      ,     seed=snake_case      ,     dtype=snake_case				)


							def     _UpperCamelCase							(      self   :    Optional[Any]      ,     snake_case   :    jax.random.PRNGKey      ,     snake_case   :    Tuple      ,     snake_case   :    FrozenDict = None				):
													'''simple docstring'''





													A__    :	Union[str, Any]			=	jnp.zeros(input_shape[0]      ,     dtype="""i4"""				)
													A__    :	Dict			=	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(snake_case				).shape[-1]				)      ,     input_shape[0]				)
													A__    :	List[Any]			=	jnp.ones_like(snake_case				)
													A__    :	Optional[Any]			=	jnp.ones_like(snake_case				)

													A__    :	List[Any]			=	jax.random.normal(snake_case      ,     input_shape[1]				)

													A__	,	A__    :	Tuple			=	jax.random.split(snake_case				)
													A__    :	Optional[int]			=	{"""params""": params_rng, """dropout""": dropout_rng}

													return self.module.init(snake_case      ,     snake_case      ,     snake_case      ,     snake_case      ,     snake_case      ,     snake_case				)["params"]


							def __call__(      self   :    List[str]      ,     snake_case   :    Optional[Any]      ,     snake_case   :    Union[str, Any]      ,     snake_case   :    Tuple=None      ,     snake_case   :    Union[str, Any]=None      ,     snake_case   :    Tuple=None      ,     snake_case   :    dict = None      ,     snake_case   :    jax.random.PRNGKey = None      ,     snake_case   :    bool = False      ,     snake_case   :    Optional[bool] = None      ,     snake_case   :    Optional[bool] = None      ,     snake_case   :    Optional[bool] = None      ,     ):
													'''simple docstring'''





													A__    :	Union[str, Any]			=	output_attentions if output_attentions is not None else self.config.output_attentions
													A__    :	Optional[Any]			=	(
													    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
													)
													A__    :	List[Any]			=	return_dict if return_dict is not None else self.config.return_dict

													if position_ids is None:
																			A__    :	List[str]			=	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(snake_case				).shape[-1]				)      ,     input_ids.shape				)

													if token_type_ids is None:
																			A__    :	Dict			=	jnp.zeros_like(snake_case				)

													if attention_mask is None:
																			A__    :	Tuple			=	jnp.ones_like(snake_case				)

													# Handle any PRNG if needed
													A__    :	Any			=	{}
													if dropout_rng is not None:
																			A__    :	int			=	dropout_rng

													return self.module.apply(
													    {"""params""": params or self.params}      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     jnp.array(snake_case      ,     dtype=jnp.floataa				)      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     not train      ,     snake_case      ,     snake_case      ,     snake_case      ,     rngs=snake_case      ,     )


							def     _UpperCamelCase							(      self   :    Optional[Any]      ,     snake_case   :    Optional[Any]      ,     snake_case   :    Dict=None      ,     snake_case   :    List[str]=None      ,     snake_case   :    Tuple=None      ,     snake_case   :    dict = None      ,     snake_case   :    jax.random.PRNGKey = None      ,     snake_case   :    str=False      ,     ):
													'''simple docstring'''





													if position_ids is None:
																			A__    :	Union[str, Any]			=	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(snake_case				).shape[-1]				)      ,     input_ids.shape				)

													if token_type_ids is None:
																			A__    :	Dict			=	jnp.zeros_like(snake_case				)

													if attention_mask is None:
																			A__    :	Union[str, Any]			=	jnp.ones_like(snake_case				)

													# Handle any PRNG if needed
													A__    :	Tuple			=	{}
													if dropout_rng is not None:
																			A__    :	Dict			=	dropout_rng

													def _get_features(snake_case   :    int      ,     snake_case   :    Optional[int]      ,     snake_case   :    Optional[int]      ,     snake_case   :    List[str]      ,     snake_case   :    int      ,     snake_case   :    Optional[Any]				):
																			A__    :	str			=	module.text_model(
																			    input_ids=snake_case      ,     attention_mask=snake_case      ,     position_ids=snake_case      ,     token_type_ids=snake_case      ,     deterministic=snake_case      ,     )
																			A__    :	Union[str, Any]			=	text_outputs[1]
																			A__    :	Optional[Any]			=	module.text_projection(snake_case				)
																			return text_features

													return self.module.apply(
													    {"""params""": params or self.params}      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     jnp.array(snake_case      ,     dtype="""i4"""				)      ,     not train      ,     method=_get_features      ,     rngs=snake_case      ,     )


							def     _UpperCamelCase							(      self   :    Union[str, Any]      ,     snake_case   :    List[str]      ,     snake_case   :    dict = None      ,     snake_case   :    jax.random.PRNGKey = None      ,     snake_case   :    str=False				):
													'''simple docstring'''





													A__    :	Union[str, Any]			=	{}
													if dropout_rng is not None:
																			A__    :	Dict			=	dropout_rng

													def _get_features(snake_case   :    str      ,     snake_case   :    Any      ,     snake_case   :    Union[str, Any]				):
																			A__    :	Union[str, Any]			=	module.vision_model(pixel_values=snake_case      ,     deterministic=snake_case				)
																			A__    :	Tuple			=	vision_outputs[1]  # pooled_output
																			A__    :	Optional[Any]			=	module.visual_projection(snake_case				)
																			return image_features

													return self.module.apply(
													    {"""params""": params or self.params}      ,     jnp.array(snake_case      ,     dtype=jnp.floataa				)      ,     not train      ,     method=_get_features      ,     rngs=snake_case      ,     )


							@classmethod
							def     _UpperCamelCase							(      cls   :    Dict      ,     snake_case   :    str = None      ,     snake_case   :    str = None      ,     *snake_case   :    List[str]      ,     **snake_case   :    Optional[int]      ,     ):
													'''simple docstring'''





													A__    :	Optional[int]			=	{
													    argument[len("""text_"""				) :]: value for argument, value in kwargs.items() if argument.startswith("""text_"""				)
													}

													A__    :	Union[str, Any]			=	{
													    argument[len("""vision_"""				) :]: value for argument, value in kwargs.items() if argument.startswith("""vision_"""				)
													}

													# remove text, vision kwargs from kwargs
													for key in kwargs_text.keys():
																			del kwargs["text_" + key]
													for key in kwargs_vision.keys():
																			del kwargs["vision_" + key]

													# Load and initialize the text and vision model
													A__    :	Any			=	kwargs_text.pop("""model"""      ,     snake_case				)
													if text_model is None:
																			assert (
																			    text_model_name_or_path is not None
																			), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
																			from transformers import FlaxAutoModel

																			if "config" not in kwargs_text:
																									from transformers import AutoConfig

																									A__    :	List[Any]			=	AutoConfig.from_pretrained(snake_case				)
																									A__    :	Tuple			=	text_config

																			A__    :	Any			=	FlaxAutoModel.from_pretrained(snake_case      ,     *snake_case      ,     **snake_case				)

													A__    :	List[Any]			=	kwargs_vision.pop("""model"""      ,     snake_case				)
													if vision_model is None:
																			assert (
																			    vision_model_name_or_path is not None
																			), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
																			from transformers import FlaxAutoModel

																			if "config" not in kwargs_vision:
																									from transformers import AutoConfig

																									A__    :	Dict			=	AutoConfig.from_pretrained(snake_case				)
																									A__    :	Dict			=	vision_config

																			A__    :	int			=	FlaxAutoModel.from_pretrained(snake_case      ,     *snake_case      ,     **snake_case				)

													# instantiate config with corresponding kwargs
													A__    :	List[Any]			=	kwargs.pop("""dtype"""      ,     jnp.floataa				)
													A__    :	Union[str, Any]			=	HybridCLIPConfig.from_text_vision_configs(text_model.config      ,     vision_model.config      ,     **snake_case				)

													# init model
													A__    :	Dict			=	cls(snake_case      ,     *snake_case      ,     dtype=snake_case      ,     **snake_case				)

													if vision_config.model_type == "clip":
																			A__    :	str			=	vision_model.params["""vision_model"""]
																			A__    :	Optional[Any]			=	vision_model.params["""visual_projection"""]["""kernel"""]
													else:
																			A__    :	List[Any]			=	vision_model.params

													A__    :	Optional[Any]			=	text_model.params

													return model













import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
      import torch

      from transformers import GPTaLMHeadModel

@require_torch
@require_sentencepiece
@require_tokenizers
class 	__snake_case					(      unittest.TestCase ):






    '''simple docstring'''





    @slow
    @unittest.skip('''Model is not available.'''				)
    def 						A		( self    :		str				):

      """simple docstring"""





      __lowerCAmelCase								=					'''nvidia/megatron-gpt2-345m/'''
      if "MYDIR" in os.environ:
        __lowerCAmelCase								=					os.path.join(os.environ['''MYDIR''']      ,    snake_case				)
      __lowerCAmelCase								=					GPTaLMHeadModel.from_pretrained(snake_case				)
      model.to(snake_case				)
      model.half()

      __lowerCAmelCase								=					torch.tensor(
          [[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]      ,    device=snake_case      ,    dtype=torch.long      ,    )

      with torch.no_grad():
        __lowerCAmelCase								=					model(snake_case				).logits

      __lowerCAmelCase								=					torch.Size((1, 9, 50_257)				)
      self.assertEqual(output.shape      ,    snake_case				)

      __lowerCAmelCase								=					torch.tensor(
          [
              4.9414,
              -0.2920,
              -1.2148,
              -4.0273,
              -0.5161,
              -5.2109,
              -1.2412,
              -1.8301,
              -1.7734,
              -4.7148,
              -0.2317,
              -1.0811,
              -2.1777,
              0.4141,
              -3.7969,
              -4.0586,
              -2.5332,
              -3.3809,
              4.3867,
          ]      ,    device=snake_case      ,    dtype=torch.half      ,    )

      for i in range(19				):
        __lowerCAmelCase						,    __lowerCAmelCase								=					8 * i // 17, 2_792 * i  # along the diagonal
        __lowerCAmelCase						,    __lowerCAmelCase								=					output[0, r, c], expected_diag[i]
        __lowerCAmelCase								=					f'''row={r} col={c} computed={computed} expected={expected}'''
        self.assertAlmostEqual(snake_case      ,    snake_case      ,    delta=1e-4      ,    msg=snake_case				)
