



from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin





@dataclass
class    __lowerCAmelCase					(	lowerCamelCase__			):
				__lowerCamelCase					=							42





class    __lowerCAmelCase					(	lowerCamelCase__			,		lowerCamelCase__			):




				@register_to_config
				def __init__(				self		,							_snake_case = 16		,							_snake_case = 88		,							_snake_case = None		,							_snake_case = None		,							_snake_case = 1		,							_snake_case = 0.0		,							_snake_case = 32		,							_snake_case = None		,							_snake_case = False		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = "geglu"		,							_snake_case = None		,							_snake_case = False		,							_snake_case = False		,							_snake_case = False		,							_snake_case = "layer_norm"		,							_snake_case = True		,							):



									"""simple docstring"""





									super().__init__()
									_lowerCAmelCase         =       use_linear_projection
									_lowerCAmelCase         =       num_attention_heads
									_lowerCAmelCase         =       attention_head_dim
									_lowerCAmelCase         =       num_attention_heads * attention_head_dim

									# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
									# Define whether input is continuous or discrete depending on configuration
									_lowerCAmelCase         =       (in_channels is not None) and (patch_size is None)
									_lowerCAmelCase         =       num_vector_embeds is not None
									_lowerCAmelCase         =       in_channels is not None and patch_size is not None

									if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
														_lowerCAmelCase         =       (
														    F'The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'
														    """ incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."""
														    """ Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"""
														    """ results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"""
														    """ would be very nice if you could open a Pull request for the `transformer/config.json` file"""
														)
														deprecate("""norm_type!=num_embeds_ada_norm"""		,							"""1.0.0"""		,							_UpperCAmelCase		,							standard_warn=_UpperCAmelCase						)
														_lowerCAmelCase         =       """ada_norm"""

									if self.is_input_continuous and self.is_input_vectorized:
														raise ValueError(
														    F'Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'
														    """ sure that either `in_channels` or `num_vector_embeds` is None."""						)
									elif self.is_input_vectorized and self.is_input_patches:
														raise ValueError(
														    F'Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'
														    """ sure that either `num_vector_embeds` or `num_patches` is None."""						)
									elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
														raise ValueError(
														    F'Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'
														    F' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'						)

									# 2. Define input layers
									if self.is_input_continuous:
														_lowerCAmelCase         =       in_channels

														_lowerCAmelCase         =       torch.nn.GroupNorm(num_groups=_UpperCAmelCase		,							num_channels=_UpperCAmelCase		,							eps=1e-6		,							affine=_UpperCAmelCase						)
														if use_linear_projection:
																			_lowerCAmelCase         =       nn.Linear(_UpperCAmelCase		,							_UpperCAmelCase						)
														else:
																			_lowerCAmelCase         =       nn.Convad(_UpperCAmelCase		,							_UpperCAmelCase		,							kernel_size=1		,							stride=1		,							padding=0						)
									elif self.is_input_vectorized:
														assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
														assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

														_lowerCAmelCase         =       sample_size
														_lowerCAmelCase         =       sample_size
														_lowerCAmelCase         =       num_vector_embeds
														_lowerCAmelCase         =       self.height * self.width

														_lowerCAmelCase         =       ImagePositionalEmbeddings(
														    num_embed=_UpperCAmelCase		,							embed_dim=_UpperCAmelCase		,							height=self.height		,							width=self.width						)
									elif self.is_input_patches:
														assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

														_lowerCAmelCase         =       sample_size
														_lowerCAmelCase         =       sample_size

														_lowerCAmelCase         =       patch_size
														_lowerCAmelCase         =       PatchEmbed(
														    height=_UpperCAmelCase		,							width=_UpperCAmelCase		,							patch_size=_UpperCAmelCase		,							in_channels=_UpperCAmelCase		,							embed_dim=_UpperCAmelCase		,							)

									# 3. Define transformers blocks
									_lowerCAmelCase         =       nn.ModuleList(
									    [
									        BasicTransformerBlock(
									            _UpperCAmelCase		,							_UpperCAmelCase		,							_UpperCAmelCase		,							dropout=_UpperCAmelCase		,							cross_attention_dim=_UpperCAmelCase		,							activation_fn=_UpperCAmelCase		,							num_embeds_ada_norm=_UpperCAmelCase		,							attention_bias=_UpperCAmelCase		,							only_cross_attention=_UpperCAmelCase		,							upcast_attention=_UpperCAmelCase		,							norm_type=_UpperCAmelCase		,							norm_elementwise_affine=_UpperCAmelCase		,							)
									        for d in range(_UpperCAmelCase						)
									    ]						)

									# 4. Define output layers
									_lowerCAmelCase         =       in_channels if out_channels is None else out_channels
									if self.is_input_continuous:
														# TODO: should use out_channels for continuous projections
														if use_linear_projection:
																			_lowerCAmelCase         =       nn.Linear(_UpperCAmelCase		,							_UpperCAmelCase						)
														else:
																			_lowerCAmelCase         =       nn.Convad(_UpperCAmelCase		,							_UpperCAmelCase		,							kernel_size=1		,							stride=1		,							padding=0						)
									elif self.is_input_vectorized:
														_lowerCAmelCase         =       nn.LayerNorm(_UpperCAmelCase						)
														_lowerCAmelCase         =       nn.Linear(_UpperCAmelCase		,							self.num_vector_embeds - 1						)
									elif self.is_input_patches:
														_lowerCAmelCase         =       nn.LayerNorm(_UpperCAmelCase		,							elementwise_affine=_UpperCAmelCase		,							eps=1e-6						)
														_lowerCAmelCase         =       nn.Linear(_UpperCAmelCase		,							2 * inner_dim						)
														_lowerCAmelCase         =       nn.Linear(_UpperCAmelCase		,							patch_size * patch_size * self.out_channels						)




				def 				snake_case  (				self		,							_snake_case		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = True		,							):



									"""simple docstring"""





									if attention_mask is not None and attention_mask.ndim == 2:
														# assume that mask is expressed as:
														#   (1 = keep,      0 = discard)
														# convert mask into a bias that can be added to attention scores:
														#       (keep = +0,     discard = -10000.0)
														_lowerCAmelCase         =       (1 - attention_mask.to(hidden_states.dtype						)) * -1_0000.0
														_lowerCAmelCase         =       attention_mask.unsqueeze(1						)

									# convert encoder_attention_mask to a bias the same way we do for attention_mask
									if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
														_lowerCAmelCase         =       (1 - encoder_attention_mask.to(hidden_states.dtype						)) * -1_0000.0
														_lowerCAmelCase         =       encoder_attention_mask.unsqueeze(1						)

									# 1. Input
									if self.is_input_continuous:
														_lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase         =       hidden_states.shape
														_lowerCAmelCase         =       hidden_states

														_lowerCAmelCase         =       self.norm(_UpperCAmelCase						)
														if not self.use_linear_projection:
																			_lowerCAmelCase         =       self.proj_in(_UpperCAmelCase						)
																			_lowerCAmelCase         =       hidden_states.shape[1]
																			_lowerCAmelCase         =       hidden_states.permute(0		,							2		,							3		,							1						).reshape(_UpperCAmelCase		,							height * width		,							_UpperCAmelCase						)
														else:
																			_lowerCAmelCase         =       hidden_states.shape[1]
																			_lowerCAmelCase         =       hidden_states.permute(0		,							2		,							3		,							1						).reshape(_UpperCAmelCase		,							height * width		,							_UpperCAmelCase						)
																			_lowerCAmelCase         =       self.proj_in(_UpperCAmelCase						)
									elif self.is_input_vectorized:
														_lowerCAmelCase         =       self.latent_image_embedding(_UpperCAmelCase						)
									elif self.is_input_patches:
														_lowerCAmelCase         =       self.pos_embed(_UpperCAmelCase						)

									# 2. Blocks
									for block in self.transformer_blocks:
														_lowerCAmelCase         =       block(
														    _UpperCAmelCase		,							attention_mask=_UpperCAmelCase		,							encoder_hidden_states=_UpperCAmelCase		,							encoder_attention_mask=_UpperCAmelCase		,							timestep=_UpperCAmelCase		,							cross_attention_kwargs=_UpperCAmelCase		,							class_labels=_UpperCAmelCase		,							)

									# 3. Output
									if self.is_input_continuous:
														if not self.use_linear_projection:
																			_lowerCAmelCase         =       hidden_states.reshape(_UpperCAmelCase		,							_UpperCAmelCase		,							_UpperCAmelCase		,							_UpperCAmelCase						).permute(0		,							3		,							1		,							2						).contiguous()
																			_lowerCAmelCase         =       self.proj_out(_UpperCAmelCase						)
														else:
																			_lowerCAmelCase         =       self.proj_out(_UpperCAmelCase						)
																			_lowerCAmelCase         =       hidden_states.reshape(_UpperCAmelCase		,							_UpperCAmelCase		,							_UpperCAmelCase		,							_UpperCAmelCase						).permute(0		,							3		,							1		,							2						).contiguous()

														_lowerCAmelCase         =       hidden_states + residual
									elif self.is_input_vectorized:
														_lowerCAmelCase         =       self.norm_out(_UpperCAmelCase						)
														_lowerCAmelCase         =       self.out(_UpperCAmelCase						)
														# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
														_lowerCAmelCase         =       logits.permute(0		,							2		,							1						)

														# log(p(x_0))
														_lowerCAmelCase         =       F.log_softmax(logits.double()		,							dim=1						).float()
									elif self.is_input_patches:
														# TODO: cleanup!
														_lowerCAmelCase         =       self.transformer_blocks[0].norma.emb(
														    _UpperCAmelCase		,							_UpperCAmelCase		,							hidden_dtype=hidden_states.dtype						)
														_lowerCAmelCase      ,      _lowerCAmelCase         =       self.proj_out_a(F.silu(_UpperCAmelCase						)						).chunk(2		,							dim=1						)
														_lowerCAmelCase         =       self.norm_out(_UpperCAmelCase						) * (1 + scale[:, None]) + shift[:, None]
														_lowerCAmelCase         =       self.proj_out_a(_UpperCAmelCase						)

														# unpatchify
														_lowerCAmelCase         =       _lowerCAmelCase         =       int(hidden_states.shape[1] ** 0.5						)
														_lowerCAmelCase         =       hidden_states.reshape(
														    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)						)
														_lowerCAmelCase         =       torch.einsum("""nhwpqc->nchpwq"""		,							_UpperCAmelCase						)
														_lowerCAmelCase         =       hidden_states.reshape(
														    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)						)

									if not return_dict:
														return (output,)

									return TransformeraDModelOutput(sample=_UpperCAmelCase						)











import tempfile
import unittest

import transformers
from transformers import XGLMConfig, XGLMTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
   import jax
   import jax.numpy as jnp
   import numpy as np

   from transformers.modeling_flax_pytorch_utils import (
       convert_pytorch_state_dict_to_flax,
       load_flax_weights_in_pytorch_model,
   )
   from transformers.models.xglm.modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel


if is_torch_available():
   import torch


@require_flax
class 		lowerCAmelCase_				:





     def __init__(	self							, _UpperCAmelCase							, _UpperCAmelCase=14							, _UpperCAmelCase=7							, _UpperCAmelCase=True							, _UpperCAmelCase=True							, _UpperCAmelCase=True							, _UpperCAmelCase=99							, _UpperCAmelCase=32							, _UpperCAmelCase=5							, _UpperCAmelCase=4							, _UpperCAmelCase=37							, _UpperCAmelCase="gelu"							, _UpperCAmelCase=0.1							, _UpperCAmelCase=0.1							, _UpperCAmelCase=512							, _UpperCAmelCase=0.02							, _UpperCAmelCase=None							, )		->				str:
        lowerCamelCase									=parent
        lowerCamelCase									=batch_size
        lowerCamelCase									=seq_length
        lowerCamelCase									=is_training
        lowerCamelCase									=use_input_mask
        lowerCamelCase									=use_labels
        lowerCamelCase									=vocab_size
        lowerCamelCase									=d_model
        lowerCamelCase									=num_hidden_layers
        lowerCamelCase									=num_attention_heads
        lowerCamelCase									=ffn_dim
        lowerCamelCase									=activation_function
        lowerCamelCase									=activation_dropout
        lowerCamelCase									=attention_dropout
        lowerCamelCase									=max_position_embeddings
        lowerCamelCase									=initializer_range
        lowerCamelCase									=None
        lowerCamelCase									=0
        lowerCamelCase									=2
        lowerCamelCase									=1





     def 						lowerCamelCase							(	self  )		->				List[str]:
        lowerCamelCase									=np.clip(ids_tensor([self.batch_size, self.seq_length]							, self.vocab_size  )							, 3							, self.vocab_size  )

        lowerCamelCase									=None
        if self.use_input_mask:
           lowerCamelCase									=random_attention_mask([self.batch_size, self.seq_length]  )

        lowerCamelCase									=XGLMConfig(
            vocab_size=self.vocab_size							, d_model=self.hidden_size							, num_layers=self.num_hidden_layers							, attention_heads=self.num_attention_heads							, ffn_dim=self.ffn_dim							, activation_function=self.activation_function							, activation_dropout=self.activation_dropout							, attention_dropout=self.attention_dropout							, max_position_embeddings=self.max_position_embeddings							, initializer_range=self.initializer_range							, use_cache=_UpperCAmelCase							, bos_token_id=self.bos_token_id							, eos_token_id=self.eos_token_id							, pad_token_id=self.pad_token_id							, )

        return (config, input_ids, input_mask)





     def 						lowerCamelCase							(	self  )		->				Tuple:
        lowerCamelCase									=self.prepare_config_and_inputs()
        lowerCamelCase				,       lowerCamelCase				,       lowerCamelCase									=config_and_inputs
        lowerCamelCase									={"""input_ids""": input_ids, """attention_mask""": attention_mask}
        return config, inputs_dict





     def 						lowerCamelCase							(	self  )		->				int:
        lowerCamelCase				,       lowerCamelCase				,       lowerCamelCase									=self.prepare_config_and_inputs()

        lowerCamelCase									=floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
        lowerCamelCase									=ids_tensor([self.batch_size, self.seq_length]							, vocab_size=2  )

        return (
            config,
            input_ids,
            attention_mask,
            encoder_hidden_states,
            encoder_attention_mask,
        )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase  )		->				Optional[Any]:
        lowerCamelCase									=20
        lowerCamelCase									=model_class_name(_UpperCAmelCase  )

        lowerCamelCase									=model.init_cache(input_ids.shape[0]							, _UpperCAmelCase  )
        lowerCamelCase									=jnp.ones((input_ids.shape[0], max_decoder_length)							, dtype="""i4"""  )

        lowerCamelCase									=jnp.broadcast_to(
            jnp.arange(input_ids.shape[-1] - 1  )[None, :]							, (input_ids.shape[0], input_ids.shape[-1] - 1)  )
        lowerCamelCase									=model(
            input_ids[:, :-1]							, attention_mask=_UpperCAmelCase							, past_key_values=_UpperCAmelCase							, position_ids=_UpperCAmelCase							, )

        lowerCamelCase									=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]							, dtype="""i4"""  )
        lowerCamelCase									=model(
            input_ids[:, -1:]							, attention_mask=_UpperCAmelCase							, past_key_values=outputs_cache.past_key_values							, position_ids=_UpperCAmelCase							, )

        lowerCamelCase									=model(_UpperCAmelCase  )

        lowerCamelCase									=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])  )  )
        self.parent.assertTrue(diff < 1E-3							, msg=F'''Max diff is {diff}'''  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase  )		->				List[Any]:
        lowerCamelCase									=20
        lowerCamelCase									=model_class_name(_UpperCAmelCase  )

        lowerCamelCase									=jnp.concatenate(
            [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])  )]							, axis=-1							, )

        lowerCamelCase									=model.init_cache(input_ids.shape[0]							, _UpperCAmelCase  )
        lowerCamelCase									=jnp.broadcast_to(
            jnp.arange(input_ids.shape[-1] - 1  )[None, :]							, (input_ids.shape[0], input_ids.shape[-1] - 1)  )

        lowerCamelCase									=model(
            input_ids[:, :-1]							, attention_mask=_UpperCAmelCase							, past_key_values=_UpperCAmelCase							, position_ids=_UpperCAmelCase							, )
        lowerCamelCase									=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]							, dtype="""i4"""  )
        lowerCamelCase									=model(
            input_ids[:, -1:]							, past_key_values=outputs_cache.past_key_values							, attention_mask=_UpperCAmelCase							, position_ids=_UpperCAmelCase							, )

        lowerCamelCase									=model(_UpperCAmelCase							, attention_mask=_UpperCAmelCase  )
        lowerCamelCase									=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])  )  )
        self.parent.assertTrue(diff < 1E-3							, msg=F'''Max diff is {diff}'''  )




@require_sentencepiece
@require_flax
class 		lowerCAmelCase_				(  UpperCamelCase				,     UpperCamelCase				,     unittest.TestCase ):
     __lowerCamelCase       :    Tuple 	=     (FlaxXGLMModel, FlaxXGLMForCausalLM) if is_flax_available() else ()
     __lowerCamelCase       :    Dict 	=     (FlaxXGLMForCausalLM,) if is_flax_available() else ()





     def 						lowerCamelCase							(	self  )		->				List[str]:
        lowerCamelCase									=FlaxXGLMModelTester(self  )





     def 						lowerCamelCase							(	self  )		->				List[str]:
        for model_class_name in self.all_model_classes:
           lowerCamelCase				,       lowerCamelCase				,       lowerCamelCase									=self.model_tester.prepare_config_and_inputs()
           self.model_tester.check_use_cache_forward(_UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase  )





     def 						lowerCamelCase							(	self  )		->				Tuple:
        for model_class_name in self.all_model_classes:
           lowerCamelCase				,       lowerCamelCase				,       lowerCamelCase									=self.model_tester.prepare_config_and_inputs()
           self.model_tester.check_use_cache_forward_with_attn_mask(
               _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase  )





     @slow
     def 						lowerCamelCase							(	self  )		->				Tuple:
        lowerCamelCase									=XGLMTokenizer.from_pretrained("""XGLM"""							, padding_side="""left"""  )
        lowerCamelCase									=tokenizer(["""Hello this is a long string""", """Hey"""]							, return_tensors="""np"""							, padding=_UpperCAmelCase							, truncation=_UpperCAmelCase  )

        lowerCamelCase									=FlaxXGLMForCausalLM.from_pretrained("""facebook/xglm-564M"""  )
        lowerCamelCase									=1
        lowerCamelCase									=False

        lowerCamelCase									=jax.jit(model.generate  )

        lowerCamelCase									=jit_generate(inputs["""input_ids"""]							, attention_mask=inputs["""attention_mask"""]  ).sequences

        lowerCamelCase									=tokenizer.batch_decode(_UpperCAmelCase							, skip_special_tokens=_UpperCAmelCase  )

        lowerCamelCase									=[
            """Hello this is a long string of questions, but I'm not sure if I'm""",
            """Hey, I'm a newbie to the forum and I'""",
        ]

        self.assertListEqual(_UpperCAmelCase							, _UpperCAmelCase  )





     @is_pt_flax_cross_test
     def 						lowerCamelCase							(	self  )		->				Dict:
        lowerCamelCase				,       lowerCamelCase									=self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__  ):
              # prepare inputs
              lowerCamelCase									=self._prepare_for_class(_UpperCAmelCase							, _UpperCAmelCase  )
              lowerCamelCase									={k: torch.tensor(v.tolist()  ) for k, v in prepared_inputs_dict.items()}

              # load corresponding PyTorch class
              lowerCamelCase									=model_class.__name__[4:]  # Skip the "Flax" at the beginning
              lowerCamelCase									=getattr(_UpperCAmelCase							, _UpperCAmelCase  )

              lowerCamelCase				,       lowerCamelCase									=pt_inputs["""input_ids"""].shape
              lowerCamelCase									=np.random.randint(0							, seq_length - 1							, size=(batch_size,)  )
              for batch_idx, start_index in enumerate(_UpperCAmelCase  ):
                 lowerCamelCase									=0
                 lowerCamelCase									=1
                 lowerCamelCase									=0
                 lowerCamelCase									=1
              lowerCamelCase									=pt_model_class(_UpperCAmelCase  ).eval()
              # Flax models don't use the `use_cache` option and cache is not returned as a default.
              # So we disable `use_cache` here for PyTorch model.
              lowerCamelCase									=False
              lowerCamelCase									=model_class(_UpperCAmelCase							, dtype=jnp.floataa  )
              lowerCamelCase									=convert_pytorch_state_dict_to_flax(pt_model.state_dict()							, _UpperCAmelCase  )
              lowerCamelCase									=fx_state

              with torch.no_grad():
                 lowerCamelCase									=pt_model(**_UpperCAmelCase  ).to_tuple()

              lowerCamelCase									=fx_model(**_UpperCAmelCase  ).to_tuple()
              self.assertEqual(len(_UpperCAmelCase  )							, len(_UpperCAmelCase  )							, """Output lengths differ between Flax and PyTorch"""  )
              for fx_output, pt_output in zip(_UpperCAmelCase							, _UpperCAmelCase  ):
                 self.assert_almost_equals(fx_output[:, -1]							, pt_output[:, -1].numpy()							, 4E-2  )

              with tempfile.TemporaryDirectory() as tmpdirname:
                 pt_model.save_pretrained(_UpperCAmelCase  )
                 lowerCamelCase									=model_class.from_pretrained(_UpperCAmelCase							, from_pt=_UpperCAmelCase  )

              lowerCamelCase									=fx_model_loaded(**_UpperCAmelCase  ).to_tuple()
              self.assertEqual(
                  len(_UpperCAmelCase  )							, len(_UpperCAmelCase  )							, """Output lengths differ between Flax and PyTorch"""  )
              for fx_output_loaded, pt_output in zip(_UpperCAmelCase							, _UpperCAmelCase  ):
                 self.assert_almost_equals(fx_output_loaded[:, -1]							, pt_output[:, -1].numpy()							, 4E-2  )





     @is_pt_flax_cross_test
     def 						lowerCamelCase							(	self  )		->				Optional[Any]:
        lowerCamelCase				,       lowerCamelCase									=self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__  ):
              # prepare inputs
              lowerCamelCase									=self._prepare_for_class(_UpperCAmelCase							, _UpperCAmelCase  )
              lowerCamelCase									={k: torch.tensor(v.tolist()  ) for k, v in prepared_inputs_dict.items()}

              # load corresponding PyTorch class
              lowerCamelCase									=model_class.__name__[4:]  # Skip the "Flax" at the beginning
              lowerCamelCase									=getattr(_UpperCAmelCase							, _UpperCAmelCase  )

              lowerCamelCase									=pt_model_class(_UpperCAmelCase  ).eval()
              lowerCamelCase									=False
              lowerCamelCase									=model_class(_UpperCAmelCase							, dtype=jnp.floataa  )

              lowerCamelCase									=load_flax_weights_in_pytorch_model(_UpperCAmelCase							, fx_model.params  )
              lowerCamelCase				,       lowerCamelCase									=pt_inputs["""input_ids"""].shape
              lowerCamelCase									=np.random.randint(0							, seq_length - 1							, size=(batch_size,)  )
              for batch_idx, start_index in enumerate(_UpperCAmelCase  ):
                 lowerCamelCase									=0
                 lowerCamelCase									=1
                 lowerCamelCase									=0
                 lowerCamelCase									=1

              # make sure weights are tied in PyTorch
              pt_model.tie_weights()

              with torch.no_grad():
                 lowerCamelCase									=pt_model(**_UpperCAmelCase  ).to_tuple()

              lowerCamelCase									=fx_model(**_UpperCAmelCase  ).to_tuple()
              self.assertEqual(len(_UpperCAmelCase  )							, len(_UpperCAmelCase  )							, """Output lengths differ between Flax and PyTorch"""  )
              for fx_output, pt_output in zip(_UpperCAmelCase							, _UpperCAmelCase  ):
                 self.assert_almost_equals(fx_output[:, -1]							, pt_output[:, -1].numpy()							, 4E-2  )

              with tempfile.TemporaryDirectory() as tmpdirname:
                 fx_model.save_pretrained(_UpperCAmelCase  )
                 lowerCamelCase									=pt_model_class.from_pretrained(_UpperCAmelCase							, from_flax=_UpperCAmelCase  )

              with torch.no_grad():
                 lowerCamelCase									=pt_model_loaded(**_UpperCAmelCase  ).to_tuple()

              self.assertEqual(
                  len(_UpperCAmelCase  )							, len(_UpperCAmelCase  )							, """Output lengths differ between Flax and PyTorch"""  )
              for fx_output, pt_output in zip(_UpperCAmelCase							, _UpperCAmelCase  ):
                 self.assert_almost_equals(fx_output[:, -1]							, pt_output[:, -1].numpy()							, 4E-2  )





     @slow
     def 						lowerCamelCase							(	self  )		->				int:
        for model_class_name in self.all_model_classes:
           lowerCamelCase									=model_class_name.from_pretrained("""facebook/xglm-564M"""  )
           lowerCamelCase									=model(np.ones((1, 1)  )  )
           self.assertIsNotNone(_UpperCAmelCase  )