



import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaTokenizer, GPTNeoConfig, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
      import jax
      import jax.numpy as jnp

      from transformers.modeling_flax_pytorch_utils import (
          convert_pytorch_state_dict_to_flax,
          load_flax_weights_in_pytorch_model,
      )
      from transformers.models.gpt_neo.modeling_flax_gpt_neo import FlaxGPTNeoForCausalLM, FlaxGPTNeoModel

if is_torch_available():
      import torch

class    _SCREAMING_SNAKE_CASE						:





     def __init__(							self			:				Optional[Any]  ,	UpperCamelCase			:				Union[str, Any]  ,	UpperCamelCase			:				str=1_4  ,	UpperCamelCase			:				str=7  ,	UpperCamelCase			:				str=True  ,	UpperCamelCase			:				Union[str, Any]=True  ,	UpperCamelCase			:				Dict=False  ,	UpperCamelCase			:				Optional[Any]=True  ,	UpperCamelCase			:				Any=9_9  ,	UpperCamelCase			:				Optional[Any]=3_2  ,	UpperCamelCase			:				Union[str, Any]=4  ,	UpperCamelCase			:				List[Any]=4  ,	UpperCamelCase			:				Tuple=[[["global", "local"], 2]]  ,	UpperCamelCase			:				str=3_7  ,	UpperCamelCase			:				int="gelu"  ,	UpperCamelCase			:				Union[str, Any]=0.1  ,	UpperCamelCase			:				Union[str, Any]=0.1  ,	UpperCamelCase			:				Dict=5_1_2  ,	UpperCamelCase			:				Union[str, Any]=7  ,	UpperCamelCase			:				Optional[Any]=0.0_2  ,	)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE							:     Dict							=						parent
      __SCREAMING_SNAKE_CASE							:     Dict							=						batch_size
      __SCREAMING_SNAKE_CASE							:     Optional[int]							=						seq_length
      __SCREAMING_SNAKE_CASE							:     List[Any]							=						is_training
      __SCREAMING_SNAKE_CASE							:     int							=						use_input_mask
      __SCREAMING_SNAKE_CASE							:     int							=						use_token_type_ids
      __SCREAMING_SNAKE_CASE							:     Any							=						use_labels
      __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						vocab_size
      __SCREAMING_SNAKE_CASE							:     List[str]							=						hidden_size
      __SCREAMING_SNAKE_CASE							:     int							=						num_hidden_layers
      __SCREAMING_SNAKE_CASE							:     Optional[int]							=						num_attention_heads
      __SCREAMING_SNAKE_CASE							:     str							=						attention_types
      __SCREAMING_SNAKE_CASE							:     Dict							=						intermediate_size
      __SCREAMING_SNAKE_CASE							:     Tuple							=						hidden_act
      __SCREAMING_SNAKE_CASE							:     Tuple							=						hidden_dropout_prob
      __SCREAMING_SNAKE_CASE							:     Dict							=						attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE							:     Any							=						max_position_embeddings
      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						window_size
      __SCREAMING_SNAKE_CASE							:     int							=						initializer_range
      __SCREAMING_SNAKE_CASE							:     List[str]							=						None
      __SCREAMING_SNAKE_CASE							:     Tuple							=						vocab_size - 1
      __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						vocab_size - 1
      __SCREAMING_SNAKE_CASE							:     str							=						vocab_size - 1





     def 						__snake_case     (							self			:				Dict					)->Optional[Any]:
      __SCREAMING_SNAKE_CASE							:     str							=						ids_tensor([self.batch_size, self.seq_length]  ,	self.vocab_size					)

      __SCREAMING_SNAKE_CASE							:     List[str]							=						None
      if self.use_input_mask:
       __SCREAMING_SNAKE_CASE							:     List[str]							=						random_attention_mask([self.batch_size, self.seq_length]					)

      __SCREAMING_SNAKE_CASE							:     Tuple							=						GPTNeoConfig(
          vocab_size=self.vocab_size  ,	hidden_size=self.hidden_size  ,	num_layers=self.num_hidden_layers  ,	num_heads=self.num_attention_heads  ,	max_position_embeddings=self.max_position_embeddings  ,	use_cache=UpperCamelCase  ,	bos_token_id=self.bos_token_id  ,	eos_token_id=self.eos_token_id  ,	pad_token_id=self.pad_token_id  ,	window_size=self.window_size  ,	attention_types=self.attention_types  ,	)

      return (config, input_ids, input_mask)





     def 						__snake_case     (							self			:				List[str]					)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						self.prepare_config_and_inputs()
      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     List[Any]							=						config_and_inputs
      __SCREAMING_SNAKE_CASE							:     int							=						{"input_ids": input_ids, "attention_mask": attention_mask}
      return config, inputs_dict





     def 						__snake_case     (							self			:				Optional[int]  ,	UpperCamelCase			:				List[str]  ,	UpperCamelCase			:				int  ,	UpperCamelCase			:				str  ,	UpperCamelCase			:				Tuple					)->int:
      __SCREAMING_SNAKE_CASE							:     List[Any]							=						2_0
      __SCREAMING_SNAKE_CASE							:     Tuple							=						model_class_name(UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     List[Any]							=						model.init_cache(input_ids.shape[0]  ,	UpperCamelCase					)
      __SCREAMING_SNAKE_CASE							:     List[Any]							=						jnp.ones((input_ids.shape[0], max_decoder_length)  ,	dtype="i4"					)

      __SCREAMING_SNAKE_CASE							:     int							=						jnp.broadcast_to(
          jnp.arange(input_ids.shape[-1] - 1					)[None, :]  ,	(input_ids.shape[0], input_ids.shape[-1] - 1)					)
      __SCREAMING_SNAKE_CASE							:     List[str]							=						model(
          input_ids[:, :-1]  ,	attention_mask=UpperCamelCase  ,	past_key_values=UpperCamelCase  ,	position_ids=UpperCamelCase  ,	)

      __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]  ,	dtype="i4"					)
      __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						model(
          input_ids[:, -1:]  ,	attention_mask=UpperCamelCase  ,	past_key_values=outputs_cache.past_key_values  ,	position_ids=UpperCamelCase  ,	)

      __SCREAMING_SNAKE_CASE							:     Dict							=						model(UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     str							=						np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])					)					)
      self.parent.assertTrue(diff < 1E-3  ,	msg=F"""Max diff is {diff}"""					)






     def 						__snake_case     (							self			:				str  ,	UpperCamelCase			:				Optional[int]  ,	UpperCamelCase			:				int  ,	UpperCamelCase			:				Dict  ,	UpperCamelCase			:				Optional[Any]					)->List[str]:
      __SCREAMING_SNAKE_CASE							:     Tuple							=						2_0
      __SCREAMING_SNAKE_CASE							:     Any							=						model_class_name(UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     List[Any]							=						jnp.concatenate(
          [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])					)]  ,	axis=-1  ,	)

      __SCREAMING_SNAKE_CASE							:     List[str]							=						model.init_cache(input_ids.shape[0]  ,	UpperCamelCase					)
      __SCREAMING_SNAKE_CASE							:     str							=						jnp.broadcast_to(
          jnp.arange(input_ids.shape[-1] - 1					)[None, :]  ,	(input_ids.shape[0], input_ids.shape[-1] - 1)					)

      __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						model(
          input_ids[:, :-1]  ,	attention_mask=UpperCamelCase  ,	past_key_values=UpperCamelCase  ,	position_ids=UpperCamelCase  ,	)
      __SCREAMING_SNAKE_CASE							:     str							=						jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]  ,	dtype="i4"					)
      __SCREAMING_SNAKE_CASE							:     Optional[int]							=						model(
          input_ids[:, -1:]  ,	past_key_values=outputs_cache.past_key_values  ,	attention_mask=UpperCamelCase  ,	position_ids=UpperCamelCase  ,	)

      __SCREAMING_SNAKE_CASE							:     Any							=						model(UpperCamelCase  ,	attention_mask=UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     Any							=						np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])					)					)
      self.parent.assertTrue(diff < 1E-3  ,	msg=F"""Max diff is {diff}"""					)






@require_flax
class    _SCREAMING_SNAKE_CASE						(UpperCamelCase      ,     UpperCamelCase      ,     unittest.TestCase			):
     lowerCAmelCase       =      (FlaxGPTNeoModel, FlaxGPTNeoForCausalLM) if is_flax_available() else ()
     lowerCAmelCase       =      (FlaxGPTNeoForCausalLM,) if is_flax_available() else ()





     def 						__snake_case     (							self			:				Any					)->List[Any]:
      __SCREAMING_SNAKE_CASE							:     Any							=						FlaxGPTNeoModelTester(self					)





     def 						__snake_case     (							self			:				List[str]					)->Optional[int]:
      for model_class_name in self.all_model_classes:
       __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     List[str]							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.check_use_cache_forward(UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase					)





     def 						__snake_case     (							self			:				List[Any]					)->Optional[int]:
      for model_class_name in self.all_model_classes:
       __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     Optional[Any]							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.check_use_cache_forward_with_attn_mask(
           UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase					)





     @slow
     def 						__snake_case     (							self			:				str					)->Optional[int]:
      __SCREAMING_SNAKE_CASE							:     str							=						GPTaTokenizer.from_pretrained("gpt2"  ,	pad_token="<|endoftext|>"  ,	padding_side="left"					)
      __SCREAMING_SNAKE_CASE							:     str							=						tokenizer(["Hello this is a long string", "Hey"]  ,	return_tensors="np"  ,	padding=UpperCamelCase  ,	truncation=UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     str							=						FlaxGPTNeoForCausalLM.from_pretrained("EleutherAI/gpt-neo-125M"					)
      __SCREAMING_SNAKE_CASE							:     Tuple							=						False
      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						model.config.eos_token_id

      __SCREAMING_SNAKE_CASE							:     Tuple							=						jax.jit(model.generate					)

      __SCREAMING_SNAKE_CASE							:     List[Any]							=						jit_generate(
          inputs["input_ids"]  ,	attention_mask=inputs["attention_mask"]  ,	pad_token_id=tokenizer.pad_token_id					).sequences

      __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						tokenizer.batch_decode(UpperCamelCase  ,	skip_special_tokens=UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     List[Any]							=						[
          "Hello this is a long string of text.\n\nI'm trying to get the text of the",
          "Hey, I'm a little late to the party. I'm going to",
      ]

      self.assertListEqual(UpperCamelCase  ,	UpperCamelCase					)





     @is_pt_flax_cross_test
     def 						__snake_case     (							self			:				Optional[int]					)->Optional[int]:
      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     List[str]							=						self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       with self.subTest(model_class.__name__					):
        # prepare inputs
        __SCREAMING_SNAKE_CASE							:     str							=						self._prepare_for_class(UpperCamelCase  ,	UpperCamelCase					)
        __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						{k: torch.tensor(v.tolist()					) for k, v in prepared_inputs_dict.items()}

        # load corresponding PyTorch class
        __SCREAMING_SNAKE_CASE							:     int							=						model_class.__name__[4:]  # Skip the "Flax" at the beginning
        __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						getattr(UpperCamelCase  ,	UpperCamelCase					)

        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     Tuple							=						pt_inputs["input_ids"].shape
        __SCREAMING_SNAKE_CASE							:     Optional[int]							=						np.random.randint(0  ,	seq_length - 1  ,	size=(batch_size,)					)
        for batch_idx, start_index in enumerate(UpperCamelCase					):
         __SCREAMING_SNAKE_CASE							:     int							=						0
         __SCREAMING_SNAKE_CASE							:     str							=						1
         __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						0
         __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						1
        __SCREAMING_SNAKE_CASE							:     int							=						pt_model_class(UpperCamelCase					).eval()
        __SCREAMING_SNAKE_CASE							:     int							=						model_class(UpperCamelCase  ,	dtype=jnp.floataa					)

        __SCREAMING_SNAKE_CASE							:     List[str]							=						convert_pytorch_state_dict_to_flax(pt_model.state_dict()  ,	UpperCamelCase					)
        __SCREAMING_SNAKE_CASE							:     List[str]							=						fx_state

        with torch.no_grad():
         __SCREAMING_SNAKE_CASE							:     int							=						pt_model(**UpperCamelCase					).to_tuple()

        __SCREAMING_SNAKE_CASE							:     List[str]							=						fx_model(**UpperCamelCase					).to_tuple()
        self.assertEqual(len(UpperCamelCase					)  ,	len(UpperCamelCase					)  ,	"Output lengths differ between Flax and PyTorch"					)
        for fx_output, pt_output in zip(UpperCamelCase  ,	UpperCamelCase					):
         self.assert_almost_equals(fx_output[:, -1]  ,	pt_output[:, -1].numpy()  ,	4E-2					)

        with tempfile.TemporaryDirectory() as tmpdirname:
         pt_model.save_pretrained(UpperCamelCase					)
         __SCREAMING_SNAKE_CASE							:     List[Any]							=						model_class.from_pretrained(UpperCamelCase  ,	from_pt=UpperCamelCase					)

        __SCREAMING_SNAKE_CASE							:     List[Any]							=						fx_model_loaded(**UpperCamelCase					).to_tuple()
        self.assertEqual(
            len(UpperCamelCase					)  ,	len(UpperCamelCase					)  ,	"Output lengths differ between Flax and PyTorch"					)
        for fx_output_loaded, pt_output in zip(UpperCamelCase  ,	UpperCamelCase					):
         self.assert_almost_equals(fx_output_loaded[:, -1]  ,	pt_output[:, -1].numpy()  ,	4E-2					)





     @is_pt_flax_cross_test
     def 						__snake_case     (							self			:				str					)->Dict:
      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     List[str]							=						self.model_tester.prepare_config_and_inputs_for_common()
      for model_class in self.all_model_classes:
       with self.subTest(model_class.__name__					):
        # prepare inputs
        __SCREAMING_SNAKE_CASE							:     List[str]							=						self._prepare_for_class(UpperCamelCase  ,	UpperCamelCase					)
        __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						{k: torch.tensor(v.tolist()					) for k, v in prepared_inputs_dict.items()}

        # load corresponding PyTorch class
        __SCREAMING_SNAKE_CASE							:     int							=						model_class.__name__[4:]  # Skip the "Flax" at the beginning
        __SCREAMING_SNAKE_CASE							:     str							=						getattr(UpperCamelCase  ,	UpperCamelCase					)

        __SCREAMING_SNAKE_CASE							:     Any							=						pt_model_class(UpperCamelCase					).eval()
        __SCREAMING_SNAKE_CASE							:     List[str]							=						model_class(UpperCamelCase  ,	dtype=jnp.floataa					)

        __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						load_flax_weights_in_pytorch_model(UpperCamelCase  ,	fx_model.params					)
        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     str							=						pt_inputs["input_ids"].shape
        __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						np.random.randint(0  ,	seq_length - 1  ,	size=(batch_size,)					)
        for batch_idx, start_index in enumerate(UpperCamelCase					):
         __SCREAMING_SNAKE_CASE							:     Any							=						0
         __SCREAMING_SNAKE_CASE							:     Optional[int]							=						1
         __SCREAMING_SNAKE_CASE							:     str							=						0
         __SCREAMING_SNAKE_CASE							:     Dict							=						1

        # make sure weights are tied in PyTorch
        pt_model.tie_weights()

        with torch.no_grad():
         __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						pt_model(**UpperCamelCase					).to_tuple()

        __SCREAMING_SNAKE_CASE							:     Tuple							=						fx_model(**UpperCamelCase					).to_tuple()
        self.assertEqual(len(UpperCamelCase					)  ,	len(UpperCamelCase					)  ,	"Output lengths differ between Flax and PyTorch"					)
        for fx_output, pt_output in zip(UpperCamelCase  ,	UpperCamelCase					):
         self.assert_almost_equals(fx_output[:, -1]  ,	pt_output[:, -1].numpy()  ,	4E-2					)

        with tempfile.TemporaryDirectory() as tmpdirname:
         fx_model.save_pretrained(UpperCamelCase					)
         __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						pt_model_class.from_pretrained(UpperCamelCase  ,	from_flax=UpperCamelCase					)

        with torch.no_grad():
         __SCREAMING_SNAKE_CASE							:     Dict							=						pt_model_loaded(**UpperCamelCase					).to_tuple()

        self.assertEqual(
            len(UpperCamelCase					)  ,	len(UpperCamelCase					)  ,	"Output lengths differ between Flax and PyTorch"					)
        for fx_output, pt_output in zip(UpperCamelCase  ,	UpperCamelCase					):
         self.assert_almost_equals(fx_output[:, -1]  ,	pt_output[:, -1].numpy()  ,	4E-2					)






     @slow
     def 						__snake_case     (							self			:				Dict					)->Any:
      for model_class_name in self.all_model_classes:
       __SCREAMING_SNAKE_CASE							:     int							=						model_class_name.from_pretrained("EleutherAI/gpt-neo-125M"					)
       __SCREAMING_SNAKE_CASE							:     List[str]							=						model(np.ones((1, 1)					)					)
       self.assertIsNotNone(UpperCamelCase					)












from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFXLMForMultipleChoice,
		    TFXLMForQuestionAnsweringSimple,
		    TFXLMForSequenceClassification,
		    TFXLMForTokenClassification,
		    TFXLMModel,
		    TFXLMWithLMHeadModel,
		    XLMConfig,
		)




class        snake_case__				:
				"""simple docstring"""



				def __init__( self							:	Optional[Any]   , __lowerCamelCase							:	Any   , )   ->	Any:
							a       =     parent
							a       =     13
							a       =     7
							a       =     True
							a       =     True
							a       =     True
							a       =     True
							a       =     True
							a       =     False
							a       =     False
							a       =     False
							a       =     2
							a       =     99
							a       =     0
							a       =     32
							a       =     2
							a       =     4
							a       =     0.1
							a       =     0.1
							a       =     5_12
							a       =     16
							a       =     2
							a       =     0.02
							a       =     3
							a       =     4
							a       =     "last"
							a       =     True
							a       =     None
							a       =     0





				def 				__UpperCAmelCase				( self							:	int				)   ->	Tuple:
							a       =     ids_tensor([self.batch_size, self.seq_length]   , self.vocab_size				)
							a       =     random_attention_mask([self.batch_size, self.seq_length]   , dtype=tf.floataa				)

							a       =     None
							if self.use_input_lengths:
										a       =     (
										    ids_tensor([self.batch_size]   , vocab_size=2				) + self.seq_length - 2
										)  # small variation of seq_length

							a       =     None
							if self.use_token_type_ids:
										a       =     ids_tensor([self.batch_size, self.seq_length]   , self.n_langs				)

							a       =     None
							a       =     None
							a       =     None
							if self.use_labels:
										a       =     ids_tensor([self.batch_size]   , self.type_sequence_label_size				)
										a       =     ids_tensor([self.batch_size, self.seq_length]   , self.num_labels				)
										a       =     ids_tensor([self.batch_size]   , 2   , dtype=tf.floataa				)
										a       =     ids_tensor([self.batch_size]   , self.num_choices				)

							a       =     XLMConfig(
							    vocab_size=self.vocab_size   , n_special=self.n_special   , emb_dim=self.hidden_size   , n_layers=self.num_hidden_layers   , n_heads=self.num_attention_heads   , dropout=self.hidden_dropout_prob   , attention_dropout=self.attention_probs_dropout_prob   , gelu_activation=self.gelu_activation   , sinusoidal_embeddings=self.sinusoidal_embeddings   , asm=self.asm   , causal=self.causal   , n_langs=self.n_langs   , max_position_embeddings=self.max_position_embeddings   , initializer_range=self.initializer_range   , summary_type=self.summary_type   , use_proj=self.use_proj   , bos_token_id=self.bos_token_id   , )

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_lengths,
							    sequence_labels,
							    token_labels,
							    is_impossible_labels,
							    choice_labels,
							    input_mask,
							)





				def 				__UpperCAmelCase				( self							:	List[str]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	Any   , )   ->	Union[str, Any]:
							a       =     TFXLMModel(config=__lowerCamelCase				)
							a       =     {"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
							a       =     model(__lowerCamelCase				)

							a       =     [input_ids, input_mask]
							a       =     model(__lowerCamelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape   , (self.batch_size, self.seq_length, self.hidden_size)				)





				def 				__UpperCAmelCase				( self							:	Union[str, Any]   , __lowerCamelCase							:	Any   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	Dict   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Any   , )   ->	Optional[Any]:
							a       =     TFXLMWithLMHeadModel(__lowerCamelCase				)

							a       =     {"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
							a       =     model(__lowerCamelCase				)

							a       =     outputs

							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.seq_length, self.vocab_size)				)





				def 				__UpperCAmelCase				( self							:	Tuple   , __lowerCamelCase							:	Dict   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	Any   , __lowerCamelCase							:	List[Any]   , )   ->	int:
							a       =     TFXLMForQuestionAnsweringSimple(__lowerCamelCase				)

							a       =     {"input_ids": input_ids, "lengths": input_lengths}

							a       =     model(__lowerCamelCase				)

							self.parent.assertEqual(result.start_logits.shape   , (self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape   , (self.batch_size, self.seq_length)				)





				def 				__UpperCAmelCase				( self							:	Dict   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	Optional[int]   , )   ->	Optional[int]:
							a       =     TFXLMForSequenceClassification(__lowerCamelCase				)

							a       =     {"input_ids": input_ids, "lengths": input_lengths}

							a       =     model(__lowerCamelCase				)

							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.type_sequence_label_size)				)





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Union[str, Any]   , )   ->	Optional[Any]:
							a       =     self.num_labels
							a       =     TFXLMForTokenClassification(config=__lowerCamelCase				)
							a       =     {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
							a       =     model(__lowerCamelCase				)
							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.seq_length, self.num_labels)				)





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Dict   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Dict   , )   ->	List[Any]:
							a       =     self.num_choices
							a       =     TFXLMForMultipleChoice(config=__lowerCamelCase				)
							a       =     tf.tile(tf.expand_dims(__lowerCamelCase   , 1				)   , (1, self.num_choices, 1)				)
							a       =     tf.tile(tf.expand_dims(__lowerCamelCase   , 1				)   , (1, self.num_choices, 1)				)
							a       =     tf.tile(tf.expand_dims(__lowerCamelCase   , 1				)   , (1, self.num_choices, 1)				)
							a       =     {
							    "input_ids": multiple_choice_inputs_ids,
							    "attention_mask": multiple_choice_input_mask,
							    "token_type_ids": multiple_choice_token_type_ids,
							}
							a       =     model(__lowerCamelCase				)
							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.num_choices)				)





				def 				__UpperCAmelCase				( self							:	Optional[int]				)   ->	str:
							a       =     self.prepare_config_and_inputs()
							(
							    (
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						
							)       =     config_and_inputs
							a       =     {
							    "input_ids": input_ids,
							    "token_type_ids": token_type_ids,
							    "langs": token_type_ids,
							    "lengths": input_lengths,
							}
							return config, inputs_dict




@require_tf
class        snake_case__				(_UpperCamelCase       ,			_UpperCamelCase       ,			unittest.TestCase  ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			:				int									=   (
				    (
				        TFXLMModel,
				        TFXLMWithLMHeadModel,
				        TFXLMForSequenceClassification,
				        TFXLMForQuestionAnsweringSimple,
				        TFXLMForTokenClassification,
				        TFXLMForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				SCREAMING_SNAKE_CASE_			:				List[str]									=   (
				    (TFXLMWithLMHeadModel,) if is_tf_available() else ()
				)  # TODO (PVP): Check other models whether language generation is also applicable
				SCREAMING_SNAKE_CASE_			:				List[Any]									=   (
				    {
				        """feature-extraction""": TFXLMModel,
				        """fill-mask""": TFXLMWithLMHeadModel,
				        """question-answering""": TFXLMForQuestionAnsweringSimple,
				        """text-classification""": TFXLMForSequenceClassification,
				        """text-generation""": TFXLMWithLMHeadModel,
				        """token-classification""": TFXLMForTokenClassification,
				        """zero-shot""": TFXLMForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				SCREAMING_SNAKE_CASE_			:				Tuple									=   False
				SCREAMING_SNAKE_CASE_			:				Dict									=   False





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	Any   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	List[Any]				)   ->	Dict:
							if (
							    pipeline_test_casse_name == "QAPipelineTests"
							    and tokenizer_name is not None
							    and not tokenizer_name.endswith("Fast"				)
							):
										# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
										# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
										# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
										return True

							return False





				def 				__UpperCAmelCase				( self							:	Tuple				)   ->	Tuple:
							a       =     TFXLMModelTester(self				)
							a       =     ConfigTester(self   , config_class=__lowerCamelCase   , emb_dim=37				)





				def 				__UpperCAmelCase				( self							:	List[str]				)   ->	Optional[int]:
							self.config_tester.run_common_tests()





				def 				__UpperCAmelCase				( self							:	Dict				)   ->	List[Any]:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_model(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Tuple				)   ->	Dict:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_lm_head(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	Dict:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_qa(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	List[Any]				)   ->	Optional[int]:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_sequence_classif(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Optional[int]				)   ->	Optional[Any]:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_for_token_classification(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	List[Any]				)   ->	str:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_for_multiple_choice(*__lowerCamelCase				)





				@slow
				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	List[Any]:
							for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										a       =     TFXLMModel.from_pretrained(__lowerCamelCase				)
										self.assertIsNotNone(__lowerCamelCase				)




@require_tf
class        snake_case__				(unittest.TestCase  ):
				"""simple docstring"""



				@slow
				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	str:
							a       =     TFXLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"				)
							a       =     tf.convert_to_tensor([[14, 4_47]]   , dtype=tf.intaa				)  # the president
							a       =     [
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							]  # the president the president the president the president the president the president the president the president the president the president
							# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
							a       =     model.generate(__lowerCamelCase   , do_sample=__lowerCamelCase				)
							self.assertListEqual(output_ids[0].numpy().tolist()   , __lowerCamelCase				)






