



import gc
import os
import tempfile
import unittest

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline, UNetaDConditionModel
from diffusers.loaders import AttnProcsLayers, LoraLoaderMixin
from diffusers.models.attention_processor import (
    Attention,
    AttnProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRAAttnProcessor2_0,
    LoRAXFormersAttnProcessor,
    XFormersAttnProcessor,
)
from diffusers.utils import TEXT_ENCODER_ATTN_MODULE, floats_tensor, torch_device
def 						snake_case__							(       __lowercase							)		->		Dict:







						"""simple docstring"""

						A__						:				Union[str, Any]         =			{}
						for name in unet.attn_processors.keys():
												A__						:				List[Any]         =			None if name.endswith("attn1.processor"							) else unet.config.cross_attention_dim
												if name.startswith("mid_block"							):
																		A__						:				Optional[int]         =			unet.config.block_out_channels[-1]
												elif name.startswith("up_blocks"							):
																		A__						:				Optional[Any]         =			int(name[len("up_blocks."							)]							)
																		A__						:				List[Any]         =			list(reversed(unet.config.block_out_channels							)							)[block_id]
												elif name.startswith("down_blocks"							):
																		A__						:				Union[str, Any]         =			int(name[len("down_blocks."							)]							)
																		A__						:				Tuple         =			unet.config.block_out_channels[block_id]
												A__						:				int         =			(
												    LoRAAttnProcessor2_0 if hasattr(_a				,  "scaled_dot_product_attention"							) else LoRAAttnProcessor
												)
												A__						:				Tuple         =			lora_attn_processor_class(
												    hidden_size=_a				,  cross_attention_dim=_a							)
						A__						:				List[str]         =			AttnProcsLayers(_a							)
						return lora_attn_procs, unet_lora_layers
def 						snake_case__							(       __lowercase							)		->		int:







						"""simple docstring"""

						A__						:				Optional[int]         =			{}
						A__						:				List[str]         =			(
						    LoRAAttnProcessor2_0 if hasattr(_a				,  "scaled_dot_product_attention"							) else LoRAAttnProcessor
						)
						for name, module in text_encoder.named_modules():
												if name.endswith(_a							):
																		A__						:				List[Any]         =			lora_attn_processor_class(
																		    hidden_size=module.out_proj.out_features				,  cross_attention_dim=_a							)
						return text_lora_attn_procs
def 						snake_case__							(       __lowercase							)		->		Dict:







						"""simple docstring"""

						A__						:				Optional[Any]         =			create_text_encoder_lora_attn_procs(_a							)
						A__						:				Any         =			AttnProcsLayers(_a							)
						return text_encoder_lora_layers
def 						snake_case__							(       __lowercase				,  __lowercase=False							)		->		Union[str, Any]:







						"""simple docstring"""

						for _, attn_proc in text_lora_attn_procs.items():
												# set up.weights
												for layer_name, layer_module in attn_proc.named_modules():
																		if layer_name.endswith("_lora"							):
																								A__						:				List[str]         =			(
																								    torch.randn_like(layer_module.up.weight							)
																								    if randn_weight
																								    else torch.zeros_like(layer_module.up.weight							)
																								)
																								A__						:				Any         =			torch.nn.Parameter(_a							)




class 				lowerCAmelCase__	(						unittest.TestCase    ):







					def 			_lowercase      (							self   :  Tuple):
											torch.manual_seed(0)
											A__						:				Union[str, Any]         =			UNetaDConditionModel(
											    block_out_channels=(32, 64)      ,		layers_per_block=2      ,		sample_size=32      ,		in_channels=4      ,		out_channels=4      ,		down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,		up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")      ,		cross_attention_dim=32      ,		)
											A__						:				int         =			DDIMScheduler(
											    beta_start=0.0_00_85      ,		beta_end=0.0_12      ,		beta_schedule="scaled_linear"      ,		clip_sample=_lowerCamelCase      ,		set_alpha_to_one=_lowerCamelCase      ,		steps_offset=1      ,		)
											torch.manual_seed(0)
											A__						:				Union[str, Any]         =			AutoencoderKL(
											    block_out_channels=[32, 64]      ,		in_channels=3      ,		out_channels=3      ,		down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]      ,		up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]      ,		latent_channels=4      ,		)
											A__						:				str         =			CLIPTextConfig(
											    bos_token_id=0      ,		eos_token_id=2      ,		hidden_size=32      ,		intermediate_size=37      ,		layer_norm_eps=1e-05      ,		num_attention_heads=4      ,		num_hidden_layers=5      ,		pad_token_id=1      ,		vocab_size=1000      ,		)
											A__						:				List[str]         =			CLIPTextModel(_lowerCamelCase)
											A__						:				int         =			CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

											A__						:				Union[str, Any]         =			create_unet_lora_layers(_lowerCamelCase)
											A__						:				Any         =			create_text_encoder_lora_layers(_lowerCamelCase)

											A__						:				Dict         =			{
											    'unet': unet,
											    'scheduler': scheduler,
											    'vae': vae,
											    'text_encoder': text_encoder,
											    'tokenizer': tokenizer,
											    'safety_checker': None,
											    'feature_extractor': None,
											}
											A__						:				List[str]         =			{
											    'unet_lora_layers': unet_lora_layers,
											    'text_encoder_lora_layers': text_encoder_lora_layers,
											    'unet_lora_attn_procs': unet_lora_attn_procs,
											}
											return pipeline_components, lora_components







					def 			_lowercase      (							self   :  str):
											A__						:				Any         =			1
											A__						:				Union[str, Any]         =			10
											A__						:				Dict         =			4
											A__						:				Optional[Any]         =			(32, 32)

											A__						:				Any         =			torch.manual_seed(0)
											A__						:				Union[str, Any]         =			floats_tensor((batch_size, num_channels) + sizes)
											A__						:				List[str]         =			torch.randint(1      ,		_lowerCamelCase      ,		size=(batch_size, sequence_length)      ,		generator=_lowerCamelCase)

											A__						:				Any         =			{
											    'prompt': 'A painting of a squirrel eating a burger',
											    'generator': generator,
											    'num_inference_steps': 2,
											    'guidance_scale': 6.0,
											    'output_type': 'numpy',
											}

											return noise, input_ids, pipeline_inputs

											# copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb







					def 			_lowercase      (							self   :  Optional[Any]):
											A__						:				int         =			77

											A__						:				Any         =			torch.randint(2      ,		56      ,		size=(1, max_seq_length)      ,		generator=torch.manual_seed(0))

											A__						:				List[str]         =			{}
											A__						:				Optional[int]         =			inputs
											return prepared_inputs







					def 			_lowercase      (							self   :  Tuple      ,		_A   :  Union[str, Any]):
											A__						:				int         =			self.get_dummy_components()
											LoraLoaderMixin.save_lora_weights(
											    save_directory=_lowerCamelCase      ,		unet_lora_layers=lora_components["unet_lora_layers"]      ,		text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]      ,		)
											self.assertTrue(os.path.isfile(os.path.join(_lowerCamelCase      ,		"pytorch_lora_weights.bin")))







					def 			_lowercase      (							self   :  int):
											A__						:				List[str]         =			self.get_dummy_components()
											A__						:				List[str]         =			StableDiffusionPipeline(**_lowerCamelCase)
											A__						:				Dict         =			sd_pipe.to(_lowerCamelCase)
											sd_pipe.set_progress_bar_config(disable=_lowerCamelCase)

											A__						:				Union[str, Any]         =			self.get_dummy_inputs()

											A__						:				Any         =			sd_pipe(**_lowerCamelCase).images
											A__						:				int         =			original_images[0, -3:, -3:, -1]

											with tempfile.TemporaryDirectory() as tmpdirname:
																	LoraLoaderMixin.save_lora_weights(
																	    save_directory=_lowerCamelCase      ,		unet_lora_layers=lora_components["unet_lora_layers"]      ,		text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]      ,		)
																	self.assertTrue(os.path.isfile(os.path.join(_lowerCamelCase      ,		"pytorch_lora_weights.bin")))
																	sd_pipe.load_lora_weights(_lowerCamelCase)

											A__						:				Optional[int]         =			sd_pipe(**_lowerCamelCase).images
											A__						:				Optional[int]         =			lora_images[0, -3:, -3:, -1]

											# Outputs shouldn't match.
											self.assertFalse(torch.allclose(torch.from_numpy(_lowerCamelCase)      ,		torch.from_numpy(_lowerCamelCase)))







					def 			_lowercase      (							self   :  Dict):
											A__						:				Tuple         =			self.get_dummy_components()
											A__						:				str         =			StableDiffusionPipeline(**_lowerCamelCase)
											A__						:				Union[str, Any]         =			sd_pipe.to(_lowerCamelCase)
											sd_pipe.set_progress_bar_config(disable=_lowerCamelCase)

											A__						:				Tuple         =			self.get_dummy_inputs()

											A__						:				Optional[Any]         =			sd_pipe(**_lowerCamelCase).images
											A__						:				Optional[int]         =			original_images[0, -3:, -3:, -1]

											with tempfile.TemporaryDirectory() as tmpdirname:
																	LoraLoaderMixin.save_lora_weights(
																	    save_directory=_lowerCamelCase      ,		unet_lora_layers=lora_components["unet_lora_layers"]      ,		text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]      ,		safe_serialization=_lowerCamelCase      ,		)
																	self.assertTrue(os.path.isfile(os.path.join(_lowerCamelCase      ,		"pytorch_lora_weights.safetensors")))
																	sd_pipe.load_lora_weights(_lowerCamelCase)

											A__						:				Dict         =			sd_pipe(**_lowerCamelCase).images
											A__						:				Dict         =			lora_images[0, -3:, -3:, -1]

											# Outputs shouldn't match.
											self.assertFalse(torch.allclose(torch.from_numpy(_lowerCamelCase)      ,		torch.from_numpy(_lowerCamelCase)))







					def 			_lowercase      (							self   :  Dict):
											A__						:				List[str]         =			self.get_dummy_components()
											A__						:				List[Any]         =			lora_components['unet_lora_attn_procs']
											A__						:				Union[str, Any]         =			StableDiffusionPipeline(**_lowerCamelCase)
											A__						:				int         =			sd_pipe.to(_lowerCamelCase)
											sd_pipe.set_progress_bar_config(disable=_lowerCamelCase)

											A__						:				str         =			self.get_dummy_inputs()

											A__						:				Dict         =			sd_pipe(**_lowerCamelCase).images
											A__						:				Dict         =			original_images[0, -3:, -3:, -1]

											with tempfile.TemporaryDirectory() as tmpdirname:
																	A__						:				Union[str, Any]         =			sd_pipe.unet
																	unet.set_attn_processor(_lowerCamelCase)
																	unet.save_attn_procs(_lowerCamelCase)
																	self.assertTrue(os.path.isfile(os.path.join(_lowerCamelCase      ,		"pytorch_lora_weights.bin")))
																	sd_pipe.load_lora_weights(_lowerCamelCase)

											A__						:				Union[str, Any]         =			sd_pipe(**_lowerCamelCase).images
											A__						:				Optional[Any]         =			lora_images[0, -3:, -3:, -1]

											# Outputs shouldn't match.
											self.assertFalse(torch.allclose(torch.from_numpy(_lowerCamelCase)      ,		torch.from_numpy(_lowerCamelCase)))







					def 			_lowercase      (							self   :  Dict):
											A__						:				Dict         =			self.get_dummy_components()
											A__						:				Any         =			StableDiffusionPipeline(**_lowerCamelCase)

											A__						:				Any         =			self.get_dummy_tokens()

											# inference without lora
											A__						:				List[str]         =			pipe.text_encoder(**_lowerCamelCase)[0]
											assert outputs_without_lora.shape == (1, 77, 32)

											# create lora_attn_procs with zeroed out up.weights
											A__						:				Any         =			create_text_encoder_lora_attn_procs(pipe.text_encoder)
											set_lora_up_weights(_lowerCamelCase      ,		randn_weight=_lowerCamelCase)

											# monkey patch
											pipe._modify_text_encoder(_lowerCamelCase)

											# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
											del text_attn_procs
											gc.collect()

											# inference with lora
											A__						:				Tuple         =			pipe.text_encoder(**_lowerCamelCase)[0]
											assert outputs_with_lora.shape == (1, 77, 32)

											assert torch.allclose(
											    _lowerCamelCase      ,		_lowerCamelCase), "lora_up_weight are all zero, so the lora outputs should be the same to without lora outputs"

											# create lora_attn_procs with randn up.weights
											A__						:				Optional[int]         =			create_text_encoder_lora_attn_procs(pipe.text_encoder)
											set_lora_up_weights(_lowerCamelCase      ,		randn_weight=_lowerCamelCase)

											# monkey patch
											pipe._modify_text_encoder(_lowerCamelCase)

											# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
											del text_attn_procs
											gc.collect()

											# inference with lora
											A__						:				Optional[int]         =			pipe.text_encoder(**_lowerCamelCase)[0]
											assert outputs_with_lora.shape == (1, 77, 32)

											assert not torch.allclose(
											    _lowerCamelCase      ,		_lowerCamelCase), "lora_up_weight are not zero, so the lora outputs should be different to without lora outputs"







					def 			_lowercase      (							self   :  List[str]):
											A__						:				Optional[Any]         =			self.get_dummy_components()
											A__						:				Any         =			StableDiffusionPipeline(**_lowerCamelCase)

											A__						:				List[Any]         =			self.get_dummy_tokens()

											# inference without lora
											A__						:				str         =			pipe.text_encoder(**_lowerCamelCase)[0]
											assert outputs_without_lora.shape == (1, 77, 32)

											# create lora_attn_procs with randn up.weights
											A__						:				Any         =			create_text_encoder_lora_attn_procs(pipe.text_encoder)
											set_lora_up_weights(_lowerCamelCase      ,		randn_weight=_lowerCamelCase)

											# monkey patch
											pipe._modify_text_encoder(_lowerCamelCase)

											# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
											del text_attn_procs
											gc.collect()

											# inference with lora
											A__						:				Any         =			pipe.text_encoder(**_lowerCamelCase)[0]
											assert outputs_with_lora.shape == (1, 77, 32)

											assert not torch.allclose(
											    _lowerCamelCase      ,		_lowerCamelCase), "lora outputs should be different to without lora outputs"

											# remove monkey patch
											pipe._remove_text_encoder_monkey_patch()

											# inference with removed lora
											A__						:				Dict         =			pipe.text_encoder(**_lowerCamelCase)[0]
											assert outputs_without_lora_removed.shape == (1, 77, 32)

											assert torch.allclose(
											    _lowerCamelCase      ,		_lowerCamelCase), "remove lora monkey patch should restore the original outputs"







					def 			_lowercase      (							self   :  Tuple):
											A__						:				Optional[int]         =			self.get_dummy_components()
											A__						:				Any         =			StableDiffusionPipeline(**_lowerCamelCase)
											A__						:				List[str]         =			sd_pipe.to(_lowerCamelCase)
											sd_pipe.set_progress_bar_config(disable=_lowerCamelCase)

											A__						:				int         =			self.get_dummy_inputs()

											with tempfile.TemporaryDirectory() as tmpdirname:
																	LoraLoaderMixin.save_lora_weights(
																	    save_directory=_lowerCamelCase      ,		unet_lora_layers=lora_components["unet_lora_layers"]      ,		text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]      ,		)
																	self.assertTrue(os.path.isfile(os.path.join(_lowerCamelCase      ,		"pytorch_lora_weights.bin")))
																	sd_pipe.load_lora_weights(_lowerCamelCase)

											A__						:				Optional[int]         =			sd_pipe(**_lowerCamelCase).images
											A__						:				List[Any]         =			lora_images[0, -3:, -3:, -1]

											A__						:				List[str]         =			sd_pipe(**_lowerCamelCase      ,		cross_attention_kwargs={"scale": 0.5}).images
											A__						:				Union[str, Any]         =			lora_images_with_scale[0, -3:, -3:, -1]

											# Outputs shouldn't match.
											self.assertFalse(
											    torch.allclose(torch.from_numpy(_lowerCamelCase)      ,		torch.from_numpy(_lowerCamelCase)))







					def 			_lowercase      (							self   :  int):
											with tempfile.TemporaryDirectory() as tmpdirname:
																	self.create_lora_weight_file(_lowerCamelCase)

																	A__						:				Dict         =			self.get_dummy_components()
																	A__						:				Union[str, Any]         =			StableDiffusionPipeline(**_lowerCamelCase)
																	A__						:				Tuple         =			sd_pipe.to(_lowerCamelCase)
																	sd_pipe.set_progress_bar_config(disable=_lowerCamelCase)

																	# check if vanilla attention processors are used
																	for _, module in sd_pipe.unet.named_modules():
																							if isinstance(_lowerCamelCase      ,		_lowerCamelCase):
																													self.assertIsInstance(module.processor      ,		(AttnProcessor, AttnProcessor2_0))

            # load LoRA weight file
																	sd_pipe.load_lora_weights(_lowerCamelCase)

																	# check if lora attention processors are used
																	for _, module in sd_pipe.unet.named_modules():
																							if isinstance(_lowerCamelCase      ,		_lowerCamelCase):
																													A__						:				Union[str, Any]         =			(
																													    LoRAAttnProcessor2_0 if hasattr(_lowerCamelCase      ,		"scaled_dot_product_attention") else LoRAAttnProcessor
																													)
																													self.assertIsInstance(module.processor      ,		_lowerCamelCase)







					@unittest.skipIf(torch_device != "cuda"      ,		"This test is supposed to run on GPU")
					def 			_lowercase      (							self   :  int):
											with tempfile.TemporaryDirectory() as tmpdirname:
																	self.create_lora_weight_file(_lowerCamelCase)

																	A__						:				Optional[Any]         =			self.get_dummy_components()
																	A__						:				Dict         =			StableDiffusionPipeline(**_lowerCamelCase)
																	A__						:				str         =			sd_pipe.to(_lowerCamelCase)
																	sd_pipe.set_progress_bar_config(disable=_lowerCamelCase)

																	# enable XFormers
																	sd_pipe.enable_xformers_memory_efficient_attention()

																	# check if xFormers attention processors are used
																	for _, module in sd_pipe.unet.named_modules():
																							if isinstance(_lowerCamelCase      ,		_lowerCamelCase):
																													self.assertIsInstance(module.processor      ,		_lowerCamelCase)

            # load LoRA weight file
																	sd_pipe.load_lora_weights(_lowerCamelCase)

																	# check if lora attention processors are used
																	for _, module in sd_pipe.unet.named_modules():
																							if isinstance(_lowerCamelCase      ,		_lowerCamelCase):
																													self.assertIsInstance(module.processor      ,		_lowerCamelCase)







					@unittest.skipIf(torch_device != "cuda"      ,		"This test is supposed to run on GPU")
					def 			_lowercase      (							self   :  Any):
											A__						:				Union[str, Any]         =			self.get_dummy_components()
											A__						:				Dict         =			StableDiffusionPipeline(**_lowerCamelCase)
											A__						:				str         =			sd_pipe.to(_lowerCamelCase)
											sd_pipe.set_progress_bar_config(disable=_lowerCamelCase)

											A__						:				Optional[int]         =			self.get_dummy_inputs()

											# enable XFormers
											sd_pipe.enable_xformers_memory_efficient_attention()

											A__						:				Optional[Any]         =			sd_pipe(**_lowerCamelCase).images
											A__						:				Optional[int]         =			original_images[0, -3:, -3:, -1]

											with tempfile.TemporaryDirectory() as tmpdirname:
																	LoraLoaderMixin.save_lora_weights(
																	    save_directory=_lowerCamelCase      ,		unet_lora_layers=lora_components["unet_lora_layers"]      ,		text_encoder_lora_layers=lora_components["text_encoder_lora_layers"]      ,		)
																	self.assertTrue(os.path.isfile(os.path.join(_lowerCamelCase      ,		"pytorch_lora_weights.bin")))
																	sd_pipe.load_lora_weights(_lowerCamelCase)

											A__						:				Union[str, Any]         =			sd_pipe(**_lowerCamelCase).images
											A__						:				List[Any]         =			lora_images[0, -3:, -3:, -1]

											# Outputs shouldn't match.
											self.assertFalse(torch.allclose(torch.from_numpy(_lowerCamelCase)      ,		torch.from_numpy(_lowerCamelCase)))
'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLMForMultipleChoice,
      TFXLMForQuestionAnsweringSimple,
      TFXLMForSequenceClassification,
      TFXLMForTokenClassification,
      TFXLMModel,
      TFXLMWithLMHeadModel,
      XLMConfig,
  )






class       __lowercase		:







     """simple docstring"""






     def __init__(		self  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						str		     =	parent
         __UpperCamelCase						:						Tuple		     =	1_3
         __UpperCamelCase						:						str		     =	7
         __UpperCamelCase						:						List[str]		     =	True
         __UpperCamelCase						:						Optional[Any]		     =	True
         __UpperCamelCase						:						Tuple		     =	True
         __UpperCamelCase						:						Tuple		     =	True
         __UpperCamelCase						:						int		     =	True
         __UpperCamelCase						:						int		     =	False
         __UpperCamelCase						:						Optional[Any]		     =	False
         __UpperCamelCase						:						Any		     =	False
         __UpperCamelCase						:						Any		     =	2
         __UpperCamelCase						:						str		     =	9_9
         __UpperCamelCase						:						int		     =	0
         __UpperCamelCase						:						Any		     =	3_2
         __UpperCamelCase						:						Tuple		     =	2
         __UpperCamelCase						:						Optional[int]		     =	4
         __UpperCamelCase						:						List[str]		     =	0.1
         __UpperCamelCase						:						List[Any]		     =	0.1
         __UpperCamelCase						:						List[str]		     =	5_1_2
         __UpperCamelCase						:						int		     =	1_6
         __UpperCamelCase						:						Any		     =	2
         __UpperCamelCase						:						Dict		     =	0.0_2
         __UpperCamelCase						:						Any		     =	3
         __UpperCamelCase						:						str		     =	4
         __UpperCamelCase						:						Any		     =	'last'
         __UpperCamelCase						:						int		     =	True
         __UpperCamelCase						:						List[str]		     =	None
         __UpperCamelCase						:						Union[str, Any]		     =	0







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[str]		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.vocab_size     )
         __UpperCamelCase						:						str		     =	random_attention_mask([self.batch_size, self.seq_length]  ,      dtype=tf.floataa     )

         __UpperCamelCase						:						Optional[Any]		     =	None
         if self.use_input_lengths:
             __UpperCamelCase						:						Optional[int]		     =	(
                 ids_tensor([self.batch_size]  ,      vocab_size=2     ) + self.seq_length - 2
             )  # small variation of seq_length

         __UpperCamelCase						:						int		     =	None
         if self.use_token_type_ids:
             __UpperCamelCase						:						Union[str, Any]		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.n_langs     )

         __UpperCamelCase						:						Optional[Any]		     =	None
         __UpperCamelCase						:						Optional[Any]		     =	None
         __UpperCamelCase						:						Tuple		     =	None
         if self.use_labels:
             __UpperCamelCase						:						Tuple		     =	ids_tensor([self.batch_size]  ,      self.type_sequence_label_size     )
             __UpperCamelCase						:						Dict		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.num_labels     )
             __UpperCamelCase						:						Optional[int]		     =	ids_tensor([self.batch_size]  ,      2  ,      dtype=tf.floataa     )
             __UpperCamelCase						:						int		     =	ids_tensor([self.batch_size]  ,      self.num_choices     )

         __UpperCamelCase						:						int		     =	XLMConfig(
             vocab_size=self.vocab_size  ,      n_special=self.n_special  ,      emb_dim=self.hidden_size  ,      n_layers=self.num_hidden_layers  ,      n_heads=self.num_attention_heads  ,      dropout=self.hidden_dropout_prob  ,      attention_dropout=self.attention_probs_dropout_prob  ,      gelu_activation=self.gelu_activation  ,      sinusoidal_embeddings=self.sinusoidal_embeddings  ,      asm=self.asm  ,      causal=self.causal  ,      n_langs=self.n_langs  ,      max_position_embeddings=self.max_position_embeddings  ,      initializer_range=self.initializer_range  ,      summary_type=self.summary_type  ,      use_proj=self.use_proj  ,      bos_token_id=self.bos_token_id  ,      )

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						str		     =	TFXLMModel(config=_lowerCamelCase     )
         __UpperCamelCase						:						Optional[Any]		     =	{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
         __UpperCamelCase						:						str		     =	model(_lowerCamelCase     )

         __UpperCamelCase						:						Dict		     =	[input_ids, input_mask]
         __UpperCamelCase						:						Tuple		     =	model(_lowerCamelCase     )
         self.parent.assertEqual(result.last_hidden_state.shape  ,      (self.batch_size, self.seq_length, self.hidden_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						int		     =	TFXLMWithLMHeadModel(_lowerCamelCase     )

         __UpperCamelCase						:						List[str]		     =	{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
         __UpperCamelCase						:						List[str]		     =	model(_lowerCamelCase     )

         __UpperCamelCase						:						Tuple		     =	outputs

         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.vocab_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						Tuple		     =	TFXLMForQuestionAnsweringSimple(_lowerCamelCase     )

         __UpperCamelCase						:						Dict		     =	{'input_ids': input_ids, 'lengths': input_lengths}

         __UpperCamelCase						:						Optional[Any]		     =	model(_lowerCamelCase     )

         self.parent.assertEqual(result.start_logits.shape  ,      (self.batch_size, self.seq_length)     )
         self.parent.assertEqual(result.end_logits.shape  ,      (self.batch_size, self.seq_length)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						Optional[Any]		     =	TFXLMForSequenceClassification(_lowerCamelCase     )

         __UpperCamelCase						:						Union[str, Any]		     =	{'input_ids': input_ids, 'lengths': input_lengths}

         __UpperCamelCase						:						Dict		     =	model(_lowerCamelCase     )

         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.type_sequence_label_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						int		     =	self.num_labels
         __UpperCamelCase						:						int		     =	TFXLMForTokenClassification(config=_lowerCamelCase     )
         __UpperCamelCase						:						List[str]		     =	{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
         __UpperCamelCase						:						List[str]		     =	model(_lowerCamelCase     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.num_labels)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						Any		     =	self.num_choices
         __UpperCamelCase						:						List[Any]		     =	TFXLMForMultipleChoice(config=_lowerCamelCase     )
         __UpperCamelCase						:						Tuple		     =	tf.tile(tf.expand_dims(_lowerCamelCase  ,      1     )  ,      (1, self.num_choices, 1)     )
         __UpperCamelCase						:						Optional[int]		     =	tf.tile(tf.expand_dims(_lowerCamelCase  ,      1     )  ,      (1, self.num_choices, 1)     )
         __UpperCamelCase						:						Union[str, Any]		     =	tf.tile(tf.expand_dims(_lowerCamelCase  ,      1     )  ,      (1, self.num_choices, 1)     )
         __UpperCamelCase						:						str		     =	{
             'input_ids': multiple_choice_inputs_ids,
             'attention_mask': multiple_choice_input_mask,
             'token_type_ids': multiple_choice_token_type_ids,
         }
         __UpperCamelCase						:						Dict		     =	model(_lowerCamelCase     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.num_choices)     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[str]		     =	self.prepare_config_and_inputs()
         (
             (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       
         )						:						Any		     =	config_and_inputs
         __UpperCamelCase						:						Union[str, Any]		     =	{
             'input_ids': input_ids,
             'token_type_ids': token_type_ids,
             'langs': token_type_ids,
             'lengths': input_lengths,
         }
         return config, inputs_dict






@require_tf
class       __lowercase		(    _lowerCamelCase      ,				_lowerCamelCase      ,				unittest.TestCase    ):







     """simple docstring"""






     SCREAMING_SNAKE_CASE__		 =    (
         (
             TFXLMModel,
             TFXLMWithLMHeadModel,
             TFXLMForSequenceClassification,
             TFXLMForQuestionAnsweringSimple,
             TFXLMForTokenClassification,
             TFXLMForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     SCREAMING_SNAKE_CASE__		 =    (
         (TFXLMWithLMHeadModel,) if is_tf_available() else ()
     )  # TODO (PVP): Check other models whether language generation is also applicable
     SCREAMING_SNAKE_CASE__		 =    (
         {
             '''feature-extraction''': TFXLMModel,
             '''fill-mask''': TFXLMWithLMHeadModel,
             '''question-answering''': TFXLMForQuestionAnsweringSimple,
             '''text-classification''': TFXLMForSequenceClassification,
             '''text-generation''': TFXLMWithLMHeadModel,
             '''token-classification''': TFXLMForTokenClassification,
             '''zero-shot''': TFXLMForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     SCREAMING_SNAKE_CASE__		 =    False
     SCREAMING_SNAKE_CASE__		 =    False







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith('Fast'     )
         ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

         return False







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	TFXLMModelTester(self     )
         __UpperCamelCase						:						Union[str, Any]		     =	ConfigTester(self  ,      config_class=_lowerCamelCase  ,      emb_dim=3_7     )







     def lowerCAmelCase  (		self     ):
         self.config_tester.run_common_tests()







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[int]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_model(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[int]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_lm_head(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_qa(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_sequence_classif(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Any		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_token_classification(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Dict		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_multiple_choice(*_lowerCamelCase     )







     @slow
     def lowerCAmelCase  (		self     ):
         for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __UpperCamelCase						:						List[Any]		     =	TFXLMModel.from_pretrained(_lowerCamelCase     )
             self.assertIsNotNone(_lowerCamelCase     )






@require_tf
class       __lowercase		(    unittest.TestCase    ):







     """simple docstring"""






     @slow
     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Union[str, Any]		     =	TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'     )
         __UpperCamelCase						:						Union[str, Any]		     =	tf.convert_to_tensor([[1_4, 4_4_7]]  ,      dtype=tf.intaa     )  # the president
         __UpperCamelCase						:						List[Any]		     =	[
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
         ]  # the president the president the president the president the president the president the president the president the president the president
         # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
         __UpperCamelCase						:						str		     =	model.generate(_lowerCamelCase  ,      do_sample=_lowerCamelCase     )
         self.assertListEqual(output_ids[0].numpy().tolist()  ,      _lowerCamelCase     )



