


from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFTransfoXLForSequenceClassification,
						    TFTransfoXLLMHeadModel,
						    TFTransfoXLModel,
						)







class        snake_case_   :
					def __init__(      self   :Tuple			,__snake_case   :Union[str, Any]			,)					->    str:
												a__             =     parent
												a__             =     13
												a__             =     7
												a__             =     30
												a__             =     self.seq_length + self.mem_len
												a__             =     15
												a__             =     True
												a__             =     True
												a__             =     99
												a__             =     [10, 50, 80]
												a__             =     32
												a__             =     32
												a__             =     4
												a__             =     8
												a__             =     1_28
												a__             =     2
												a__             =     2
												a__             =     None
												a__             =     1
												a__             =     0
												a__             =     3
												a__             =     self.vocab_size - 1
												a__             =     0.01
					def  lowerCamelCase__(      self   :int      )					->    str:
												a__             =     ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size      )
												a__             =     ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size      )

												a__             =     None
												if self.use_labels:
																			a__             =     ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size      )

												a__             =     TransfoXLConfig(
												    vocab_size=self.vocab_size			,mem_len=self.mem_len			,clamp_len=self.clamp_len			,cutoffs=self.cutoffs			,d_model=self.hidden_size			,d_embed=self.d_embed			,n_head=self.num_attention_heads			,d_head=self.d_head			,d_inner=self.d_inner			,div_val=self.div_val			,n_layer=self.num_hidden_layers			,eos_token_id=self.eos_token_id			,pad_token_id=self.vocab_size - 1			,init_range=self.init_range			,num_labels=self.num_labels			,)

												return (config, input_ids_a, input_ids_a, lm_labels)
					def  lowerCamelCase__(      self   :Optional[Any]      )					->    Optional[Any]:
												random.seed(self.seed      )
												tf.random.set_seed(self.seed      )
					def  lowerCamelCase__(      self   :Tuple			,__snake_case   :List[Any]			,__snake_case   :Tuple			,__snake_case   :str			,__snake_case   :int      )					->    Optional[Any]:
												a__             =     TFTransfoXLModel(__a      )

												a__      ,					a__             =     model(__a      ).to_tuple()

												a__             =     {'input_ids': input_ids_a, 'mems': mems_a}

												a__      ,					a__             =     model(__a      ).to_tuple()

												self.parent.assertEqual(hidden_states_a.shape			,(self.batch_size, self.seq_length, self.hidden_size)      )
												self.parent.assertEqual(hidden_states_a.shape			,(self.batch_size, self.seq_length, self.hidden_size)      )
												self.parent.assertListEqual(
												    [mem.shape for mem in mems_a]			,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers			,)
												self.parent.assertListEqual(
												    [mem.shape for mem in mems_a]			,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers			,)
					def  lowerCamelCase__(      self   :Tuple			,__snake_case   :Tuple			,__snake_case   :Union[str, Any]			,__snake_case   :List[str]			,__snake_case   :Union[str, Any]      )					->    str:
												a__             =     TFTransfoXLLMHeadModel(__a      )

												a__      ,					a__             =     model(__a      ).to_tuple()

												a__             =     {'input_ids': input_ids_a, 'labels': lm_labels}
												a__      ,					a__             =     model(__a      ).to_tuple()

												a__      ,					a__             =     model([input_ids_a, mems_a]      ).to_tuple()

												a__             =     {'input_ids': input_ids_a, 'mems': mems_a, 'labels': lm_labels}

												a__      ,					a__             =     model(__a      ).to_tuple()

												self.parent.assertEqual(lm_logits_a.shape			,(self.batch_size, self.seq_length, self.vocab_size)      )
												self.parent.assertListEqual(
												    [mem.shape for mem in mems_a]			,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers			,)

												self.parent.assertEqual(lm_logits_a.shape			,(self.batch_size, self.seq_length, self.vocab_size)      )
												self.parent.assertListEqual(
												    [mem.shape for mem in mems_a]			,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers			,)
					def  lowerCamelCase__(      self   :Dict			,__snake_case   :List[Any]			,__snake_case   :List[str]			,__snake_case   :Any			,__snake_case   :str      )					->    Optional[int]:
												a__             =     TFTransfoXLForSequenceClassification(__a      )
												a__             =     model(__a      )
												self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_labels)      )



					def  lowerCamelCase__(      self   :str      )					->    List[Any]:
												a__             =     self.prepare_config_and_inputs()
												((a__)      ,					(a__)      ,					(a__)      ,					(a__))             =     config_and_inputs
												a__             =     {'input_ids': input_ids_a}
												return config, inputs_dict







@require_tf
class        snake_case_   (lowerCamelCase_	,						lowerCamelCase_	,						unittest.TestCase      ):
					UpperCAmelCase__					:    Optional[int]          =			(
					    (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
					)
					UpperCAmelCase__					:    List[str]          =			() if is_tf_available() else ()
					UpperCAmelCase__					:    Tuple          =			(
					    {
					        '''feature-extraction''': TFTransfoXLModel,
					        '''text-classification''': TFTransfoXLForSequenceClassification,
					        '''text-generation''': TFTransfoXLLMHeadModel,
					        '''zero-shot''': TFTransfoXLForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					# TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
					UpperCAmelCase__					:    str          =			False
					UpperCAmelCase__					:    List[str]          =			False
					UpperCAmelCase__					:    Any          =			False
					UpperCAmelCase__					:    int          =			False
					def  lowerCamelCase__(      self   :Union[str, Any]			,__snake_case   :Union[str, Any]			,__snake_case   :List[str]			,__snake_case   :Tuple			,__snake_case   :Optional[Any]			,__snake_case   :Tuple      )					->    List[Any]:
												if pipeline_test_casse_name == "TextGenerationPipelineTests":
																			# Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
																			# `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
																			# tokenizer.
																			return True

												return False
					def  lowerCamelCase__(      self   :str      )					->    List[str]:
												a__             =     TFTransfoXLModelTester(self      )
												a__             =     ConfigTester(self			,config_class=__a			,d_embed=37      )
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    Any:
												self.config_tester.run_common_tests()
					def  lowerCamelCase__(      self   :Any      )					->    str:
												self.model_tester.set_seed()
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_transfo_xl_model(*__a      )
					def  lowerCamelCase__(      self   :str      )					->    List[str]:
												self.model_tester.set_seed()
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_transfo_xl_lm_head(*__a      )
					def  lowerCamelCase__(      self   :Optional[Any]      )					->    Optional[Any]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*__a      )
					def  lowerCamelCase__(      self   :Optional[Any]      )					->    List[str]:
												a__      ,					a__             =     self.model_tester.prepare_config_and_inputs_for_common()
												a__             =     [TFTransfoXLForSequenceClassification]

												for model_class in self.all_model_classes:
																			a__             =     model_class(__a      )
																			assert isinstance(model.get_input_embeddings()			,tf.keras.layers.Layer      )
																			if model_class in list_other_models_with_output_ebd:
																										a__             =     model.get_output_embeddings()
																										assert isinstance(__a			,tf.keras.layers.Layer      )
																										a__             =     model.get_bias()
																										assert name is None
																			else:
																										a__             =     model.get_output_embeddings()
																										assert x is None
																										a__             =     model.get_bias()
																										assert name is None
					def  lowerCamelCase__(      self   :Optional[int]      )					->    Optional[int]:
												# TODO JP: Make TransfoXL XLA compliant
												pass
					@slow
					def  lowerCamelCase__(      self   :Tuple      )					->    Tuple:
												for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			a__             =     TFTransfoXLModel.from_pretrained(__a      )
																			self.assertIsNotNone(__a      )



					@unittest.skip(reason='This model doesn\'t play well with fit() due to not returning a single loss.'      )
					def  lowerCamelCase__(      self   :Tuple      )					->    Dict:
												pass







@require_tf
class        snake_case_   (unittest.TestCase      ):
					@unittest.skip('Skip test until #12651 is resolved.'      )
					@slow
					def  lowerCamelCase__(      self   :Optional[int]      )					->    Dict:
												a__             =     TFTransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103'      )
												# fmt: off
												a__             =     tf.convert_to_tensor([[33,12_97,2,1,10_09,4,11_09,1_17_39,47_62,3_58,5,25,2_45,22,17_06,17,2_00_98,5,32_15,21,37,11_10,3,13,10_41,4,24,6_03,4_90,2,7_14_77,2_00_98,10_44_47,2,2_09_61,1,26_04,4,1,3_29,3,62_24,8_31,1_60_02,2,8,6_03,7_89_67,2_95_46,23,8_03,20,25,4_16,5,8,2_32,4,2_77,6,18_55,46_01,3,2_95_46,54,8,36_09,5,5_72_11,49,4,1,2_77,18,8,17_55,1_56_91,3,3_41,25,4_16,6_93,4_25_73,71,17,4_01,94,31,1_79_19,2,2_95_46,78_73,18,1,4_35,23,1_10_11,7_55,5,51_67,3,79_83,98,84,2,2_95_46,32_67,8,36_09,4,1,48_65,10_75,2,60_87,71,6,3_46,8,58_54,3,2_95_46,8_24,14_00,18_68,2,19,1_60,2,3_11,8,54_96,2,2_09_20,17,25,1_50_97,3,24,24,0]]			,dtype=tf.intaa      )  # noqa: E231
												# fmt: on
												#  In 1991 , the remains of Russian Tsar Nicholas II and his family
												#  ( except for Alexei and Maria ) are discovered .
												#  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
												#  remainder of the story . 1883 Western Siberia ,
												#  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
												#  Rasputin has a vision and denounces one of the men as a horse thief . Although his
												#  father initially slaps him for making such an accusation , Rasputin watches as the
												#  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
												#  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
												#  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

												# fmt: off
												a__             =     [33,12_97,2,1,10_09,4,11_09,1_17_39,47_62,3_58,5,25,2_45,22,17_06,17,2_00_98,5,32_15,21,37,11_10,3,13,10_41,4,24,6_03,4_90,2,7_14_77,2_00_98,10_44_47,2,2_09_61,1,26_04,4,1,3_29,3,62_24,8_31,1_60_02,2,8,6_03,7_89_67,2_95_46,23,8_03,20,25,4_16,5,8,2_32,4,2_77,6,18_55,46_01,3,2_95_46,54,8,36_09,5,5_72_11,49,4,1,2_77,18,8,17_55,1_56_91,3,3_41,25,4_16,6_93,4_25_73,71,17,4_01,94,31,1_79_19,2,2_95_46,78_73,18,1,4_35,23,1_10_11,7_55,5,51_67,3,79_83,98,84,2,2_95_46,32_67,8,36_09,4,1,48_65,10_75,2,60_87,71,6,3_46,8,58_54,3,2_95_46,8_24,14_00,18_68,2,19,1_60,2,3_11,8,54_96,2,2_09_20,17,25,1_50_97,3,24,24,0,33,1,18_57,2,1,10_09,4,11_09,1_17_39,47_62,3_58,5,25,2_45,28,11_10,3,13,10_41,4,24,6_03,4_90,2,7_14_77,2_00_98,10_44_47,2,2_09_61,1,26_04,4,1,3_29,3,0]  # noqa: E231
												# fmt: on
												#  In 1991, the remains of Russian Tsar Nicholas II and his family (
												#  except for Alexei and Maria ) are discovered. The voice of young son,
												#  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
												#  1883 Western Siberia, a young Grigori Rasputin is asked by his father
												#  and a group of men to perform magic. Rasputin has a vision and
												#  denounces one of the men as a horse thief. Although his father initially
												#  slaps him for making such an accusation, Rasputin watches as the man
												#  is chased outside and beaten. Twenty years later, Rasputin sees a vision
												#  of the Virgin Mary, prompting him to become a priest.
												#  Rasputin quickly becomes famous, with people, even a bishop, begging for
												#  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
												# Nicholas II and his family were discovered. The voice of <unk> young son,
												# Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

												a__             =     model.generate(__a			,max_length=2_00			,do_sample=__a      )
												self.assertListEqual(output_ids[0].numpy().tolist()			,__a      )











'''simple docstring'''







import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            =['image_processor', 'tokenizer']
				__a            ='LayoutLMv3ImageProcessor'
				__a            =('LayoutLMv3Tokenizer', 'LayoutLMv3TokenizerFast')




				def __init__(  self     :					Tuple					,				__a     :					int=None					,				__a     :					Union[str, Any]=None					,				**__a     :					Optional[Any]					):
								_a								= None
								if "feature_extractor" in kwargs:
												warnings.warn(
												    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
												    " instead."					,				__a					,				)
												_a								= kwargs.pop("feature_extractor"					)

								_a								= image_processor if image_processor is not None else feature_extractor
								if image_processor is None:
												raise ValueError("You need to specify an `image_processor`."					)
								if tokenizer is None:
												raise ValueError("You need to specify a `tokenizer`."					)

								super().__init__(__a					,				__a					)




				def __call__(  self     :					Any					,				__a     :					List[str]					,				__a     :					Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None					,				__a     :					Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None					,				__a     :					Union[List[List[int]], List[List[List[int]]]] = None					,				__a     :					Optional[Union[List[int], List[List[int]]]] = None					,				__a     :					bool = True					,				__a     :					Union[bool, str, PaddingStrategy] = False					,				__a     :					Union[bool, str, TruncationStrategy] = None					,				__a     :					Optional[int] = None					,				__a     :					int = 0					,				__a     :					Optional[int] = None					,				__a     :					Optional[bool] = None					,				__a     :					Optional[bool] = None					,				__a     :					bool = False					,				__a     :					bool = False					,				__a     :					bool = False					,				__a     :					bool = False					,				__a     :					bool = True					,				__a     :					Optional[Union[str, TensorType]] = None					,				**__a     :					Dict					,				):
								# verify input
								if self.image_processor.apply_ocr and (boxes is not None):
												raise ValueError(
												    "You cannot provide bounding boxes if you initialized the image processor with apply_ocr set to True."					)

								if self.image_processor.apply_ocr and (word_labels is not None):
												raise ValueError(
												    "You cannot provide word labels if you initialized the image processor with apply_ocr set to True."					)

								# first, apply the image processor
								_a								= self.image_processor(images=__a					,				return_tensors=__a					)

								# second, apply the tokenizer
								if text is not None and self.image_processor.apply_ocr and text_pair is None:
												if isinstance(__a					,				__a					):
																_a								= [text]  # add batch dimension (as the image processor always adds a batch dimension)
												_a								= features["words"]

								_a								= self.tokenizer(
								    text=text if text is not None else features["words"]					,				text_pair=text_pair if text_pair is not None else None					,				boxes=boxes if boxes is not None else features["boxes"]					,				word_labels=__a					,				add_special_tokens=__a					,				padding=__a					,				truncation=__a					,				max_length=__a					,				stride=__a					,				pad_to_multiple_of=__a					,				return_token_type_ids=__a					,				return_attention_mask=__a					,				return_overflowing_tokens=__a					,				return_special_tokens_mask=__a					,				return_offsets_mapping=__a					,				return_length=__a					,				verbose=__a					,				return_tensors=__a					,				**__a					,				)

								# add pixel values
								_a								= features.pop("pixel_values"					)
								if return_overflowing_tokens is True:
												_a								= self.get_overflowing_images(__a					,				encoded_inputs["overflow_to_sample_mapping"]					)
								_a								= images

								return encoded_inputs




				def   UpperCamelCase__   (  self     :					Optional[int]					,				__a     :					str					,				__a     :					List[Any]					):
								# in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
								_a								= []
								for sample_idx in overflow_to_sample_mapping:
												images_with_overflow.append(images[sample_idx]					)

								if len(__a					) != len(__a					):
												raise ValueError(
												    "Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"
												    f' {len(__a					)} and {len(__a					)}'					)

								return images_with_overflow




				def   UpperCamelCase__   (  self     :					int					,				*__a     :					str					,				**__a     :					Tuple					):
								return self.tokenizer.batch_decode(*__a					,				**__a					)




				def   UpperCamelCase__   (  self     :					str					,				*__a     :					List[Any]					,				**__a     :					List[str]					):
								return self.tokenizer.decode(*__a					,				**__a					)




				@property
				def   UpperCamelCase__   (  self     :					Tuple					):
								return ["input_ids", "bbox", "attention_mask", "pixel_values"]




				@property
				def   UpperCamelCase__   (  self     :					int					):
								warnings.warn(
								    "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."					,				__a					,				)
								return self.image_processor_class




				@property
				def   UpperCamelCase__   (  self     :					List[str]					):
								warnings.warn(
								    "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."					,				__a					,				)
								return self.image_processor
