
"""simple docstring"""


from __future__ import annotations

import unittest

from transformers import AutoTokenizer, GPTJConfig, is_tf_available
from transformers.testing_utils import require_tf, slow, tooslow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers.models.gptj.modeling_tf_gptj import (
			    TFGPTJForCausalLM,
			    TFGPTJForQuestionAnswering,
			    TFGPTJForSequenceClassification,
			    TFGPTJModel,
			    shape_list,
			)







class 						lowerCamelCase__   :





			'''simple docstring'''



			def __init__(				self	,	__UpperCAmelCase    )					->	Tuple:
				_lowerCAmelCase								=parent
				_lowerCAmelCase								=13
				_lowerCAmelCase								=7
				_lowerCAmelCase								=True
				_lowerCAmelCase								=True
				_lowerCAmelCase								=True
				_lowerCAmelCase								=True
				_lowerCAmelCase								=True
				_lowerCAmelCase								=99
				_lowerCAmelCase								=32
				_lowerCAmelCase								=4
				_lowerCAmelCase								=2
				_lowerCAmelCase								=4
				_lowerCAmelCase								=37
				_lowerCAmelCase								="""gelu"""
				_lowerCAmelCase								=0.1
				_lowerCAmelCase								=0.1
				_lowerCAmelCase								=5_12
				_lowerCAmelCase								=16
				_lowerCAmelCase								=2
				_lowerCAmelCase								=0.0_2
				_lowerCAmelCase								=3
				_lowerCAmelCase								=4
				_lowerCAmelCase								=None
				_lowerCAmelCase								=self.vocab_size - 1
				_lowerCAmelCase								=self.vocab_size - 1
				_lowerCAmelCase								=self.vocab_size - 1



			def    _lowerCAmelCase						(				self    )					->	str:
				_lowerCAmelCase								=ids_tensor([self.batch_size, self.seq_length]	,	self.vocab_size    )

				_lowerCAmelCase								=None
				if self.use_input_mask:
					_lowerCAmelCase								=random_attention_mask([self.batch_size, self.seq_length]    )

				_lowerCAmelCase								=None
				if self.use_token_type_ids:
					_lowerCAmelCase								=ids_tensor([self.batch_size, self.seq_length]	,	self.type_vocab_size    )

				_lowerCAmelCase								=None
				if self.use_mc_token_ids:
					_lowerCAmelCase								=ids_tensor([self.batch_size, self.num_choices]	,	self.seq_length    )

				_lowerCAmelCase								=None
				_lowerCAmelCase								=None
				_lowerCAmelCase								=None
				if self.use_labels:
					_lowerCAmelCase								=ids_tensor([self.batch_size]	,	self.type_sequence_label_size    )
					_lowerCAmelCase								=ids_tensor([self.batch_size, self.seq_length]	,	self.num_labels    )
					_lowerCAmelCase								=ids_tensor([self.batch_size]	,	self.num_choices    )

				_lowerCAmelCase								=GPTJConfig(
				    vocab_size=self.vocab_size	,	n_embd=self.hidden_size	,	n_layer=self.num_hidden_layers	,	n_head=self.num_attention_heads	,	intermediate_size=self.intermediate_size	,	hidden_act=self.hidden_act	,	hidden_dropout_prob=self.hidden_dropout_prob	,	attention_probs_dropout_prob=self.attention_probs_dropout_prob	,	n_positions=self.max_position_embeddings	,	type_vocab_size=self.type_vocab_size	,	initializer_range=self.initializer_range	,	bos_token_id=self.bos_token_id	,	eos_token_id=self.eos_token_id	,	pad_token_id=self.pad_token_id	,	rotary_dim=self.rotary_dim	,	return_dict=__UpperCAmelCase	,	)

				_lowerCAmelCase								=ids_tensor([self.num_hidden_layers, self.num_attention_heads]	,	2    )

				return (
				    config,
				    input_ids,
				    input_mask,
				    head_mask,
				    token_type_ids,
				    mc_token_ids,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				)



			def    _lowerCAmelCase						(				self	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	*__UpperCAmelCase    )					->	int:
				_lowerCAmelCase								=TFGPTJModel(config=__UpperCAmelCase    )
				_lowerCAmelCase								={
				    """input_ids""": input_ids,
				    """attention_mask""": input_mask,
				    """token_type_ids""": token_type_ids,
				}
				_lowerCAmelCase								=model(__UpperCAmelCase    )

				_lowerCAmelCase								=[input_ids, None, input_mask]  # None is the input for 'past'
				_lowerCAmelCase								=model(__UpperCAmelCase    )

				_lowerCAmelCase								=model(__UpperCAmelCase    )

				self.parent.assertEqual(result.last_hidden_state.shape	,	(self.batch_size, self.seq_length, self.hidden_size)    )



			def    _lowerCAmelCase						(				self	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	*__UpperCAmelCase    )					->	Any:
				_lowerCAmelCase								=TFGPTJModel(config=__UpperCAmelCase    )

				# first forward pass
				_lowerCAmelCase								=model(__UpperCAmelCase	,	token_type_ids=__UpperCAmelCase	,	use_cache=__UpperCAmelCase    )
				_lowerCAmelCase								=model(__UpperCAmelCase	,	token_type_ids=__UpperCAmelCase    )
				_lowerCAmelCase								=model(__UpperCAmelCase	,	token_type_ids=__UpperCAmelCase	,	use_cache=__UpperCAmelCase    )

				self.parent.assertTrue(len(__UpperCAmelCase    ) == len(__UpperCAmelCase    )    )
				self.parent.assertTrue(len(__UpperCAmelCase    ) == len(__UpperCAmelCase    ) + 1    )

				_lowerCAmelCase		,				_lowerCAmelCase								=outputs.to_tuple()

				# create hypothetical next token and extent to next_input_ids
				_lowerCAmelCase								=ids_tensor((self.batch_size, 1)	,	config.vocab_size    )
				_lowerCAmelCase								=ids_tensor([self.batch_size, 1]	,	self.type_vocab_size    )

				# append to next input_ids and token_type_ids
				_lowerCAmelCase								=tf.concat([input_ids, next_tokens]	,	axis=-1    )
				_lowerCAmelCase								=tf.concat([token_type_ids, next_token_types]	,	axis=-1    )

				_lowerCAmelCase								=model(__UpperCAmelCase	,	token_type_ids=__UpperCAmelCase    )["""last_hidden_state"""]
				_lowerCAmelCase								=model(__UpperCAmelCase	,	token_type_ids=__UpperCAmelCase	,	past_key_values=__UpperCAmelCase    )[
				    """last_hidden_state"""
				]

				# select random slice
				_lowerCAmelCase								=int(ids_tensor((1,)	,	shape_list(__UpperCAmelCase    )[-1]    )    )
				_lowerCAmelCase								=output_from_no_past[:, -1, random_slice_idx]
				_lowerCAmelCase								=output_from_past[:, 0, random_slice_idx]

				# test that outputs are equal for slice
				tf.debugging.assert_near(__UpperCAmelCase	,	__UpperCAmelCase	,	rtol=1e-6    )



			def    _lowerCAmelCase						(				self	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	*__UpperCAmelCase    )					->	Union[str, Any]:
				_lowerCAmelCase								=TFGPTJModel(config=__UpperCAmelCase    )

				# create attention mask
				_lowerCAmelCase								=self.seq_length // 2
				_lowerCAmelCase								=tf.ones((self.batch_size, half_seq_length)	,	dtype=tf.intaa    )
				_lowerCAmelCase								=tf.zeros((self.batch_size, self.seq_length - half_seq_length)	,	dtype=tf.intaa    )
				_lowerCAmelCase								=tf.concat([attn_mask_begin, attn_mask_end]	,	axis=1    )

				# first forward pass
				_lowerCAmelCase		,				_lowerCAmelCase								=model(__UpperCAmelCase	,	attention_mask=__UpperCAmelCase    ).to_tuple()

				# create hypothetical next token and extent to next_input_ids
				_lowerCAmelCase								=ids_tensor((self.batch_size, 1)	,	config.vocab_size    )

				# change a random masked slice from input_ids
				_lowerCAmelCase								=ids_tensor((1,)	,	__UpperCAmelCase    ).numpy() + 1
				_lowerCAmelCase								=ids_tensor((self.batch_size, self.seq_length)	,	config.vocab_size    )
				_lowerCAmelCase								=tf.range(self.seq_length    ) == (self.seq_length - random_seq_idx_to_change)
				_lowerCAmelCase								=tf.transpose(
				    tf.broadcast_to(tf.expand_dims(__UpperCAmelCase	,	-1    )	,	(self.seq_length, self.batch_size)    )    )
				_lowerCAmelCase								=tf.where(__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase    )

				# append to next input_ids and attn_mask
				_lowerCAmelCase								=tf.concat([input_ids, next_tokens]	,	axis=-1    )
				_lowerCAmelCase								=tf.concat([attn_mask, tf.ones((shape_list(__UpperCAmelCase    )[0], 1)	,	dtype=tf.intaa    )]	,	axis=1    )

				# get two different outputs
				_lowerCAmelCase								=model(__UpperCAmelCase	,	attention_mask=__UpperCAmelCase    )["""last_hidden_state"""]
				_lowerCAmelCase								=model(__UpperCAmelCase	,	past_key_values=__UpperCAmelCase	,	attention_mask=__UpperCAmelCase    )[
				    """last_hidden_state"""
				]

				# select random slice
				_lowerCAmelCase								=int(ids_tensor((1,)	,	shape_list(__UpperCAmelCase    )[-1]    )    )
				_lowerCAmelCase								=output_from_no_past[:, -1, random_slice_idx]
				_lowerCAmelCase								=output_from_past[:, 0, random_slice_idx]

				# test that outputs are equal for slice
				tf.debugging.assert_near(__UpperCAmelCase	,	__UpperCAmelCase	,	rtol=1e-12    )



			def    _lowerCAmelCase						(				self	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	*__UpperCAmelCase    )					->	List[Any]:
				_lowerCAmelCase								=TFGPTJModel(config=__UpperCAmelCase    )

				_lowerCAmelCase								=input_ids[:1, :]
				_lowerCAmelCase								=input_mask[:1, :]
				_lowerCAmelCase								=token_type_ids[:1, :]
				_lowerCAmelCase								=1

				# first forward pass
				_lowerCAmelCase								=model(__UpperCAmelCase	,	attention_mask=__UpperCAmelCase	,	token_type_ids=__UpperCAmelCase	,	use_cache=__UpperCAmelCase    )

				_lowerCAmelCase		,				_lowerCAmelCase								=outputs.to_tuple()

				# create hypothetical next token and extent to next_input_ids
				_lowerCAmelCase								=ids_tensor((self.batch_size, 3)	,	config.vocab_size    )
				_lowerCAmelCase								=ids_tensor((self.batch_size, 3)	,	2    )
				_lowerCAmelCase								=ids_tensor((self.batch_size, 3)	,	self.type_vocab_size    )

				# append to next input_ids and token_type_ids
				_lowerCAmelCase								=tf.concat([input_ids, next_tokens]	,	axis=-1    )
				_lowerCAmelCase								=tf.concat([input_mask, next_attn_mask]	,	axis=-1    )
				_lowerCAmelCase								=tf.concat([token_type_ids, next_token_types]	,	axis=-1    )

				_lowerCAmelCase								=model(
				    __UpperCAmelCase	,	token_type_ids=__UpperCAmelCase	,	attention_mask=__UpperCAmelCase    )["""last_hidden_state"""]
				_lowerCAmelCase								=model(
				    __UpperCAmelCase	,	token_type_ids=__UpperCAmelCase	,	attention_mask=__UpperCAmelCase	,	past_key_values=__UpperCAmelCase	,	)["""last_hidden_state"""]
				self.parent.assertTrue(output_from_past.shape[1] == next_tokens.shape[1]    )

				# select random slice
				_lowerCAmelCase								=int(ids_tensor((1,)	,	shape_list(__UpperCAmelCase    )[-1]    )    )
				_lowerCAmelCase								=output_from_no_past[:, -3:, random_slice_idx]
				_lowerCAmelCase								=output_from_past[:, :, random_slice_idx]

				# test that outputs are equal for slice
				tf.debugging.assert_near(__UpperCAmelCase	,	__UpperCAmelCase	,	rtol=1e-3    )



			def    _lowerCAmelCase						(				self	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	*__UpperCAmelCase    )					->	Dict:
				_lowerCAmelCase								=TFGPTJForCausalLM(config=__UpperCAmelCase    )
				_lowerCAmelCase								={
				    """input_ids""": input_ids,
				    """attention_mask""": input_mask,
				    """token_type_ids""": token_type_ids,
				}
				_lowerCAmelCase								=model(__UpperCAmelCase    )
				self.parent.assertEqual(result.logits.shape	,	(self.batch_size, self.seq_length, self.vocab_size)    )



			def    _lowerCAmelCase						(				self    )					->	List[str]:
				_lowerCAmelCase								=self.prepare_config_and_inputs()

				(
				    (
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				(
				    _lowerCAmelCase
				)		,				
				)								=config_and_inputs

				_lowerCAmelCase								={
				    """input_ids""": input_ids,
				    """token_type_ids""": token_type_ids,
				    """attention_mask""": input_mask,
				}
				return config, inputs_dict







@require_tf
class 						lowerCamelCase__   (   __magic_name__					,      __magic_name__					,      __magic_name__					,      unittest.TestCase							):





			'''simple docstring'''
			lowerCamelCase       =						(
			    (TFGPTJForCausalLM, TFGPTJForSequenceClassification, TFGPTJForQuestionAnswering, TFGPTJModel)
			    if is_tf_available()
			    else ()
			)

			lowerCamelCase       =						(TFGPTJForCausalLM,) if is_tf_available() else ()
			lowerCamelCase       =						(
			    {
			        '''feature-extraction''': TFGPTJModel,
			        '''question-answering''': TFGPTJForQuestionAnswering,
			        '''text-classification''': TFGPTJForSequenceClassification,
			        '''text-generation''': TFGPTJForCausalLM,
			        '''zero-shot''': TFGPTJForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)
			lowerCamelCase       =						False
			lowerCamelCase       =						False
			lowerCamelCase       =						False
			lowerCamelCase       =						False



			def    _lowerCAmelCase						(				self	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase    )					->	Optional[int]:
				if (
				    pipeline_test_casse_name == "QAPipelineTests"
				    and tokenizer_name is not None
				    and not tokenizer_name.endswith("""Fast"""    )
				):
					# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
					# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
					# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
					return True

				return False



			def    _lowerCAmelCase						(				self    )					->	Tuple:
				_lowerCAmelCase								=TFGPTJModelTester(self    )
				_lowerCAmelCase								=ConfigTester(self	,	config_class=__UpperCAmelCase	,	n_embd=37    )



			def    _lowerCAmelCase						(				self    )					->	Any:
				self.config_tester.run_common_tests()



			def    _lowerCAmelCase						(				self    )					->	Tuple:
				_lowerCAmelCase								=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_gptj_model(*__UpperCAmelCase    )



			def    _lowerCAmelCase						(				self    )					->	List[str]:
				_lowerCAmelCase								=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_gptj_model_past(*__UpperCAmelCase    )



			def    _lowerCAmelCase						(				self    )					->	Optional[Any]:
				_lowerCAmelCase								=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_gptj_model_attention_mask_past(*__UpperCAmelCase    )



			def    _lowerCAmelCase						(				self    )					->	Tuple:
				_lowerCAmelCase								=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_gptj_model_past_large_inputs(*__UpperCAmelCase    )



			def    _lowerCAmelCase						(				self    )					->	List[str]:
				_lowerCAmelCase								=self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_gptj_lm_head_model(*__UpperCAmelCase    )



			@slow
			@unittest.skipIf(
			    not is_tf_available() or len(tf.config.list_physical_devices("""GPU"""    )    ) > 0	,	"""skip testing on GPU for now to avoid GPU OOM."""	,	)
			def    _lowerCAmelCase						(				self    )					->	Any:
				_lowerCAmelCase								=TFGPTJModel.from_pretrained("""EleutherAI/gpt-j-6B"""	,	from_pt=__UpperCAmelCase    )
				self.assertIsNotNone(__UpperCAmelCase    )



			@unittest.skip(reason="""Currently, model embeddings are going to undergo a major refactor."""    )
			def    _lowerCAmelCase						(				self    )					->	List[Any]:
				super().test_resize_token_embeddings()







@require_tf
@tooslow
# Marked as @tooslow due to GPU OOM -- but still useful to run locally. Requires ~39GB of RAM.
class 						lowerCamelCase__   (   unittest.TestCase							):





			'''simple docstring'''



			def    _lowerCAmelCase						(				self    )					->	List[str]:
				_lowerCAmelCase								=TFGPTJForCausalLM.from_pretrained("""EleutherAI/gpt-j-6B"""	,	from_pt=__UpperCAmelCase    )
				_lowerCAmelCase								=tf.convert_to_tensor([[4_64, 32_90]]	,	dtype=tf.intaa    )  # The dog
				# fmt: off
				# The dog is a man's best friend. It is a loyal companion, and it is a friend
				_lowerCAmelCase								=[4_64, 32_90, 3_18, 2_57, 5_82, 3_38, 12_66, 15_45, 13, 6_32, 3_18, 2_57, 91_12, 1_51_85, 11, 2_90, 3_40, 3_18, 2_57, 15_45]
				# fmt: on
				_lowerCAmelCase								=model.generate(__UpperCAmelCase	,	do_sample=__UpperCAmelCase    )
				self.assertListEqual(output_ids[0].numpy().tolist()	,	__UpperCAmelCase    )



			def    _lowerCAmelCase						(				self    )					->	Dict:
				_lowerCAmelCase								=AutoTokenizer.from_pretrained("""EleutherAI/gpt-j-6B"""	,	revision="""float16"""    )
				_lowerCAmelCase								=TFGPTJForCausalLM.from_pretrained("""EleutherAI/gpt-j-6B"""	,	revision="""float16"""	,	from_pt=__UpperCAmelCase    )

				_lowerCAmelCase								=tokenizer("""Today is a nice day and"""	,	return_tensors="""tf"""    )
				# forces the generation to happen on CPU, to avoid GPU-related quirks
				with tf.device(""":/CPU:0"""    ):
					_lowerCAmelCase								=model.generate(**__UpperCAmelCase	,	do_sample=__UpperCAmelCase	,	seed=[42, 0]    )
				_lowerCAmelCase								=tokenizer.decode(output_ids[0]	,	skip_special_tokens=__UpperCAmelCase    )

				_lowerCAmelCase								="""Today is a nice day and I’m going to go for a walk. I’"""
				self.assertEqual(__UpperCAmelCase	,	__UpperCAmelCase    )



			def    _lowerCAmelCase						(				self    )					->	int:
				_lowerCAmelCase								=TFGPTJForCausalLM.from_pretrained("""EleutherAI/gpt-j-6B"""	,	revision="""float16"""	,	from_pt=__UpperCAmelCase    )
				_lowerCAmelCase								=AutoTokenizer.from_pretrained("""EleutherAI/gpt-j-6B"""	,	revision="""float16"""    )

				_lowerCAmelCase								="""left"""

				# Define PAD Token = EOS Token = 50256
				_lowerCAmelCase								=tokenizer.eos_token
				_lowerCAmelCase								=model.config.eos_token_id

				# use different length sentences to test batching
				_lowerCAmelCase								=[
				    """Hello, my dog is a little""",
				    """Today, I""",
				]
				_lowerCAmelCase								=[
				    """Hello, my dog is a little over a year old and has been diagnosed with hip dysplasia""",
				    """Today, I’m going to be talking about a topic that’""",
				]
				return model, tokenizer, sentences, expected_output_sentences



			def    _lowerCAmelCase						(				self    )					->	Union[str, Any]:
				# Confirms that we get the expected results with left-padded beam search
				_lowerCAmelCase		,				_lowerCAmelCase		,				_lowerCAmelCase		,				_lowerCAmelCase								=self._get_beam_search_test_objects()

				_lowerCAmelCase								=tokenizer(__UpperCAmelCase	,	return_tensors="""tf"""	,	padding=__UpperCAmelCase    )
				_lowerCAmelCase								=model.generate(**__UpperCAmelCase	,	do_sample=__UpperCAmelCase	,	num_beams=2    )
				_lowerCAmelCase								=tokenizer.batch_decode(__UpperCAmelCase	,	skip_special_tokens=__UpperCAmelCase    )
				self.assertListEqual(__UpperCAmelCase	,	__UpperCAmelCase    )



			def    _lowerCAmelCase						(				self    )					->	List[str]:
				# Confirms that left-padding is working properly
				_lowerCAmelCase		,				_lowerCAmelCase		,				_lowerCAmelCase		,				_lowerCAmelCase								=self._get_beam_search_test_objects()

				_lowerCAmelCase								=tokenizer(__UpperCAmelCase	,	return_tensors="""tf"""	,	padding=__UpperCAmelCase    )
				_lowerCAmelCase								=tokenizer(sentences[0]	,	return_tensors="""tf"""    )
				_lowerCAmelCase								=model.generate(**__UpperCAmelCase	,	do_sample=__UpperCAmelCase	,	num_beams=2    )
				_lowerCAmelCase								=(
				    shape_list(inputs_non_padded["""input_ids"""]    )[-1]
				    - tf.reduce_sum(tf.cast(inputs["""attention_mask"""][-1]	,	tf.intaa    )    ).numpy()
				)
				_lowerCAmelCase								=tokenizer(sentences[1]	,	return_tensors="""tf"""    )
				_lowerCAmelCase								=model.generate(
				    **__UpperCAmelCase	,	do_sample=__UpperCAmelCase	,	num_beams=2	,	max_length=model.config.max_length - num_paddings    )
				_lowerCAmelCase								=tokenizer.decode(output_non_padded[0]	,	skip_special_tokens=__UpperCAmelCase    )
				_lowerCAmelCase								=tokenizer.decode(output_padded[0]	,	skip_special_tokens=__UpperCAmelCase    )
				self.assertListEqual(__UpperCAmelCase	,	[non_padded_sentence, padded_sentence]    )



			def    _lowerCAmelCase						(				self    )					->	Tuple:
				# Confirms that XLA is working properly
				_lowerCAmelCase		,				_lowerCAmelCase		,				_lowerCAmelCase		,				_lowerCAmelCase								=self._get_beam_search_test_objects()

				_lowerCAmelCase								=tokenizer(__UpperCAmelCase	,	return_tensors="""tf"""	,	padding=__UpperCAmelCase    )
				_lowerCAmelCase								=tf.function(model.generate	,	jit_compile=__UpperCAmelCase    )
				_lowerCAmelCase								=xla_generate(**__UpperCAmelCase	,	do_sample=__UpperCAmelCase	,	num_beams=2    )
				_lowerCAmelCase								=tokenizer.batch_decode(__UpperCAmelCase	,	skip_special_tokens=__UpperCAmelCase    )
				self.assertListEqual(__UpperCAmelCase	,	__UpperCAmelCase    )





"""simple docstring"""


import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


__A					=			logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
__A					=			direct_transformers_import(Path(__file__).parent)


__A					=			{
    'AutoTokenizer': 'PreTrainedTokenizerBase',
    'AutoFeatureExtractor': 'FeatureExtractionMixin',
    'AutoImageProcessor': 'ImageProcessingMixin',
}







class 						lowerCamelCase__   (   __magic_name__							):





			'''simple docstring'''
			lowerCamelCase       =						['''feature_extractor''', '''tokenizer''']
			# Names need to be attr_class for attr in attributes
			lowerCamelCase       =						None
			lowerCamelCase       =						None
			lowerCamelCase       =						None



			def __init__(				self	,	*__UpperCAmelCase	,	**__UpperCAmelCase    )					->	Dict:
				# Sanitize args and kwargs
				for key in kwargs:
					if key not in self.attributes:
						raise TypeError(f'''Unexpected keyword argument {key}.'''    )
				for arg, attribute_name in zip(__UpperCAmelCase	,	self.attributes    ):
					if attribute_name in kwargs:
						raise TypeError(f'''Got multiple values for argument {attribute_name}.'''    )
					else:
						_lowerCAmelCase								=arg

				if len(__UpperCAmelCase    ) != len(self.attributes    ):
					raise ValueError(
					    f'''This processor requires {len(self.attributes    )} arguments: {', '.join(self.attributes    )}. Got '''
					    f'''{len(__UpperCAmelCase    )} arguments instead.'''    )

				# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
				for attribute_name, arg in kwargs.items():
					_lowerCAmelCase								=getattr(self	,	f'''{attribute_name}_class'''    )
					# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
					_lowerCAmelCase								=AUTO_TO_BASE_CLASS_MAPPING.get(__UpperCAmelCase	,	__UpperCAmelCase    )
					if isinstance(__UpperCAmelCase	,	__UpperCAmelCase    ):
						_lowerCAmelCase								=tuple(getattr(__UpperCAmelCase	,	__UpperCAmelCase    ) for n in class_name if n is not None    )
					else:
						_lowerCAmelCase								=getattr(__UpperCAmelCase	,	__UpperCAmelCase    )

					if not isinstance(__UpperCAmelCase	,	__UpperCAmelCase    ):
						raise ValueError(
						    f'''Received a {type(__UpperCAmelCase    ).__name__} for argument {attribute_name}, but a {class_name} was expected.'''    )

					setattr(self	,	__UpperCAmelCase	,	__UpperCAmelCase    )



			def __repr__(				self    )					->	Optional[int]:
				_lowerCAmelCase								=[f'''- {name}: {repr(getattr(self	,	__UpperCAmelCase    )    )}''' for name in self.attributes]
				_lowerCAmelCase								="""\n""".join(__UpperCAmelCase    )
				return f'''{self.__class__.__name__}:\n{attributes_repr}'''



			def    _lowerCAmelCase						(				self	,	__UpperCAmelCase	,	__UpperCAmelCase = False	,	**__UpperCAmelCase    )					->	Union[str, Any]:
				os.makedirs(__UpperCAmelCase	,	exist_ok=__UpperCAmelCase    )

				if push_to_hub:
					_lowerCAmelCase								=kwargs.pop("""commit_message"""	,	__UpperCAmelCase    )
					_lowerCAmelCase								=kwargs.pop("""repo_id"""	,	save_directory.split(os.path.sep    )[-1]    )
					_lowerCAmelCase								=self._create_repo(__UpperCAmelCase	,	**__UpperCAmelCase    )
					_lowerCAmelCase								=self._get_files_timestamps(__UpperCAmelCase    )
				# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
				# loaded from the Hub.
				if self._auto_class is not None:
					_lowerCAmelCase								=[getattr(self	,	__UpperCAmelCase    ) for attribute_name in self.attributes]
					_lowerCAmelCase								=[(a.init_kwargs if isinstance(__UpperCAmelCase	,	__UpperCAmelCase    ) else a) for a in attrs]
					custom_object_save(self	,	__UpperCAmelCase	,	config=__UpperCAmelCase    )

				for attribute_name in self.attributes:
					_lowerCAmelCase								=getattr(self	,	__UpperCAmelCase    )
					# Include the processor class in the attribute config so this processor can then be reloaded with the
					# `AutoProcessor` API.
					if hasattr(__UpperCAmelCase	,	"""_set_processor_class"""    ):
						attribute._set_processor_class(self.__class__.__name__    )
					attribute.save_pretrained(__UpperCAmelCase    )

				if self._auto_class is not None:
					# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
					for attribute_name in self.attributes:
						_lowerCAmelCase								=getattr(self	,	__UpperCAmelCase    )
						if isinstance(__UpperCAmelCase	,	__UpperCAmelCase    ):
							del attribute.init_kwargs["auto_map"]

				if push_to_hub:
					self._upload_modified_files(
					    __UpperCAmelCase	,	__UpperCAmelCase	,	__UpperCAmelCase	,	commit_message=__UpperCAmelCase	,	token=kwargs.get("""use_auth_token"""    )	,	)



			@classmethod
			def    _lowerCAmelCase						(				cls	,	__UpperCAmelCase	,	__UpperCAmelCase = None	,	__UpperCAmelCase = False	,	__UpperCAmelCase = False	,	__UpperCAmelCase = None	,	__UpperCAmelCase = "main"	,	**__UpperCAmelCase	,	)					->	Any:
				_lowerCAmelCase								=cache_dir
				_lowerCAmelCase								=force_download
				_lowerCAmelCase								=local_files_only
				_lowerCAmelCase								=revision

				_lowerCAmelCase								=kwargs.pop("""use_auth_token"""	,	__UpperCAmelCase    )
				if use_auth_token is not None:
					warnings.warn(
					    """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""	,	__UpperCAmelCase    )
					if token is not None:
						raise ValueError(
						    """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""    )
					_lowerCAmelCase								=use_auth_token

				if token is not None:
					# change to `token` in a follow-up PR
					_lowerCAmelCase								=token

				_lowerCAmelCase								=cls._get_arguments_from_pretrained(__UpperCAmelCase	,	**__UpperCAmelCase    )
				return cls(*__UpperCAmelCase    )



			@classmethod
			def    _lowerCAmelCase						(				cls	,	__UpperCAmelCase="AutoProcessor"    )					->	Optional[int]:
				if not isinstance(__UpperCAmelCase	,	__UpperCAmelCase    ):
					_lowerCAmelCase								=auto_class.__name__

				import transformers.models.auto as auto_module

				if not hasattr(__UpperCAmelCase	,	__UpperCAmelCase    ):
					raise ValueError(f'''{auto_class} is not a valid auto class.'''    )

				_lowerCAmelCase								=auto_class



			@classmethod
			def    _lowerCAmelCase						(				cls	,	__UpperCAmelCase	,	**__UpperCAmelCase    )					->	Optional[Any]:
				_lowerCAmelCase								=[]
				for attribute_name in cls.attributes:
					_lowerCAmelCase								=getattr(cls	,	f'''{attribute_name}_class'''    )
					if isinstance(__UpperCAmelCase	,	__UpperCAmelCase    ):
						_lowerCAmelCase								=tuple(getattr(__UpperCAmelCase	,	__UpperCAmelCase    ) if n is not None else None for n in class_name    )
						_lowerCAmelCase								=kwargs.get("""use_fast"""	,	__UpperCAmelCase    )
						if use_fast and classes[1] is not None:
							_lowerCAmelCase								=classes[1]
						else:
							_lowerCAmelCase								=classes[0]
					else:
						_lowerCAmelCase								=getattr(__UpperCAmelCase	,	__UpperCAmelCase    )

					args.append(attribute_class.from_pretrained(__UpperCAmelCase	,	**__UpperCAmelCase    )    )
				return args



			@property
			def    _lowerCAmelCase						(				self    )					->	Optional[int]:
				_lowerCAmelCase								=getattr(self	,	self.attributes[0]    )
				return getattr(__UpperCAmelCase	,	"""model_input_names"""	,	__UpperCAmelCase    )







__A					=			copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
			__A					=			ProcessorMixin.push_to_hub.__doc__.format(
			    object='processor', object_class='AutoProcessor', object_files='processor files'
			)



