



from __future__ import annotations

import unittest

from transformers import AutoTokenizer, GPTJConfig, is_tf_available
from transformers.testing_utils import require_tf, slow, tooslow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers.models.gptj.modeling_tf_gptj import (
		    TFGPTJForCausalLM,
		    TFGPTJForQuestionAnswering,
		    TFGPTJForSequenceClassification,
		    TFGPTJModel,
		    shape_list,
		)


class 	a__       :


					"""simple docstring"""



					def __init__(		self     :List[Any]      ,  lowercase__     :Tuple    ):
										lowercase						   =       parent
										lowercase						   =       13
										lowercase						   =       7
										lowercase						   =       True
										lowercase						   =       True
										lowercase						   =       True
										lowercase						   =       True
										lowercase						   =       True
										lowercase						   =       99
										lowercase						   =       32
										lowercase						   =       4
										lowercase						   =       2
										lowercase						   =       4
										lowercase						   =       37
										lowercase						   =       'gelu'
										lowercase						   =       0.1
										lowercase						   =       0.1
										lowercase						   =       512
										lowercase						   =       16
										lowercase						   =       2
										lowercase						   =       0.02
										lowercase						   =       3
										lowercase						   =       4
										lowercase						   =       None
										lowercase						   =       self.vocab_size - 1
										lowercase						   =       self.vocab_size - 1
										lowercase						   =       self.vocab_size - 1

					def 				__UpperCAmelCase		(		self     :List[Any]    ):
										lowercase						   =       ids_tensor([self.batch_size, self.seq_length]      ,  self.vocab_size    )

										lowercase						   =       None
										if self.use_input_mask:
															lowercase						   =       random_attention_mask([self.batch_size, self.seq_length]    )

										lowercase						   =       None
										if self.use_token_type_ids:
															lowercase						   =       ids_tensor([self.batch_size, self.seq_length]      ,  self.type_vocab_size    )

										lowercase						   =       None
										if self.use_mc_token_ids:
															lowercase						   =       ids_tensor([self.batch_size, self.num_choices]      ,  self.seq_length    )

										lowercase						   =       None
										lowercase						   =       None
										lowercase						   =       None
										if self.use_labels:
															lowercase						   =       ids_tensor([self.batch_size]      ,  self.type_sequence_label_size    )
															lowercase						   =       ids_tensor([self.batch_size, self.seq_length]      ,  self.num_labels    )
															lowercase						   =       ids_tensor([self.batch_size]      ,  self.num_choices    )

										lowercase						   =       GPTJConfig(
										    vocab_size=self.vocab_size      ,  n_embd=self.hidden_size      ,  n_layer=self.num_hidden_layers      ,  n_head=self.num_attention_heads      ,  intermediate_size=self.intermediate_size      ,  hidden_act=self.hidden_act      ,  hidden_dropout_prob=self.hidden_dropout_prob      ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,  n_positions=self.max_position_embeddings      ,  type_vocab_size=self.type_vocab_size      ,  initializer_range=self.initializer_range      ,  bos_token_id=self.bos_token_id      ,  eos_token_id=self.eos_token_id      ,  pad_token_id=self.pad_token_id      ,  rotary_dim=self.rotary_dim      ,  return_dict=_lowercase      ,  )

										lowercase						   =       ids_tensor([self.num_hidden_layers, self.num_attention_heads]      ,  2    )

										return (
										    config,
										    input_ids,
										    input_mask,
										    head_mask,
										    token_type_ids,
										    mc_token_ids,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										)

					def 				__UpperCAmelCase		(		self     :Dict      ,  lowercase__     :Dict      ,  lowercase__     :str      ,  lowercase__     :Optional[int]      ,  lowercase__     :Optional[Any]      ,  lowercase__     :str      ,  *lowercase__     :str    ):
										lowercase						   =       TFGPTJModel(config=_lowercase    )
										lowercase						   =       {
										    'input_ids': input_ids,
										    'attention_mask': input_mask,
										    'token_type_ids': token_type_ids,
										}
										lowercase						   =       model(_lowercase    )

										lowercase						   =       [input_ids, None, input_mask]  # None is the input for 'past'
										lowercase						   =       model(_lowercase    )

										lowercase						   =       model(_lowercase    )

										self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, self.seq_length, self.hidden_size)    )

					def 				__UpperCAmelCase		(		self     :str      ,  lowercase__     :int      ,  lowercase__     :Dict      ,  lowercase__     :Tuple      ,  lowercase__     :Tuple      ,  lowercase__     :List[str]      ,  *lowercase__     :List[Any]    ):
										lowercase						   =       TFGPTJModel(config=_lowercase    )

										# first forward pass
										lowercase						   =       model(_lowercase      ,  token_type_ids=_lowercase      ,  use_cache=_lowercase    )
										lowercase						   =       model(_lowercase      ,  token_type_ids=_lowercase    )
										lowercase						   =       model(_lowercase      ,  token_type_ids=_lowercase      ,  use_cache=_lowercase    )

										self.parent.assertTrue(len(_lowercase    ) == len(_lowercase    )    )
										self.parent.assertTrue(len(_lowercase    ) == len(_lowercase    ) + 1    )

										lowercase   ,      lowercase						   =       outputs.to_tuple()

										# create hypothetical next token and extent to next_input_ids
										lowercase						   =       ids_tensor((self.batch_size, 1)      ,  config.vocab_size    )
										lowercase						   =       ids_tensor([self.batch_size, 1]      ,  self.type_vocab_size    )

										# append to next input_ids and token_type_ids
										lowercase						   =       tf.concat([input_ids, next_tokens]      ,  axis=-1    )
										lowercase						   =       tf.concat([token_type_ids, next_token_types]      ,  axis=-1    )

										lowercase						   =       model(_lowercase      ,  token_type_ids=_lowercase    )['last_hidden_state']
										lowercase						   =       model(_lowercase      ,  token_type_ids=_lowercase      ,  past_key_values=_lowercase    )[
										    'last_hidden_state'
										]

										# select random slice
										lowercase						   =       int(ids_tensor((1,)      ,  shape_list(_lowercase    )[-1]    )    )
										lowercase						   =       output_from_no_past[:, -1, random_slice_idx]
										lowercase						   =       output_from_past[:, 0, random_slice_idx]

										# test that outputs are equal for slice
										tf.debugging.assert_near(_lowercase      ,  _lowercase      ,  rtol=1E-6    )

					def 				__UpperCAmelCase		(		self     :int      ,  lowercase__     :Optional[int]      ,  lowercase__     :int      ,  lowercase__     :Union[str, Any]      ,  lowercase__     :str      ,  lowercase__     :List[str]      ,  *lowercase__     :Optional[int]    ):
										lowercase						   =       TFGPTJModel(config=_lowercase    )

										# create attention mask
										lowercase						   =       self.seq_length // 2
										lowercase						   =       tf.ones((self.batch_size, half_seq_length)      ,  dtype=tf.intaa    )
										lowercase						   =       tf.zeros((self.batch_size, self.seq_length - half_seq_length)      ,  dtype=tf.intaa    )
										lowercase						   =       tf.concat([attn_mask_begin, attn_mask_end]      ,  axis=1    )

										# first forward pass
										lowercase   ,      lowercase						   =       model(_lowercase      ,  attention_mask=_lowercase    ).to_tuple()

										# create hypothetical next token and extent to next_input_ids
										lowercase						   =       ids_tensor((self.batch_size, 1)      ,  config.vocab_size    )

										# change a random masked slice from input_ids
										lowercase						   =       ids_tensor((1,)      ,  _lowercase    ).numpy() + 1
										lowercase						   =       ids_tensor((self.batch_size, self.seq_length)      ,  config.vocab_size    )
										lowercase						   =       tf.range(self.seq_length    ) == (self.seq_length - random_seq_idx_to_change)
										lowercase						   =       tf.transpose(
										    tf.broadcast_to(tf.expand_dims(_lowercase      ,  -1    )      ,  (self.seq_length, self.batch_size)    )    )
										lowercase						   =       tf.where(_lowercase      ,  _lowercase      ,  _lowercase    )

										# append to next input_ids and attn_mask
										lowercase						   =       tf.concat([input_ids, next_tokens]      ,  axis=-1    )
										lowercase						   =       tf.concat([attn_mask, tf.ones((shape_list(_lowercase    )[0], 1)      ,  dtype=tf.intaa    )]      ,  axis=1    )

										# get two different outputs
										lowercase						   =       model(_lowercase      ,  attention_mask=_lowercase    )['last_hidden_state']
										lowercase						   =       model(_lowercase      ,  past_key_values=_lowercase      ,  attention_mask=_lowercase    )[
										    'last_hidden_state'
										]

										# select random slice
										lowercase						   =       int(ids_tensor((1,)      ,  shape_list(_lowercase    )[-1]    )    )
										lowercase						   =       output_from_no_past[:, -1, random_slice_idx]
										lowercase						   =       output_from_past[:, 0, random_slice_idx]

										# test that outputs are equal for slice
										tf.debugging.assert_near(_lowercase      ,  _lowercase      ,  rtol=1E-12    )

					def 				__UpperCAmelCase		(		self     :List[str]      ,  lowercase__     :str      ,  lowercase__     :Optional[Any]      ,  lowercase__     :Union[str, Any]      ,  lowercase__     :List[str]      ,  lowercase__     :Optional[int]      ,  *lowercase__     :Union[str, Any]    ):
										lowercase						   =       TFGPTJModel(config=_lowercase    )

										lowercase						   =       input_ids[:1, :]
										lowercase						   =       input_mask[:1, :]
										lowercase						   =       token_type_ids[:1, :]
										lowercase						   =       1

										# first forward pass
										lowercase						   =       model(_lowercase      ,  attention_mask=_lowercase      ,  token_type_ids=_lowercase      ,  use_cache=_lowercase    )

										lowercase   ,      lowercase						   =       outputs.to_tuple()

										# create hypothetical next token and extent to next_input_ids
										lowercase						   =       ids_tensor((self.batch_size, 3)      ,  config.vocab_size    )
										lowercase						   =       ids_tensor((self.batch_size, 3)      ,  2    )
										lowercase						   =       ids_tensor((self.batch_size, 3)      ,  self.type_vocab_size    )

										# append to next input_ids and token_type_ids
										lowercase						   =       tf.concat([input_ids, next_tokens]      ,  axis=-1    )
										lowercase						   =       tf.concat([input_mask, next_attn_mask]      ,  axis=-1    )
										lowercase						   =       tf.concat([token_type_ids, next_token_types]      ,  axis=-1    )

										lowercase						   =       model(
										    _lowercase      ,  token_type_ids=_lowercase      ,  attention_mask=_lowercase    )['last_hidden_state']
										lowercase						   =       model(
										    _lowercase      ,  token_type_ids=_lowercase      ,  attention_mask=_lowercase      ,  past_key_values=_lowercase      ,  )['last_hidden_state']
										self.parent.assertTrue(output_from_past.shape[1] == next_tokens.shape[1]    )

										# select random slice
										lowercase						   =       int(ids_tensor((1,)      ,  shape_list(_lowercase    )[-1]    )    )
										lowercase						   =       output_from_no_past[:, -3:, random_slice_idx]
										lowercase						   =       output_from_past[:, :, random_slice_idx]

										# test that outputs are equal for slice
										tf.debugging.assert_near(_lowercase      ,  _lowercase      ,  rtol=1E-3    )

					def 				__UpperCAmelCase		(		self     :Union[str, Any]      ,  lowercase__     :int      ,  lowercase__     :Optional[int]      ,  lowercase__     :List[Any]      ,  lowercase__     :Dict      ,  lowercase__     :Any      ,  *lowercase__     :Dict    ):
										lowercase						   =       TFGPTJForCausalLM(config=_lowercase    )
										lowercase						   =       {
										    'input_ids': input_ids,
										    'attention_mask': input_mask,
										    'token_type_ids': token_type_ids,
										}
										lowercase						   =       model(_lowercase    )
										self.parent.assertEqual(result.logits.shape      ,  (self.batch_size, self.seq_length, self.vocab_size)    )



					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										lowercase						   =       self.prepare_config_and_inputs()

										(
										    (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,      (
										    lowercase   
										)   ,         
										)						   =       config_and_inputs

										lowercase						   =       {
										    'input_ids': input_ids,
										    'token_type_ids': token_type_ids,
										    'attention_mask': input_mask,
										}
										return config, inputs_dict




@require_tf
class 	a__       (       __lowerCamelCase  ,			__lowerCamelCase  ,			__lowerCamelCase  ,			unittest.TestCase       ):


					"""simple docstring"""



					A__  :				Tuple					 =				(
					    (TFGPTJForCausalLM, TFGPTJForSequenceClassification, TFGPTJForQuestionAnswering, TFGPTJModel)
					    if is_tf_available()
					    else ()
					)

					A__  :				Dict					 =				(TFGPTJForCausalLM,) if is_tf_available() else ()
					A__  :				Optional[int]					 =				(
					    {
					        '''feature-extraction''': TFGPTJModel,
					        '''question-answering''': TFGPTJForQuestionAnswering,
					        '''text-classification''': TFGPTJForSequenceClassification,
					        '''text-generation''': TFGPTJForCausalLM,
					        '''zero-shot''': TFGPTJForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					A__  :				Tuple					 =				False
					A__  :				Dict					 =				False
					A__  :				Any					 =				False
					A__  :				List[Any]					 =				False

					def 				__UpperCAmelCase		(		self     :Union[str, Any]      ,  lowercase__     :Union[str, Any]      ,  lowercase__     :str      ,  lowercase__     :Optional[Any]      ,  lowercase__     :str      ,  lowercase__     :Union[str, Any]    ):
										if (
										    pipeline_test_casse_name == "QAPipelineTests"
										    and tokenizer_name is not None
										    and not tokenizer_name.endswith('Fast'    )
										):
															# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
															# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
															# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
															return True

										return False

					def 				__UpperCAmelCase		(		self     :Optional[int]    ):
										lowercase						   =       TFGPTJModelTester(self    )
										lowercase						   =       ConfigTester(self      ,  config_class=_lowercase      ,  n_embd=37    )

					def 				__UpperCAmelCase		(		self     :List[Any]    ):
										self.config_tester.run_common_tests()

					def 				__UpperCAmelCase		(		self     :str    ):
										lowercase						   =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_gptj_model(*_lowercase    )

					def 				__UpperCAmelCase		(		self     :Optional[int]    ):
										lowercase						   =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_gptj_model_past(*_lowercase    )

					def 				__UpperCAmelCase		(		self     :List[Any]    ):
										lowercase						   =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_gptj_model_attention_mask_past(*_lowercase    )

					def 				__UpperCAmelCase		(		self     :int    ):
										lowercase						   =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_gptj_model_past_large_inputs(*_lowercase    )

					def 				__UpperCAmelCase		(		self     :int    ):
										lowercase						   =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_gptj_lm_head_model(*_lowercase    )

					@slow
					@unittest.skipIf(
					    not is_tf_available() or len(tf.config.list_physical_devices('GPU'    )    ) > 0      ,  'skip testing on GPU for now to avoid GPU OOM.'      ,  )
					def 				__UpperCAmelCase		(		self     :int    ):
										lowercase						   =       TFGPTJModel.from_pretrained('EleutherAI/gpt-j-6B'      ,  from_pt=_lowercase    )
										self.assertIsNotNone(_lowercase    )



					@unittest.skip(reason='Currently, model embeddings are going to undergo a major refactor.'    )
					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										super().test_resize_token_embeddings()




@require_tf
@tooslow
# Marked as @tooslow due to GPU OOM -- but still useful to run locally. Requires ~39GB of RAM.
class 	a__       (       unittest.TestCase       ):


					"""simple docstring"""



					def 				__UpperCAmelCase		(		self     :List[Any]    ):
										lowercase						   =       TFGPTJForCausalLM.from_pretrained('EleutherAI/gpt-j-6B'      ,  from_pt=_lowercase    )
										lowercase						   =       tf.convert_to_tensor([[464, 3290]]      ,  dtype=tf.intaa    )  # The dog
										# fmt: off
										# The dog is a man's best friend. It is a loyal companion, and it is a friend
										lowercase						   =       [464, 3290, 318, 257, 582, 338, 1266, 1545, 13, 632, 318, 257, 9112, 1_5185, 11, 290, 340, 318, 257, 1545]
										# fmt: on
										lowercase						   =       model.generate(_lowercase      ,  do_sample=_lowercase    )
										self.assertListEqual(output_ids[0].numpy().tolist()      ,  _lowercase    )

					def 				__UpperCAmelCase		(		self     :List[Any]    ):
										lowercase						   =       AutoTokenizer.from_pretrained('EleutherAI/gpt-j-6B'      ,  revision='float16'    )
										lowercase						   =       TFGPTJForCausalLM.from_pretrained('EleutherAI/gpt-j-6B'      ,  revision='float16'      ,  from_pt=_lowercase    )

										lowercase						   =       tokenizer('Today is a nice day and'      ,  return_tensors='tf'    )
										# forces the generation to happen on CPU, to avoid GPU-related quirks
										with tf.device(':/CPU:0'    ):
															lowercase						   =       model.generate(**_lowercase      ,  do_sample=_lowercase      ,  seed=[42, 0]    )
										lowercase						   =       tokenizer.decode(output_ids[0]      ,  skip_special_tokens=_lowercase    )

										lowercase						   =       'Today is a nice day and I’m going to go for a walk. I’'
										self.assertEqual(_lowercase      ,  _lowercase    )

					def 				__UpperCAmelCase		(		self     :Any    ):
										lowercase						   =       TFGPTJForCausalLM.from_pretrained('EleutherAI/gpt-j-6B'      ,  revision='float16'      ,  from_pt=_lowercase    )
										lowercase						   =       AutoTokenizer.from_pretrained('EleutherAI/gpt-j-6B'      ,  revision='float16'    )

										lowercase						   =       'left'

										# Define PAD Token = EOS Token = 50256
										lowercase						   =       tokenizer.eos_token
										lowercase						   =       model.config.eos_token_id

										# use different length sentences to test batching
										lowercase						   =       [
										    'Hello, my dog is a little',
										    'Today, I',
										]
										lowercase						   =       [
										    'Hello, my dog is a little over a year old and has been diagnosed with hip dysplasia',
										    'Today, I’m going to be talking about a topic that’',
										]
										return model, tokenizer, sentences, expected_output_sentences

					def 				__UpperCAmelCase		(		self     :List[str]    ):
										lowercase   ,      lowercase   ,      lowercase   ,      lowercase						   =       self._get_beam_search_test_objects()

										lowercase						   =       tokenizer(_lowercase      ,  return_tensors='tf'      ,  padding=_lowercase    )
										lowercase						   =       model.generate(**_lowercase      ,  do_sample=_lowercase      ,  num_beams=2    )
										lowercase						   =       tokenizer.batch_decode(_lowercase      ,  skip_special_tokens=_lowercase    )
										self.assertListEqual(_lowercase      ,  _lowercase    )

					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										lowercase   ,      lowercase   ,      lowercase   ,      lowercase						   =       self._get_beam_search_test_objects()

										lowercase						   =       tokenizer(_lowercase      ,  return_tensors='tf'      ,  padding=_lowercase    )
										lowercase						   =       tokenizer(sentences[0]      ,  return_tensors='tf'    )
										lowercase						   =       model.generate(**_lowercase      ,  do_sample=_lowercase      ,  num_beams=2    )
										lowercase						   =       (
										    shape_list(inputs_non_padded['input_ids']    )[-1]
										    - tf.reduce_sum(tf.cast(inputs['attention_mask'][-1]      ,  tf.intaa    )    ).numpy()
										)
										lowercase						   =       tokenizer(sentences[1]      ,  return_tensors='tf'    )
										lowercase						   =       model.generate(
										    **_lowercase      ,  do_sample=_lowercase      ,  num_beams=2      ,  max_length=model.config.max_length - num_paddings    )
										lowercase						   =       tokenizer.decode(output_non_padded[0]      ,  skip_special_tokens=_lowercase    )
										lowercase						   =       tokenizer.decode(output_padded[0]      ,  skip_special_tokens=_lowercase    )
										self.assertListEqual(_lowercase      ,  [non_padded_sentence, padded_sentence]    )



					def 				__UpperCAmelCase		(		self     :str    ):
										lowercase   ,      lowercase   ,      lowercase   ,      lowercase						   =       self._get_beam_search_test_objects()

										lowercase						   =       tokenizer(_lowercase      ,  return_tensors='tf'      ,  padding=_lowercase    )
										lowercase						   =       tf.function(model.generate      ,  jit_compile=_lowercase    )
										lowercase						   =       xla_generate(**_lowercase      ,  do_sample=_lowercase      ,  num_beams=2    )
										lowercase						   =       tokenizer.batch_decode(_lowercase      ,  skip_special_tokens=_lowercase    )
										self.assertListEqual(_lowercase      ,  _lowercase    )







"""simple docstring"""







import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


UpperCamelCase_							:							Optional[int]				 =		logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
UpperCamelCase_							:							Optional[Any]				 =		direct_transformers_import(Path(__file__).parent)


UpperCamelCase_							:							Optional[int]				 =		{
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}


class       _lowercase		( __lowerCamelCase	):
				lowerCamelCase__							=							['feature_extractor', 'tokenizer']
				# Names need to be attr_class for attr in attributes
				lowerCamelCase__							=							None
				lowerCamelCase__							=							None
				lowerCamelCase__							=							None

				def __init__(  self							:     Any							,     *_lowercase							:     Optional[int]							,     **_lowercase							:     int  ):


										'''simple docstring'''
										# Sanitize args and kwargs
										for key in kwargs:
																if key not in self.attributes:
																						raise TypeError(f'Unexpected keyword argument {key}.'  )
										for arg, attribute_name in zip(_lowercase							,     self.attributes  ):
																if attribute_name in kwargs:
																						raise TypeError(f'Got multiple values for argument {attribute_name}.'  )
																else:
																						UpperCAmelCase__       						=		arg

										if len(_lowercase  ) != len(self.attributes  ):
																raise ValueError(
																    f'This processor requires {len(self.attributes  )} arguments: {", ".join(self.attributes  )}. Got '
																    f'{len(_lowercase  )} arguments instead.'  )

										# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
										for attribute_name, arg in kwargs.items():
																UpperCAmelCase__       						=		getattr(self							,     f'{attribute_name}_class'  )
																# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
																UpperCAmelCase__       						=		AUTO_TO_BASE_CLASS_MAPPING.get(_lowercase							,     _lowercase  )
																if isinstance(_lowercase							,     _lowercase  ):
																						UpperCAmelCase__       						=		tuple(getattr(_lowercase							,     _lowercase  ) for n in class_name if n is not None  )
																else:
																						UpperCAmelCase__       						=		getattr(_lowercase							,     _lowercase  )

																if not isinstance(_lowercase							,     _lowercase  ):
																						raise ValueError(
																						    f'Received a {type(_lowercase  ).__name__} for argument {attribute_name}, but a {class_name} was expected.'  )

																setattr(self							,     _lowercase							,     _lowercase  )

				def __repr__(  self							:     Tuple  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[f'- {name}: {repr(getattr(self							,     _lowercase  )  )}' for name in self.attributes]
										UpperCAmelCase__       						=		"""\n""".join(_lowercase  )
										return f'{self.__class__.__name__}:\n{attributes_repr}'

				def 							A			(  self							:     Tuple							,     _lowercase							:     Dict							,     _lowercase							:     bool = False							,     **_lowercase							:     List[str]  ):


										'''simple docstring'''
										os.makedirs(_lowercase							,     exist_ok=_lowercase  )

										if push_to_hub:
																UpperCAmelCase__       						=		kwargs.pop("""commit_message"""							,     _lowercase  )
																UpperCAmelCase__       						=		kwargs.pop("""repo_id"""							,     save_directory.split(os.path.sep  )[-1]  )
																UpperCAmelCase__       						=		self._create_repo(_lowercase							,     **_lowercase  )
																UpperCAmelCase__       						=		self._get_files_timestamps(_lowercase  )
										# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
										# loaded from the Hub.
										if self._auto_class is not None:
																UpperCAmelCase__       						=		[getattr(self							,     _lowercase  ) for attribute_name in self.attributes]
																UpperCAmelCase__       						=		[(a.init_kwargs if isinstance(_lowercase							,     _lowercase  ) else a) for a in attrs]
																custom_object_save(self							,     _lowercase							,     config=_lowercase  )

										for attribute_name in self.attributes:
																UpperCAmelCase__       						=		getattr(self							,     _lowercase  )
																# Include the processor class in the attribute config so this processor can then be reloaded with the
																# `AutoProcessor` API.
																if hasattr(_lowercase							,     """_set_processor_class"""  ):
																						attribute._set_processor_class(self.__class__.__name__  )
																attribute.save_pretrained(_lowercase  )

										if self._auto_class is not None:
																# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
																for attribute_name in self.attributes:
																						UpperCAmelCase__       						=		getattr(self							,     _lowercase  )
																						if isinstance(_lowercase							,     _lowercase  ):
																												del attribute.init_kwargs["auto_map"]

										if push_to_hub:
																self._upload_modified_files(
																    _lowercase							,     _lowercase							,     _lowercase							,     commit_message=_lowercase							,     token=kwargs.get("""use_auth_token"""  )							,     )

				@classmethod
				def 							A			(  cls							:     Dict							,     _lowercase							:     Union[str, os.PathLike]							,     _lowercase							:     Optional[Union[str, os.PathLike]] = None							,     _lowercase							:     bool = False							,     _lowercase							:     bool = False							,     _lowercase							:     Optional[Union[str, bool]] = None							,     _lowercase							:     str = "main"							,     **_lowercase							:     Dict							,     ):


										'''simple docstring'''
										UpperCAmelCase__       						=		cache_dir
										UpperCAmelCase__       						=		force_download
										UpperCAmelCase__       						=		local_files_only
										UpperCAmelCase__       						=		revision

										UpperCAmelCase__       						=		kwargs.pop("""use_auth_token"""							,     _lowercase  )
										if use_auth_token is not None:
																warnings.warn(
																    """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""							,     _lowercase  )
																if token is not None:
																						raise ValueError(
																						    """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""  )
																UpperCAmelCase__       						=		use_auth_token

										if token is not None:
																# change to `token` in a follow-up PR
																UpperCAmelCase__       						=		token

										UpperCAmelCase__       						=		cls._get_arguments_from_pretrained(_lowercase							,     **_lowercase  )
										return cls(*_lowercase  )

				@classmethod
				def 							A			(  cls							:     List[Any]							,     _lowercase							:     Dict="AutoProcessor"  ):


										'''simple docstring'''
										if not isinstance(_lowercase							,     _lowercase  ):
																UpperCAmelCase__       						=		auto_class.__name__

										import transformers.models.auto as auto_module

										if not hasattr(_lowercase							,     _lowercase  ):
																raise ValueError(f'{auto_class} is not a valid auto class.'  )

										UpperCAmelCase__       						=		auto_class

				@classmethod
				def 							A			(  cls							:     Any							,     _lowercase							:     Tuple							,     **_lowercase							:     List[Any]  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[]
										for attribute_name in cls.attributes:
																UpperCAmelCase__       						=		getattr(cls							,     f'{attribute_name}_class'  )
																if isinstance(_lowercase							,     _lowercase  ):
																						UpperCAmelCase__       						=		tuple(getattr(_lowercase							,     _lowercase  ) if n is not None else None for n in class_name  )
																						UpperCAmelCase__       						=		kwargs.get("""use_fast"""							,     _lowercase  )
																						if use_fast and classes[1] is not None:
																												UpperCAmelCase__       						=		classes[1]
																						else:
																												UpperCAmelCase__       						=		classes[0]
																else:
																						UpperCAmelCase__       						=		getattr(_lowercase							,     _lowercase  )

																args.append(attribute_class.from_pretrained(_lowercase							,     **_lowercase  )  )
										return args

				@property
				def 							A			(  self							:     int  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		getattr(self							,     self.attributes[0]  )
										return getattr(_lowercase							,     """model_input_names"""							,     _lowercase  )
UpperCamelCase_							:							List[Any]				 =		copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
						UpperCamelCase_							:							Dict				 =		ProcessorMixin.push_to_hub.__doc__.format(
						    object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
						)

