


from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
			from transformers import (
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelForTableQuestionAnswering,
			    TFAutoModelForTokenClassification,
			    TFAutoModelWithLMHead,
			    TFBertForMaskedLM,
			    TFBertForPreTraining,
			    TFBertForQuestionAnswering,
			    TFBertForSequenceClassification,
			    TFBertModel,
			    TFFunnelBaseModel,
			    TFFunnelModel,
			    TFGPTaLMHeadModel,
			    TFRobertaForMaskedLM,
			    TFTaForConditionalGeneration,
			    TFTapasForQuestionAnswering,
			)
			from transformers.models.auto.modeling_tf_auto import (
			    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
			    TF_MODEL_FOR_MASKED_LM_MAPPING,
			    TF_MODEL_FOR_PRETRAINING_MAPPING,
			    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
			    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
			    TF_MODEL_MAPPING,
			)
			from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST


class        _a							(			UpperCamelCase__			):
							_lowercase	:			List[str]        = '''new-model'''


if is_tf_available():


			class        _a							(			UpperCamelCase__			):
										_lowercase	:			str        = NewModelConfig


@require_tf
class        _a							(			unittest.TestCase			):

							@slow
							def       lowerCamelCase_ (							self:	Union[str, Any]							)			->   Optional[Any]:





									"""simple docstring"""







									lowercase__            =      '''bert-base-cased'''
									lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
									self.assertIsNotNone(UpperCamelCase_							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

									lowercase__            =      TFAutoModel.from_pretrained(UpperCamelCase_							)
									self.assertIsNotNone(UpperCamelCase_							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	str							)			->   Dict:





									"""simple docstring"""







									lowercase__            =      '''bert-base-cased'''
									lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
									self.assertIsNotNone(UpperCamelCase_							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

									lowercase__            =      TFAutoModelForPreTraining.from_pretrained(UpperCamelCase_							)
									self.assertIsNotNone(UpperCamelCase_							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	Optional[Any]							)			->   Dict:





									"""simple docstring"""







									for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

											lowercase__            =      TFAutoModelForCausalLM.from_pretrained(UpperCamelCase_							)
											lowercase__     ,		lowercase__            =      TFAutoModelForCausalLM.from_pretrained(UpperCamelCase_    ,						output_loading_info=UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	Any							)			->   List[Any]:





									"""simple docstring"""







									for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

											lowercase__            =      TFAutoModelWithLMHead.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	Optional[Any]							)			->   Tuple:





									"""simple docstring"""







									for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

											lowercase__            =      TFAutoModelForMaskedLM.from_pretrained(UpperCamelCase_							)
											lowercase__     ,		lowercase__            =      TFAutoModelForMaskedLM.from_pretrained(UpperCamelCase_    ,						output_loading_info=UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	List[str]							)			->   Optional[Any]:





									"""simple docstring"""







									for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

											lowercase__            =      TFAutoModelForSeqaSeqLM.from_pretrained(UpperCamelCase_							)
											lowercase__     ,		lowercase__            =      TFAutoModelForSeqaSeqLM.from_pretrained(UpperCamelCase_    ,						output_loading_info=UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	List[Any]							)			->   Tuple:





									"""simple docstring"""







									for model_name in ["bert-base-uncased"]:
											lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

											lowercase__            =      TFAutoModelForSequenceClassification.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	Optional[int]							)			->   int:





									"""simple docstring"""







									for model_name in ["bert-base-uncased"]:
											lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

											lowercase__            =      TFAutoModelForQuestionAnswering.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							@require_tensorflow_probability
							def       lowerCamelCase_ (							self:	Tuple							)			->   Any:





									"""simple docstring"""







									for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
											lowercase__            =      AutoConfig.from_pretrained(UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

											lowercase__            =      TFAutoModelForTableQuestionAnswering.from_pretrained(UpperCamelCase_							)
											lowercase__     ,		lowercase__            =      TFAutoModelForTableQuestionAnswering.from_pretrained(
											    UpperCamelCase_    ,						output_loading_info=UpperCamelCase_							)
											self.assertIsNotNone(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							def       lowerCamelCase_ (							self:	Union[str, Any]							)			->   str:





									"""simple docstring"""







									lowercase__            =      TFAutoModelWithLMHead.from_pretrained(UpperCamelCase_							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)
									self.assertEqual(model.num_parameters()    ,						14_410							)
									self.assertEqual(model.num_parameters(only_trainable=UpperCamelCase_							)    ,						14_410							)

							def       lowerCamelCase_ (							self:	List[Any]							)			->   Optional[Any]:





									"""simple docstring"""







									lowercase__            =      TFAutoModelWithLMHead.from_pretrained(UpperCamelCase_							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)
									self.assertEqual(model.num_parameters()    ,						14_410							)
									self.assertEqual(model.num_parameters(only_trainable=UpperCamelCase_							)    ,						14_410							)

							def       lowerCamelCase_ (							self:	Optional[Any]							)			->   Any:





									"""simple docstring"""







									lowercase__            =      TFAutoModel.from_pretrained('''sgugger/funnel-random-tiny'''							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

									lowercase__            =      copy.deepcopy(model.config							)
									lowercase__            =      ['''FunnelBaseModel''']
									lowercase__            =      TFAutoModel.from_config(UpperCamelCase_							)
									self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

									with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(UpperCamelCase_							)
											lowercase__            =      TFAutoModel.from_pretrained(UpperCamelCase_							)
											self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

							def       lowerCamelCase_ (							self:	Tuple							)			->   Any:





									"""simple docstring"""







									try:
											AutoConfig.register('''new-model'''    ,						UpperCamelCase_							)

											lowercase__            =      [
											    TFAutoModel,
											    TFAutoModelForCausalLM,
											    TFAutoModelForMaskedLM,
											    TFAutoModelForPreTraining,
											    TFAutoModelForQuestionAnswering,
											    TFAutoModelForSequenceClassification,
											    TFAutoModelForTokenClassification,
											]

											for auto_class in auto_classes:
													with self.subTest(auto_class.__name__							):
															# Wrong config class will raise an error
															with self.assertRaises(UpperCamelCase_							):
																	auto_class.register(UpperCamelCase_    ,						UpperCamelCase_							)
															auto_class.register(UpperCamelCase_    ,						UpperCamelCase_							)
															# Trying to register something existing in the Transformers library will raise an error
															with self.assertRaises(UpperCamelCase_							):
																	auto_class.register(UpperCamelCase_    ,						UpperCamelCase_							)

															# Now that the config is registered, it can be used as any other config with the auto-API
															lowercase__            =      BertModelTester(self							).get_config()
															lowercase__            =      NewModelConfig(**tiny_config.to_dict()							)
															lowercase__            =      auto_class.from_config(UpperCamelCase_							)
															self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

															with tempfile.TemporaryDirectory() as tmp_dir:
																	model.save_pretrained(UpperCamelCase_							)
																	lowercase__            =      auto_class.from_pretrained(UpperCamelCase_							)
																	self.assertIsInstance(UpperCamelCase_    ,						UpperCamelCase_							)

									finally:
											if "new-model" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["new-model"]
											for mapping in (
											    TF_MODEL_MAPPING,
											    TF_MODEL_FOR_PRETRAINING_MAPPING,
											    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
											    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
											    TF_MODEL_FOR_MASKED_LM_MAPPING,
											):
													if NewModelConfig in mapping._extra_content:
															del mapping._extra_content[NewModelConfig]

							def       lowerCamelCase_ (							self:	Dict							)			->   Optional[Any]:





									"""simple docstring"""







									with self.assertRaisesRegex(
									    UpperCamelCase_    ,						'''bert-base is not a local folder and is not a valid model identifier'''							):
											lowercase__            =      TFAutoModel.from_pretrained('''bert-base'''							)

							def       lowerCamelCase_ (							self:	Optional[int]							)			->   Any:





									"""simple docstring"""







									with self.assertRaisesRegex(
									    UpperCamelCase_    ,						r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''							):
											lowercase__            =      TFAutoModel.from_pretrained(UpperCamelCase_    ,						revision='''aaaaaa'''							)

							def       lowerCamelCase_ (							self:	Optional[Any]							)			->   Dict:





									"""simple docstring"""







									with self.assertRaisesRegex(
									    UpperCamelCase_    ,						'''hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'''    ,						):
											lowercase__            =      TFAutoModel.from_pretrained('''hf-internal-testing/config-no-model'''							)

							def       lowerCamelCase_ (							self:	Any							)			->   Optional[int]:





									"""simple docstring"""







									with self.assertRaisesRegex(UpperCamelCase_    ,						'''Use `from_pt=True` to load this model'''							):
											lowercase__            =      TFAutoModel.from_pretrained('''hf-internal-testing/tiny-bert-pt-only'''							)


							def       lowerCamelCase_ (							self:	str							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''							)
									with RequestCounter() as counter:
											lowercase__            =      TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''							)
											self.assertEqual(counter.get_request_count    ,						0							)
											self.assertEqual(counter.head_request_count    ,						1							)
											self.assertEqual(counter.other_request_count    ,						0							)

									# With a sharded checkpoint
									lowercase__            =      TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''							)
									with RequestCounter() as counter:
											lowercase__            =      TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''							)
											self.assertEqual(counter.get_request_count    ,						0							)
											self.assertEqual(counter.head_request_count    ,						1							)
											self.assertEqual(counter.other_request_count    ,						0							)







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case						:Union[str, Any]      		=     logging.get_logger(__name__)

__snake_case						:Any      		=     {
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[int]											=						'''switch_transformers'''
       UpperCamelCase__  :    Optional[Any]											=						['''past_key_values''']
       UpperCamelCase__  :    Optional[Any]											=						{'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str=32_128     ,		__SCREAMING_SNAKE_CASE							:     int=768     ,		__SCREAMING_SNAKE_CASE							:     Any=64     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=2_048     ,		__SCREAMING_SNAKE_CASE							:     List[str]=64     ,		__SCREAMING_SNAKE_CASE							:     int=12     ,		__SCREAMING_SNAKE_CASE							:     Any=3     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=12     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=3     ,		__SCREAMING_SNAKE_CASE							:     Any=12     ,		__SCREAMING_SNAKE_CASE							:     Tuple=8     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=False     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=0.01     ,		__SCREAMING_SNAKE_CASE							:     Dict="float32"     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=False     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=32     ,		__SCREAMING_SNAKE_CASE							:     int=128     ,		__SCREAMING_SNAKE_CASE							:     Any=0.1     ,		__SCREAMING_SNAKE_CASE							:     int=1E-6     ,		__SCREAMING_SNAKE_CASE							:     Dict=0.0_01     ,		__SCREAMING_SNAKE_CASE							:     List[str]=0.0_01     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=1.0     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="relu"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=True     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=False     ,		__SCREAMING_SNAKE_CASE							:     str=True     ,		__SCREAMING_SNAKE_CASE							:     int=0     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=1     ,		**__SCREAMING_SNAKE_CASE							:     Dict     ,		):


           '''simple docstring'''

           __a						       =				vocab_size
           __a						       =				d_model
           __a						       =				d_kv
           __a						       =				d_ff

           __a						       =				num_sparse_encoder_layers

           __a						       =				num_layers
           __a						       =				(
               num_decoder_layers if num_decoder_layers is not None else self.num_layers
           )  # default = symmetry
           __a						       =				num_sparse_decoder_layers

           # This tells us, each how many encoder layer we'll have to set a sparse layer.
           if self.num_sparse_encoder_layers > 0:
               __a						       =				self.num_layers // self.num_sparse_encoder_layers
           else:
               __a						       =				self.num_layers  # HACK: this will create 0 sparse layers

           # This tells us, each how many encoder layer we'll have to set a sparse layer.
           if self.num_sparse_decoder_layers > 0:
               __a						       =				self.num_decoder_layers // self.num_sparse_decoder_layers
           else:
               __a						       =				self.num_decoder_layers  # HACK: this will create 0 sparse layers

           __a						       =				num_heads
           __a						       =				num_experts
           __a						       =				expert_capacity
           __a						       =				router_bias
           __a						       =				router_jitter_noise
           if router_dtype not in ["float32", "float16", "bfloat16"]:
               raise ValueError(F'`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}')
           __a						       =				router_dtype

           __a						       =				router_ignore_padding_tokens
           __a						       =				relative_attention_num_buckets
           __a						       =				relative_attention_max_distance

           __a						       =				dropout_rate
           __a						       =				layer_norm_epsilon
           __a						       =				initializer_factor
           __a						       =				feed_forward_proj
           __a						       =				use_cache
           __a						       =				add_router_probs

           __a						       =				router_z_loss_coef
           __a						       =				router_aux_loss_coef

           __a						       =				self.feed_forward_proj.split('''-''')
           __a						       =				act_info[-1]
           __a						       =				act_info[0] == '''gated'''

           if len(__SCREAMING_SNAKE_CASE) > 1 and act_info[0] != "gated" or len(__SCREAMING_SNAKE_CASE) > 2:
               raise ValueError(
                   F'`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'
                   '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
                   '''\'gated-gelu\' or \'relu\'''')

           # for backwards compatibility
           if feed_forward_proj == "gated-gelu":
               __a						       =				'''gelu_new'''

           super().__init__(
               pad_token_id=__SCREAMING_SNAKE_CASE     ,		eos_token_id=__SCREAMING_SNAKE_CASE     ,		is_encoder_decoder=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)





