



"""simple docstring"""







import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse('0.8.3'):
							raise Exception('requires gluonnlp == 0.8.3')

if version.parse(mx.__version__) != version.parse('1.5.0'):
							raise Exception('requires mxnet == 1.5.0')

logging.set_verbosity_info()
lowerCAmelCase_   						=   logging.get_logger(__name__)

lowerCAmelCase_   						=   'The Nymphenburg Palace is a beautiful palace in Munich!'



def      __UpperCAmelCase      (    __lowerCamelCase						,     __lowerCamelCase	)							->      List[str]:
		lowercase__   :    Tuple				=			{
		    '''attention_cell''': '''multi_head''',
		    '''num_layers''': 4,
		    '''units''': 10_24,
		    '''hidden_size''': 7_68,
		    '''max_length''': 5_12,
		    '''num_heads''': 8,
		    '''scaled''': True,
		    '''dropout''': 0.1,
		    '''use_residual''': True,
		    '''embed_size''': 10_24,
		    '''embed_dropout''': 0.1,
		    '''word_embed''': None,
		    '''layer_norm_eps''': 1E-5,
		    '''token_type_vocab_size''': 2,
		}

		lowercase__   :    Optional[Any]				=			bort_4_8_768_1024_hparams

		# Let's construct the original Bort model here
		# Taken from official BERT implementation, see:
		# https://github.com/alexa/bort/blob/master/bort/bort.py
		lowercase__   :    Tuple				=			BERTEncoder(
		    attention_cell=predefined_args['''attention_cell''']						,     num_layers=predefined_args['''num_layers''']						,     units=predefined_args['''units''']						,     hidden_size=predefined_args['''hidden_size''']						,     max_length=predefined_args['''max_length''']						,     num_heads=predefined_args['''num_heads''']						,     scaled=predefined_args['''scaled''']						,     dropout=predefined_args['''dropout''']						,     output_attention=__lowerCamelCase						,     output_all_encodings=__lowerCamelCase						,     use_residual=predefined_args['''use_residual''']						,     activation=predefined_args.get('''activation'''						,     '''gelu'''	)						,     layer_norm_eps=predefined_args.get('''layer_norm_eps'''						,     __lowerCamelCase	)						,     )

		# Vocab information needs to be fetched first
		# It's the same as RoBERTa, so RobertaTokenizer can be used later
		lowercase__   :    List[str]				=			'''openwebtext_ccnews_stories_books_cased'''

		# Specify download folder to Gluonnlp's vocab
		lowercase__   :    Union[str, Any]				=			os.path.join(get_home_dir()						,     '''models'''	)
		lowercase__   :    List[str]				=			_load_vocab(__lowerCamelCase						,     __lowerCamelCase						,     __lowerCamelCase						,     cls=__lowerCamelCase	)

		lowercase__   :    str				=			nlp.model.BERTModel(
		    __lowerCamelCase						,     len(__lowerCamelCase	)						,     units=predefined_args['''units''']						,     embed_size=predefined_args['''embed_size''']						,     embed_dropout=predefined_args['''embed_dropout''']						,     word_embed=predefined_args['''word_embed''']						,     use_pooler=__lowerCamelCase						,     use_token_type_embed=__lowerCamelCase						,     token_type_vocab_size=predefined_args['''token_type_vocab_size''']						,     use_classifier=__lowerCamelCase						,     use_decoder=__lowerCamelCase						,     )

		original_bort.load_parameters(__lowerCamelCase						,     cast_dtype=__lowerCamelCase						,     ignore_extra=__lowerCamelCase	)
		lowercase__   :    List[Any]				=			original_bort._collect_params_with_prefix()

		# Build our config 🤗
		lowercase__   :    Optional[int]				=			{
		    '''architectures''': ['''BertForMaskedLM'''],
		    '''attention_probs_dropout_prob''': predefined_args['''dropout'''],
		    '''hidden_act''': '''gelu''',
		    '''hidden_dropout_prob''': predefined_args['''dropout'''],
		    '''hidden_size''': predefined_args['''embed_size'''],
		    '''initializer_range''': 0.0_2,
		    '''intermediate_size''': predefined_args['''hidden_size'''],
		    '''layer_norm_eps''': predefined_args['''layer_norm_eps'''],
		    '''max_position_embeddings''': predefined_args['''max_length'''],
		    '''model_type''': '''bort''',
		    '''num_attention_heads''': predefined_args['''num_heads'''],
		    '''num_hidden_layers''': predefined_args['''num_layers'''],
		    '''pad_token_id''': 1,  # 2 = BERT, 1 = RoBERTa
		    '''type_vocab_size''': 1,  # 2 = BERT, 1 = RoBERTa
		    '''vocab_size''': len(__lowerCamelCase	),
		}

		lowercase__   :    str				=			BertConfig.from_dict(__lowerCamelCase	)
		lowercase__   :    str				=			BertForMaskedLM(__lowerCamelCase	)
		hf_bort_model.eval()

		# Parameter mapping table (Gluonnlp to Transformers)
		# * denotes layer index
		#
		# | Gluon Parameter                                                | Transformers Parameter
		# | -------------------------------------------------------------- | ----------------------
		# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
		# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
		# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
		# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
		# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
		# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
		# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
		# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
		# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
		# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
		# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
		# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
		# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
		# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

		# Helper function to convert MXNET Arrays to PyTorch
		def to_torch(__lowerCamelCase	) -> nn.Parameter:
				return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()	)	)

		# Check param shapes and map new HF param back
		def check_and_map_params(__lowerCamelCase						,     __lowerCamelCase	):
				lowercase__   :    str				=			hf_param.shape

				lowercase__   :    int				=			to_torch(params[gluon_param]	)
				lowercase__   :    Optional[Any]				=			gluon_param.shape

				assert (
				    shape_hf == shape_gluon
				), f"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

				return gluon_param

		lowercase__   :    str				=			check_and_map_params(
		    hf_bort_model.bert.embeddings.word_embeddings.weight						,     '''word_embed.0.weight'''	)
		lowercase__   :    int				=			check_and_map_params(
		    hf_bort_model.bert.embeddings.position_embeddings.weight						,     '''encoder.position_weight'''	)
		lowercase__   :    int				=			check_and_map_params(
		    hf_bort_model.bert.embeddings.LayerNorm.bias						,     '''encoder.layer_norm.beta'''	)
		lowercase__   :    Optional[int]				=			check_and_map_params(
		    hf_bort_model.bert.embeddings.LayerNorm.weight						,     '''encoder.layer_norm.gamma'''	)

		# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
		lowercase__   :    int				=			torch.zeros_like(
		    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data	)

		for i in range(hf_bort_config.num_hidden_layers	):
				lowercase__   :    BertLayer				=			hf_bort_model.bert.encoder.layer[i]

				# self attention
				lowercase__   :    BertSelfAttention				=			layer.attention.self

				lowercase__   :    Any				=			check_and_map_params(
				    self_attn.key.bias.data						,     f"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""	)

				lowercase__   :    int				=			check_and_map_params(
				    self_attn.key.weight.data						,     f"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""	)
				lowercase__   :    Optional[int]				=			check_and_map_params(
				    self_attn.query.bias.data						,     f"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""	)
				lowercase__   :    int				=			check_and_map_params(
				    self_attn.query.weight.data						,     f"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""	)
				lowercase__   :    Optional[int]				=			check_and_map_params(
				    self_attn.value.bias.data						,     f"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""	)
				lowercase__   :    Optional[Any]				=			check_and_map_params(
				    self_attn.value.weight.data						,     f"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""	)

				# self attention output
				lowercase__   :    BertSelfOutput				=			layer.attention.output

				lowercase__   :    Union[str, Any]				=			check_and_map_params(
				    self_output.dense.bias						,     f"""encoder.transformer_cells.{i}.proj.bias"""	)
				lowercase__   :    List[str]				=			check_and_map_params(
				    self_output.dense.weight						,     f"""encoder.transformer_cells.{i}.proj.weight"""	)
				lowercase__   :    List[str]				=			check_and_map_params(
				    self_output.LayerNorm.bias						,     f"""encoder.transformer_cells.{i}.layer_norm.beta"""	)
				lowercase__   :    Any				=			check_and_map_params(
				    self_output.LayerNorm.weight						,     f"""encoder.transformer_cells.{i}.layer_norm.gamma"""	)

				# intermediate
				lowercase__   :    BertIntermediate				=			layer.intermediate

				lowercase__   :    Dict				=			check_and_map_params(
				    intermediate.dense.bias						,     f"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""	)
				lowercase__   :    Union[str, Any]				=			check_and_map_params(
				    intermediate.dense.weight						,     f"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""	)

				# output
				lowercase__   :    BertOutput				=			layer.output

				lowercase__   :    str				=			check_and_map_params(
				    bert_output.dense.bias						,     f"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""	)
				lowercase__   :    Any				=			check_and_map_params(
				    bert_output.dense.weight						,     f"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""	)
				lowercase__   :    int				=			check_and_map_params(
				    bert_output.LayerNorm.bias						,     f"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""	)
				lowercase__   :    Any				=			check_and_map_params(
				    bert_output.LayerNorm.weight						,     f"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""	)

		# Save space and energy 🎄
		hf_bort_model.half()

		# Compare output of both models
		lowercase__   :    str				=			RobertaTokenizer.from_pretrained('''roberta-base'''	)

		lowercase__   :    Tuple				=			tokenizer.encode_plus(__lowerCamelCase	)['''input_ids''']

		# Get gluon output
		lowercase__   :    List[str]				=			mx.nd.array([input_ids]	)
		lowercase__   :    Union[str, Any]				=			original_bort(inputs=__lowerCamelCase						,     token_types=[]	)

		# Get Transformer output (save and reload model again)
		hf_bort_model.save_pretrained(__lowerCamelCase	)
		lowercase__   :    List[str]				=			BertModel.from_pretrained(__lowerCamelCase	)
		hf_bort_model.eval()

		lowercase__   :    Tuple				=			tokenizer.encode_plus(__lowerCamelCase						,     return_tensors='''pt'''	)
		lowercase__   :    Dict				=			hf_bort_model(**__lowerCamelCase	)[0]

		lowercase__   :    List[str]				=			output_gluon[0].asnumpy()
		lowercase__   :    Optional[Any]				=			output_hf[0].detach().numpy()

		lowercase__   :    List[str]				=			np.max(np.abs(hf_layer - gluon_layer	)	).item()
		lowercase__   :    List[str]				=			np.allclose(__lowerCamelCase						,     __lowerCamelCase						,     atol=1E-3	)

		if success:
				print('''✔️ Both model do output the same tensors'''	)
		else:
				print('''❌ Both model do **NOT** output the same tensors'''	)
				print('''Absolute difference is:'''						,     __lowerCamelCase	)


if __name__ == "__main__":
							lowerCAmelCase_   						=   argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--bort_checkpoint_path', default=None, type=str, required=True, help='Path the official Bort params file.'
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							lowerCAmelCase_   						=   parser.parse_args()
							convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)





"""simple docstring"""







from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
							from transformers import (
							    AutoConfig,
							    BertConfig,
							    GPTaConfig,
							    TaConfig,
							    TFAutoModel,
							    TFAutoModelForCausalLM,
							    TFAutoModelForMaskedLM,
							    TFAutoModelForPreTraining,
							    TFAutoModelForQuestionAnswering,
							    TFAutoModelForSeqaSeqLM,
							    TFAutoModelForSequenceClassification,
							    TFAutoModelWithLMHead,
							    TFBertForMaskedLM,
							    TFBertForPreTraining,
							    TFBertForQuestionAnswering,
							    TFBertForSequenceClassification,
							    TFBertModel,
							    TFGPTaLMHeadModel,
							    TFRobertaForMaskedLM,
							    TFTaForConditionalGeneration,
							)
							from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
							from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
							from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
							from transformers import (
							    AutoModel,
							    AutoModelForCausalLM,
							    AutoModelForMaskedLM,
							    AutoModelForPreTraining,
							    AutoModelForQuestionAnswering,
							    AutoModelForSeqaSeqLM,
							    AutoModelForSequenceClassification,
							    AutoModelWithLMHead,
							    BertForMaskedLM,
							    BertForPreTraining,
							    BertForQuestionAnswering,
							    BertForSequenceClassification,
							    BertModel,
							    GPTaLMHeadModel,
							    RobertaForMaskedLM,
							    TaForConditionalGeneration,
							)
@is_pt_tf_cross_test
class        __A       (	unittest.TestCase    ):







		'''simple docstring'''





		@slow
		def 					UpperCAmelCase    (	self						:  Optional[Any]						)					->		Union[str, Any]:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    Tuple				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Dict				=			TFAutoModel.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			AutoModel.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Union[str, Any]						)					->		Tuple:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    Dict				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    str				=			TFAutoModelForPreTraining.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Optional[Any]				=			AutoModelForPreTraining.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Tuple						)					->		Dict:






				"""simple docstring"""



				for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    Any				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			TFAutoModelForCausalLM.from_pretrained(_snake_case			,from_pt=_snake_case						)
						lowercase__		,			lowercase__   :    Optional[Any]				=			TFAutoModelForCausalLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Union[str, Any]				=			AutoModelForCausalLM.from_pretrained(_snake_case			,from_tf=_snake_case						)
						lowercase__		,			lowercase__   :    Optional[Any]				=			AutoModelForCausalLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Any						)					->		Tuple:






				"""simple docstring"""



				for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    Any				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Optional[Any]				=			TFAutoModelWithLMHead.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			AutoModelWithLMHead.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  List[str]						)					->		Optional[Any]:






				"""simple docstring"""



				for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    str				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Union[str, Any]				=			TFAutoModelForMaskedLM.from_pretrained(_snake_case			,from_pt=_snake_case						)
						lowercase__		,			lowercase__   :    str				=			TFAutoModelForMaskedLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			AutoModelForMaskedLM.from_pretrained(_snake_case			,from_tf=_snake_case						)
						lowercase__		,			lowercase__   :    Any				=			AutoModelForMaskedLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  List[Any]						)					->		Dict:






				"""simple docstring"""



				for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    Union[str, Any]				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			TFAutoModelForSeqaSeqLM.from_pretrained(_snake_case			,from_pt=_snake_case						)
						lowercase__		,			lowercase__   :    List[str]				=			TFAutoModelForSeqaSeqLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			AutoModelForSeqaSeqLM.from_pretrained(_snake_case			,from_tf=_snake_case						)
						lowercase__		,			lowercase__   :    Optional[int]				=			AutoModelForSeqaSeqLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  List[str]						)					->		Union[str, Any]:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    Tuple				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			TFAutoModelForSequenceClassification.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Union[str, Any]				=			AutoModelForSequenceClassification.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Tuple						)					->		Optional[Any]:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    List[Any]				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    str				=			TFAutoModelForQuestionAnswering.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			AutoModelForQuestionAnswering.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		def 					UpperCAmelCase    (	self						:  Dict						)					->		Any:






				"""simple docstring"""



				lowercase__   :    Optional[Any]				=			TFAutoModelWithLMHead.from_pretrained(_snake_case			,from_pt=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)

				lowercase__   :    Union[str, Any]				=			AutoModelWithLMHead.from_pretrained(_snake_case			,from_tf=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)







		def 					UpperCAmelCase    (	self						:  int						)					->		List[Any]:






				"""simple docstring"""



				lowercase__   :    List[Any]				=			TFAutoModelWithLMHead.from_pretrained(_snake_case			,from_pt=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)

				lowercase__   :    int				=			AutoModelWithLMHead.from_pretrained(_snake_case			,from_tf=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)
