





from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
    from transformers import (
        AutoConfig,
        BertConfig,
        GPTaConfig,
        TaConfig,
        TFAutoModel,
        TFAutoModelForCausalLM,
        TFAutoModelForMaskedLM,
        TFAutoModelForPreTraining,
        TFAutoModelForQuestionAnswering,
        TFAutoModelForSeqaSeqLM,
        TFAutoModelForSequenceClassification,
        TFAutoModelWithLMHead,
        TFBertForMaskedLM,
        TFBertForPreTraining,
        TFBertForQuestionAnswering,
        TFBertForSequenceClassification,
        TFBertModel,
        TFGPTaLMHeadModel,
        TFRobertaForMaskedLM,
        TFTaForConditionalGeneration,
    )
    from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
    from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
    from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
    from transformers import (
        AutoModel,
        AutoModelForCausalLM,
        AutoModelForMaskedLM,
        AutoModelForPreTraining,
        AutoModelForQuestionAnswering,
        AutoModelForSeqaSeqLM,
        AutoModelForSequenceClassification,
        AutoModelWithLMHead,
        BertForMaskedLM,
        BertForPreTraining,
        BertForQuestionAnswering,
        BertForSequenceClassification,
        BertModel,
        GPTaLMHeadModel,
        RobertaForMaskedLM,
        TaForConditionalGeneration,
    )





@is_pt_tf_cross_test
class  A__    (							unittest.TestCase	):




   """simple docstring"""



   @slow
   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          for model_name in ["bert-base-uncased"]:
                 a__						:	str =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	str =				TFAutoModel.from_pretrained(lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Optional[Any] =				AutoModel.from_pretrained(lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   @slow
   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          for model_name in ["bert-base-uncased"]:
                 a__						:	List[str] =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Any =				TFAutoModelForPreTraining.from_pretrained(lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Optional[int] =				AutoModelForPreTraining.from_pretrained(lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   @slow
   def 							__lowercase		(					self)			->						str:







          '''simple docstring'''

          for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 a__						:	Optional[Any] =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	List[Any] =				TFAutoModelForCausalLM.from_pretrained(lowercase					,  from_pt=lowercase)
                 a__ ,							a__						:	Tuple =				TFAutoModelForCausalLM.from_pretrained(
                     lowercase					,  output_loading_info=lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Tuple =				AutoModelForCausalLM.from_pretrained(lowercase					,  from_tf=lowercase)
                 a__ ,							a__						:	Any =				AutoModelForCausalLM.from_pretrained(
                     lowercase					,  output_loading_info=lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   @slow
   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 a__						:	Optional[Any] =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	str =				TFAutoModelWithLMHead.from_pretrained(lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	int =				AutoModelWithLMHead.from_pretrained(lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   @slow
   def 							__lowercase		(					self)			->						List[str]:







          '''simple docstring'''

          for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 a__						:	str =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	List[str] =				TFAutoModelForMaskedLM.from_pretrained(lowercase					,  from_pt=lowercase)
                 a__ ,							a__						:	Optional[Any] =				TFAutoModelForMaskedLM.from_pretrained(
                     lowercase					,  output_loading_info=lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Tuple =				AutoModelForMaskedLM.from_pretrained(lowercase					,  from_tf=lowercase)
                 a__ ,							a__						:	Optional[int] =				AutoModelForMaskedLM.from_pretrained(
                     lowercase					,  output_loading_info=lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   @slow
   def 							__lowercase		(					self)			->						List[str]:







          '''simple docstring'''

          for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 a__						:	Optional[Any] =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Optional[int] =				TFAutoModelForSeqaSeqLM.from_pretrained(lowercase					,  from_pt=lowercase)
                 a__ ,							a__						:	List[str] =				TFAutoModelForSeqaSeqLM.from_pretrained(
                     lowercase					,  output_loading_info=lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Union[str, Any] =				AutoModelForSeqaSeqLM.from_pretrained(lowercase					,  from_tf=lowercase)
                 a__ ,							a__						:	Optional[int] =				AutoModelForSeqaSeqLM.from_pretrained(
                     lowercase					,  output_loading_info=lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   @slow
   def 							__lowercase		(					self)			->						List[str]:







          '''simple docstring'''

          for model_name in ["bert-base-uncased"]:
                 a__						:	Optional[Any] =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	int =				TFAutoModelForSequenceClassification.from_pretrained(lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Any =				AutoModelForSequenceClassification.from_pretrained(lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   @slow
   def 							__lowercase		(					self)			->						Optional[int]:







          '''simple docstring'''

          for model_name in ["bert-base-uncased"]:
                 a__						:	List[Any] =				AutoConfig.from_pretrained(lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	Optional[int] =				TFAutoModelForQuestionAnswering.from_pretrained(lowercase					,  from_pt=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

                 a__						:	str =				AutoModelForQuestionAnswering.from_pretrained(lowercase					,  from_tf=lowercase)
                 self.assertIsNotNone(lowercase)
                 self.assertIsInstance(lowercase					,  lowercase)

   def 							__lowercase		(					self)			->						List[str]:







          '''simple docstring'''

          a__						:	Tuple =				TFAutoModelWithLMHead.from_pretrained(lowercase					,  from_pt=lowercase)
          self.assertIsInstance(lowercase					,  lowercase)
          self.assertEqual(model.num_parameters()					,  1_4410)
          self.assertEqual(model.num_parameters(only_trainable=lowercase)					,  1_4410)

          a__						:	Any =				AutoModelWithLMHead.from_pretrained(lowercase					,  from_tf=lowercase)
          self.assertIsInstance(lowercase					,  lowercase)
          self.assertEqual(model.num_parameters()					,  1_4410)
          self.assertEqual(model.num_parameters(only_trainable=lowercase)					,  1_4410)

   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          a__						:	List[str] =				TFAutoModelWithLMHead.from_pretrained(lowercase					,  from_pt=lowercase)
          self.assertIsInstance(lowercase					,  lowercase)
          self.assertEqual(model.num_parameters()					,  1_4410)
          self.assertEqual(model.num_parameters(only_trainable=lowercase)					,  1_4410)

          a__						:	int =				AutoModelWithLMHead.from_pretrained(lowercase					,  from_tf=lowercase)
          self.assertIsInstance(lowercase					,  lowercase)
          self.assertEqual(model.num_parameters()					,  1_4410)
          self.assertEqual(model.num_parameters(only_trainable=lowercase)					,  1_4410)






import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
lowerCAmelCase__      		= object()

# For specifying empty leaf dict `{}`
lowerCAmelCase__      		= object()
def 							_UpperCAmelCase				(UpperCamelCase__     :  str				,							UpperCamelCase__     :  Union[str, Any]						):
       _A      :       str				    = tuple((re.compile(x + "$"						) for x in qs)						)
       for i in range(len(UpperCamelCase__						) - len(UpperCamelCase__						) + 1						):
              _A      :       Tuple				    = [x.match(UpperCamelCase__						) for x, y in zip(UpperCamelCase__				,							ks[i:]						)]
              if matches and all(UpperCamelCase__						):
                     return True
       return False
def 							_UpperCAmelCase				(UpperCamelCase__     :  str						):
       def replace(UpperCamelCase__     :  List[str]				,							UpperCamelCase__     :  Optional[int]						):
              for rule, replacement in rules:
                     if _match(UpperCamelCase__				,							UpperCamelCase__						):
                            return replacement
              return val

       return replace
def 							_UpperCAmelCase				():
       return [
           # embeddings
           (("transformer", "wpe", "embedding"), P("mp"				,							UpperCamelCase__						)),
           (("transformer", "wte", "embedding"), P("mp"				,							UpperCamelCase__						)),
           # atention
           (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(UpperCamelCase__				,							"mp"						)),
           (("attention", "out_proj", "kernel"), P("mp"				,							UpperCamelCase__						)),
           (("attention", "out_proj", "bias"), None),
           # mlp
           (("mlp", "c_fc", "kernel"), P(UpperCamelCase__				,							"mp"						)),
           (("mlp", "c_fc", "bias"), P("mp"						)),
           (("mlp", "c_proj", "kernel"), P("mp"				,							UpperCamelCase__						)),
           (("mlp", "c_proj", "bias"), None),
           # layer norms
           ((r"ln_\d+", "bias"), None),
           ((r"\d+", r"ln_\d+", "scale"), None),
           (("ln_f", "bias"), None),
           (("ln_f", "scale"), None),
       ]






def 							_UpperCAmelCase				(UpperCamelCase__     :  List[str]						):
       _A      :       int				    = _get_partition_rules()
       _A      :       Optional[int]				    = _replacement_rules(UpperCamelCase__						)
       _A      :       Optional[int]				    = {k: _unmatched for k in flatten_dict(UpperCamelCase__						)}
       _A      :       List[str]				    = {k: replace(UpperCamelCase__				,							UpperCamelCase__						) for k, v in initd.items()}
       assert _unmatched not in result.values(), "Incomplete partition spec."
       return freeze(unflatten_dict(UpperCamelCase__						)						)
