
"""simple docstring"""






import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
__UpperCAmelCase       =  'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
__UpperCAmelCase       =  direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
__UpperCAmelCase       =  re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
__UpperCAmelCase       =  re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__UpperCAmelCase       =  re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
__UpperCAmelCase       =  [
    ('pretraining', 'MODEL_FOR_PRETRAINING_MAPPING_NAMES', 'AutoModelForPreTraining'),
    ('feature-extraction', 'MODEL_MAPPING_NAMES', 'AutoModel'),
    ('audio-classification', 'MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForAudioClassification'),
    ('text-generation', 'MODEL_FOR_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForCausalLM'),
    ('automatic-speech-recognition', 'MODEL_FOR_CTC_MAPPING_NAMES', 'AutoModelForCTC'),
    ('image-classification', 'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForImageClassification'),
    ('image-segmentation', 'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES', 'AutoModelForImageSegmentation'),
    ('fill-mask', 'MODEL_FOR_MASKED_LM_MAPPING_NAMES', 'AutoModelForMaskedLM'),
    ('object-detection', 'MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES', 'AutoModelForObjectDetection'),
    (
        'zero-shot-object-detection',
        'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES',
        'AutoModelForZeroShotObjectDetection',
    ),
    ('question-answering', 'MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES', 'AutoModelForQuestionAnswering'),
    ('text2text-generation', 'MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForSeq2SeqLM'),
    ('text-classification', 'MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForSequenceClassification'),
    ('automatic-speech-recognition', 'MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES', 'AutoModelForSpeechSeq2Seq'),
    (
        'table-question-answering',
        'MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForTableQuestionAnswering',
    ),
    ('token-classification', 'MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForTokenClassification'),
    ('multiple-choice', 'MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES', 'AutoModelForMultipleChoice'),
    (
        'next-sentence-prediction',
        'MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES',
        'AutoModelForNextSentencePrediction',
    ),
    (
        'audio-frame-classification',
        'MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForAudioFrameClassification',
    ),
    ('audio-xvector', 'MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES', 'AutoModelForAudioXVector'),
    (
        'document-question-answering',
        'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForDocumentQuestionAnswering',
    ),
    (
        'visual-question-answering',
        'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForVisualQuestionAnswering',
    ),
    ('image-to-text', 'MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES', 'AutoModelForVision2Seq'),
    (
        'zero-shot-image-classification',
        'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForZeroShotImageClassification',
    ),
    ('depth-estimation', 'MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES', 'AutoModelForDepthEstimation'),
    ('video-classification', 'MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForVideoClassification'),
    ('mask-generation', 'MODEL_FOR_MASK_GENERATION_MAPPING_NAMES', 'AutoModelForMaskGeneration'),
]






def 						_snake_case       (	lowercase__		:  Optional[int]		)  ->				List[str]:


			'''simple docstring'''





			lowerCAmelCase_    :List[Any]			     =  re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""      ,     lowercase__		)
			return [m.group(0		) for m in matches]






def 						_snake_case       (	)  ->				str:


			'''simple docstring'''





			lowerCAmelCase_    :List[str]			     =  transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
			lowerCAmelCase_    :Optional[Any]			     =  {
			    config.replace("""Config"""      ,     """"""		): model_type for model_type, config in config_maping_names.items()
			}

			# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
			lowerCAmelCase_    :List[Any]			     =  collections.defaultdict(lowercase__		)
			lowerCAmelCase_    :Dict			     =  collections.defaultdict(lowercase__		)
			lowerCAmelCase_    :Union[str, Any]			     =  collections.defaultdict(lowercase__		)

			# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
			for attr_name in dir(lowercase__		):
						lowerCAmelCase_    :List[Any]			     =  None
						if _re_tf_models.match(lowercase__		) is not None:
									lowerCAmelCase_    :List[str]			     =  tf_models
									lowerCAmelCase_    :Dict			     =  _re_tf_models.match(lowercase__		).groups()[0]
						elif _re_flax_models.match(lowercase__		) is not None:
									lowerCAmelCase_    :Tuple			     =  flax_models
									lowerCAmelCase_    :List[str]			     =  _re_flax_models.match(lowercase__		).groups()[0]
						elif _re_pt_models.match(lowercase__		) is not None:
									lowerCAmelCase_    :List[Any]			     =  pt_models
									lowerCAmelCase_    :str			     =  _re_pt_models.match(lowercase__		).groups()[0]

						if lookup_dict is not None:
									while len(lowercase__		) > 0:
												if attr_name in model_prefix_to_model_type:
															lowerCAmelCase_    :List[Any]			     =  True
															break
												# Try again after removing the last word in the name
												lowerCAmelCase_    :Optional[int]			     =  """""".join(camel_case_split(lowercase__		)[:-1]		)

			lowerCAmelCase_    :Optional[Any]			     =  set(list(pt_models.keys()		) + list(tf_models.keys()		) + list(flax_models.keys()		)		)
			lowerCAmelCase_    :Dict			     =  list(lowercase__		)
			all_models.sort()

			lowerCAmelCase_    :str			     =  {"""model_type""": all_models}
			lowerCAmelCase_    :Optional[int]			     =  [pt_models[t] for t in all_models]
			lowerCAmelCase_    :Dict			     =  [tf_models[t] for t in all_models]
			lowerCAmelCase_    :Optional[Any]			     =  [flax_models[t] for t in all_models]

			# Now let's use the auto-mapping names to make sure
			lowerCAmelCase_    :Any			     =  {}
			for t in all_models:
						if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
									lowerCAmelCase_    :Any			     =  """AutoProcessor"""
						elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
									lowerCAmelCase_    :List[str]			     =  """AutoTokenizer"""
						elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
									lowerCAmelCase_    :List[Any]			     =  """AutoFeatureExtractor"""
						else:
									# Default to AutoTokenizer if a model has nothing, for backward compatibility.
									lowerCAmelCase_    :Union[str, Any]			     =  """AutoTokenizer"""

			lowerCAmelCase_    :Union[str, Any]			     =  [processors[t] for t in all_models]

			return pd.DataFrame(lowercase__		)






def 						_snake_case       (	lowercase__		:  Any		)  ->				str:


			'''simple docstring'''





			lowerCAmelCase_    :str			     =  [
			    transformers_module.models.auto.modeling_auto,
			    transformers_module.models.auto.modeling_tf_auto,
			    transformers_module.models.auto.modeling_flax_auto,
			]
			for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
						lowerCAmelCase_    :Dict			     =  [model_mapping, f"""TF_{model_mapping}""", f"""FLAX_{model_mapping}"""]
						lowerCAmelCase_    :Dict			     =  [auto_class, f"""TF_{auto_class}""", f"""Flax_{auto_class}"""]
						# Loop through all three frameworks
						for module, cls, mapping in zip(lowercase__      ,     lowercase__      ,     lowercase__		):
									# The type of pipeline may not exist in this framework
									if not hasattr(lowercase__      ,     lowercase__		):
												continue
									# First extract all model_names
									lowerCAmelCase_    :Optional[Any]			     =  []
									for name in getattr(lowercase__      ,     lowercase__		).values():
												if isinstance(lowercase__      ,     lowercase__		):
															model_names.append(lowercase__		)
												else:
															model_names.extend(list(lowercase__		)		)

            # Add pipeline tag and auto model class for those models
									table.update({model_name: (pipeline_tag, cls) for model_name in model_names}		)

			return table






def 						_snake_case       (	lowercase__		:  List[Any]      ,     lowercase__		:  Union[str, Any]		)  ->				Any:


			'''simple docstring'''





			lowerCAmelCase_    :Union[str, Any]			     =  get_frameworks_table()
			lowerCAmelCase_    :Optional[int]			     =  Dataset.from_pandas(lowercase__		)

			lowerCAmelCase_    :Any			     =  hf_hub_download(
			    """huggingface/transformers-metadata"""      ,     """pipeline_tags.json"""      ,     repo_type="""dataset"""      ,     token=lowercase__		)
			lowerCAmelCase_    :List[str]			     =  Dataset.from_json(lowercase__		)
			lowerCAmelCase_    :int			     =  {
			    tags_dataset[i]["""model_class"""]: (tags_dataset[i]["""pipeline_tag"""], tags_dataset[i]["""auto_class"""])
			    for i in range(len(lowercase__		)		)
			}
			lowerCAmelCase_    :Dict			     =  update_pipeline_and_auto_class_table(lowercase__		)

			# Sort the model classes to avoid some nondeterministic updates to create false update commits.
			lowerCAmelCase_    :str			     =  sorted(table.keys()		)
			lowerCAmelCase_    :List[str]			     =  pd.DataFrame(
			    {
			        """model_class""": model_classes,
			        """pipeline_tag""": [table[m][0] for m in model_classes],
			        """auto_class""": [table[m][1] for m in model_classes],
			    }		)
			lowerCAmelCase_    :List[str]			     =  Dataset.from_pandas(lowercase__		)

			with tempfile.TemporaryDirectory() as tmp_dir:
						frameworks_dataset.to_json(os.path.join(lowercase__      ,     """frameworks.json"""		)		)
						tags_dataset.to_json(os.path.join(lowercase__      ,     """pipeline_tags.json"""		)		)

						if commit_sha is not None:
									lowerCAmelCase_    :List[Any]			     =  (
									    f"""Update with commit {commit_sha}\n\nSee: """
									    f"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
									)
						else:
									lowerCAmelCase_    :int			     =  """Update"""

						upload_folder(
						    repo_id="""huggingface/transformers-metadata"""      ,     folder_path=lowercase__      ,     repo_type="""dataset"""      ,     token=lowercase__      ,     commit_message=lowercase__      ,     )






def 						_snake_case       (	)  ->				str:


			'''simple docstring'''





			lowerCAmelCase_    :Tuple			     =  {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
			lowerCAmelCase_    :Optional[int]			     =  transformers_module.pipelines.SUPPORTED_TASKS
			lowerCAmelCase_    :int			     =  []
			for key in pipeline_tasks:
						if key not in in_table:
									lowerCAmelCase_    :Any			     =  pipeline_tasks[key]["""pt"""]
									if isinstance(lowercase__      ,     (list, tuple)		):
												lowerCAmelCase_    :int			     =  model[0]
									lowerCAmelCase_    :List[Any]			     =  model.__name__
									if model not in in_table.values():
												missing.append(lowercase__		)

			if len(lowercase__		) > 0:
						lowerCAmelCase_    :Optional[int]			     =  """, """.join(lowercase__		)
						raise ValueError(
						    """The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside """
						    f"""`utils/update_metadata.py`: {msg}. Please add them!"""		)


if __name__ == "__main__":
				__UpperCAmelCase       =  argparse.ArgumentParser()
				parser.add_argument('--token', type=str, help='The token to use to push to the transformers-metadata dataset.')
				parser.add_argument('--commit_sha', type=str, help='The sha of the commit going with this update.')
				parser.add_argument('--check-only', action='store_true', help='Activate to just check all pipelines are present.')
				__UpperCAmelCase       =  parser.parse_args()

				if args.check_only:
								check_pipeline_tags()
				else:
								update_metadata(args.token, args.commit_sha)








import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
					import os

					from flax.core.frozen_dict import unfreeze
					from flax.traverse_util import flatten_dict

					from transformers import FlaxBertModel

					A     :	int    = '0.12'  # assumed parallelism: 8







@require_flax
@is_staging_test
class 			__A( unittest.TestCase			):




				@classmethod
				def 				SCREAMING_SNAKE_CASE_		(     cls				)    ->      Union[str, Any]:


								'''simple docstring'''




								__a										=    TOKEN
								HfFolder.save_token(_snake_case				)




				@classmethod
				def 				SCREAMING_SNAKE_CASE_		(     cls				)    ->      Union[str, Any]:


								'''simple docstring'''




								try:
												delete_repo(token=cls._token		,     repo_id='''test-model-flax'''				)
								except HTTPError:
												pass

								try:
												delete_repo(token=cls._token		,     repo_id='''valid_org/test-model-flax-org'''				)
								except HTTPError:
												pass




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Optional[Any]:


								'''simple docstring'''




								__a										=    BertConfig(
								    vocab_size=99		,     hidden_size=32		,     num_hidden_layers=5		,     num_attention_heads=4		,     intermediate_size=37				)
								__a										=    FlaxBertModel(_snake_case				)
								model.push_to_hub('''test-model-flax'''		,     use_auth_token=self._token				)

								__a										=    FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""				)

								__a										=    flatten_dict(unfreeze(model.params				)				)
								__a										=    flatten_dict(unfreeze(new_model.params				)				)

								for key in base_params.keys():
												__a										=    (base_params[key] - new_params[key]).sum().item()
												self.assertLessEqual(_snake_case		,     1E-3		,     msg=F"""{key} not identical"""				)

								# Reset repo
								delete_repo(token=self._token		,     repo_id='''test-model-flax'''				)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
												model.save_pretrained(_snake_case		,     repo_id='''test-model-flax'''		,     push_to_hub=_snake_case		,     use_auth_token=self._token				)

								__a										=    FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""				)

								__a										=    flatten_dict(unfreeze(model.params				)				)
								__a										=    flatten_dict(unfreeze(new_model.params				)				)

								for key in base_params.keys():
												__a										=    (base_params[key] - new_params[key]).sum().item()
												self.assertLessEqual(_snake_case		,     1E-3		,     msg=F"""{key} not identical"""				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Optional[int]:


								'''simple docstring'''




								__a										=    BertConfig(
								    vocab_size=99		,     hidden_size=32		,     num_hidden_layers=5		,     num_attention_heads=4		,     intermediate_size=37				)
								__a										=    FlaxBertModel(_snake_case				)
								model.push_to_hub('''valid_org/test-model-flax-org'''		,     use_auth_token=self._token				)

								__a										=    FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''				)

								__a										=    flatten_dict(unfreeze(model.params				)				)
								__a										=    flatten_dict(unfreeze(new_model.params				)				)

								for key in base_params.keys():
												__a										=    (base_params[key] - new_params[key]).sum().item()
												self.assertLessEqual(_snake_case		,     1E-3		,     msg=F"""{key} not identical"""				)

								# Reset repo
								delete_repo(token=self._token		,     repo_id='''valid_org/test-model-flax-org'''				)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
												model.save_pretrained(
												    _snake_case		,     repo_id='''valid_org/test-model-flax-org'''		,     push_to_hub=_snake_case		,     use_auth_token=self._token				)

								__a										=    FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''				)

								__a										=    flatten_dict(unfreeze(model.params				)				)
								__a										=    flatten_dict(unfreeze(new_model.params				)				)

								for key in base_params.keys():
												__a										=    (base_params[key] - new_params[key]).sum().item()
												self.assertLessEqual(_snake_case		,     1E-3		,     msg=F"""{key} not identical"""				)



def     __lowerCAmelCase			(			a__						,    a__  )       ->  str:
				__a										=    True
				__a										=    flatten_dict(modela.params  )
				__a										=    flatten_dict(modela.params  )
				for key in flat_params_a.keys():
								if np.sum(np.abs(flat_params_a[key] - flat_params_a[key]  )  ) > 1e-4:
												__a										=    False

				return models_are_equal







@require_flax
class 			__A( unittest.TestCase			):




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Union[str, Any]:


								'''simple docstring'''




								__a										=    BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''				)
								__a										=    FlaxBertModel(_snake_case				)

								__a										=    '''bert'''
								with tempfile.TemporaryDirectory() as tmp_dir:
												model.save_pretrained(os.path.join(_snake_case		,     _snake_case				)				)

												with self.assertRaises(_snake_case				):
																__a										=    FlaxBertModel.from_pretrained(_snake_case				)

												__a										=    FlaxBertModel.from_pretrained(_snake_case		,     subfolder=_snake_case				)

								self.assertTrue(check_models_equal(_snake_case		,     _snake_case				)				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Optional[Any]:


								'''simple docstring'''




								__a										=    BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''				)
								__a										=    FlaxBertModel(_snake_case				)

								__a										=    '''bert'''
								with tempfile.TemporaryDirectory() as tmp_dir:
												model.save_pretrained(os.path.join(_snake_case		,     _snake_case				)		,     max_shard_size='''10KB'''				)

												with self.assertRaises(_snake_case				):
																__a										=    FlaxBertModel.from_pretrained(_snake_case				)

												__a										=    FlaxBertModel.from_pretrained(_snake_case		,     subfolder=_snake_case				)

								self.assertTrue(check_models_equal(_snake_case		,     _snake_case				)				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      int:


								'''simple docstring'''




								__a										=    '''bert'''
								__a										=    '''hf-internal-testing/tiny-random-bert-subfolder'''

								with self.assertRaises(_snake_case				):
												__a										=    FlaxBertModel.from_pretrained(_snake_case				)

								__a										=    FlaxBertModel.from_pretrained(_snake_case		,     subfolder=_snake_case				)

								self.assertIsNotNone(_snake_case				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Optional[int]:


								'''simple docstring'''




								__a										=    '''bert'''
								__a										=    '''hf-internal-testing/tiny-random-bert-sharded-subfolder'''
								with self.assertRaises(_snake_case				):
												__a										=    FlaxBertModel.from_pretrained(_snake_case				)

								__a										=    FlaxBertModel.from_pretrained(_snake_case		,     subfolder=_snake_case				)

								self.assertIsNotNone(_snake_case				)