



"""simple docstring"""
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import LevitImageProcessor
class 				lowerCamelCase  (			unittest.TestCase				):






					'''simple docstring'''





					def __init__(       self:							int						,     snake_case:							int						,     snake_case:							Tuple=7						,     snake_case:							Optional[int]=3						,     snake_case:							List[Any]=18						,     snake_case:							Dict=30						,     snake_case:							Optional[int]=400						,     snake_case:							int=True						,     snake_case:							Dict=None						,     snake_case:							Dict=True						,     snake_case:							Tuple=None						,     snake_case:							Optional[Any]=True						,     snake_case:							Dict=[0.5, 0.5, 0.5]						,     snake_case:							Union[str, Any]=[0.5, 0.5, 0.5]						,     )   ->	int:
											snake_case_ :Tuple						=			size if size is not None else {"""shortest_edge""": 18}
											snake_case_ :Dict						=			crop_size if crop_size is not None else {"""height""": 18, """width""": 18}
											snake_case_ :str						=			parent
											snake_case_ :int						=			batch_size
											snake_case_ :Tuple						=			num_channels
											snake_case_ :Any						=			image_size
											snake_case_ :int						=			min_resolution
											snake_case_ :Dict						=			max_resolution
											snake_case_ :Any						=			do_resize
											snake_case_ :Optional[int]						=			size
											snake_case_ :Optional[int]						=			do_center_crop
											snake_case_ :Dict						=			crop_size
											snake_case_ :int						=			do_normalize
											snake_case_ :Optional[Any]						=			image_mean
											snake_case_ :Optional[Any]						=			image_std






					def    lowerCAmelCase_				(       self:							List[Any]						)   ->	str:
											return {
											    "image_mean": self.image_mean,
											    "image_std": self.image_std,
											    "do_normalize": self.do_normalize,
											    "do_resize": self.do_resize,
											    "do_center_crop": self.do_center_crop,
											    "size": self.size,
											    "crop_size": self.crop_size,
											}

@require_torch
@require_vision
class 				lowerCamelCase  (			_lowerCAmelCase    ,						unittest.TestCase				):






					'''simple docstring'''





					_A		:			List[str]						=     LevitImageProcessor if is_vision_available() else None






					def    lowerCAmelCase_				(       self:							List[Any]						)   ->	str:
											snake_case_ :List[str]						=			LevitImageProcessingTester(self						)






					@property
					def    lowerCAmelCase_				(       self:							str						)   ->	Dict:
											return self.image_processor_tester.prepare_image_processor_dict()






					def    lowerCAmelCase_				(       self:							List[Any]						)   ->	Any:
											snake_case_ :Any						=			self.image_processing_class(**self.image_processor_dict						)
											self.assertTrue(hasattr(snake_case						,     """image_mean"""						)						)
											self.assertTrue(hasattr(snake_case						,     """image_std"""						)						)
											self.assertTrue(hasattr(snake_case						,     """do_normalize"""						)						)
											self.assertTrue(hasattr(snake_case						,     """do_resize"""						)						)
											self.assertTrue(hasattr(snake_case						,     """do_center_crop"""						)						)
											self.assertTrue(hasattr(snake_case						,     """size"""						)						)






					def    lowerCAmelCase_				(       self:							Dict						)   ->	List[str]:
											snake_case_ :List[Any]						=			self.image_processing_class.from_dict(self.image_processor_dict						)
											self.assertEqual(image_processor.size						,     {"""shortest_edge""": 18}						)
											self.assertEqual(image_processor.crop_size						,     {"""height""": 18, """width""": 18}						)

											snake_case_ :Tuple						=			self.image_processing_class.from_dict(self.image_processor_dict						,     size=42						,     crop_size=84						)
											self.assertEqual(image_processor.size						,     {"""shortest_edge""": 42}						)
											self.assertEqual(image_processor.crop_size						,     {"""height""": 84, """width""": 84}						)






					def    lowerCAmelCase_				(       self:							Optional[int]						)   ->	List[str]:
											pass






					def    lowerCAmelCase_				(       self:							int						)   ->	Optional[Any]:
											# Initialize image_processing
											snake_case_ :str						=			self.image_processing_class(**self.image_processor_dict						)
											# create random PIL images
											snake_case_ :List[str]						=			prepare_image_inputs(self.image_processor_tester						,     equal_resolution=snake_case						)
											for image in image_inputs:
																	self.assertIsInstance(snake_case						,     Image.Image						)

											# Test not batched input
											snake_case_ :int						=			image_processing(image_inputs[0]						,     return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,     (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )						,     )

											# Test batched
											snake_case_ :Optional[Any]						=			image_processing(snake_case						,     return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,     (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )						,     )






					def    lowerCAmelCase_				(       self:							List[Any]						)   ->	List[Any]:
											# Initialize image_processing
											snake_case_ :List[str]						=			self.image_processing_class(**self.image_processor_dict						)
											# create random numpy tensors
											snake_case_ :Union[str, Any]						=			prepare_image_inputs(self.image_processor_tester						,     equal_resolution=snake_case						,     numpify=snake_case						)
											for image in image_inputs:
																	self.assertIsInstance(snake_case						,     np.ndarray						)

											# Test not batched input
											snake_case_ :Union[str, Any]						=			image_processing(image_inputs[0]						,     return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,     (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )						,     )

											# Test batched
											snake_case_ :List[str]						=			image_processing(snake_case						,     return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,     (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )						,     )






					def    lowerCAmelCase_				(       self:							List[str]						)   ->	int:
											# Initialize image_processing
											snake_case_ :Optional[int]						=			self.image_processing_class(**self.image_processor_dict						)
											# create random PyTorch tensors
											snake_case_ :Any						=			prepare_image_inputs(self.image_processor_tester						,     equal_resolution=snake_case						,     torchify=snake_case						)
											for image in image_inputs:
																	self.assertIsInstance(snake_case						,     torch.Tensor						)

											# Test not batched input
											snake_case_ :Dict						=			image_processing(image_inputs[0]						,     return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,     (
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )						,     )

											# Test batched
											snake_case_ :Optional[int]						=			image_processing(snake_case						,     return_tensors="""pt"""						).pixel_values
											self.assertEqual(
											    encoded_images.shape						,     (
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )						,     )





"""simple docstring"""
import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
__a      =				"src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
__a      =				direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
__a      =				re.compile(r"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
__a      =				re.compile(r"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__a      =				re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
__a      =				[
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]
def    A_  (			_lowercase	):







						'''simple docstring'''


						snake_case_ :Any						=			re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)""",					_lowercase	)
						return [m.group(0	) for m in matches]
def    A_  (			):







						'''simple docstring'''


						snake_case_ :int						=			transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
						snake_case_ :Dict						=			{
						    config.replace("""Config""",					""""""	): model_type for model_type, config in config_maping_names.items()
						}

						# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
						snake_case_ :Optional[Any]						=			collections.defaultdict(_lowercase	)
						snake_case_ :int						=			collections.defaultdict(_lowercase	)
						snake_case_ :List[str]						=			collections.defaultdict(_lowercase	)

						# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
						for attr_name in dir(_lowercase	):
												snake_case_ :int						=			None
												if _re_tf_models.match(_lowercase	) is not None:
																		snake_case_ :int						=			tf_models
																		snake_case_ :List[str]						=			_re_tf_models.match(_lowercase	).groups()[0]
												elif _re_flax_models.match(_lowercase	) is not None:
																		snake_case_ :List[Any]						=			flax_models
																		snake_case_ :Any						=			_re_flax_models.match(_lowercase	).groups()[0]
												elif _re_pt_models.match(_lowercase	) is not None:
																		snake_case_ :Optional[Any]						=			pt_models
																		snake_case_ :int						=			_re_pt_models.match(_lowercase	).groups()[0]

												if lookup_dict is not None:
																		while len(_lowercase	) > 0:
																								if attr_name in model_prefix_to_model_type:
																														snake_case_ :Optional[int]						=			True
																														break
																								# Try again after removing the last word in the name
																								snake_case_ :Optional[Any]						=			"""""".join(camel_case_split(_lowercase	)[:-1]	)

						snake_case_ :Optional[int]						=			set(list(pt_models.keys()	) + list(tf_models.keys()	) + list(flax_models.keys()	)	)
						snake_case_ :Optional[Any]						=			list(_lowercase	)
						all_models.sort()

						snake_case_ :Optional[int]						=			{"""model_type""": all_models}
						snake_case_ :Optional[int]						=			[pt_models[t] for t in all_models]
						snake_case_ :Any						=			[tf_models[t] for t in all_models]
						snake_case_ :Dict						=			[flax_models[t] for t in all_models]

						# Now let's use the auto-mapping names to make sure
						snake_case_ :Dict						=			{}
						for t in all_models:
												if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
																		snake_case_ :Optional[Any]						=			"""AutoProcessor"""
												elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
																		snake_case_ :Tuple						=			"""AutoTokenizer"""
												elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
																		snake_case_ :Tuple						=			"""AutoFeatureExtractor"""
												else:
																		# Default to AutoTokenizer if a model has nothing, for backward compatibility.
																		snake_case_ :str						=			"""AutoTokenizer"""

						snake_case_ :int						=			[processors[t] for t in all_models]

						return pd.DataFrame(_lowercase	)
def    A_  (			_lowercase	):







						'''simple docstring'''


						snake_case_ :List[Any]						=			[
						    transformers_module.models.auto.modeling_auto,
						    transformers_module.models.auto.modeling_tf_auto,
						    transformers_module.models.auto.modeling_flax_auto,
						]
						for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
												snake_case_ :Optional[int]						=			[model_mapping, f"""TF_{model_mapping}""", f"""FLAX_{model_mapping}"""]
												snake_case_ :List[str]						=			[auto_class, f"""TF_{auto_class}""", f"""Flax_{auto_class}"""]
												# Loop through all three frameworks
												for module, cls, mapping in zip(_lowercase,					_lowercase,					_lowercase	):
																		# The type of pipeline may not exist in this framework
																		if not hasattr(_lowercase,					_lowercase	):
																								continue
																		# First extract all model_names
																		snake_case_ :Tuple						=			[]
																		for name in getattr(_lowercase,					_lowercase	).values():
																								if isinstance(_lowercase,					_lowercase	):
																														model_names.append(_lowercase	)
																								else:
																														model_names.extend(list(_lowercase	)	)

            # Add pipeline tag and auto model class for those models
																		table.update({model_name: (pipeline_tag, cls) for model_name in model_names}	)

						return table
def    A_  (			_lowercase,					_lowercase	):







						'''simple docstring'''


						snake_case_ :List[Any]						=			get_frameworks_table()
						snake_case_ :str						=			Dataset.from_pandas(_lowercase	)

						snake_case_ :List[Any]						=			hf_hub_download(
						    """huggingface/transformers-metadata""",					"""pipeline_tags.json""",					repo_type="""dataset""",					token=_lowercase	)
						snake_case_ :List[str]						=			Dataset.from_json(_lowercase	)
						snake_case_ :int						=			{
						    tags_dataset[i]["""model_class"""]: (tags_dataset[i]["""pipeline_tag"""], tags_dataset[i]["""auto_class"""])
						    for i in range(len(_lowercase	)	)
						}
						snake_case_ :Optional[int]						=			update_pipeline_and_auto_class_table(_lowercase	)

						# Sort the model classes to avoid some nondeterministic updates to create false update commits.
						snake_case_ :Tuple						=			sorted(table.keys()	)
						snake_case_ :Tuple						=			pd.DataFrame(
						    {
						        """model_class""": model_classes,
						        """pipeline_tag""": [table[m][0] for m in model_classes],
						        """auto_class""": [table[m][1] for m in model_classes],
						    }	)
						snake_case_ :Union[str, Any]						=			Dataset.from_pandas(_lowercase	)

						with tempfile.TemporaryDirectory() as tmp_dir:
												frameworks_dataset.to_json(os.path.join(_lowercase,					"""frameworks.json"""	)	)
												tags_dataset.to_json(os.path.join(_lowercase,					"""pipeline_tags.json"""	)	)

												if commit_sha is not None:
																		snake_case_ :Union[str, Any]						=			(
																		    f"""Update with commit {commit_sha}\n\nSee: """
																		    f"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
																		)
												else:
																		snake_case_ :List[Any]						=			"""Update"""

												upload_folder(
												    repo_id="""huggingface/transformers-metadata""",					folder_path=_lowercase,					repo_type="""dataset""",					token=_lowercase,					commit_message=_lowercase,					)
def    A_  (			):







						'''simple docstring'''


						snake_case_ :List[Any]						=			{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
						snake_case_ :Dict						=			transformers_module.pipelines.SUPPORTED_TASKS
						snake_case_ :List[str]						=			[]
						for key in pipeline_tasks:
												if key not in in_table:
																		snake_case_ :int						=			pipeline_tasks[key]["""pt"""]
																		if isinstance(_lowercase,					(list, tuple)	):
																								snake_case_ :Any						=			model[0]
																		snake_case_ :str						=			model.__name__
																		if model not in in_table.values():
																								missing.append(_lowercase	)

						if len(_lowercase	) > 0:
												snake_case_ :Optional[int]						=			""", """.join(_lowercase	)
												raise ValueError(
												    """The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside """
												    f"""`utils/update_metadata.py`: {msg}. Please add them!"""	)


if __name__ == "__main__":
						__a      =				argparse.ArgumentParser()
						parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
						parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
						parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
						__a      =				parser.parse_args()

						if args.check_only:
												check_pipeline_tags()
						else:
												update_metadata(args.token, args.commit_sha)
