




import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


UpperCAmelCase_    ={}
for task, _ in pipeline_test_mapping.items():
		UpperCAmelCase_    ={"""pt""": None, """tf""": None}


# DO **NOT** add item to this set (unless the reason is approved)
UpperCAmelCase_    ={
    """tests/models/esm/test_modeling_esmfold.py""",  # The pipeline test mapping is added to `test_modeling_esm.py`
}



def    UpperCAmelCase							(  _snake_case				):

							if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
														return "pt"
							elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
														return "tf"
							elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
														return "flax"
							else:
														return None



def    UpperCAmelCase							(  _snake_case    ,			_snake_case				):
							# Use the cached results
							if PIPELINE_TEST_MAPPING[task].get(_snake_case    ,			_snake_case				) is not None:
														return PIPELINE_TEST_MAPPING[task][framework]

							lowerCAmelCase =		pipeline_test_mapping[task]['''test''']
							lowerCAmelCase =		None

							if framework == "pt":
														lowerCAmelCase =		getattr(_snake_case    ,			'''model_mapping'''    ,			_snake_case				)
							elif framework == "tf":
														lowerCAmelCase =		getattr(_snake_case    ,			'''tf_model_mapping'''    ,			_snake_case				)

							if mapping is not None:
														lowerCAmelCase =		dict(mapping.items()				)

							# cache the results
							lowerCAmelCase =		mapping
							return mapping



def    UpperCAmelCase							(  _snake_case    ,			_snake_case				):
							lowerCAmelCase =		get_framework(_snake_case				)
							if framework is None:
														return None
							lowerCAmelCase =		get_mapping_for_task(_snake_case    ,			_snake_case				)
							if mapping is None:
														return None

							lowerCAmelCase =		list({model_class.config_class for model_class in test_class.all_model_classes}				)
							if len(_snake_case				) != 1:
														raise ValueError('''There should be exactly one configuration class from `test_class.all_model_classes`.'''				)

							# This could be a list/tuple of model classes, but it's rare.
							lowerCAmelCase =		mapping.get(config_classes[0]    ,			_snake_case				)
							if isinstance(_snake_case    ,			(tuple, list)				):
														lowerCAmelCase =		sorted(_snake_case    ,			key=lambda _snake_case				: x.__name__				)

							return model_class



def    UpperCAmelCase							(  _snake_case				):
							lowerCAmelCase =		[(task, get_model_for_pipeline_test(_snake_case    ,			_snake_case				)) for task in pipeline_test_mapping]
							lowerCAmelCase =		sorted([(task, model) for task, model in mapping if model is not None]    ,			key=lambda _snake_case				: x[0]				)

							return dict(_snake_case				)



def    UpperCAmelCase							(  _snake_case				):
							lowerCAmelCase =		get_framework(_snake_case				)
							if framework == "pt":
														lowerCAmelCase =		'''torch'''
							lowerCAmelCase =		'''{}'''

							lowerCAmelCase =		get_pipeline_model_mapping(_snake_case				)
							if len(_snake_case				) == 0:
														return ""

							lowerCAmelCase =		[]
							for task, model_classes in mapping.items():
														if isinstance(_snake_case    ,			(tuple, list)				):
																					# A list/tuple of model classes
																					lowerCAmelCase =		'''(''' + ''', '''.join([x.__name__ for x in model_classes]				) + ''')'''
														else:
																					# A single model class
																					lowerCAmelCase =		model_classes.__name__
														texts.append(F"""\"{task}\": {value}"""				)
							lowerCAmelCase =		'''{''' + ''', '''.join(_snake_case				) + '''}'''
							lowerCAmelCase =		F"""pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}"""

							return text



def    UpperCAmelCase							(  _snake_case				):
							lowerCAmelCase =		{'''ModelTesterMixin''', '''TFModelTesterMixin''', '''FlaxModelTesterMixin'''}
							if not issubclass(_snake_case    ,			unittest.TestCase				):
														return False
							return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]				)				) > 0



def    UpperCAmelCase							(  _snake_case				):
							lowerCAmelCase =		[x for x in get_test_classes(_snake_case				) if is_valid_test_class(_snake_case				)]

							lowerCAmelCase =		None
							for test_class in test_classes:
														# If a test class has defined `pipeline_model_mapping`, let's take it
														if getattr(_snake_case    ,			'''pipeline_model_mapping'''    ,			_snake_case				) is not None:
																					lowerCAmelCase =		test_class
																					break
    # Take the test class with the shortest name (just a heuristic)
							if target_test_class is None and len(_snake_case				) > 0:
														lowerCAmelCase =		sorted(_snake_case    ,			key=lambda _snake_case				: (len(x.__name__				), x.__name__)				)[0]

							return target_test_class



def    UpperCAmelCase							(  _snake_case    ,			_snake_case    ,			_snake_case				):
							lowerCAmelCase =		start_idx
							for idx, line in enumerate(lines[start_idx:]				):
														lowerCAmelCase =		len(_snake_case				) - len(line.lstrip()				)
														if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
																					lowerCAmelCase =		start_idx + idx
														elif idx > 0 and indent <= indent_level:
																					# Outside the definition block of `pipeline_model_mapping`
																					break

							return end_idx



def    UpperCAmelCase							(  _snake_case    ,			_snake_case=False				):
							if getattr(_snake_case    ,			'''pipeline_model_mapping'''    ,			_snake_case				) is not None:
														if not overwrite:
																					return "", -1

							lowerCAmelCase =		get_pipeline_model_mapping_string(_snake_case				)
							if len(_snake_case				) == 0:
														return "", -1
							lowerCAmelCase =		line_to_add + '''\n'''

							# The code defined the class `test_class`
							lowerCAmelCase ,  lowerCAmelCase =		inspect.getsourcelines(_snake_case				)
							# `inspect` gives the code for an object, including decorator(s) if any.
							# We (only) need the exact line of the class definition.
							for idx, line in enumerate(_snake_case				):
														if line.lstrip().startswith('''class '''				):
																					lowerCAmelCase =		class_lines[idx:]
																					class_start_line_no += idx
																					break
							lowerCAmelCase =		class_start_line_no + len(_snake_case				) - 1

							# The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
							# `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
							# block that appears in a `test_class`.
							lowerCAmelCase =		None
							# The indent level of the line at `class_lines[start_idx]` (if defined)
							lowerCAmelCase =		0
							# To record if `pipeline_model_mapping` is found in `test_class`.
							lowerCAmelCase =		None
							for idx, line in enumerate(_snake_case				):
														if line.strip().startswith('''all_model_classes = '''				):
																					lowerCAmelCase =		len(_snake_case				) - len(line.lstrip()				)
																					lowerCAmelCase =		idx
														elif line.strip().startswith('''all_generative_model_classes = '''				):
																					lowerCAmelCase =		len(_snake_case				) - len(line.lstrip()				)
																					lowerCAmelCase =		idx
														elif line.strip().startswith('''pipeline_model_mapping = '''				):
																					lowerCAmelCase =		len(_snake_case				) - len(line.lstrip()				)
																					lowerCAmelCase =		idx
																					lowerCAmelCase =		line
																					break

							if start_idx is None:
														return "", -1
							# Find the ending index (inclusive) of the above found block.
							lowerCAmelCase =		find_block_ending(_snake_case    ,			_snake_case    ,			_snake_case				)

							# Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
							# `is_timm_available()` instead of `is_torch_available()`.
							# Keep leading and trailing whitespaces
							lowerCAmelCase =		re.compile(R'''\s(is_\S+?_available\(\))\s'''				)
							for line in class_lines[start_idx : end_idx + 1]:
														lowerCAmelCase =		r.search(_snake_case				)
														if backend_condition is not None:
																					# replace the leading and trailing whitespaces to the space character " ".
																					lowerCAmelCase =		''' ''' + backend_condition[0][1:-1] + ''' '''
																					lowerCAmelCase =		r.sub(_snake_case    ,			_snake_case				)
																					break

							if def_line is None:
														# `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
														# `all_model_classes` or `all_generative_model_classes`.
														lowerCAmelCase =		end_idx
							else:
														# `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
														lowerCAmelCase =		start_idx - 1
														# mark the lines of the currently existing `pipeline_model_mapping` to be removed.
														for idx in range(_snake_case    ,			end_idx + 1				):
																					# These lines are going to be removed before writing to the test file.
																					lowerCAmelCase =		None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
							lowerCAmelCase =		[x.__name__ for x in test_class.__bases__]
							if "PipelineTesterMixin" not in parent_classes:
														# Put `PipelineTesterMixin` just before `unittest.TestCase`
														lowerCAmelCase =		[x for x in parent_classes if x != '''TestCase'''] + ['''PipelineTesterMixin''']
														if "TestCase" in parent_classes:
																					# Here we **assume** the original string is always with `unittest.TestCase`.
																					_parent_classes.append('''unittest.TestCase'''				)
														lowerCAmelCase =		''', '''.join(_parent_classes				)
														for idx, line in enumerate(_snake_case				):
																					# Find the ending of the declaration of `test_class`
																					if line.strip().endswith('''):'''				):
																												# mark the lines of the declaration of `test_class` to be removed
																												for _idx in range(idx + 1				):
																																			lowerCAmelCase =		None  # noqa
																												break
        # Add the new, one-line, class declaration for `test_class`
														lowerCAmelCase =		F"""class {test_class.__name__}({parent_classes}):\n"""

							# Add indentation
							lowerCAmelCase =		''' ''' * indent_level + line_to_add
							# Insert `pipeline_model_mapping` to `class_lines`.
							# (The line at `target_idx` should be kept by definition!)
							lowerCAmelCase =		class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
							# Remove the lines that are marked to be removed
							lowerCAmelCase =		[x for x in class_lines if x is not None]

							# Move from test class to module (in order to write to the test file)
							lowerCAmelCase =		inspect.getsourcelines(inspect.getmodule(_snake_case				)				)[0]
							# Be careful with the 1-off between line numbers and array indices
							lowerCAmelCase =		module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
							lowerCAmelCase =		''''''.join(_snake_case				)

							lowerCAmelCase =		inspect.getsourcefile(_snake_case				)
							with open(_snake_case    ,			'''w'''    ,			encoding='''UTF-8'''    ,			newline='''\n'''				) as fp:
														fp.write(_snake_case				)

							return line_to_add



def    UpperCAmelCase							(  _snake_case    ,			_snake_case=False				):
							lowerCAmelCase =		find_test_class(_snake_case				)
							if test_class:
														add_pipeline_model_mapping(_snake_case    ,			overwrite=_snake_case				)


if __name__ == "__main__":
		UpperCAmelCase_    =argparse.ArgumentParser()
		parser.add_argument(
		    """--test_file""", type=str, help="""A path to the test file, starting with the repository's `tests` directory."""
		)
		parser.add_argument(
		    """--all""",
		    action="""store_true""",
		    help="""If to check and modify all test files.""",
		)
		parser.add_argument(
		    """--overwrite""",
		    action="""store_true""",
		    help="""If to overwrite a test class if it has already defined `pipeline_model_mapping`.""",
		)
		UpperCAmelCase_    =parser.parse_args()

		if not args.all and not args.test_file:
				raise ValueError("""Please specify either `test_file` or pass `--all` to check/modify all test files.""")
		elif args.all and args.test_file:
				raise ValueError("""Only one of `--test_file` and `--all` could be specified.""")

		UpperCAmelCase_    =[]
		if args.test_file:
				UpperCAmelCase_    =[args.test_file]
		else:
				UpperCAmelCase_    =os.path.join("""tests""", """models""", """**""", """test_modeling_*.py""")
				for test_file in glob.glob(pattern):
						# `Flax` is not concerned at this moment
						if not test_file.startswith("""test_modeling_flax_"""):
								test_files.append(test_file)

		for test_file in test_files:
				if test_file in TEST_FILE_TO_IGNORE:
						print(F'''[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.''')
						continue
				add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)









import argparse

from transformers import CanineConfig, CanineModel, CanineTokenizer, load_tf_weights_in_canine
from transformers.utils import logging


logging.set_verbosity_info()



def    UpperCAmelCase							(  _snake_case    ,			_snake_case				):
							# Initialize PyTorch model
							lowerCAmelCase =		CanineConfig()
							lowerCAmelCase =		CanineModel(_snake_case				)
							model.eval()

							print(F"""Building PyTorch model from configuration: {config}"""				)

							# Load weights from tf checkpoint
							load_tf_weights_in_canine(_snake_case    ,			_snake_case    ,			_snake_case				)

							# Save pytorch-model (weights and configuration)
							print(F"""Save PyTorch model to {pytorch_dump_path}"""				)
							model.save_pretrained(_snake_case				)

							# Save tokenizer files
							lowerCAmelCase =		CanineTokenizer()
							print(F"""Save tokenizer files to {pytorch_dump_path}"""				)
							tokenizer.save_pretrained(_snake_case				)


if __name__ == "__main__":
		UpperCAmelCase_    =argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    """--tf_checkpoint_path""",
		    default=None,
		    type=str,
		    required=True,
		    help="""Path to the TensorFlow checkpoint. Should end with model.ckpt""",
		)
		parser.add_argument(
		    """--pytorch_dump_path""",
		    default=None,
		    type=str,
		    required=True,
		    help="""Path to a folder where the PyTorch model will be placed.""",
		)
		UpperCAmelCase_    =parser.parse_args()
		convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.pytorch_dump_path)



