


"""simple docstring"""







import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


__a      			= {}
for task, _ in pipeline_test_mapping.items():
						__a      			= {"pt": None, "tf": None}


# DO **NOT** add item to this set (unless the reason is approved)
__a      			= {
    "tests/models/esm/test_modeling_esmfold.py",  # The pipeline test mapping is added to `test_modeling_esm.py`
}






def        a__			(							_UpperCamelCase			):





						"""simple docstring"""

						if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
												return "pt"
						elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
												return "tf"
						elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
												return "flax"
						else:
												return None






def        a__			(							_UpperCamelCase						,      _UpperCamelCase			):





						"""simple docstring"""

						if PIPELINE_TEST_MAPPING[task].get(_UpperCamelCase						,      _UpperCamelCase			) is not None:
												return PIPELINE_TEST_MAPPING[task][framework]

						UpperCAmelCase__												=     pipeline_test_mapping[task]['''test''']
						UpperCAmelCase__												=     None

						if framework == "pt":
												UpperCAmelCase__												=     getattr(_UpperCamelCase						,      '''model_mapping'''						,      _UpperCamelCase			)
						elif framework == "tf":
												UpperCAmelCase__												=     getattr(_UpperCamelCase						,      '''tf_model_mapping'''						,      _UpperCamelCase			)

						if mapping is not None:
												UpperCAmelCase__												=     dict(mapping.items()			)

						# cache the results
						UpperCAmelCase__												=     mapping
						return mapping






def        a__			(							_UpperCamelCase						,      _UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     get_framework(_UpperCamelCase			)
						if framework is None:
												return None
						UpperCAmelCase__												=     get_mapping_for_task(_UpperCamelCase						,      _UpperCamelCase			)
						if mapping is None:
												return None

						UpperCAmelCase__												=     list({model_class.config_class for model_class in test_class.all_model_classes}			)
						if len(_UpperCamelCase			) != 1:
												raise ValueError('''There should be exactly one configuration class from `test_class.all_model_classes`.'''			)

						# This could be a list/tuple of model classes, but it's rare.
						UpperCAmelCase__												=     mapping.get(config_classes[0]						,      _UpperCamelCase			)
						if isinstance(_UpperCamelCase						,      (tuple, list)			):
												UpperCAmelCase__												=     sorted(_UpperCamelCase						,      key=lambda _UpperCamelCase			: x.__name__			)

						return model_class






def        a__			(							_UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     [(task, get_model_for_pipeline_test(_UpperCamelCase						,      _UpperCamelCase			)) for task in pipeline_test_mapping]
						UpperCAmelCase__												=     sorted([(task, model) for task, model in mapping if model is not None]						,      key=lambda _UpperCamelCase			: x[0]			)

						return dict(_UpperCamelCase			)






def        a__			(							_UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     get_framework(_UpperCamelCase			)
						if framework == "pt":
												UpperCAmelCase__												=     '''torch'''
						UpperCAmelCase__												=     '''{}'''

						UpperCAmelCase__												=     get_pipeline_model_mapping(_UpperCamelCase			)
						if len(_UpperCamelCase			) == 0:
												return ""

						UpperCAmelCase__												=     []
						for task, model_classes in mapping.items():
												if isinstance(_UpperCamelCase						,      (tuple, list)			):
																		# A list/tuple of model classes
																		UpperCAmelCase__												=     '''(''' + ''', '''.join([x.__name__ for x in model_classes]			) + ''')'''
												else:
																		# A single model class
																		UpperCAmelCase__												=     model_classes.__name__
												texts.append(F"""\"{task}\": {value}"""			)
						UpperCAmelCase__												=     '''{''' + ''', '''.join(_UpperCamelCase			) + '''}'''
						UpperCAmelCase__												=     F"""pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}"""

						return text






def        a__			(							_UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     {'''ModelTesterMixin''', '''TFModelTesterMixin''', '''FlaxModelTesterMixin'''}
						if not issubclass(_UpperCamelCase						,      unittest.TestCase			):
												return False
						return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]			)			) > 0






def        a__			(							_UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     [x for x in get_test_classes(_UpperCamelCase			) if is_valid_test_class(_UpperCamelCase			)]

						UpperCAmelCase__												=     None
						for test_class in test_classes:
												# If a test class has defined `pipeline_model_mapping`, let's take it
												if getattr(_UpperCamelCase						,      '''pipeline_model_mapping'''						,      _UpperCamelCase			) is not None:
																		UpperCAmelCase__												=     test_class
																		break
    # Take the test class with the shortest name (just a heuristic)
						if target_test_class is None and len(_UpperCamelCase			) > 0:
												UpperCAmelCase__												=     sorted(_UpperCamelCase						,      key=lambda _UpperCamelCase			: (len(x.__name__			), x.__name__)			)[0]

						return target_test_class






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     start_idx
						for idx, line in enumerate(lines[start_idx:]			):
												UpperCAmelCase__												=     len(_UpperCamelCase			) - len(line.lstrip()			)
												if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
																		UpperCAmelCase__												=     start_idx + idx
												elif idx > 0 and indent <= indent_level:
																		# Outside the definition block of `pipeline_model_mapping`
																		break

						return end_idx






def        a__			(							_UpperCamelCase						,      _UpperCamelCase=False			):





						"""simple docstring"""

						if getattr(_UpperCamelCase						,      '''pipeline_model_mapping'''						,      _UpperCamelCase			) is not None:
												if not overwrite:
																		return "", -1

						UpperCAmelCase__												=     get_pipeline_model_mapping_string(_UpperCamelCase			)
						if len(_UpperCamelCase			) == 0:
												return "", -1
						UpperCAmelCase__												=     line_to_add + '''\n'''

						# The code defined the class `test_class`
						UpperCAmelCase__						,					UpperCAmelCase__												=     inspect.getsourcelines(_UpperCamelCase			)
						# `inspect` gives the code for an object, including decorator(s) if any.
						# We (only) need the exact line of the class definition.
						for idx, line in enumerate(_UpperCamelCase			):
												if line.lstrip().startswith('''class '''			):
																		UpperCAmelCase__												=     class_lines[idx:]
																		class_start_line_no += idx
																		break
						UpperCAmelCase__												=     class_start_line_no + len(_UpperCamelCase			) - 1

						# The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
						# `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
						# block that appears in a `test_class`.
						UpperCAmelCase__												=     None
						# The indent level of the line at `class_lines[start_idx]` (if defined)
						UpperCAmelCase__												=     0
						# To record if `pipeline_model_mapping` is found in `test_class`.
						UpperCAmelCase__												=     None
						for idx, line in enumerate(_UpperCamelCase			):
												if line.strip().startswith('''all_model_classes = '''			):
																		UpperCAmelCase__												=     len(_UpperCamelCase			) - len(line.lstrip()			)
																		UpperCAmelCase__												=     idx
												elif line.strip().startswith('''all_generative_model_classes = '''			):
																		UpperCAmelCase__												=     len(_UpperCamelCase			) - len(line.lstrip()			)
																		UpperCAmelCase__												=     idx
												elif line.strip().startswith('''pipeline_model_mapping = '''			):
																		UpperCAmelCase__												=     len(_UpperCamelCase			) - len(line.lstrip()			)
																		UpperCAmelCase__												=     idx
																		UpperCAmelCase__												=     line
																		break

						if start_idx is None:
												return "", -1
						# Find the ending index (inclusive) of the above found block.
						UpperCAmelCase__												=     find_block_ending(_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase			)

						# Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
						# `is_timm_available()` instead of `is_torch_available()`.
						# Keep leading and trailing whitespaces
						UpperCAmelCase__												=     re.compile(R'''\s(is_\S+?_available\(\))\s'''			)
						for line in class_lines[start_idx : end_idx + 1]:
												UpperCAmelCase__												=     r.search(_UpperCamelCase			)
												if backend_condition is not None:
																		# replace the leading and trailing whitespaces to the space character " ".
																		UpperCAmelCase__												=     ''' ''' + backend_condition[0][1:-1] + ''' '''
																		UpperCAmelCase__												=     r.sub(_UpperCamelCase						,      _UpperCamelCase			)
																		break

						if def_line is None:
												# `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
												# `all_model_classes` or `all_generative_model_classes`.
												UpperCAmelCase__												=     end_idx
						else:
												# `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
												UpperCAmelCase__												=     start_idx - 1
												# mark the lines of the currently existing `pipeline_model_mapping` to be removed.
												for idx in range(_UpperCamelCase						,      end_idx + 1			):
																		# These lines are going to be removed before writing to the test file.
																		UpperCAmelCase__												=     None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
						UpperCAmelCase__												=     [x.__name__ for x in test_class.__bases__]
						if "PipelineTesterMixin" not in parent_classes:
												# Put `PipelineTesterMixin` just before `unittest.TestCase`
												UpperCAmelCase__												=     [x for x in parent_classes if x != '''TestCase'''] + ['''PipelineTesterMixin''']
												if "TestCase" in parent_classes:
																		# Here we **assume** the original string is always with `unittest.TestCase`.
																		_parent_classes.append('''unittest.TestCase'''			)
												UpperCAmelCase__												=     ''', '''.join(_parent_classes			)
												for idx, line in enumerate(_UpperCamelCase			):
																		# Find the ending of the declaration of `test_class`
																		if line.strip().endswith('''):'''			):
																								# mark the lines of the declaration of `test_class` to be removed
																								for _idx in range(idx + 1			):
																														UpperCAmelCase__												=     None  # noqa
																								break
        # Add the new, one-line, class declaration for `test_class`
												UpperCAmelCase__												=     F"""class {test_class.__name__}({parent_classes}):\n"""

						# Add indentation
						UpperCAmelCase__												=     ''' ''' * indent_level + line_to_add
						# Insert `pipeline_model_mapping` to `class_lines`.
						# (The line at `target_idx` should be kept by definition!)
						UpperCAmelCase__												=     class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
						# Remove the lines that are marked to be removed
						UpperCAmelCase__												=     [x for x in class_lines if x is not None]

						# Move from test class to module (in order to write to the test file)
						UpperCAmelCase__												=     inspect.getsourcelines(inspect.getmodule(_UpperCamelCase			)			)[0]
						# Be careful with the 1-off between line numbers and array indices
						UpperCAmelCase__												=     module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
						UpperCAmelCase__												=     ''''''.join(_UpperCamelCase			)

						UpperCAmelCase__												=     inspect.getsourcefile(_UpperCamelCase			)
						with open(_UpperCamelCase						,      '''w'''						,      encoding='''UTF-8'''						,      newline='''\n'''			) as fp:
												fp.write(_UpperCamelCase			)

						return line_to_add






def        a__			(							_UpperCamelCase						,      _UpperCamelCase=False			):





						"""simple docstring"""

						UpperCAmelCase__												=     find_test_class(_UpperCamelCase			)
						if test_class:
												add_pipeline_model_mapping(_UpperCamelCase						,      overwrite=_UpperCamelCase			)


if __name__ == "__main__":
						__a      			= argparse.ArgumentParser()
						parser.add_argument(
						    "--test_file", type=str, help="A path to the test file, starting with the repository's `tests` directory."
						)
						parser.add_argument(
						    "--all",
						    action="store_true",
						    help="If to check and modify all test files.",
						)
						parser.add_argument(
						    "--overwrite",
						    action="store_true",
						    help="If to overwrite a test class if it has already defined `pipeline_model_mapping`.",
						)
						__a      			= parser.parse_args()

						if not args.all and not args.test_file:
												raise ValueError("Please specify either `test_file` or pass `--all` to check/modify all test files.")
						elif args.all and args.test_file:
												raise ValueError("Only one of `--test_file` and `--all` could be specified.")

						__a      			= []
						if args.test_file:
												__a      			= [args.test_file]
						else:
												__a      			= os.path.join("tests", "models", "**", "test_modeling_*.py")
												for test_file in glob.glob(pattern):
																		# `Flax` is not concerned at this moment
																		if not test_file.startswith("test_modeling_flax_"):
																								test_files.append(test_file)

						for test_file in test_files:
												if test_file in TEST_FILE_TO_IGNORE:
																		print(f'''[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.''')
																		continue
												add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)



"""simple docstring"""







import inspect
import json
import os
import tempfile
import unittest

from transformers.models.speech_to_text_a import SpeechaTextaTokenizer
from transformers.models.speech_to_text_a.tokenization_speech_to_text_a import VOCAB_FILES_NAMES

from ...test_tokenization_common import TokenizerTesterMixin
class 				lowerCamelCase							(      __SCREAMING_SNAKE_CASE					,       unittest.TestCase			):


							"""simple docstring"""
							SCREAMING_SNAKE_CASE              =   SpeechaTextaTokenizer
							SCREAMING_SNAKE_CASE              =   False






							def        lowerCAmelCase  (    self			:  Dict					):
													super().setUp()

													UpperCAmelCase__												=     '''<s> <pad> </s> <unk> here@@ a couple of@@ words for the he@@ re@@ vocab'''.split(''' '''					)
													UpperCAmelCase__												=     ['''he re</w> 123''', '''here a 1456''']
													UpperCAmelCase__												=     dict(zip(lowerCamelCase_		,			range(len(lowerCamelCase_					)					)					)					)

													UpperCAmelCase__												=     {'''pad_token''': '''<pad>''', '''unk_token''': '''<unk>''', '''bos_token''': '''<s>''', '''eos_token''': '''</s>'''}

													UpperCAmelCase__												=     tempfile.mkdtemp()
													UpperCAmelCase__												=     os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''vocab_file''']					)
													UpperCAmelCase__												=     os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''merges_file''']					)

													with open(self.vocab_file		,			'''w'''		,			encoding='''utf-8'''					) as fp:
																			fp.write(json.dumps(lowerCamelCase_					) + '''\n'''					)

													with open(self.merges_file		,			'''w'''					) as fp:
																			fp.write('''\n'''.join(lowerCamelCase_					)					)






							def        lowerCAmelCase  (    self			:  Union[str, Any]					):
													UpperCAmelCase__												=     list(self.get_tokenizer().get_vocab().keys()					)

													self.assertEqual(vocab_keys[0]		,			'''<s>'''					)
													self.assertEqual(vocab_keys[1]		,			'''<pad>'''					)
													self.assertEqual(vocab_keys[-1]		,			'''vocab'''					)
													self.assertEqual(len(lowerCamelCase_					)		,			14					)






							def        lowerCAmelCase  (    self			:  Optional[Any]					):
													self.assertEqual(self.get_tokenizer().vocab_size		,			14					)






							def        lowerCAmelCase  (    self			:  Any					):
													UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(self.tmpdirname					)

													# make sure @@ is correctly concatenated
													UpperCAmelCase__												=     [4, 6, 8, 7, 10]  # ["here@@", "couple", "words", "of@@", "the"]
													UpperCAmelCase__												=     tokenizer.decode(lowerCamelCase_					)

													self.assertTrue(output_string == '''herecouple words ofthe'''					)






							def        lowerCAmelCase  (    self			:  int					):
													UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(self.tmpdirname					)

													with tempfile.TemporaryDirectory() as tmp_dirname:
																			tokenizer.save_pretrained(lowerCamelCase_					)
																			os.remove(os.path.join(lowerCamelCase_		,			'''merges.txt'''					)					)

																			# load tokenizer without merges file should not throw an error
																			UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(lowerCamelCase_					)

													with tempfile.TemporaryDirectory() as tmp_dirname:
																			# save tokenizer and load again
																			tokenizer.save_pretrained(lowerCamelCase_					)
																			UpperCAmelCase__												=     SpeechaTextaTokenizer.from_pretrained(lowerCamelCase_					)

													self.assertIsNotNone(lowerCamelCase_					)






							def        lowerCAmelCase  (    self			:  int					):
													if not self.test_slow_tokenizer:
																			return

													UpperCAmelCase__												=     inspect.signature(self.tokenizer_class.__init__					)
													UpperCAmelCase__												=     self.get_tokenizer()

													for parameter_name, parameter in signature.parameters.items():
																			if parameter.default != inspect.Parameter.empty and parameter_name != "merges_file":
																									self.assertIn(lowerCamelCase_		,			tokenizer.init_kwargs					)