



'''simple docstring'''




import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


a  :     Tuple       =       {}
for task, _ in pipeline_test_mapping.items():
							a  :     Dict       =       {"""pt""": None, """tf""": None}


# DO **NOT** add item to this set (unless the reason is approved)
a  :     Dict       =       {
    """tests/models/esm/test_modeling_esmfold.py""",  # The pipeline test mapping is added to `test_modeling_esm.py`
}



def  __lowerCamelCase    (   _lowercase						) ->     List[Any]:

							if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
														return "pt"
							elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
														return "tf"
							elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
														return "flax"
							else:
														return None



def  __lowerCamelCase    (   _lowercase      ,							_lowercase						) ->     int:
							# Use the cached results
							if PIPELINE_TEST_MAPPING[task].get(_lowercase      ,							_lowercase						) is not None:
														return PIPELINE_TEST_MAPPING[task][framework]

							UpperCAmelCase	:						List[Any]					=						pipeline_test_mapping[task]["""test"""]
							UpperCAmelCase	:						Optional[int]					=						None

							if framework == "pt":
														UpperCAmelCase	:						int					=						getattr(_lowercase      ,							"""model_mapping"""      ,							_lowercase						)
							elif framework == "tf":
														UpperCAmelCase	:						Optional[Any]					=						getattr(_lowercase      ,							"""tf_model_mapping"""      ,							_lowercase						)

							if mapping is not None:
														UpperCAmelCase	:						str					=						dict(mapping.items()						)

							# cache the results
							UpperCAmelCase	:						Optional[Any]					=						mapping
							return mapping



def  __lowerCamelCase    (   _lowercase      ,							_lowercase						) ->     Tuple:
							UpperCAmelCase	:						str					=						get_framework(_lowercase						)
							if framework is None:
														return None
							UpperCAmelCase	:						Optional[Any]					=						get_mapping_for_task(_lowercase      ,							_lowercase						)
							if mapping is None:
														return None

							UpperCAmelCase	:						List[Any]					=						list({model_class.config_class for model_class in test_class.all_model_classes}						)
							if len(_lowercase						) != 1:
														raise ValueError("""There should be exactly one configuration class from `test_class.all_model_classes`."""						)

							# This could be a list/tuple of model classes, but it's rare.
							UpperCAmelCase	:						Optional[Any]					=						mapping.get(config_classes[0]      ,							_lowercase						)
							if isinstance(_lowercase      ,							(tuple, list)						):
														UpperCAmelCase	:						Any					=						sorted(_lowercase      ,							key=lambda _lowercase						: x.__name__						)

							return model_class



def  __lowerCamelCase    (   _lowercase						) ->     List[str]:
							UpperCAmelCase	:						List[str]					=						[(task, get_model_for_pipeline_test(_lowercase      ,							_lowercase						)) for task in pipeline_test_mapping]
							UpperCAmelCase	:						Optional[Any]					=						sorted([(task, model) for task, model in mapping if model is not None]      ,							key=lambda _lowercase						: x[0]						)

							return dict(_lowercase						)



def  __lowerCamelCase    (   _lowercase						) ->     int:
							UpperCAmelCase	:						Union[str, Any]					=						get_framework(_lowercase						)
							if framework == "pt":
														UpperCAmelCase	:						Optional[Any]					=						"""torch"""
							UpperCAmelCase	:						Dict					=						"""{}"""

							UpperCAmelCase	:						Any					=						get_pipeline_model_mapping(_lowercase						)
							if len(_lowercase						) == 0:
														return ""

							UpperCAmelCase	:						List[Any]					=						[]
							for task, model_classes in mapping.items():
														if isinstance(_lowercase      ,							(tuple, list)						):
																					# A list/tuple of model classes
																					UpperCAmelCase	:						Tuple					=						"""(""" + """, """.join([x.__name__ for x in model_classes]						) + """)"""
														else:
																					# A single model class
																					UpperCAmelCase	:						Optional[int]					=						model_classes.__name__
														texts.append(F'''"{task}": {value}'''						)
							UpperCAmelCase	:						Tuple					=						"""{""" + """, """.join(_lowercase						) + """}"""
							UpperCAmelCase	:						str					=						F'''pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}'''

							return text



def  __lowerCamelCase    (   _lowercase						) ->     Union[str, Any]:
							UpperCAmelCase	:						List[Any]					=						{"""ModelTesterMixin""", """TFModelTesterMixin""", """FlaxModelTesterMixin"""}
							if not issubclass(_lowercase      ,							unittest.TestCase						):
														return False
							return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]						)						) > 0



def  __lowerCamelCase    (   _lowercase						) ->     List[str]:
							UpperCAmelCase	:						List[Any]					=						[x for x in get_test_classes(_lowercase						) if is_valid_test_class(_lowercase						)]

							UpperCAmelCase	:						Dict					=						None
							for test_class in test_classes:
														# If a test class has defined `pipeline_model_mapping`, let's take it
														if getattr(_lowercase      ,							"""pipeline_model_mapping"""      ,							_lowercase						) is not None:
																					UpperCAmelCase	:						List[Any]					=						test_class
																					break
    # Take the test class with the shortest name (just a heuristic)
							if target_test_class is None and len(_lowercase						) > 0:
														UpperCAmelCase	:						List[Any]					=						sorted(_lowercase      ,							key=lambda _lowercase						: (len(x.__name__						), x.__name__)						)[0]

							return target_test_class



def  __lowerCamelCase    (   _lowercase      ,							_lowercase      ,							_lowercase						) ->     Optional[Any]:
							UpperCAmelCase	:						Union[str, Any]					=						start_idx
							for idx, line in enumerate(lines[start_idx:]						):
														UpperCAmelCase	:						int					=						len(_lowercase						) - len(line.lstrip()						)
														if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
																					UpperCAmelCase	:						str					=						start_idx + idx
														elif idx > 0 and indent <= indent_level:
																					# Outside the definition block of `pipeline_model_mapping`
																					break

							return end_idx



def  __lowerCamelCase    (   _lowercase      ,							_lowercase=False						) ->     List[Any]:
							if getattr(_lowercase      ,							"""pipeline_model_mapping"""      ,							_lowercase						) is not None:
														if not overwrite:
																					return "", -1

							UpperCAmelCase	:						Tuple					=						get_pipeline_model_mapping_string(_lowercase						)
							if len(_lowercase						) == 0:
														return "", -1
							UpperCAmelCase	:						List[str]					=						line_to_add + """\n"""

							# The code defined the class `test_class`
							UpperCAmelCase	,	UpperCAmelCase	:						List[Any]					=						inspect.getsourcelines(_lowercase						)
							# `inspect` gives the code for an object, including decorator(s) if any.
							# We (only) need the exact line of the class definition.
							for idx, line in enumerate(_lowercase						):
														if line.lstrip().startswith("""class """						):
																					UpperCAmelCase	:						Optional[int]					=						class_lines[idx:]
																					class_start_line_no += idx
																					break
							UpperCAmelCase	:						Any					=						class_start_line_no + len(_lowercase						) - 1

							# The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
							# `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
							# block that appears in a `test_class`.
							UpperCAmelCase	:						Any					=						None
							# The indent level of the line at `class_lines[start_idx]` (if defined)
							UpperCAmelCase	:						str					=						0
							# To record if `pipeline_model_mapping` is found in `test_class`.
							UpperCAmelCase	:						int					=						None
							for idx, line in enumerate(_lowercase						):
														if line.strip().startswith("""all_model_classes = """						):
																					UpperCAmelCase	:						Optional[int]					=						len(_lowercase						) - len(line.lstrip()						)
																					UpperCAmelCase	:						str					=						idx
														elif line.strip().startswith("""all_generative_model_classes = """						):
																					UpperCAmelCase	:						List[str]					=						len(_lowercase						) - len(line.lstrip()						)
																					UpperCAmelCase	:						str					=						idx
														elif line.strip().startswith("""pipeline_model_mapping = """						):
																					UpperCAmelCase	:						Optional[int]					=						len(_lowercase						) - len(line.lstrip()						)
																					UpperCAmelCase	:						Tuple					=						idx
																					UpperCAmelCase	:						Dict					=						line
																					break

							if start_idx is None:
														return "", -1
							# Find the ending index (inclusive) of the above found block.
							UpperCAmelCase	:						str					=						find_block_ending(_lowercase      ,							_lowercase      ,							_lowercase						)

							# Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
							# `is_timm_available()` instead of `is_torch_available()`.
							# Keep leading and trailing whitespaces
							UpperCAmelCase	:						Optional[Any]					=						re.compile(R"""\s(is_\S+?_available\(\))\s"""						)
							for line in class_lines[start_idx : end_idx + 1]:
														UpperCAmelCase	:						Any					=						r.search(_lowercase						)
														if backend_condition is not None:
																					# replace the leading and trailing whitespaces to the space character " ".
																					UpperCAmelCase	:						int					=						""" """ + backend_condition[0][1:-1] + """ """
																					UpperCAmelCase	:						Union[str, Any]					=						r.sub(_lowercase      ,							_lowercase						)
																					break

							if def_line is None:
														# `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
														# `all_model_classes` or `all_generative_model_classes`.
														UpperCAmelCase	:						Any					=						end_idx
							else:
														# `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
														UpperCAmelCase	:						Optional[int]					=						start_idx - 1
														# mark the lines of the currently existing `pipeline_model_mapping` to be removed.
														for idx in range(_lowercase      ,							end_idx + 1						):
																					# These lines are going to be removed before writing to the test file.
																					UpperCAmelCase	:						Optional[int]					=						None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
							UpperCAmelCase	:						int					=						[x.__name__ for x in test_class.__bases__]
							if "PipelineTesterMixin" not in parent_classes:
														# Put `PipelineTesterMixin` just before `unittest.TestCase`
														UpperCAmelCase	:						Tuple					=						[x for x in parent_classes if x != """TestCase"""] + ["""PipelineTesterMixin"""]
														if "TestCase" in parent_classes:
																					# Here we **assume** the original string is always with `unittest.TestCase`.
																					_parent_classes.append("""unittest.TestCase"""						)
														UpperCAmelCase	:						List[Any]					=						""", """.join(_parent_classes						)
														for idx, line in enumerate(_lowercase						):
																					# Find the ending of the declaration of `test_class`
																					if line.strip().endswith("""):"""						):
																												# mark the lines of the declaration of `test_class` to be removed
																												for _idx in range(idx + 1						):
																																			UpperCAmelCase	:						Optional[Any]					=						None  # noqa
																												break
        # Add the new, one-line, class declaration for `test_class`
														UpperCAmelCase	:						Optional[Any]					=						F'''class {test_class.__name__}({parent_classes}):\n'''

							# Add indentation
							UpperCAmelCase	:						int					=						""" """ * indent_level + line_to_add
							# Insert `pipeline_model_mapping` to `class_lines`.
							# (The line at `target_idx` should be kept by definition!)
							UpperCAmelCase	:						List[str]					=						class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
							# Remove the lines that are marked to be removed
							UpperCAmelCase	:						List[Any]					=						[x for x in class_lines if x is not None]

							# Move from test class to module (in order to write to the test file)
							UpperCAmelCase	:						Optional[Any]					=						inspect.getsourcelines(inspect.getmodule(_lowercase						)						)[0]
							# Be careful with the 1-off between line numbers and array indices
							UpperCAmelCase	:						Optional[int]					=						module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
							UpperCAmelCase	:						int					=						"""""".join(_lowercase						)

							UpperCAmelCase	:						Tuple					=						inspect.getsourcefile(_lowercase						)
							with open(_lowercase      ,							"""w"""      ,							encoding="""UTF-8"""      ,							newline="""\n"""						) as fp:
														fp.write(_lowercase						)

							return line_to_add



def  __lowerCamelCase    (   _lowercase      ,							_lowercase=False						) ->     Dict:
							UpperCAmelCase	:						List[str]					=						find_test_class(_lowercase						)
							if test_class:
														add_pipeline_model_mapping(_lowercase      ,							overwrite=_lowercase						)


if __name__ == "__main__":
							a  :     Optional[Any]       =       argparse.ArgumentParser()
							parser.add_argument(
							    """--test_file""", type=str, help="""A path to the test file, starting with the repository's `tests` directory."""
							)
							parser.add_argument(
							    """--all""",
							    action="""store_true""",
							    help="""If to check and modify all test files.""",
							)
							parser.add_argument(
							    """--overwrite""",
							    action="""store_true""",
							    help="""If to overwrite a test class if it has already defined `pipeline_model_mapping`.""",
							)
							a  :     Dict       =       parser.parse_args()

							if not args.all and not args.test_file:
														raise ValueError("""Please specify either `test_file` or pass `--all` to check/modify all test files.""")
							elif args.all and args.test_file:
														raise ValueError("""Only one of `--test_file` and `--all` could be specified.""")

							a  :     Union[str, Any]       =       []
							if args.test_file:
														a  :     List[str]       =       [args.test_file]
							else:
														a  :     Optional[int]       =       os.path.join("""tests""", """models""", """**""", """test_modeling_*.py""")
														for test_file in glob.glob(pattern):
																					# `Flax` is not concerned at this moment
																					if not test_file.startswith("""test_modeling_flax_"""):
																												test_files.append(test_file)

							for test_file in test_files:
														if test_file in TEST_FILE_TO_IGNORE:
																					print(F'''[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.''')
																					continue
														add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)






'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


a  :     Any       =       logging.get_logger(__name__)

a  :     int       =       {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

a  :     List[Any]       =       {
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json""",
    },
}

class      UpperCamelCase_					(				__magic_name__     ):





	def __init__(    self						,						A=None						,						A=None						,						A=False						,						A=True						,						A=False						,						A="<s>"						,						A="</s>"						,						A="<unk>"						,						A="<sep>"						,						A="<pad>"						,						A="<cls>"						,						A="<mask>"						,						A=["<eop>", "<eod>"]						,						**A						,						)     ->     Tuple:
								UpperCAmelCase	:						str					=						AddedToken(A						,						lstrip=A						,						rstrip=A       ) if isinstance(A						,						A       ) else mask_token

								super().__init__(
								    vocab_file=A						,						tokenizer_file=A						,						do_lower_case=A						,						remove_space=A						,						keep_accents=A						,						bos_token=A						,						eos_token=A						,						unk_token=A						,						sep_token=A						,						pad_token=A						,						cls_token=A						,						mask_token=A						,						additional_special_tokens=A						,						**A						,						)

								UpperCAmelCase	:						List[Any]					=						3
								UpperCAmelCase	:						Tuple					=						do_lower_case
								UpperCAmelCase	:						Union[str, Any]					=						remove_space
								UpperCAmelCase	:						Tuple					=						keep_accents
								UpperCAmelCase	:						Any					=						vocab_file
								UpperCAmelCase	:						Tuple					=						False if not self.vocab_file else True

								try:
															import jieba
								except ModuleNotFoundError as error:
															raise error.__class__(
															    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
															    """See https://pypi.org/project/jieba/ for installation."""       )
								UpperCAmelCase	:						int					=						jieba
								UpperCAmelCase	:						Optional[int]					=						str.maketrans(""" \n"""						,						"""\u2582\u2583"""       )





	def 					_lowercase(    self						,						A						,						A = None       )     ->     List[int]:
								UpperCAmelCase	:						Optional[int]					=						[self.sep_token_id]
								UpperCAmelCase	:						str					=						[self.cls_token_id]
								if token_ids_a is None:
															return token_ids_a + sep + cls
								return token_ids_a + sep + token_ids_a + sep + cls





	def 					_lowercase(    self						,						A						,						A = None       )     ->     List[int]:
								UpperCAmelCase	:						Union[str, Any]					=						[self.sep_token_id]
								UpperCAmelCase	:						str					=						[2]

								if token_ids_a is None:
															return len(token_ids_a + sep       ) * [0] + cls_segment_id
								return len(token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1] + cls_segment_id





	def 					_lowercase(    self						,						A						,						A = None       )     ->     Tuple[str]:
								if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""       )

								if not os.path.isdir(A       ):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
															return
								UpperCAmelCase	:						Tuple					=						os.path.join(
								    A						,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

								if os.path.abspath(self.vocab_file       ) != os.path.abspath(A       ):
															copyfile(self.vocab_file						,						A       )

								return (out_vocab_file,)





	def 					_lowercase(    self						,						A						,						*A						,						**A       )     ->     List[Any]:
								UpperCAmelCase	:						Optional[Any]					=						[
								    """ """.join([x.translate(self.translator       ) for x in self.jieba.cut(A						,						cut_all=A       )]       )
								    for text in batch_text_or_text_pairs
								]
								return super()._batch_encode_plus(A						,						*A						,						**A       )






	def 					_lowercase(    self						,						*A						,						**A       )     ->     str:
								UpperCAmelCase	:						Optional[int]					=						super()._decode(*A						,						**A       )
								UpperCAmelCase	:						int					=						text.replace(""" """						,						""""""       ).replace("""\u2582"""						,						""" """       ).replace("""\u2583"""						,						"""\n"""       )
								return text

