




"""simple docstring"""

import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


UpperCamelCase__			:     Optional[int]			       =				logging.get_logger(__name__)

UpperCamelCase__			:     Union[str, Any]			       =				OrderedDict(
    [
        ("align", "AlignProcessor"),
        ("altclip", "AltCLIPProcessor"),
        ("bark", "BarkProcessor"),
        ("blip", "BlipProcessor"),
        ("blip-2", "Blip2Processor"),
        ("bridgetower", "BridgeTowerProcessor"),
        ("chinese_clip", "ChineseCLIPProcessor"),
        ("clap", "ClapProcessor"),
        ("clip", "CLIPProcessor"),
        ("clipseg", "CLIPSegProcessor"),
        ("flava", "FlavaProcessor"),
        ("git", "GitProcessor"),
        ("groupvit", "CLIPProcessor"),
        ("hubert", "Wav2Vec2Processor"),
        ("instructblip", "InstructBlipProcessor"),
        ("layoutlmv2", "LayoutLMv2Processor"),
        ("layoutlmv3", "LayoutLMv3Processor"),
        ("markuplm", "MarkupLMProcessor"),
        ("mctct", "MCTCTProcessor"),
        ("mgp-str", "MgpstrProcessor"),
        ("oneformer", "OneFormerProcessor"),
        ("owlvit", "OwlViTProcessor"),
        ("pix2struct", "Pix2StructProcessor"),
        ("sam", "SamProcessor"),
        ("sew", "Wav2Vec2Processor"),
        ("sew-d", "Wav2Vec2Processor"),
        ("speech_to_text", "Speech2TextProcessor"),
        ("speech_to_text_2", "Speech2Text2Processor"),
        ("speecht5", "SpeechT5Processor"),
        ("trocr", "TrOCRProcessor"),
        ("tvlt", "TvltProcessor"),
        ("unispeech", "Wav2Vec2Processor"),
        ("unispeech-sat", "Wav2Vec2Processor"),
        ("vilt", "ViltProcessor"),
        ("vision-text-dual-encoder", "VisionTextDualEncoderProcessor"),
        ("wav2vec2", "Wav2Vec2Processor"),
        ("wav2vec2-conformer", "Wav2Vec2Processor"),
        ("wavlm", "Wav2Vec2Processor"),
        ("whisper", "WhisperProcessor"),
        ("xclip", "XCLIPProcessor"),
    ]
)

UpperCamelCase__			:     Optional[int]			       =				_LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)




def    snake_case__						(							lowerCAmelCase :		str		):
					for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
										if class_name in processors:
															__snake_case        =						model_type_to_module_name(lowerCAmelCase		)

															__snake_case        =						importlib.import_module(F'''.{module_name}'''					,       """transformers.models"""		)
															try:
																				return getattr(lowerCAmelCase					,       lowerCAmelCase		)
															except AttributeError:
																				continue

					for processor in PROCESSOR_MAPPING._extra_content.values():
										if getattr(lowerCAmelCase					,       """__name__"""					,       lowerCAmelCase		) == class_name:
															return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
					__snake_case        =						importlib.import_module("""transformers"""		)
					if hasattr(lowerCAmelCase					,       lowerCAmelCase		):
										return getattr(lowerCAmelCase					,       lowerCAmelCase		)

					return None

class      A__   :




	def __init__( self	:						Any						)       ->			Any:
						raise EnvironmentError(
						    """AutoProcessor is designed to be instantiated """
						    """using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method."""						)




	@classmethod
	@replace_list_option_in_docstrings(lowercase						)
	def 				a_		( cls	:						Any   ,     lowercase	:						Any   ,     **lowercase	:						List[str]						)       ->			List[str]:
						__snake_case        =						kwargs.pop("""config"""   ,     lowercase						)
						__snake_case        =						kwargs.pop("""trust_remote_code"""   ,     lowercase						)
						__snake_case        =						True

						__snake_case        =						None
						__snake_case        =						None

						# First, let's see if we have a preprocessor config.
						# Filter the kwargs for `get_file_from_repo`.
						__snake_case        =						{
						    key: kwargs[key] for key in inspect.signature(lowercase						).parameters.keys() if key in kwargs
						}
						# Let's start by checking whether the processor class is saved in an image processor
						__snake_case        =						get_file_from_repo(
						    lowercase   ,     lowercase   ,     **lowercase						)
						if preprocessor_config_file is not None:
											__snake_case   ,   __snake_case        =						ImageProcessingMixin.get_image_processor_dict(lowercase   ,     **lowercase						)
											__snake_case        =						config_dict.get("""processor_class"""   ,     lowercase						)
											if "AutoProcessor" in config_dict.get("""auto_map"""   ,     {}						):
																__snake_case        =						config_dict["""auto_map"""]["""AutoProcessor"""]

        # If not found, let's check whether the processor class is saved in a feature extractor config
						if preprocessor_config_file is not None and processor_class is None:
											__snake_case   ,   __snake_case        =						FeatureExtractionMixin.get_feature_extractor_dict(lowercase   ,     **lowercase						)
											__snake_case        =						config_dict.get("""processor_class"""   ,     lowercase						)
											if "AutoProcessor" in config_dict.get("""auto_map"""   ,     {}						):
																__snake_case        =						config_dict["""auto_map"""]["""AutoProcessor"""]

						if processor_class is None:
											# Next, let's check whether the processor class is saved in a tokenizer
											__snake_case        =						get_file_from_repo(
											    lowercase   ,     lowercase   ,     **lowercase						)
											if tokenizer_config_file is not None:
																with open(lowercase   ,     encoding="""utf-8"""						) as reader:
																					__snake_case        =						json.load(lowercase						)

																__snake_case        =						config_dict.get("""processor_class"""   ,     lowercase						)
																if "AutoProcessor" in config_dict.get("""auto_map"""   ,     {}						):
																					__snake_case        =						config_dict["""auto_map"""]["""AutoProcessor"""]

						if processor_class is None:
											# Otherwise, load config, if it can be loaded.
											if not isinstance(lowercase   ,     lowercase						):
																__snake_case        =						AutoConfig.from_pretrained(
																    lowercase   ,     trust_remote_code=lowercase   ,     **lowercase						)

											# And check if the config contains the processor class.
											__snake_case        =						getattr(lowercase   ,     """processor_class"""   ,     lowercase						)
											if hasattr(lowercase   ,     """auto_map"""						) and "AutoProcessor" in config.auto_map:
																__snake_case        =						config.auto_map["""AutoProcessor"""]

						if processor_class is not None:
											__snake_case        =						processor_class_from_name(lowercase						)

						__snake_case        =						processor_auto_map is not None
						__snake_case        =						processor_class is not None or type(lowercase						) in PROCESSOR_MAPPING
						__snake_case        =						resolve_trust_remote_code(
						    lowercase   ,     lowercase   ,     lowercase   ,     lowercase						)

						if has_remote_code and trust_remote_code:
											__snake_case        =						get_class_from_dynamic_module(
											    lowercase   ,     lowercase   ,     **lowercase						)
											__snake_case        =						kwargs.pop("""code_revision"""   ,     lowercase						)
											if os.path.isdir(lowercase						):
																processor_class.register_for_auto_class()
											return processor_class.from_pretrained(
											    lowercase   ,     trust_remote_code=lowercase   ,     **lowercase						)
						elif processor_class is not None:
											return processor_class.from_pretrained(
											    lowercase   ,     trust_remote_code=lowercase   ,     **lowercase						)
						# Last try: we use the PROCESSOR_MAPPING.
						elif type(lowercase						) in PROCESSOR_MAPPING:
											return PROCESSOR_MAPPING[type(lowercase						)].from_pretrained(lowercase   ,     **lowercase						)

						# At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
						# tokenizer.
						try:
											return AutoTokenizer.from_pretrained(
											    lowercase   ,     trust_remote_code=lowercase   ,     **lowercase						)
						except Exception:
											try:
																return AutoImageProcessor.from_pretrained(
																    lowercase   ,     trust_remote_code=lowercase   ,     **lowercase						)
											except Exception:
																pass

											try:
																return AutoFeatureExtractor.from_pretrained(
																    lowercase   ,     trust_remote_code=lowercase   ,     **lowercase						)
											except Exception:
																pass

						raise ValueError(
						    f'''Unrecognized processing class in {pretrained_model_name_or_path}. Can\'t instantiate a processor, a '''
						    """tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains"""
						    """the files of at least one of those processing classes."""						)




	@staticmethod
	def 				a_		( lowercase	:						Optional[int]   ,     lowercase	:						str						)       ->			Optional[Any]:
						PROCESSOR_MAPPING.register(lowercase   ,     lowercase						)








"""simple docstring"""

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__			:     str			       =				logging.get_logger(__name__)


UpperCamelCase__			:     Any			       =				{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase__			:     str			       =				{
    "vocab_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json",
    },
    "added_tokens.json": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json",
    },
    "merges_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt",
    },
}

UpperCamelCase__			:     str			       =				{
    "RUCAIBox/mvp": 10_24,
}




@lru_cache()
def    snake_case__						(							):
					__snake_case        =						(
					    list(range(ord("""!"""		)					,       ord("""~"""		) + 1		)		) + list(range(ord("""¡"""		)					,       ord("""¬"""		) + 1		)		) + list(range(ord("""®"""		)					,       ord("""ÿ"""		) + 1		)		)
					)
					__snake_case        =						bs[:]
					__snake_case        =						0
					for b in range(2**8		):
										if b not in bs:
															bs.append(lowerCAmelCase		)
															cs.append(2**8 + n		)
															n += 1
					__snake_case        =						[chr(lowerCAmelCase		) for n in cs]
					return dict(zip(lowerCAmelCase					,       lowerCAmelCase		)		)




def    snake_case__						(							lowerCAmelCase :		Optional[int]		):
					__snake_case        =						set()
					__snake_case        =						word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)		)
										__snake_case        =						char
					return pairs

class      A__   (   lowercase_  ):
	A_				:					List[str]      =   VOCAB_FILES_NAMES
	A_				:					Any      =   PRETRAINED_VOCAB_FILES_MAP
	A_				:					Optional[Any]      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A_				:					Union[str, Any]      =   ["""input_ids""", """attention_mask"""]




	def __init__( self	:						Tuple   ,     lowercase	:						Optional[int]   ,     lowercase	:						Union[str, Any]   ,     lowercase	:						Union[str, Any]="replace"   ,     lowercase	:						Dict="<s>"   ,     lowercase	:						Any="</s>"   ,     lowercase	:						List[Any]="</s>"   ,     lowercase	:						List[Any]="<s>"   ,     lowercase	:						Optional[Any]="<unk>"   ,     lowercase	:						int="<pad>"   ,     lowercase	:						List[str]="<mask>"   ,     lowercase	:						Any=False   ,     **lowercase	:						Optional[int]   ,     )       ->			Tuple:
						__snake_case        =						AddedToken(lowercase   ,     lstrip=lowercase   ,     rstrip=lowercase						) if isinstance(lowercase   ,     lowercase						) else bos_token
						__snake_case        =						AddedToken(lowercase   ,     lstrip=lowercase   ,     rstrip=lowercase						) if isinstance(lowercase   ,     lowercase						) else eos_token
						__snake_case        =						AddedToken(lowercase   ,     lstrip=lowercase   ,     rstrip=lowercase						) if isinstance(lowercase   ,     lowercase						) else sep_token
						__snake_case        =						AddedToken(lowercase   ,     lstrip=lowercase   ,     rstrip=lowercase						) if isinstance(lowercase   ,     lowercase						) else cls_token
						__snake_case        =						AddedToken(lowercase   ,     lstrip=lowercase   ,     rstrip=lowercase						) if isinstance(lowercase   ,     lowercase						) else unk_token
						__snake_case        =						AddedToken(lowercase   ,     lstrip=lowercase   ,     rstrip=lowercase						) if isinstance(lowercase   ,     lowercase						) else pad_token

						# Mask token behave like a normal word, i.e. include the space before it
						__snake_case        =						AddedToken(lowercase   ,     lstrip=lowercase   ,     rstrip=lowercase						) if isinstance(lowercase   ,     lowercase						) else mask_token

						super().__init__(
						    errors=lowercase   ,     bos_token=lowercase   ,     eos_token=lowercase   ,     unk_token=lowercase   ,     sep_token=lowercase   ,     cls_token=lowercase   ,     pad_token=lowercase   ,     mask_token=lowercase   ,     add_prefix_space=lowercase   ,     **lowercase   ,     )
						with open(lowercase   ,     encoding="""utf-8"""						) as vocab_handle:
											__snake_case        =						json.load(lowercase						)
						__snake_case        =						{v: k for k, v in self.encoder.items()}
						__snake_case        =						errors  # how to handle errors in decoding
						__snake_case        =						bytes_to_unicode()
						__snake_case        =						{v: k for k, v in self.byte_encoder.items()}
						with open(lowercase   ,     encoding="""utf-8"""						) as merges_handle:
											__snake_case        =						merges_handle.read().split("""\n"""						)[1:-1]
						__snake_case        =						[tuple(merge.split()						) for merge in bpe_merges]
						__snake_case        =						dict(zip(lowercase   ,     range(len(lowercase						)						)						)						)
						__snake_case        =						{}
						__snake_case        =						add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						__snake_case        =						re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""						)




	@property
	def 				a_		( self	:						List[str]						)       ->			Any:
						return len(self.encoder						)




	def 				a_		( self	:						str						)       ->			str:
						return dict(self.encoder   ,     **self.added_tokens_encoder						)




	def 				a_		( self	:						int   ,     lowercase	:						Optional[Any]						)       ->			int:
						if token in self.cache:
											return self.cache[token]
						__snake_case        =						tuple(lowercase						)
						__snake_case        =						get_pairs(lowercase						)

						if not pairs:
											return token

						while True:
											__snake_case        =						min(lowercase   ,     key=lambda lowercase						: self.bpe_ranks.get(lowercase   ,     float("""inf"""						)						)						)
											if bigram not in self.bpe_ranks:
																break
											__snake_case   ,   __snake_case        =						bigram
											__snake_case        =						[]
											__snake_case        =						0
											while i < len(lowercase						):
																try:
																					__snake_case        =						word.index(lowercase   ,     lowercase						)
																except ValueError:
																					new_word.extend(word[i:]						)
																					break
																else:
																					new_word.extend(word[i:j]						)
																					__snake_case        =						j

																if word[i] == first and i < len(lowercase						) - 1 and word[i + 1] == second:
																					new_word.append(first + second						)
																					i += 2
																else:
																					new_word.append(word[i]						)
																					i += 1
											__snake_case        =						tuple(lowercase						)
											__snake_case        =						new_word
											if len(lowercase						) == 1:
																break
											else:
																__snake_case        =						get_pairs(lowercase						)
						__snake_case        =						""" """.join(lowercase						)
						__snake_case        =						word
						return word




	def 				a_		( self	:						List[str]   ,     lowercase	:						Optional[int]						)       ->			List[str]:
						__snake_case        =						[]
						for token in re.findall(self.pat   ,     lowercase						):
											__snake_case        =						"""""".join(
											    self.byte_encoder[b] for b in token.encode("""utf-8"""						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase						).split(""" """						)						)
						return bpe_tokens




	def 				a_		( self	:						Dict   ,     lowercase	:						List[Any]						)       ->			List[Any]:
						return self.encoder.get(lowercase   ,     self.encoder.get(self.unk_token						)						)




	def 				a_		( self	:						Optional[int]   ,     lowercase	:						Optional[Any]						)       ->			List[str]:
						return self.decoder.get(lowercase						)




	def 				a_		( self	:						Tuple   ,     lowercase	:						int						)       ->			Any:
						__snake_case        =						"""""".join(lowercase						)
						__snake_case        =						bytearray([self.byte_decoder[c] for c in text]						).decode("""utf-8"""   ,     errors=self.errors						)
						return text




	def 				a_		( self	:						List[Any]   ,     lowercase	:						str   ,     lowercase	:						Optional[str] = None						)       ->			Tuple[str]:
						if not os.path.isdir(lowercase						):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
											return
						__snake_case        =						os.path.join(
						    lowercase   ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
						__snake_case        =						os.path.join(
						    lowercase   ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

						with open(lowercase   ,     """w"""   ,     encoding="""utf-8"""						) as f:
											f.write(json.dumps(self.encoder   ,     indent=2   ,     sort_keys=lowercase   ,     ensure_ascii=lowercase						) + """\n"""						)

						__snake_case        =						0
						with open(lowercase   ,     """w"""   ,     encoding="""utf-8"""						) as writer:
											writer.write("""#version: 0.2\n"""						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,     key=lambda lowercase						: kv[1]						):
																if index != token_index:
																					logger.warning(
																					    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																					    """ Please check that the tokenizer is not corrupted!"""						)
																					__snake_case        =						token_index
																writer.write(""" """.join(lowercase						) + """\n"""						)
																index += 1

						return vocab_file, merge_file




	def 				a_		( self	:						List[Any]   ,     lowercase	:						List[int]   ,     lowercase	:						Optional[List[int]] = None						)       ->			List[int]:
						if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__snake_case        =						[self.cls_token_id]
						__snake_case        =						[self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep




	def 				a_		( self	:						Optional[Any]   ,     lowercase	:						List[int]   ,     lowercase	:						Optional[List[int]] = None   ,     lowercase	:						bool = False						)       ->			List[int]:
						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowercase   ,     token_ids_a=lowercase   ,     already_has_special_tokens=lowercase						)

						if token_ids_a is None:
											return [1] + ([0] * len(lowercase						)) + [1]
						return [1] + ([0] * len(lowercase						)) + [1, 1] + ([0] * len(lowercase						)) + [1]




	def 				a_		( self	:						Tuple   ,     lowercase	:						List[int]   ,     lowercase	:						Optional[List[int]] = None						)       ->			List[int]:
						__snake_case        =						[self.sep_token_id]
						__snake_case        =						[self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep						) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]




	def 				a_		( self	:						Union[str, Any]   ,     lowercase	:						List[Any]   ,     lowercase	:						int=False   ,     **lowercase	:						str						)       ->			Tuple:
						__snake_case        =						kwargs.pop("""add_prefix_space"""   ,     self.add_prefix_space						)
						if (is_split_into_words or add_prefix_space) and (len(lowercase						) > 0 and not text[0].isspace()):
											__snake_case        =						""" """ + text
						return (text, kwargs)


