




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_    =logging.get_logger(__name__)


UpperCAmelCase_    ={"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCAmelCase_    ={
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

UpperCAmelCase_    ={
    """RUCAIBox/mvp""": 1024,
}



@lru_cache()
def    UpperCAmelCase							(  ):
							lowerCAmelCase =		(
							    list(range(ord('''!'''				)    ,			ord('''~'''				) + 1				)				) + list(range(ord('''¡'''				)    ,			ord('''¬'''				) + 1				)				) + list(range(ord('''®'''				)    ,			ord('''ÿ'''				) + 1				)				)
							)
							lowerCAmelCase =		bs[:]
							lowerCAmelCase =		0
							for b in range(2**8				):
														if b not in bs:
																					bs.append(_snake_case				)
																					cs.append(2**8 + n				)
																					n += 1
							lowerCAmelCase =		[chr(_snake_case				) for n in cs]
							return dict(zip(_snake_case    ,			_snake_case				)				)



def    UpperCAmelCase							(  _snake_case				):
							lowerCAmelCase =		set()
							lowerCAmelCase =		word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)				)
														lowerCAmelCase =		char
							return pairs


class 				__UpperCamelCase      (			__UpperCAmelCase	):





				'''simple docstring'''



				__a				:			Union[str, Any]					    =VOCAB_FILES_NAMES
				__a				:			Any					    =PRETRAINED_VOCAB_FILES_MAP
				__a				:			List[str]					    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a				:			Optional[Any]					    =["""input_ids""", """attention_mask"""]



				def __init__(			self							,			UpperCAmelCase_							,			UpperCAmelCase_							,			UpperCAmelCase_="replace"							,			UpperCAmelCase_="<s>"							,			UpperCAmelCase_="</s>"							,			UpperCAmelCase_="</s>"							,			UpperCAmelCase_="<s>"							,			UpperCAmelCase_="<unk>"							,			UpperCAmelCase_="<pad>"							,			UpperCAmelCase_="<mask>"							,			UpperCAmelCase_=False							,			**UpperCAmelCase_							,			):
											lowerCAmelCase =		AddedToken(UpperCAmelCase_							,			lstrip=UpperCAmelCase_							,			rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_							,			UpperCAmelCase_    ) else bos_token
											lowerCAmelCase =		AddedToken(UpperCAmelCase_							,			lstrip=UpperCAmelCase_							,			rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_							,			UpperCAmelCase_    ) else eos_token
											lowerCAmelCase =		AddedToken(UpperCAmelCase_							,			lstrip=UpperCAmelCase_							,			rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_							,			UpperCAmelCase_    ) else sep_token
											lowerCAmelCase =		AddedToken(UpperCAmelCase_							,			lstrip=UpperCAmelCase_							,			rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_							,			UpperCAmelCase_    ) else cls_token
											lowerCAmelCase =		AddedToken(UpperCAmelCase_							,			lstrip=UpperCAmelCase_							,			rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_							,			UpperCAmelCase_    ) else unk_token
											lowerCAmelCase =		AddedToken(UpperCAmelCase_							,			lstrip=UpperCAmelCase_							,			rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_							,			UpperCAmelCase_    ) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											lowerCAmelCase =		AddedToken(UpperCAmelCase_							,			lstrip=UpperCAmelCase_							,			rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_							,			UpperCAmelCase_    ) else mask_token

											super().__init__(
											    errors=UpperCAmelCase_							,			bos_token=UpperCAmelCase_							,			eos_token=UpperCAmelCase_							,			unk_token=UpperCAmelCase_							,			sep_token=UpperCAmelCase_							,			cls_token=UpperCAmelCase_							,			pad_token=UpperCAmelCase_							,			mask_token=UpperCAmelCase_							,			add_prefix_space=UpperCAmelCase_							,			**UpperCAmelCase_							,			)
											with open(UpperCAmelCase_							,			encoding='''utf-8'''    ) as vocab_handle:
																		lowerCAmelCase =		json.load(UpperCAmelCase_    )
											lowerCAmelCase =		{v: k for k, v in self.encoder.items()}
											lowerCAmelCase =		errors  # how to handle errors in decoding
											lowerCAmelCase =		bytes_to_unicode()
											lowerCAmelCase =		{v: k for k, v in self.byte_encoder.items()}
											with open(UpperCAmelCase_							,			encoding='''utf-8'''    ) as merges_handle:
																		lowerCAmelCase =		merges_handle.read().split('''\n'''    )[1:-1]
											lowerCAmelCase =		[tuple(merge.split()    ) for merge in bpe_merges]
											lowerCAmelCase =		dict(zip(UpperCAmelCase_							,			range(len(UpperCAmelCase_    )    )    )    )
											lowerCAmelCase =		{}
											lowerCAmelCase =		add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											lowerCAmelCase =		re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )



				@property
				def    __snake_case		(			self    ):
											return len(self.encoder    )



				def    __snake_case		(			self    ):
											return dict(self.encoder							,			**self.added_tokens_encoder    )



				def    __snake_case		(			self							,			UpperCAmelCase_    ):
											if token in self.cache:
																		return self.cache[token]
											lowerCAmelCase =		tuple(UpperCAmelCase_    )
											lowerCAmelCase =		get_pairs(UpperCAmelCase_    )

											if not pairs:
																		return token

											while True:
																		lowerCAmelCase =		min(UpperCAmelCase_							,			key=lambda UpperCAmelCase_    : self.bpe_ranks.get(UpperCAmelCase_							,			float('''inf'''    )    )    )
																		if bigram not in self.bpe_ranks:
																									break
																		lowerCAmelCase ,  lowerCAmelCase =		bigram
																		lowerCAmelCase =		[]
																		lowerCAmelCase =		0
																		while i < len(UpperCAmelCase_    ):
																									try:
																																lowerCAmelCase =		word.index(UpperCAmelCase_							,			UpperCAmelCase_    )
																									except ValueError:
																																new_word.extend(word[i:]    )
																																break
																									else:
																																new_word.extend(word[i:j]    )
																																lowerCAmelCase =		j

																									if word[i] == first and i < len(UpperCAmelCase_    ) - 1 and word[i + 1] == second:
																																new_word.append(first + second    )
																																i += 2
																									else:
																																new_word.append(word[i]    )
																																i += 1
																		lowerCAmelCase =		tuple(UpperCAmelCase_    )
																		lowerCAmelCase =		new_word
																		if len(UpperCAmelCase_    ) == 1:
																									break
																		else:
																									lowerCAmelCase =		get_pairs(UpperCAmelCase_    )
											lowerCAmelCase =		''' '''.join(UpperCAmelCase_    )
											lowerCAmelCase =		word
											return word



				def    __snake_case		(			self							,			UpperCAmelCase_    ):
											lowerCAmelCase =		[]
											for token in re.findall(self.pat							,			UpperCAmelCase_    ):
																		lowerCAmelCase =		''''''.join(
																		    self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																		bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase_    ).split(''' '''    )    )
											return bpe_tokens



				def    __snake_case		(			self							,			UpperCAmelCase_    ):
											return self.encoder.get(UpperCAmelCase_							,			self.encoder.get(self.unk_token    )    )



				def    __snake_case		(			self							,			UpperCAmelCase_    ):
											return self.decoder.get(UpperCAmelCase_    )



				def    __snake_case		(			self							,			UpperCAmelCase_    ):
											lowerCAmelCase =		''''''.join(UpperCAmelCase_    )
											lowerCAmelCase =		bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''							,			errors=self.errors    )
											return text



				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_ = None    ):
											if not os.path.isdir(UpperCAmelCase_    ):
																		logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
																		return
											lowerCAmelCase =		os.path.join(
											    UpperCAmelCase_							,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
											lowerCAmelCase =		os.path.join(
											    UpperCAmelCase_							,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

											with open(UpperCAmelCase_							,			'''w'''							,			encoding='''utf-8'''    ) as f:
																		f.write(json.dumps(self.encoder							,			indent=2							,			sort_keys=UpperCAmelCase_							,			ensure_ascii=UpperCAmelCase_    ) + '''\n'''    )

											lowerCAmelCase =		0
											with open(UpperCAmelCase_							,			'''w'''							,			encoding='''utf-8'''    ) as writer:
																		writer.write('''#version: 0.2\n'''    )
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,			key=lambda UpperCAmelCase_    : kv[1]    ):
																									if index != token_index:
																																logger.warning(
																																    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																																    ''' Please check that the tokenizer is not corrupted!'''    )
																																lowerCAmelCase =		token_index
																									writer.write(''' '''.join(UpperCAmelCase_    ) + '''\n'''    )
																									index += 1

											return vocab_file, merge_file



				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_ = None    ):
											if token_ids_a is None:
																		return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											lowerCAmelCase =		[self.cls_token_id]
											lowerCAmelCase =		[self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep



				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_ = None							,			UpperCAmelCase_ = False    ):
											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=UpperCAmelCase_							,			token_ids_a=UpperCAmelCase_							,			already_has_special_tokens=UpperCAmelCase_    )

											if token_ids_a is None:
																		return [1] + ([0] * len(UpperCAmelCase_    )) + [1]
											return [1] + ([0] * len(UpperCAmelCase_    )) + [1, 1] + ([0] * len(UpperCAmelCase_    )) + [1]



				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_ = None    ):
											lowerCAmelCase =		[self.sep_token_id]
											lowerCAmelCase =		[self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]






				def    __snake_case		(			self							,			UpperCAmelCase_							,			UpperCAmelCase_=False							,			**UpperCAmelCase_    ):
											lowerCAmelCase =		kwargs.pop('''add_prefix_space'''							,			self.add_prefix_space    )
											if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase_    ) > 0 and not text[0].isspace()):
																		lowerCAmelCase =		''' ''' + text
											return (text, kwargs)









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase_    ={"""configuration_vilt""": ["""VILT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ViltConfig"""]}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =["""ViltFeatureExtractor"""]
		UpperCAmelCase_    =["""ViltImageProcessor"""]
		UpperCAmelCase_    =["""ViltProcessor"""]

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_    =[
		    """VILT_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """ViltForImageAndTextRetrieval""",
		    """ViltForImagesAndTextClassification""",
		    """ViltForTokenClassification""",
		    """ViltForMaskedLM""",
		    """ViltForQuestionAnswering""",
		    """ViltLayer""",
		    """ViltModel""",
		    """ViltPreTrainedModel""",
		]


if TYPE_CHECKING:
		from .configuration_vilt import VILT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViltConfig

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_vilt import ViltFeatureExtractor
				from .image_processing_vilt import ViltImageProcessor
				from .processing_vilt import ViltProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_vilt import (
				    VILT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ViltForImageAndTextRetrieval,
				    ViltForImagesAndTextClassification,
				    ViltForMaskedLM,
				    ViltForQuestionAnswering,
				    ViltForTokenClassification,
				    ViltLayer,
				    ViltModel,
				    ViltPreTrainedModel,
				)


else:
		import sys

		UpperCAmelCase_    =_LazyModule(__name__, globals()["""__file__"""], _import_structure)



