
'''simple docstring'''







import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

_lowercase					:    str       				=			logging.get_logger(__name__)

_lowercase					:    int       				=			{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

_lowercase					:    Optional[Any]       				=			{
    "vocab_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"
        ),
    },
}

_lowercase					:    Tuple       				=			{
    "microsoft/deberta-base": 5_1_2,
    "microsoft/deberta-large": 5_1_2,
    "microsoft/deberta-xlarge": 5_1_2,
    "microsoft/deberta-base-mnli": 5_1_2,
    "microsoft/deberta-large-mnli": 5_1_2,
    "microsoft/deberta-xlarge-mnli": 5_1_2,
}

_lowercase					:    Optional[Any]       				=			{
    "microsoft/deberta-base": {"do_lower_case": False},
    "microsoft/deberta-large": {"do_lower_case": False},
}
def    snake_case_						(							):



						"""simple docstring"""





						lowercase_      :	str						=			(
						    list(range(ord('''!'''				) , ord('''~'''				) + 1				)				) + list(range(ord('''¡'''				) , ord('''¬'''				) + 1				)				) + list(range(ord('''®'''				) , ord('''ÿ'''				) + 1				)				)
						)
						lowercase_      :	Union[str, Any]						=			bs[:]
						lowercase_      :	str						=			0
						for b in range(2**8				):
												if b not in bs:
																		bs.append(__SCREAMING_SNAKE_CASE				)
																		cs.append(2**8 + n				)
																		n += 1
						lowercase_      :	Optional[int]						=			[chr(__SCREAMING_SNAKE_CASE				) for n in cs]
						return dict(zip(__SCREAMING_SNAKE_CASE , __SCREAMING_SNAKE_CASE				)				)
def    snake_case_						(							__SCREAMING_SNAKE_CASE   :					List[str]				):



						"""simple docstring"""





						lowercase_      :	List[str]						=			set()
						lowercase_      :	Union[str, Any]						=			word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)				)
												lowercase_      :	Tuple						=			char
						return pairs






class 					lowerCAmelCase__  (    lowerCamelCase_	):
					lowerCAmelCase_	  =	VOCAB_FILES_NAMES
					lowerCAmelCase_	  =	PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_	  =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_	  =	['''input_ids''', '''attention_mask''', '''token_type_ids''']






					def __init__(  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE="replace"							,				__SCREAMING_SNAKE_CASE="[CLS]"							,				__SCREAMING_SNAKE_CASE="[SEP]"							,				__SCREAMING_SNAKE_CASE="[SEP]"							,				__SCREAMING_SNAKE_CASE="[CLS]"							,				__SCREAMING_SNAKE_CASE="[UNK]"							,				__SCREAMING_SNAKE_CASE="[PAD]"							,				__SCREAMING_SNAKE_CASE="[MASK]"							,				__SCREAMING_SNAKE_CASE=False							,				__SCREAMING_SNAKE_CASE=False							,				**__SCREAMING_SNAKE_CASE							,				):


											"""simple docstring"""





											lowercase_      :	List[str]						=			AddedToken(__SCREAMING_SNAKE_CASE							,				lstrip=__SCREAMING_SNAKE_CASE							,				rstrip=__SCREAMING_SNAKE_CASE						) if isinstance(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						) else bos_token
											lowercase_      :	List[Any]						=			AddedToken(__SCREAMING_SNAKE_CASE							,				lstrip=__SCREAMING_SNAKE_CASE							,				rstrip=__SCREAMING_SNAKE_CASE						) if isinstance(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						) else eos_token
											lowercase_      :	Union[str, Any]						=			AddedToken(__SCREAMING_SNAKE_CASE							,				lstrip=__SCREAMING_SNAKE_CASE							,				rstrip=__SCREAMING_SNAKE_CASE						) if isinstance(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						) else sep_token
											lowercase_      :	Optional[int]						=			AddedToken(__SCREAMING_SNAKE_CASE							,				lstrip=__SCREAMING_SNAKE_CASE							,				rstrip=__SCREAMING_SNAKE_CASE						) if isinstance(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						) else cls_token
											lowercase_      :	Optional[Any]						=			AddedToken(__SCREAMING_SNAKE_CASE							,				lstrip=__SCREAMING_SNAKE_CASE							,				rstrip=__SCREAMING_SNAKE_CASE						) if isinstance(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						) else unk_token
											lowercase_      :	Dict						=			AddedToken(__SCREAMING_SNAKE_CASE							,				lstrip=__SCREAMING_SNAKE_CASE							,				rstrip=__SCREAMING_SNAKE_CASE						) if isinstance(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											lowercase_      :	Dict						=			AddedToken(__SCREAMING_SNAKE_CASE							,				lstrip=__SCREAMING_SNAKE_CASE							,				rstrip=__SCREAMING_SNAKE_CASE						) if isinstance(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						) else mask_token

											super().__init__(
											    errors=__SCREAMING_SNAKE_CASE							,				bos_token=__SCREAMING_SNAKE_CASE							,				eos_token=__SCREAMING_SNAKE_CASE							,				unk_token=__SCREAMING_SNAKE_CASE							,				sep_token=__SCREAMING_SNAKE_CASE							,				cls_token=__SCREAMING_SNAKE_CASE							,				pad_token=__SCREAMING_SNAKE_CASE							,				mask_token=__SCREAMING_SNAKE_CASE							,				add_prefix_space=__SCREAMING_SNAKE_CASE							,				add_bos_token=__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE							,				)
											lowercase_      :	int						=			add_bos_token

											with open(__SCREAMING_SNAKE_CASE							,				encoding='''utf-8'''						) as vocab_handle:
																	lowercase_      :	Tuple						=			json.load(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Union[str, Any]						=			{v: k for k, v in self.encoder.items()}
											lowercase_      :	Any						=			errors  # how to handle errors in decoding
											lowercase_      :	List[Any]						=			bytes_to_unicode()
											lowercase_      :	Any						=			{v: k for k, v in self.byte_encoder.items()}
											with open(__SCREAMING_SNAKE_CASE							,				encoding='''utf-8'''						) as merges_handle:
																	lowercase_      :	int						=			merges_handle.read().split('''\n'''						)[1:-1]
											lowercase_      :	Optional[Any]						=			[tuple(merge.split()						) for merge in bpe_merges]
											lowercase_      :	Optional[int]						=			dict(zip(__SCREAMING_SNAKE_CASE							,				range(len(__SCREAMING_SNAKE_CASE						)						)						)						)
											lowercase_      :	List[str]						=			{}
											lowercase_      :	Optional[int]						=			add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											lowercase_      :	List[Any]						=			re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)






					@property
					# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
					def       _snake_case       (  self						):


											"""simple docstring"""





											return len(self.encoder						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											return dict(self.encoder							,				**self.added_tokens_encoder						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											if token in self.cache:
																	return self.cache[token]
											lowercase_      :	Union[str, Any]						=			tuple(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	List[Any]						=			get_pairs(__SCREAMING_SNAKE_CASE						)

											if not pairs:
																	return token

											while True:
																	lowercase_      :	Tuple						=			min(__SCREAMING_SNAKE_CASE							,				key=lambda __SCREAMING_SNAKE_CASE						: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE							,				float('''inf'''						)						)						)
																	if bigram not in self.bpe_ranks:
																							break
																	lowercase_		,					lowercase_      :	Tuple						=			bigram
																	lowercase_      :	str						=			[]
																	lowercase_      :	List[Any]						=			0
																	while i < len(__SCREAMING_SNAKE_CASE						):
																							try:
																													lowercase_      :	Tuple						=			word.index(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)
																							except ValueError:
																													new_word.extend(word[i:]						)
																													break
																							else:
																													new_word.extend(word[i:j]						)
																													lowercase_      :	List[str]						=			j

																							if word[i] == first and i < len(__SCREAMING_SNAKE_CASE						) - 1 and word[i + 1] == second:
																													new_word.append(first + second						)
																													i += 2
																							else:
																													new_word.append(word[i]						)
																													i += 1
																	lowercase_      :	Tuple						=			tuple(__SCREAMING_SNAKE_CASE						)
																	lowercase_      :	Dict						=			new_word
																	if len(__SCREAMING_SNAKE_CASE						) == 1:
																							break
																	else:
																							lowercase_      :	List[str]						=			get_pairs(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Tuple						=			''' '''.join(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Union[str, Any]						=			word
											return word






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE = None						):


											"""simple docstring"""





											if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											lowercase_      :	List[Any]						=			[self.cls_token_id]
											lowercase_      :	Dict						=			[self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE = None							,				__SCREAMING_SNAKE_CASE = False						):


											"""simple docstring"""





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__SCREAMING_SNAKE_CASE							,				token_ids_a=__SCREAMING_SNAKE_CASE							,				already_has_special_tokens=__SCREAMING_SNAKE_CASE						)

											if token_ids_a is None:
																	return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]
											return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE = None						):


											"""simple docstring"""





											lowercase_      :	Dict						=			[self.sep_token_id]
											lowercase_      :	int						=			[self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Optional[Any]						=			[]
											for token in re.findall(self.pat							,				__SCREAMING_SNAKE_CASE						):
																	lowercase_      :	List[str]						=			''''''.join(
																	    self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__SCREAMING_SNAKE_CASE						).split(''' '''						)						)
											return bpe_tokens






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											return self.encoder.get(__SCREAMING_SNAKE_CASE							,				self.encoder.get(self.unk_token						)						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											return self.decoder.get(__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Any						=			''''''.join(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	int						=			bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8'''							,				errors=self.errors						)
											return text






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE = None						):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE						):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
																	return
											lowercase_      :	Tuple						=			os.path.join(
											    __SCREAMING_SNAKE_CASE							,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
											lowercase_      :	Any						=			os.path.join(
											    __SCREAMING_SNAKE_CASE							,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

											with open(__SCREAMING_SNAKE_CASE							,				'''w'''							,				encoding='''utf-8'''						) as f:
																	f.write(json.dumps(self.encoder							,				indent=2							,				sort_keys=__SCREAMING_SNAKE_CASE							,				ensure_ascii=__SCREAMING_SNAKE_CASE						) + '''\n'''						)

											lowercase_      :	Optional[Any]						=			0
											with open(__SCREAMING_SNAKE_CASE							,				'''w'''							,				encoding='''utf-8'''						) as writer:
																	writer.write('''#version: 0.2\n'''						)
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,				key=lambda __SCREAMING_SNAKE_CASE						: kv[1]						):
																							if index != token_index:
																													logger.warning(
																													    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																													    ''' Please check that the tokenizer is not corrupted!'''						)
																													lowercase_      :	int						=			token_index
																							writer.write(''' '''.join(__SCREAMING_SNAKE_CASE						) + '''\n'''						)
																							index += 1

											return vocab_file, merge_file






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE=False							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Tuple						=			kwargs.pop('''add_prefix_space'''							,				self.add_prefix_space						)
											if (is_split_into_words or add_prefix_space) and (len(__SCREAMING_SNAKE_CASE						) > 0 and not text[0].isspace()):
																	lowercase_      :	int						=			''' ''' + text
											return (text, kwargs)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Tuple						=			[]
											for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(__SCREAMING_SNAKE_CASE							,				add_special_tokens=__SCREAMING_SNAKE_CASE						) + [self.eos_token_id]						)
											if len(__SCREAMING_SNAKE_CASE						) > self.model_max_length:
																	lowercase_      :	List[Any]						=			input_ids[-self.model_max_length :]
											return input_ids





'''simple docstring'''







from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class 					lowerCAmelCase__  (    lowerCamelCase_	):
					lowerCAmelCase_	  =	['''image_processor''', '''tokenizer''']
					lowerCAmelCase_	  =	'''EfficientNetImageProcessor'''
					lowerCAmelCase_	  =	('''BertTokenizer''', '''BertTokenizerFast''')






					def __init__(  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											super().__init__(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)






					def __call__(  self							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE="max_length"							,				__SCREAMING_SNAKE_CASE=64							,				__SCREAMING_SNAKE_CASE=None							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											if text is None and images is None:
																	raise ValueError('''You have to specify either text or images. Both cannot be none.'''						)

											if text is not None:
																	lowercase_      :	str						=			self.tokenizer(
																	    __SCREAMING_SNAKE_CASE							,				padding=__SCREAMING_SNAKE_CASE							,				max_length=__SCREAMING_SNAKE_CASE							,				return_tensors=__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)

											if images is not None:
																	lowercase_      :	int						=			self.image_processor(__SCREAMING_SNAKE_CASE							,				return_tensors=__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)

											if text is not None and images is not None:
																	lowercase_      :	Any						=			image_features.pixel_values
																	return encoding
											elif text is not None:
																	return encoding
											else:
																	return BatchEncoding(data=dict(**__SCREAMING_SNAKE_CASE						)							,				tensor_type=__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											return self.tokenizer.batch_decode(*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											return self.tokenizer.decode(*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						)






					@property
					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	Dict						=			self.tokenizer.model_input_names
											lowercase_      :	Optional[int]						=			self.image_processor.model_input_names
											return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names						)						)



