





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__ :			Optional[int]       	=			logging.get_logger(__name__)


UpperCamelCase__ :			Optional[Any]       	=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase__ :			Optional[int]       	=			{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

UpperCamelCase__ :			Any       	=			{
    """RUCAIBox/mvp""": 10_24,
}




@lru_cache()
def  A_(      ):
	UpperCAmelCase_		    =  (
	    list(range(ord("""!""" )					,   ord("""~""" ) + 1 ) ) + list(range(ord("""¡""" )					,   ord("""¬""" ) + 1 ) ) + list(range(ord("""®""" )					,   ord("""ÿ""" ) + 1 ) )
	)
	UpperCAmelCase_		    =  bs[:]
	UpperCAmelCase_		    =  0
	for b in range(2**8 ):
		if b not in bs:
			bs.append(A )
			cs.append(2**8 + n )
			n += 1
	UpperCAmelCase_		    =  [chr(A ) for n in cs]
	return dict(zip(A					,   A ) )




def  A_(      A ):
	UpperCAmelCase_		    =  set()
	UpperCAmelCase_		    =  word[0]
	for char in word[1:]:
		pairs.add((prev_char, char) )
		UpperCAmelCase_		    =  char
	return pairs







class 	_UpperCamelCase	(							A_	):







					'''simple docstring'''

					lowerCamelCase  :  Any           =							VOCAB_FILES_NAMES
					lowerCamelCase  :  str           =							PRETRAINED_VOCAB_FILES_MAP
					lowerCamelCase  :  Union[str, Any]           =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCamelCase  :  Any           =							['input_ids', 'attention_mask']





					def __init__(    self     :	List[str]			,       __lowercase     :	str			,       __lowercase     :	Any			,       __lowercase     :	Tuple="replace"			,       __lowercase     :	Optional[int]="<s>"			,       __lowercase     :	Any="</s>"			,       __lowercase     :	int="</s>"			,       __lowercase     :	Dict="<s>"			,       __lowercase     :	Dict="<unk>"			,       __lowercase     :	int="<pad>"			,       __lowercase     :	List[str]="<mask>"			,       __lowercase     :	Union[str, Any]=False			,       **__lowercase     :	str			,       ):






						'''simple docstring'''







						UpperCAmelCase_		    =  AddedToken(__lowercase			,       lstrip=__lowercase			,       rstrip=__lowercase		) if isinstance(__lowercase			,       __lowercase		) else bos_token
						UpperCAmelCase_		    =  AddedToken(__lowercase			,       lstrip=__lowercase			,       rstrip=__lowercase		) if isinstance(__lowercase			,       __lowercase		) else eos_token
						UpperCAmelCase_		    =  AddedToken(__lowercase			,       lstrip=__lowercase			,       rstrip=__lowercase		) if isinstance(__lowercase			,       __lowercase		) else sep_token
						UpperCAmelCase_		    =  AddedToken(__lowercase			,       lstrip=__lowercase			,       rstrip=__lowercase		) if isinstance(__lowercase			,       __lowercase		) else cls_token
						UpperCAmelCase_		    =  AddedToken(__lowercase			,       lstrip=__lowercase			,       rstrip=__lowercase		) if isinstance(__lowercase			,       __lowercase		) else unk_token
						UpperCAmelCase_		    =  AddedToken(__lowercase			,       lstrip=__lowercase			,       rstrip=__lowercase		) if isinstance(__lowercase			,       __lowercase		) else pad_token

						# Mask token behave like a normal word, i.e. include the space before it
						UpperCAmelCase_		    =  AddedToken(__lowercase			,       lstrip=__lowercase			,       rstrip=__lowercase		) if isinstance(__lowercase			,       __lowercase		) else mask_token

						super().__init__(
						    errors=__lowercase			,       bos_token=__lowercase			,       eos_token=__lowercase			,       unk_token=__lowercase			,       sep_token=__lowercase			,       cls_token=__lowercase			,       pad_token=__lowercase			,       mask_token=__lowercase			,       add_prefix_space=__lowercase			,       **__lowercase			,       )
						with open(__lowercase			,       encoding="""utf-8"""		) as vocab_handle:
							UpperCAmelCase_		    =  json.load(__lowercase		)
						UpperCAmelCase_		    =  {v: k for k, v in self.encoder.items()}
						UpperCAmelCase_		    =  errors  # how to handle errors in decoding
						UpperCAmelCase_		    =  bytes_to_unicode()
						UpperCAmelCase_		    =  {v: k for k, v in self.byte_encoder.items()}
						with open(__lowercase			,       encoding="""utf-8"""		) as merges_handle:
							UpperCAmelCase_		    =  merges_handle.read().split("""\n"""		)[1:-1]
						UpperCAmelCase_		    =  [tuple(merge.split()		) for merge in bpe_merges]
						UpperCAmelCase_		    =  dict(zip(__lowercase			,       range(len(__lowercase		)		)		)		)
						UpperCAmelCase_		    =  {}
						UpperCAmelCase_		    =  add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						UpperCAmelCase_		    =  re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""		)





					@property
					def 							SCREAMING_SNAKE_CASE				(    self     :	Tuple		):






						'''simple docstring'''







						return len(self.encoder		)





					def 							SCREAMING_SNAKE_CASE				(    self     :	List[str]		):






						'''simple docstring'''







						return dict(self.encoder			,       **self.added_tokens_encoder		)





					def 							SCREAMING_SNAKE_CASE				(    self     :	List[Any]			,       __lowercase     :	int		):






						'''simple docstring'''







						if token in self.cache:
							return self.cache[token]
						UpperCAmelCase_		    =  tuple(__lowercase		)
						UpperCAmelCase_		    =  get_pairs(__lowercase		)

						if not pairs:
							return token

						while True:
							UpperCAmelCase_		    =  min(__lowercase			,       key=lambda __lowercase		: self.bpe_ranks.get(__lowercase			,       float("""inf"""		)		)		)
							if bigram not in self.bpe_ranks:
								break
							UpperCAmelCase_    ,						UpperCAmelCase_		    =  bigram
							UpperCAmelCase_		    =  []
							UpperCAmelCase_		    =  0
							while i < len(__lowercase		):
								try:
									UpperCAmelCase_		    =  word.index(__lowercase			,       __lowercase		)
								except ValueError:
									new_word.extend(word[i:]		)
									break
								else:
									new_word.extend(word[i:j]		)
									UpperCAmelCase_		    =  j

								if word[i] == first and i < len(__lowercase		) - 1 and word[i + 1] == second:
									new_word.append(first + second		)
									i += 2
								else:
									new_word.append(word[i]		)
									i += 1
							UpperCAmelCase_		    =  tuple(__lowercase		)
							UpperCAmelCase_		    =  new_word
							if len(__lowercase		) == 1:
								break
							else:
								UpperCAmelCase_		    =  get_pairs(__lowercase		)
						UpperCAmelCase_		    =  """ """.join(__lowercase		)
						UpperCAmelCase_		    =  word
						return word





					def 							SCREAMING_SNAKE_CASE				(    self     :	Any			,       __lowercase     :	Optional[int]		):






						'''simple docstring'''







						UpperCAmelCase_		    =  []
						for token in re.findall(self.pat			,       __lowercase		):
							UpperCAmelCase_		    =  """""".join(
							    self.byte_encoder[b] for b in token.encode("""utf-8"""		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowercase		).split(""" """		)		)
						return bpe_tokens





					def 							SCREAMING_SNAKE_CASE				(    self     :	Dict			,       __lowercase     :	str		):






						'''simple docstring'''







						return self.encoder.get(__lowercase			,       self.encoder.get(self.unk_token		)		)





					def 							SCREAMING_SNAKE_CASE				(    self     :	Optional[Any]			,       __lowercase     :	Any		):






						'''simple docstring'''







						return self.decoder.get(__lowercase		)





					def 							SCREAMING_SNAKE_CASE				(    self     :	Tuple			,       __lowercase     :	List[str]		):






						'''simple docstring'''







						UpperCAmelCase_		    =  """""".join(__lowercase		)
						UpperCAmelCase_		    =  bytearray([self.byte_decoder[c] for c in text]		).decode("""utf-8"""			,       errors=self.errors		)
						return text





					def 							SCREAMING_SNAKE_CASE				(    self     :	Optional[int]			,       __lowercase     :	str			,       __lowercase     :	Optional[str] = None		):






						'''simple docstring'''







						if not os.path.isdir(__lowercase		):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
							return
						UpperCAmelCase_		    =  os.path.join(
						    __lowercase			,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
						UpperCAmelCase_		    =  os.path.join(
						    __lowercase			,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

						with open(__lowercase			,       """w"""			,       encoding="""utf-8"""		) as f:
							f.write(json.dumps(self.encoder			,       indent=2			,       sort_keys=__lowercase			,       ensure_ascii=__lowercase		) + """\n"""		)

						UpperCAmelCase_		    =  0
						with open(__lowercase			,       """w"""			,       encoding="""utf-8"""		) as writer:
							writer.write("""#version: 0.2\n"""		)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,       key=lambda __lowercase		: kv[1]		):
								if index != token_index:
									logger.warning(
									    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
									    """ Please check that the tokenizer is not corrupted!"""		)
									UpperCAmelCase_		    =  token_index
								writer.write(""" """.join(__lowercase		) + """\n"""		)
								index += 1

						return vocab_file, merge_file





					def 							SCREAMING_SNAKE_CASE				(    self     :	Optional[Any]			,       __lowercase     :	List[int]			,       __lowercase     :	Optional[List[int]] = None		):






						'''simple docstring'''







						if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						UpperCAmelCase_		    =  [self.cls_token_id]
						UpperCAmelCase_		    =  [self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep





					def 							SCREAMING_SNAKE_CASE				(    self     :	int			,       __lowercase     :	List[int]			,       __lowercase     :	Optional[List[int]] = None			,       __lowercase     :	bool = False		):






						'''simple docstring'''







						if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=__lowercase			,       token_ids_a=__lowercase			,       already_has_special_tokens=__lowercase		)

						if token_ids_a is None:
							return [1] + ([0] * len(__lowercase		)) + [1]
						return [1] + ([0] * len(__lowercase		)) + [1, 1] + ([0] * len(__lowercase		)) + [1]





					def 							SCREAMING_SNAKE_CASE				(    self     :	Optional[Any]			,       __lowercase     :	List[int]			,       __lowercase     :	Optional[List[int]] = None		):






						'''simple docstring'''







						UpperCAmelCase_		    =  [self.sep_token_id]
						UpperCAmelCase_		    =  [self.cls_token_id]

						if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]





					def 							SCREAMING_SNAKE_CASE				(    self     :	Optional[int]			,       __lowercase     :	List[Any]			,       __lowercase     :	Optional[Any]=False			,       **__lowercase     :	Dict		):






						'''simple docstring'''







						UpperCAmelCase_		    =  kwargs.pop("""add_prefix_space"""			,       self.add_prefix_space		)
						if (is_split_into_words or add_prefix_space) and (len(__lowercase		) > 0 and not text[0].isspace()):
							UpperCAmelCase_		    =  """ """ + text
						return (text, kwargs)









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCamelCase__ :			Any       	=			{
    """configuration_splinter""": ["""SPLINTER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """SplinterConfig"""],
    """tokenization_splinter""": ["""SplinterTokenizer"""],
}

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				UpperCamelCase__ :			Any       	=			["""SplinterTokenizerFast"""]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				UpperCamelCase__ :			Union[str, Any]       	=			[
				    """SPLINTER_PRETRAINED_MODEL_ARCHIVE_LIST""",
				    """SplinterForQuestionAnswering""",
				    """SplinterForPreTraining""",
				    """SplinterLayer""",
				    """SplinterModel""",
				    """SplinterPreTrainedModel""",
				]


if TYPE_CHECKING:
				from .configuration_splinter import SPLINTER_PRETRAINED_CONFIG_ARCHIVE_MAP, SplinterConfig
				from .tokenization_splinter import SplinterTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_splinter_fast import SplinterTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_splinter import (
								    SPLINTER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    SplinterForPreTraining,
								    SplinterForQuestionAnswering,
								    SplinterLayer,
								    SplinterModel,
								    SplinterPreTrainedModel,
								)


else:
				import sys

				UpperCamelCase__ :			List[str]       	=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


