




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__lowerCamelCase :    Optional[Any]      						=      logging.get_logger(__name__)

__lowerCamelCase :    int      						=      {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowerCamelCase :    int      						=      {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

__lowerCamelCase :    Optional[int]      						=      {
    '''openai-gpt''': 5_12,
}







def 		lowercase__ (		__A:						Optional[int]					):





					'''simple docstring'''






					__magic_name__				:						Union[str, Any]     				=				text.strip()
					if not text:
										return []
					__magic_name__				:						List[Any]     				=				text.split()
					return tokens


class  lowerCamelCase     (		_lowerCamelCase    ):






			'''simple docstring'''





			def __init__(			self      : int							,    lowerCamelCase_      : Union[str, Any]=True							,    lowerCamelCase_      : Dict=None							,    lowerCamelCase_      : str=True							,    lowerCamelCase_      : str=None							,    lowerCamelCase_      : Optional[int]=True							,    )    -> Any:
								if never_split is None:
													__magic_name__				:						List[str]     				=				[]
								__magic_name__				:						List[Any]     				=				do_lower_case
								__magic_name__				:						Union[str, Any]     				=				set(lowerCamelCase_	)
								__magic_name__				:						Any     				=				tokenize_chinese_chars
								__magic_name__				:						List[str]     				=				strip_accents
								__magic_name__				:						List[str]     				=				do_split_on_punc

			def 				UpperCAmelCase__							(			self      : Optional[int]							,    lowerCamelCase_      : Optional[int]							,    lowerCamelCase_      : Union[str, Any]=None	)    -> Optional[Any]:
								__magic_name__				:						List[str]     				=				self.never_split.union(set(lowerCamelCase_	)	) if never_split else self.never_split
								__magic_name__				:						Union[str, Any]     				=				self._clean_text(lowerCamelCase_	)

								# This was added on November 1st, 2018 for the multilingual and Chinese
								# models. This is also applied to the English models now, but it doesn't
								# matter since the English models were not trained on any Chinese data
								# and generally don't have any Chinese data in them (there are Chinese
								# characters in the vocabulary because Wikipedia does have some Chinese
								# words in the English Wikipedia.).
								if self.tokenize_chinese_chars:
													__magic_name__				:						List[Any]     				=				self._tokenize_chinese_chars(lowerCamelCase_	)
								# prevents treating the same character with different unicode codepoints as different characters
								__magic_name__				:						Tuple     				=				unicodedata.normalize('''NFC'''							,    lowerCamelCase_	)
								__magic_name__				:						Union[str, Any]     				=				whitespace_tokenize(lowerCamelCase_	)
								__magic_name__				:						Dict     				=				[]
								for token in orig_tokens:
													if token not in never_split:
																		if self.do_lower_case:
																							__magic_name__				:						int     				=				token.lower()
																							if self.strip_accents is not False:
																												__magic_name__				:						Any     				=				self._run_strip_accents(lowerCamelCase_	)
																		elif self.strip_accents:
																							__magic_name__				:						Optional[Any]     				=				self._run_strip_accents(lowerCamelCase_	)
													split_tokens.extend(self._run_split_on_punc(lowerCamelCase_							,    lowerCamelCase_	)	)

								__magic_name__				:						Optional[Any]     				=				whitespace_tokenize(''' '''.join(lowerCamelCase_	)	)
								return output_tokens

			def 				UpperCAmelCase__							(			self      : Any							,    lowerCamelCase_      : int	)    -> Tuple:
								__magic_name__				:						List[str]     				=				unicodedata.normalize('''NFD'''							,    lowerCamelCase_	)
								__magic_name__				:						Dict     				=				[]
								for char in text:
													__magic_name__				:						Any     				=				unicodedata.category(lowerCamelCase_	)
													if cat == "Mn":
																		continue
													output.append(lowerCamelCase_	)
								return "".join(lowerCamelCase_	)

			def 				UpperCAmelCase__							(			self      : Any							,    lowerCamelCase_      : Dict							,    lowerCamelCase_      : Union[str, Any]=None	)    -> List[str]:
								if not self.do_split_on_punc or (never_split is not None and text in never_split):
													return [text]
								__magic_name__				:						List[Any]     				=				list(lowerCamelCase_	)
								__magic_name__				:						List[Any]     				=				0
								__magic_name__				:						str     				=				True
								__magic_name__				:						int     				=				[]
								while i < len(lowerCamelCase_	):
													__magic_name__				:						List[Any]     				=				chars[i]
													if _is_punctuation(lowerCamelCase_	):
																		output.append([char]	)
																		__magic_name__				:						Tuple     				=				True
													else:
																		if start_new_word:
																							output.append([]	)
																		__magic_name__				:						List[str]     				=				False
																		output[-1].append(lowerCamelCase_	)
													i += 1

								return ["".join(lowerCamelCase_	) for x in output]

			def 				UpperCAmelCase__							(			self      : Optional[Any]							,    lowerCamelCase_      : Optional[int]	)    -> Optional[Any]:
								__magic_name__				:						Union[str, Any]     				=				[]
								for char in text:
													__magic_name__				:						Any     				=				ord(lowerCamelCase_	)
													if self._is_chinese_char(lowerCamelCase_	):
																		output.append(''' '''	)
																		output.append(lowerCamelCase_	)
																		output.append(''' '''	)
													else:
																		output.append(lowerCamelCase_	)
								return "".join(lowerCamelCase_	)

			def 				UpperCAmelCase__							(			self      : Dict							,    lowerCamelCase_      : str	)    -> Union[str, Any]:
								# This defines a "chinese character" as anything in the CJK Unicode block:
								#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
								#
								# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
								# despite its name. The modern Korean Hangul alphabet is a different block,
								# as is Japanese Hiragana and Katakana. Those alphabets are used to write
								# space-separated words, so they are not treated specially and handled
								# like the all of the other languages.
								if (
								    (cp >= 0X4E00 and cp <= 0X9FFF)
								    or (cp >= 0X3400 and cp <= 0X4DBF)  #
								    or (cp >= 0X20000 and cp <= 0X2A6DF)  #
								    or (cp >= 0X2A700 and cp <= 0X2B73F)  #
								    or (cp >= 0X2B740 and cp <= 0X2B81F)  #
								    or (cp >= 0X2B820 and cp <= 0X2CEAF)  #
								    or (cp >= 0XF900 and cp <= 0XFAFF)
								    or (cp >= 0X2F800 and cp <= 0X2FA1F)  #
								):  #
													return True

								return False


			def 				UpperCAmelCase__							(			self      : List[str]							,    lowerCamelCase_      : List[str]	)    -> Optional[Any]:
								__magic_name__				:						int     				=				[]
								for char in text:
													__magic_name__				:						List[Any]     				=				ord(lowerCamelCase_	)
													if cp == 0 or cp == 0XFFFD or _is_control(lowerCamelCase_	):
																		continue
													if _is_whitespace(lowerCamelCase_	):
																		output.append(''' '''	)
													else:
																		output.append(lowerCamelCase_	)
								return "".join(lowerCamelCase_	)







def 		lowercase__ (		__A:						List[Any]					):





					'''simple docstring'''






					__magic_name__				:						Optional[int]     				=				set()
					__magic_name__				:						int     				=				word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)					)
										__magic_name__				:						List[Any]     				=				char
					return pairs







def 		lowercase__ (		__A:						List[Any]					):





					'''simple docstring'''






					__magic_name__				:						Union[str, Any]     				=				text.replace('''—'''				,'''-'''					)
					__magic_name__				:						List[Any]     				=				text.replace('''–'''				,'''-'''					)
					__magic_name__				:						int     				=				text.replace('''―'''				,'''-'''					)
					__magic_name__				:						List[Any]     				=				text.replace('''…'''				,'''...'''					)
					__magic_name__				:						Union[str, Any]     				=				text.replace('''´'''				,'''\''''					)
					__magic_name__				:						Dict     				=				re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''				,r''' \1 '''				,__A					)
					__magic_name__				:						List[str]     				=				re.sub(r'''\s*\n\s*'''				,''' \n '''				,__A					)
					__magic_name__				:						List[Any]     				=				re.sub(r'''[^\S\n]+'''				,''' '''				,__A					)
					return text.strip()


class  lowerCamelCase     (		_lowerCamelCase    ):






			'''simple docstring'''





			UpperCamelCase__										=VOCAB_FILES_NAMES
			UpperCamelCase__										=PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase__										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase__										=['''input_ids''', '''attention_mask''']

			def __init__(			self      : Union[str, Any]							,    lowerCamelCase_      : Dict							,    lowerCamelCase_      : List[Any]							,    lowerCamelCase_      : Optional[int]="<unk>"							,    **lowerCamelCase_      : Union[str, Any]	)    -> Union[str, Any]:
								super().__init__(unk_token=lowerCamelCase_							,    **lowerCamelCase_	)

								try:
													import ftfy
													from spacy.lang.en import English

													__magic_name__				:						Dict     				=				English()
													__magic_name__				:						Optional[int]     				=				_nlp.tokenizer
													__magic_name__				:						List[str]     				=				ftfy.fix_text
								except ImportError:
													logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''	)
													__magic_name__				:						Optional[int]     				=				BasicTokenizer(do_lower_case=lowerCamelCase_	)
													__magic_name__				:						Union[str, Any]     				=				None

								with open(lowerCamelCase_							,    encoding='''utf-8'''	) as vocab_handle:
													__magic_name__				:						str     				=				json.load(lowerCamelCase_	)
								__magic_name__				:						Union[str, Any]     				=				{v: k for k, v in self.encoder.items()}
								with open(lowerCamelCase_							,    encoding='''utf-8'''	) as merges_handle:
													__magic_name__				:						Dict     				=				merges_handle.read().split('''\n'''	)[1:-1]
								__magic_name__				:						Union[str, Any]     				=				[tuple(merge.split()	) for merge in merges]
								__magic_name__				:						List[str]     				=				dict(zip(lowerCamelCase_							,    range(len(lowerCamelCase_	)	)	)	)
								__magic_name__				:						Any     				=				{}

			@property
			def 				UpperCAmelCase__							(			self      : Any	)    -> List[str]:
								return True

			@property
			def 				UpperCAmelCase__							(			self      : Any	)    -> Dict:
								return len(self.encoder	)

			def 				UpperCAmelCase__							(			self      : int	)    -> Any:
								return dict(self.encoder							,    **self.added_tokens_encoder	)

			def 				UpperCAmelCase__							(			self      : List[Any]							,    lowerCamelCase_      : List[Any]	)    -> Union[str, Any]:
								__magic_name__				:						Tuple     				=				tuple(token[:-1]	) + (token[-1] + '''</w>''',)
								if token in self.cache:
													return self.cache[token]
								__magic_name__				:						Optional[int]     				=				get_pairs(lowerCamelCase_	)

								if not pairs:
													return token + "</w>"

								while True:
													__magic_name__				:						List[Any]     				=				min(lowerCamelCase_							,    key=lambda lowerCamelCase_	: self.bpe_ranks.get(lowerCamelCase_							,    float('''inf'''	)	)	)
													if bigram not in self.bpe_ranks:
																		break
													__magic_name__				,     __magic_name__				:						List[str]     				=				bigram
													__magic_name__				:						List[str]     				=				[]
													__magic_name__				:						str     				=				0
													while i < len(lowerCamelCase_	):
																		try:
																							__magic_name__				:						Optional[int]     				=				word.index(lowerCamelCase_							,    lowerCamelCase_	)
																		except ValueError:
																							new_word.extend(word[i:]	)
																							break
																		else:
																							new_word.extend(word[i:j]	)
																							__magic_name__				:						Optional[Any]     				=				j

																		if word[i] == first and i < len(lowerCamelCase_	) - 1 and word[i + 1] == second:
																							new_word.append(first + second	)
																							i += 2
																		else:
																							new_word.append(word[i]	)
																							i += 1
													__magic_name__				:						Any     				=				tuple(lowerCamelCase_	)
													__magic_name__				:						Any     				=				new_word
													if len(lowerCamelCase_	) == 1:
																		break
													else:
																		__magic_name__				:						List[Any]     				=				get_pairs(lowerCamelCase_	)
								__magic_name__				:						List[str]     				=				''' '''.join(lowerCamelCase_	)
								if word == "\n  </w>":
													__magic_name__				:						Union[str, Any]     				=				'''\n</w>'''
								__magic_name__				:						Tuple     				=				word
								return word

			def 				UpperCAmelCase__							(			self      : Optional[Any]							,    lowerCamelCase_      : int	)    -> Optional[Any]:
								__magic_name__				:						Dict     				=				[]
								if self.fix_text is None:
													# Using BERT's BasicTokenizer
													__magic_name__				:						Dict     				=				self.nlp.tokenize(lowerCamelCase_	)
													for token in text:
																		split_tokens.extend(list(self.bpe(lowerCamelCase_	).split(''' '''	)	)	)
								else:
													# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
													__magic_name__				:						Tuple     				=				self.nlp(text_standardize(self.fix_text(lowerCamelCase_	)	)	)
													for token in text:
																		split_tokens.extend(list(self.bpe(token.text.lower()	).split(''' '''	)	)	)
								return split_tokens

			def 				UpperCAmelCase__							(			self      : List[str]							,    lowerCamelCase_      : List[Any]	)    -> Dict:
								return self.encoder.get(lowerCamelCase_							,    self.encoder.get(self.unk_token	)	)

			def 				UpperCAmelCase__							(			self      : Dict							,    lowerCamelCase_      : str	)    -> int:
								return self.decoder.get(lowerCamelCase_							,    self.unk_token	)

			def 				UpperCAmelCase__							(			self      : List[Any]							,    lowerCamelCase_      : Optional[Any]	)    -> Tuple:
								__magic_name__				:						Optional[int]     				=				''''''.join(lowerCamelCase_	).replace('''</w>'''							,    ''' '''	).strip()
								return out_string


			def 				UpperCAmelCase__							(			self      : Union[str, Any]							,    lowerCamelCase_      : str							,    lowerCamelCase_      : Optional[str] = None	)    -> Tuple[str]:
								if not os.path.isdir(lowerCamelCase_	):
													logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
													return
								__magic_name__				:						Optional[Any]     				=				os.path.join(
								    lowerCamelCase_							,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
								__magic_name__				:						str     				=				os.path.join(
								    lowerCamelCase_							,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']	)

								with open(lowerCamelCase_							,    '''w'''							,    encoding='''utf-8'''	) as f:
													f.write(json.dumps(self.encoder							,    indent=2							,    sort_keys=lowerCamelCase_							,    ensure_ascii=lowerCamelCase_	) + '''\n'''	)

								__magic_name__				:						Tuple     				=				0
								with open(lowerCamelCase_							,    '''w'''							,    encoding='''utf-8'''	) as writer:
													writer.write('''#version: 0.2\n'''	)
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,    key=lambda lowerCamelCase_	: kv[1]	):
																		if index != token_index:
																							logger.warning(
																							    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																							    ''' Please check that the tokenizer is not corrupted!'''	)
																							__magic_name__				:						List[str]     				=				token_index
																		writer.write(''' '''.join(lowerCamelCase_	) + '''\n'''	)
																		index += 1

								return vocab_file, merge_file









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


__lowerCamelCase :    List[Any]      						=      logging.get_logger(__name__)

__lowerCamelCase :    Optional[int]      						=      {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase :    List[Any]      						=      {
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

__lowerCamelCase :    int      						=      {
    '''microsoft/mpnet-base''': 5_12,
}

__lowerCamelCase :    Union[str, Any]      						=      {
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}


class  lowerCamelCase     (		_lowerCamelCase    ):






			'''simple docstring'''





			UpperCamelCase__										=VOCAB_FILES_NAMES
			UpperCamelCase__										=PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase__										=PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase__										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase__										=MPNetTokenizer
			UpperCamelCase__										=['''input_ids''', '''attention_mask''']

			def __init__(			self      : Optional[int]							,    lowerCamelCase_      : List[str]=None							,    lowerCamelCase_      : int=None							,    lowerCamelCase_      : Dict=True							,    lowerCamelCase_      : int="<s>"							,    lowerCamelCase_      : Dict="</s>"							,    lowerCamelCase_      : int="</s>"							,    lowerCamelCase_      : List[Any]="<s>"							,    lowerCamelCase_      : Union[str, Any]="[UNK]"							,    lowerCamelCase_      : Any="<pad>"							,    lowerCamelCase_      : Dict="<mask>"							,    lowerCamelCase_      : Optional[Any]=True							,    lowerCamelCase_      : List[Any]=None							,    **lowerCamelCase_      : List[Any]							,    )    -> Dict:
								super().__init__(
								    lowerCamelCase_							,    tokenizer_file=lowerCamelCase_							,    do_lower_case=lowerCamelCase_							,    bos_token=lowerCamelCase_							,    eos_token=lowerCamelCase_							,    sep_token=lowerCamelCase_							,    cls_token=lowerCamelCase_							,    unk_token=lowerCamelCase_							,    pad_token=lowerCamelCase_							,    mask_token=lowerCamelCase_							,    tokenize_chinese_chars=lowerCamelCase_							,    strip_accents=lowerCamelCase_							,    **lowerCamelCase_							,    )

								__magic_name__				:						Any     				=				json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    pre_tok_state.get('''lowercase'''							,    lowerCamelCase_	) != do_lower_case
								    or pre_tok_state.get('''strip_accents'''							,    lowerCamelCase_	) != strip_accents
								):
													__magic_name__				:						int     				=				getattr(lowerCamelCase_							,    pre_tok_state.pop('''type'''	)	)
													__magic_name__				:						str     				=				do_lower_case
													__magic_name__				:						Union[str, Any]     				=				strip_accents
													__magic_name__				:						str     				=				pre_tok_class(**lowerCamelCase_	)

								__magic_name__				:						List[Any]     				=				do_lower_case

			@property
			def 				UpperCAmelCase__							(			self      : str	)    -> str:
								if self._mask_token is None:
													if self.verbose:
																		logger.error('''Using mask_token, but it is not set yet.'''	)
													return None
								return str(self._mask_token	)

			@mask_token.setter
			def 				UpperCAmelCase__							(			self      : Any							,    lowerCamelCase_      : List[str]	)    -> Optional[int]:
								__magic_name__				:						List[str]     				=				AddedToken(lowerCamelCase_							,    lstrip=lowerCamelCase_							,    rstrip=lowerCamelCase_	) if isinstance(lowerCamelCase_							,    lowerCamelCase_	) else value
								__magic_name__				:						Tuple     				=				value

			def 				UpperCAmelCase__							(			self      : str							,    lowerCamelCase_      : Optional[Any]							,    lowerCamelCase_      : Any=None	)    -> List[Any]:
								__magic_name__				:						Optional[int]     				=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
													return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

			def 				UpperCAmelCase__							(			self      : List[str]							,    lowerCamelCase_      : List[int]							,    lowerCamelCase_      : Optional[List[int]] = None	)    -> List[int]:
								__magic_name__				:						Tuple     				=				[self.sep_token_id]
								__magic_name__				:						Any     				=				[self.cls_token_id]

								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]


			def 				UpperCAmelCase__							(			self      : Optional[int]							,    lowerCamelCase_      : str							,    lowerCamelCase_      : Optional[str] = None	)    -> Tuple[str]:
								__magic_name__				:						List[Any]     				=				self._tokenizer.model.save(lowerCamelCase_							,    name=lowerCamelCase_	)
								return tuple(lowerCamelCase_	)



