





import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_   				=  logging.get_logger(__name__)

UpperCAmelCase_   				=  {
    """src_vocab_file""": """vocab-src.json""",
    """tgt_vocab_file""": """vocab-tgt.json""",
    """merges_file""": """merges.txt""",
}

UpperCAmelCase_   				=  {
    """src_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"""
    },
    """tgt_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"""
    },
    """merges_file""": {"""stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"""},
}

UpperCAmelCase_   				=  {"""stas/tiny-wmt19-en-de""": 1024}
UpperCAmelCase_   				=  {
    """stas/tiny-wmt19-en-de""": {
        """langs""": ["""en""", """de"""],
        """model_max_length""": 1024,
        """special_tokens_map_file""": None,
        """full_tokenizer_file""": None,
    }
}





def     lowerCamelCase__  (  UpperCamelCase__      :							Any    )      ->			Dict:



					'''simple docstring'''




					_snake_case						=					set()
					_snake_case						=					word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)    )
										_snake_case						=					char
					return pairs





def     lowerCamelCase__  (  UpperCamelCase__      :							Optional[int]    )      ->			int:



					'''simple docstring'''




					_snake_case						=					text.replace('，'							,     ','    )
					_snake_case						=					re.sub(R'。\s*'							,     '. '							,     UpperCamelCase__    )
					_snake_case						=					text.replace('、'							,     ','    )
					_snake_case						=					text.replace('”'							,     '"'    )
					_snake_case						=					text.replace('“'							,     '"'    )
					_snake_case						=					text.replace('∶'							,     ':'    )
					_snake_case						=					text.replace('：'							,     ':'    )
					_snake_case						=					text.replace('？'							,     '?'    )
					_snake_case						=					text.replace('《'							,     '"'    )
					_snake_case						=					text.replace('》'							,     '"'    )
					_snake_case						=					text.replace('）'							,     ')'    )
					_snake_case						=					text.replace('！'							,     '!'    )
					_snake_case						=					text.replace('（'							,     '('    )
					_snake_case						=					text.replace('；'							,     ';'    )
					_snake_case						=					text.replace('１'							,     '1'    )
					_snake_case						=					text.replace('」'							,     '"'    )
					_snake_case						=					text.replace('「'							,     '"'    )
					_snake_case						=					text.replace('０'							,     '0'    )
					_snake_case						=					text.replace('３'							,     '3'    )
					_snake_case						=					text.replace('２'							,     '2'    )
					_snake_case						=					text.replace('５'							,     '5'    )
					_snake_case						=					text.replace('６'							,     '6'    )
					_snake_case						=					text.replace('９'							,     '9'    )
					_snake_case						=					text.replace('７'							,     '7'    )
					_snake_case						=					text.replace('８'							,     '8'    )
					_snake_case						=					text.replace('４'							,     '4'    )
					_snake_case						=					re.sub(R'．\s*'							,     '. '							,     UpperCamelCase__    )
					_snake_case						=					text.replace('～'							,     '~'    )
					_snake_case						=					text.replace('’'							,     '\''    )
					_snake_case						=					text.replace('…'							,     '...'    )
					_snake_case						=					text.replace('━'							,     '-'    )
					_snake_case						=					text.replace('〈'							,     '<'    )
					_snake_case						=					text.replace('〉'							,     '>'    )
					_snake_case						=					text.replace('【'							,     '['    )
					_snake_case						=					text.replace('】'							,     ']'    )
					_snake_case						=					text.replace('％'							,     '%'    )
					return text





def     lowerCamelCase__  (  UpperCamelCase__      :							List[str]    )      ->			List[str]:



					'''simple docstring'''




					_snake_case						=					[]
					for char in text:
										_snake_case						=					unicodedata.category(UpperCamelCase__    )
										if cat.startswith('C'    ):
															continue
										output.append(UpperCamelCase__    )
					return "".join(UpperCamelCase__    )




class       UpperCamelCase_		(	_lowerCamelCase       ):
					lowerCAmelCase_				=						VOCAB_FILES_NAMES
					lowerCAmelCase_				=						PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_				=						PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase_				=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_				=						['''input_ids''', '''attention_mask''']




					def __init__(		self					,    lowerCAmelCase_=None					,    lowerCAmelCase_=None					,    lowerCAmelCase_=None					,    lowerCAmelCase_=None					,    lowerCAmelCase_=False					,    lowerCAmelCase_="<unk>"					,    lowerCAmelCase_="<s>"					,    lowerCAmelCase_="</s>"					,    lowerCAmelCase_="<pad>"					,    **lowerCAmelCase_					,    )		->				List[str]:
										super().__init__(
										    langs=lowerCAmelCase_					,    src_vocab_file=lowerCAmelCase_					,    tgt_vocab_file=lowerCAmelCase_					,    merges_file=lowerCAmelCase_					,    do_lower_case=lowerCAmelCase_					,    unk_token=lowerCAmelCase_					,    bos_token=lowerCAmelCase_					,    sep_token=lowerCAmelCase_					,    pad_token=lowerCAmelCase_					,    **lowerCAmelCase_					,    )

										try:
															import sacremoses
										except ImportError:
															raise ImportError(
															    'You need to install sacremoses to use XLMTokenizer. '
															    'See https://pypi.org/project/sacremoses/ for installation.'	)

										_snake_case						=					sacremoses

										_snake_case						=					src_vocab_file
										_snake_case						=					tgt_vocab_file
										_snake_case						=					merges_file
										_snake_case						=					do_lower_case

										# cache of sm.MosesPunctNormalizer instance
										_snake_case						=					{}
										# cache of sm.MosesTokenizer instance
										_snake_case						=					{}
										_snake_case						=					{}

										if langs and len(lowerCAmelCase_	) == 2:
															_snake_case				,  _snake_case						=					langs
										else:
															raise ValueError(
															    F'''arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '''
															    'Usually that means that tokenizer can\'t find a mapping for the given model path '
															    'in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'	)

										with open(lowerCAmelCase_					,    encoding='utf-8'	) as src_vocab_handle:
															_snake_case						=					json.load(lowerCAmelCase_	)
										with open(lowerCAmelCase_					,    encoding='utf-8'	) as tgt_vocab_handle:
															_snake_case						=					json.load(lowerCAmelCase_	)
															_snake_case						=					{v: k for k, v in tgt_vocab.items()}
										with open(lowerCAmelCase_					,    encoding='utf-8'	) as merges_handle:
															_snake_case						=					merges_handle.read().split('\n'	)[:-1]
										_snake_case						=					[tuple(merge.split()[:2]	) for merge in merges]
										_snake_case						=					dict(zip(lowerCAmelCase_					,    range(len(lowerCAmelCase_	)	)	)	)
										_snake_case						=					{}




					def 						lowerCAmelCase      (		self	)		->				Dict[str, int]:
										return self.get_src_vocab()




					@property
					def 						lowerCAmelCase      (		self	)		->				int:
										return self.src_vocab_size




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_	)		->				List[str]:
										if lang not in self.cache_moses_punct_normalizer:
															_snake_case						=					self.sm.MosesPunctNormalizer(lang=lowerCAmelCase_	)
															_snake_case						=					punct_normalizer
										return self.cache_moses_punct_normalizer[lang].normalize(lowerCAmelCase_	)




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_	)		->				Tuple:
										if lang not in self.cache_moses_tokenizer:
															_snake_case						=					self.sm.MosesTokenizer(lang=lowerCAmelCase_	)
															_snake_case						=					moses_tokenizer
										return self.cache_moses_tokenizer[lang].tokenize(
										    lowerCAmelCase_					,    aggressive_dash_splits=lowerCAmelCase_					,    return_str=lowerCAmelCase_					,    escape=lowerCAmelCase_	)




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_	)		->				int:
										if lang not in self.cache_moses_detokenizer:
															_snake_case						=					self.sm.MosesDetokenizer(lang=lowerCAmelCase_	)
															_snake_case						=					moses_detokenizer
										return self.cache_moses_detokenizer[lang].detokenize(lowerCAmelCase_	)




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_	)		->				Tuple:
										_snake_case						=					replace_unicode_punct(lowerCAmelCase_	)
										_snake_case						=					self.moses_punct_norm(lowerCAmelCase_					,    lowerCAmelCase_	)
										_snake_case						=					remove_non_printing_char(lowerCAmelCase_	)
										return text




					@property
					def 						lowerCAmelCase      (		self	)		->				Optional[int]:
										return len(self.encoder	)




					@property
					def 						lowerCAmelCase      (		self	)		->				Dict:
										return len(self.decoder	)




					def 						lowerCAmelCase      (		self	)		->				List[str]:
										return dict(self.encoder					,    **self.added_tokens_encoder	)




					def 						lowerCAmelCase      (		self	)		->				Tuple:
										return dict(self.decoder					,    **self.added_tokens_decoder	)




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_	)		->				int:
										_snake_case						=					tuple(token[:-1]	) + (token[-1] + '</w>',)
										if token in self.cache:
															return self.cache[token]
										_snake_case						=					get_pairs(lowerCAmelCase_	)

										if not pairs:
															return token + "</w>"

										while True:
															_snake_case						=					min(lowerCAmelCase_					,    key=lambda lowerCAmelCase_	: self.bpe_ranks.get(lowerCAmelCase_					,    float('inf'	)	)	)
															if bigram not in self.bpe_ranks:
																				break
															_snake_case				,  _snake_case						=					bigram
															_snake_case						=					[]
															_snake_case						=					0
															while i < len(lowerCAmelCase_	):
																				try:
																									_snake_case						=					word.index(lowerCAmelCase_					,    lowerCAmelCase_	)
																				except ValueError:
																									new_word.extend(word[i:]	)
																									break
																				else:
																									new_word.extend(word[i:j]	)
																									_snake_case						=					j

																				if word[i] == first and i < len(lowerCAmelCase_	) - 1 and word[i + 1] == second:
																									new_word.append(first + second	)
																									i += 2
																				else:
																									new_word.append(word[i]	)
																									i += 1
															_snake_case						=					tuple(lowerCAmelCase_	)
															_snake_case						=					new_word
															if len(lowerCAmelCase_	) == 1:
																				break
															else:
																				_snake_case						=					get_pairs(lowerCAmelCase_	)
										_snake_case						=					' '.join(lowerCAmelCase_	)
										if word == "\n  </w>":
															_snake_case						=					'\n</w>'
										_snake_case						=					word
										return word




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_="en"					,    lowerCAmelCase_=False	)		->				Optional[int]:
										_snake_case						=					self.src_lang

										if self.do_lower_case:
															_snake_case						=					text.lower()

										if bypass_tokenizer:
															_snake_case						=					text.split()
										else:
															_snake_case						=					self.moses_pipeline(lowerCAmelCase_					,    lang=lowerCAmelCase_	)
															_snake_case						=					self.moses_tokenize(lowerCAmelCase_					,    lang=lowerCAmelCase_	)

										_snake_case						=					[]
										for token in text:
															if token:
																				split_tokens.extend(list(self.bpe(lowerCAmelCase_	).split(' '	)	)	)

										return split_tokens




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_	)		->				Any:
										return self.encoder.get(lowerCAmelCase_					,    self.encoder.get(self.unk_token	)	)




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_	)		->				Optional[Any]:
										return self.decoder.get(lowerCAmelCase_					,    self.unk_token	)




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_	)		->				Any:
										_snake_case						=					[t.replace(' '					,    ''	).replace('</w>'					,    ' '	) for t in tokens]
										_snake_case						=					''.join(lowerCAmelCase_	).split()
										# detokenize
										_snake_case						=					self.moses_detokenize(lowerCAmelCase_					,    self.tgt_lang	)
										return text




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_ = None	)		->				List[int]:
										_snake_case						=					[self.sep_token_id]

										# no bos used in fairseq
										if token_ids_a is None:
															return token_ids_a + sep
										return token_ids_a + sep + token_ids_a + sep




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_ = None					,    lowerCAmelCase_ = False	)		->				List[int]:

										if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCAmelCase_					,    token_ids_a=lowerCAmelCase_					,    already_has_special_tokens=lowerCAmelCase_	)
										# no bos used in fairseq
										if token_ids_a is not None:
															return ([0] * len(lowerCAmelCase_	)) + [1] + ([0] * len(lowerCAmelCase_	)) + [1]
										return ([0] * len(lowerCAmelCase_	)) + [1]




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_ = None	)		->				List[int]:
										_snake_case						=					[self.sep_token_id]

										# no bos used in fairseq
										if token_ids_a is None:
															return len(token_ids_a + sep	) * [0]
										return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]




					def 						lowerCAmelCase      (		self					,    lowerCAmelCase_					,    lowerCAmelCase_ = None	)		->				Tuple[str]:
										if not os.path.isdir(lowerCAmelCase_	):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
															return

										_snake_case						=					os.path.join(
										    lowerCAmelCase_					,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['src_vocab_file']	)
										_snake_case						=					os.path.join(
										    lowerCAmelCase_					,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['tgt_vocab_file']	)
										_snake_case						=					os.path.join(
										    lowerCAmelCase_					,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

										with open(lowerCAmelCase_					,    'w'					,    encoding='utf-8'	) as f:
															f.write(json.dumps(self.encoder					,    indent=2					,    sort_keys=lowerCAmelCase_					,    ensure_ascii=lowerCAmelCase_	) + '\n'	)

										with open(lowerCAmelCase_					,    'w'					,    encoding='utf-8'	) as f:
															_snake_case						=					{v: k for k, v in self.decoder.items()}
															f.write(json.dumps(lowerCAmelCase_					,    indent=2					,    sort_keys=lowerCAmelCase_					,    ensure_ascii=lowerCAmelCase_	) + '\n'	)

										_snake_case						=					0
										with open(lowerCAmelCase_					,    'w'					,    encoding='utf-8'	) as writer:
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,    key=lambda lowerCAmelCase_	: kv[1]	):
																				if index != token_index:
																									logger.warning(
																									    F'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
																									    ' Please check that the tokenizer is not corrupted!'	)
																									_snake_case						=					token_index
																				writer.write(' '.join(lowerCAmelCase_	) + '\n'	)
																				index += 1

										return src_vocab_file, tgt_vocab_file, merges_file




					def __getstate__(		self	)		->				Optional[Any]:
										_snake_case						=					self.__dict__.copy()
										_snake_case						=					None
										return state






					def __setstate__(		self					,    lowerCAmelCase_	)		->				int:
										_snake_case						=					d

										try:
															import sacremoses
										except ImportError:
															raise ImportError(
															    'You need to install sacremoses to use XLMTokenizer. '
															    'See https://pypi.org/project/sacremoses/ for installation.'	)

										_snake_case						=					sacremoses










import math
from timeit import timeit





def     lowerCamelCase__  (  UpperCamelCase__      :							int    )      ->			int:



					'''simple docstring'''




					_snake_case						=					0
					_snake_case						=					abs(UpperCamelCase__    )
					while True:
										_snake_case						=					n // 10
										digits += 1
										if n == 0:
															break
					return digits





def     lowerCamelCase__  (  UpperCamelCase__      :							int    )      ->			int:



					'''simple docstring'''




					return 1 if n == 0 else math.floor(math.log(abs(UpperCamelCase__    )							,     10    ) + 1    )





def     lowerCamelCase__  (  UpperCamelCase__      :							int    )      ->			int:



					'''simple docstring'''




					return len(str(abs(UpperCamelCase__    )    )    )





def     lowerCamelCase__  (  )      ->			None:



					'''simple docstring'''




					from collections.abc import Callable

					def benchmark_a_function(UpperCamelCase__      :							Callable							,     UpperCamelCase__      :							int    ) -> None:
										_snake_case						=					F'''{func.__name__}({value})'''
										_snake_case						=					timeit(F'''__main__.{call}'''							,     setup='import __main__'    )
										print(F'''{call}: {func(UpperCamelCase__    )} -- {timing} seconds'''    )

					for value in (262_144, 1_125_899_906_842_624, 1_267_650_600_228_229_401_496_703_205_376):
										for func in (num_digits, num_digits_fast, num_digits_faster):
															benchmark_a_function(UpperCamelCase__							,     UpperCamelCase__    )
										print()


if __name__ == "__main__":
					import doctest

					doctest.testmod()
					benchmark()



