




import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase						:    Union[str, Any]      =  logging.get_logger(__name__)

_UpperCAmelCase						:    Union[str, Any]      =  {
    "src_vocab_file": "vocab-src.json",
    "tgt_vocab_file": "vocab-tgt.json",
    "merges_file": "merges.txt",
}

_UpperCAmelCase						:    Optional[Any]      =  {
    "src_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"
    },
    "tgt_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"
    },
    "merges_file": {"stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"},
}

_UpperCAmelCase						:    Dict      =  {"stas/tiny-wmt19-en-de": 1024}
_UpperCAmelCase						:    List[Any]      =  {
    "stas/tiny-wmt19-en-de": {
        "langs": ["en", "de"],
        "model_max_length": 1024,
        "special_tokens_map_file": None,
        "full_tokenizer_file": None,
    }
}



def    UpperCAmelCase__	(  lowerCamelCase ):
  lowercase     :Tuple   						=    set()
  lowercase     :int   						=    word[0]
  for char in word[1:]:
    pairs.add((prev_char, char) )
    lowercase     :Any   						=    char
  return pairs



def    UpperCAmelCase__	(  lowerCamelCase ):
  lowercase     :Any   						=    text.replace("，",      "," )
  lowercase     :Any   						=    re.sub(r"。\s*",      ". ",      lowerCamelCase )
  lowercase     :Optional[Any]   						=    text.replace("、",      "," )
  lowercase     :int   						=    text.replace("”",      "\"" )
  lowercase     :Optional[int]   						=    text.replace("“",      "\"" )
  lowercase     :int   						=    text.replace("∶",      ":" )
  lowercase     :int   						=    text.replace("：",      ":" )
  lowercase     :Optional[int]   						=    text.replace("？",      "?" )
  lowercase     :Optional[Any]   						=    text.replace("《",      "\"" )
  lowercase     :Any   						=    text.replace("》",      "\"" )
  lowercase     :str   						=    text.replace("）",      ")" )
  lowercase     :Union[str, Any]   						=    text.replace("！",      "!" )
  lowercase     :Optional[int]   						=    text.replace("（",      "(" )
  lowercase     :Dict   						=    text.replace("；",      ";" )
  lowercase     :Optional[Any]   						=    text.replace("１",      "1" )
  lowercase     :Union[str, Any]   						=    text.replace("」",      "\"" )
  lowercase     :Union[str, Any]   						=    text.replace("「",      "\"" )
  lowercase     :Optional[Any]   						=    text.replace("０",      "0" )
  lowercase     :List[str]   						=    text.replace("３",      "3" )
  lowercase     :List[str]   						=    text.replace("２",      "2" )
  lowercase     :List[str]   						=    text.replace("５",      "5" )
  lowercase     :List[str]   						=    text.replace("６",      "6" )
  lowercase     :Optional[Any]   						=    text.replace("９",      "9" )
  lowercase     :Union[str, Any]   						=    text.replace("７",      "7" )
  lowercase     :Union[str, Any]   						=    text.replace("８",      "8" )
  lowercase     :Union[str, Any]   						=    text.replace("４",      "4" )
  lowercase     :Union[str, Any]   						=    re.sub(r"．\s*",      ". ",      lowerCamelCase )
  lowercase     :Optional[Any]   						=    text.replace("～",      "~" )
  lowercase     :str   						=    text.replace("’",      "'" )
  lowercase     :List[str]   						=    text.replace("…",      "..." )
  lowercase     :int   						=    text.replace("━",      "-" )
  lowercase     :Any   						=    text.replace("〈",      "<" )
  lowercase     :Optional[Any]   						=    text.replace("〉",      ">" )
  lowercase     :str   						=    text.replace("【",      "[" )
  lowercase     :Any   						=    text.replace("】",      "]" )
  lowercase     :Optional[Any]   						=    text.replace("％",      "%" )
  return text



def    UpperCAmelCase__	(  lowerCamelCase ):
  lowercase     :Any   						=    []
  for char in text:
    lowercase     :Any   						=    unicodedata.category(lowerCamelCase )
    if cat.startswith("C" ):
      continue
    output.append(lowerCamelCase )
  return "".join(lowerCamelCase )







class  __lowerCAmelCase							(				lowerCAmelCase):
       _a			       =    VOCAB_FILES_NAMES
       _a			       =    PRETRAINED_VOCAB_FILES_MAP
       _a			       =    PRETRAINED_INIT_CONFIGURATION
       _a			       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _a			       =    ['''input_ids''', '''attention_mask''']






       def __init__( self:       List[str]							,       _lowerCAmelCase:       str=None							,       _lowerCAmelCase:       Union[str, Any]=None							,       _lowerCAmelCase:       List[Any]=None							,       _lowerCAmelCase:       List[Any]=None							,       _lowerCAmelCase:       Any=False							,       _lowerCAmelCase:       Any="<unk>"							,       _lowerCAmelCase:       Any="<s>"							,       _lowerCAmelCase:       Optional[int]="</s>"							,       _lowerCAmelCase:       Optional[Any]="<pad>"							,       **_lowerCAmelCase:       int							,       ):
         super().__init__(
             langs=_lowerCAmelCase							,       src_vocab_file=_lowerCAmelCase							,       tgt_vocab_file=_lowerCAmelCase							,       merges_file=_lowerCAmelCase							,       do_lower_case=_lowerCAmelCase							,       unk_token=_lowerCAmelCase							,       bos_token=_lowerCAmelCase							,       sep_token=_lowerCAmelCase							,       pad_token=_lowerCAmelCase							,       **_lowerCAmelCase							,       )

         try:
           import sacremoses
         except ImportError:
           raise ImportError(
               "You need to install sacremoses to use XLMTokenizer. "
               "See https://pypi.org/project/sacremoses/ for installation."					)

         lowercase     :List[Any]   						=    sacremoses

         lowercase     :Union[str, Any]   						=    src_vocab_file
         lowercase     :List[str]   						=    tgt_vocab_file
         lowercase     :int   						=    merges_file
         lowercase     :Tuple   						=    do_lower_case

         # cache of sm.MosesPunctNormalizer instance
         lowercase     :str   						=    {}
         # cache of sm.MosesTokenizer instance
         lowercase     :int   						=    {}
         lowercase     :int   						=    {}

         if langs and len(_lowerCAmelCase					) == 2:
           lowercase						,     lowercase     :str   						=    langs
         else:
           raise ValueError(
               F"arg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got {langs}. "
               "Usually that means that tokenizer can't find a mapping for the given model path "
               "in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."					)

         with open(_lowerCAmelCase							,       encoding="utf-8"					) as src_vocab_handle:
           lowercase     :Any   						=    json.load(_lowerCAmelCase					)
         with open(_lowerCAmelCase							,       encoding="utf-8"					) as tgt_vocab_handle:
           lowercase     :Optional[Any]   						=    json.load(_lowerCAmelCase					)
           lowercase     :Optional[int]   						=    {v: k for k, v in tgt_vocab.items()}
         with open(_lowerCAmelCase							,       encoding="utf-8"					) as merges_handle:
           lowercase     :Optional[Any]   						=    merges_handle.read().split("\n"					)[:-1]
         lowercase     :Any   						=    [tuple(merge.split()[:2]					) for merge in merges]
         lowercase     :Tuple   						=    dict(zip(_lowerCAmelCase							,       range(len(_lowerCAmelCase					)					)					)					)
         lowercase     :Optional[int]   						=    {}






       def        SCREAMING_SNAKE_CASE     ( self:       Optional[int]					):
         return self.get_src_vocab()






       @property
       def        SCREAMING_SNAKE_CASE     ( self:       str					):
         return self.src_vocab_size






       def        SCREAMING_SNAKE_CASE     ( self:       Any							,       _lowerCAmelCase:       Any							,       _lowerCAmelCase:       Union[str, Any]					):
         if lang not in self.cache_moses_punct_normalizer:
           lowercase     :Any   						=    self.sm.MosesPunctNormalizer(lang=_lowerCAmelCase					)
           lowercase     :Optional[int]   						=    punct_normalizer
         return self.cache_moses_punct_normalizer[lang].normalize(_lowerCAmelCase					)






       def        SCREAMING_SNAKE_CASE     ( self:       Optional[int]							,       _lowerCAmelCase:       Tuple							,       _lowerCAmelCase:       List[Any]					):
         if lang not in self.cache_moses_tokenizer:
           lowercase     :Optional[int]   						=    self.sm.MosesTokenizer(lang=_lowerCAmelCase					)
           lowercase     :Tuple   						=    moses_tokenizer
         return self.cache_moses_tokenizer[lang].tokenize(
             _lowerCAmelCase							,       aggressive_dash_splits=_lowerCAmelCase							,       return_str=_lowerCAmelCase							,       escape=_lowerCAmelCase					)






       def        SCREAMING_SNAKE_CASE     ( self:       Union[str, Any]							,       _lowerCAmelCase:       Union[str, Any]							,       _lowerCAmelCase:       int					):
         if lang not in self.cache_moses_detokenizer:
           lowercase     :Dict   						=    self.sm.MosesDetokenizer(lang=_lowerCAmelCase					)
           lowercase     :Any   						=    moses_detokenizer
         return self.cache_moses_detokenizer[lang].detokenize(_lowerCAmelCase					)






       def        SCREAMING_SNAKE_CASE     ( self:       int							,       _lowerCAmelCase:       List[str]							,       _lowerCAmelCase:       Tuple					):
         lowercase     :int   						=    replace_unicode_punct(_lowerCAmelCase					)
         lowercase     :Any   						=    self.moses_punct_norm(_lowerCAmelCase							,       _lowerCAmelCase					)
         lowercase     :Union[str, Any]   						=    remove_non_printing_char(_lowerCAmelCase					)
         return text






       @property
       def        SCREAMING_SNAKE_CASE     ( self:       Optional[Any]					):
         return len(self.encoder					)






       @property
       def        SCREAMING_SNAKE_CASE     ( self:       Union[str, Any]					):
         return len(self.decoder					)






       def        SCREAMING_SNAKE_CASE     ( self:       List[str]					):
         return dict(self.encoder							,       **self.added_tokens_encoder					)






       def        SCREAMING_SNAKE_CASE     ( self:       Optional[int]					):
         return dict(self.decoder							,       **self.added_tokens_decoder					)






       def        SCREAMING_SNAKE_CASE     ( self:       Optional[Any]							,       _lowerCAmelCase:       Union[str, Any]					):
         lowercase     :str   						=    tuple(token[:-1]					) + (token[-1] + "</w>",)
         if token in self.cache:
           return self.cache[token]
         lowercase     :int   						=    get_pairs(_lowerCAmelCase					)

         if not pairs:
           return token + "</w>"

         while True:
           lowercase     :Dict   						=    min(_lowerCAmelCase							,       key=lambda _lowerCAmelCase					: self.bpe_ranks.get(_lowerCAmelCase							,       float("inf"					)					)					)
           if bigram not in self.bpe_ranks:
             break
           lowercase						,     lowercase     :List[str]   						=    bigram
           lowercase     :Dict   						=    []
           lowercase     :str   						=    0
           while i < len(_lowerCAmelCase					):
             try:
               lowercase     :str   						=    word.index(_lowerCAmelCase							,       _lowerCAmelCase					)
             except ValueError:
               new_word.extend(word[i:]					)
               break
             else:
               new_word.extend(word[i:j]					)
               lowercase     :Tuple   						=    j

             if word[i] == first and i < len(_lowerCAmelCase					) - 1 and word[i + 1] == second:
               new_word.append(first + second					)
               i += 2
             else:
               new_word.append(word[i]					)
               i += 1
           lowercase     :Optional[int]   						=    tuple(_lowerCAmelCase					)
           lowercase     :List[Any]   						=    new_word
           if len(_lowerCAmelCase					) == 1:
             break
           else:
             lowercase     :List[str]   						=    get_pairs(_lowerCAmelCase					)
         lowercase     :int   						=    " ".join(_lowerCAmelCase					)
         if word == "\n  </w>":
           lowercase     :List[Any]   						=    "\n</w>"
         lowercase     :Tuple   						=    word
         return word






       def        SCREAMING_SNAKE_CASE     ( self:       List[str]							,       _lowerCAmelCase:       str							,       _lowerCAmelCase:       str="en"							,       _lowerCAmelCase:       int=False					):
         lowercase     :List[str]   						=    self.src_lang

         if self.do_lower_case:
           lowercase     :Any   						=    text.lower()

         if bypass_tokenizer:
           lowercase     :Any   						=    text.split()
         else:
           lowercase     :Tuple   						=    self.moses_pipeline(_lowerCAmelCase							,       lang=_lowerCAmelCase					)
           lowercase     :Tuple   						=    self.moses_tokenize(_lowerCAmelCase							,       lang=_lowerCAmelCase					)

         lowercase     :List[Any]   						=    []
         for token in text:
           if token:
             split_tokens.extend(list(self.bpe(_lowerCAmelCase					).split(" "					)					)					)

         return split_tokens






       def        SCREAMING_SNAKE_CASE     ( self:       Any							,       _lowerCAmelCase:       int					):
         return self.encoder.get(_lowerCAmelCase							,       self.encoder.get(self.unk_token					)					)






       def        SCREAMING_SNAKE_CASE     ( self:       Tuple							,       _lowerCAmelCase:       Optional[Any]					):
         return self.decoder.get(_lowerCAmelCase							,       self.unk_token					)






       def        SCREAMING_SNAKE_CASE     ( self:       Optional[int]							,       _lowerCAmelCase:       Tuple					):
         lowercase     :int   						=    [t.replace(" "							,       ""					).replace("</w>"							,       " "					) for t in tokens]
         lowercase     :Union[str, Any]   						=    "".join(_lowerCAmelCase					).split()
         # detokenize
         lowercase     :Dict   						=    self.moses_detokenize(_lowerCAmelCase							,       self.tgt_lang					)
         return text






       def        SCREAMING_SNAKE_CASE     ( self:       Optional[Any]							,       _lowerCAmelCase:       List[int]							,       _lowerCAmelCase:       Optional[List[int]] = None					):
         lowercase     :Optional[int]   						=    [self.sep_token_id]

         # no bos used in fairseq
         if token_ids_a is None:
           return token_ids_a + sep
         return token_ids_a + sep + token_ids_a + sep






       def        SCREAMING_SNAKE_CASE     ( self:       Union[str, Any]							,       _lowerCAmelCase:       List[int]							,       _lowerCAmelCase:       Optional[List[int]] = None							,       _lowerCAmelCase:       bool = False					):

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_lowerCAmelCase							,       token_ids_a=_lowerCAmelCase							,       already_has_special_tokens=_lowerCAmelCase					)
         # no bos used in fairseq
         if token_ids_a is not None:
           return ([0] * len(_lowerCAmelCase					)) + [1] + ([0] * len(_lowerCAmelCase					)) + [1]
         return ([0] * len(_lowerCAmelCase					)) + [1]






       def        SCREAMING_SNAKE_CASE     ( self:       Optional[int]							,       _lowerCAmelCase:       List[int]							,       _lowerCAmelCase:       Optional[List[int]] = None					):
         lowercase     :str   						=    [self.sep_token_id]

         # no bos used in fairseq
         if token_ids_a is None:
           return len(token_ids_a + sep					) * [0]
         return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






       def        SCREAMING_SNAKE_CASE     ( self:       List[Any]							,       _lowerCAmelCase:       str							,       _lowerCAmelCase:       Optional[str] = None					):
         if not os.path.isdir(_lowerCAmelCase					):
           logger.error(F"Vocabulary path ({save_directory}) should be a directory"					)
           return

         lowercase     :Optional[int]   						=    os.path.join(
             _lowerCAmelCase							,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["src_vocab_file"]					)
         lowercase     :Any   						=    os.path.join(
             _lowerCAmelCase							,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["tgt_vocab_file"]					)
         lowercase     :Optional[int]   						=    os.path.join(
             _lowerCAmelCase							,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]					)

         with open(_lowerCAmelCase							,       "w"							,       encoding="utf-8"					) as f:
           f.write(json.dumps(self.encoder							,       indent=2							,       sort_keys=_lowerCAmelCase							,       ensure_ascii=_lowerCAmelCase					) + "\n"					)

         with open(_lowerCAmelCase							,       "w"							,       encoding="utf-8"					) as f:
           lowercase     :Optional[int]   						=    {v: k for k, v in self.decoder.items()}
           f.write(json.dumps(_lowerCAmelCase							,       indent=2							,       sort_keys=_lowerCAmelCase							,       ensure_ascii=_lowerCAmelCase					) + "\n"					)

         lowercase     :Tuple   						=    0
         with open(_lowerCAmelCase							,       "w"							,       encoding="utf-8"					) as writer:
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,       key=lambda _lowerCAmelCase					: kv[1]					):
             if index != token_index:
               logger.warning(
                   F"Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."
                   " Please check that the tokenizer is not corrupted!"					)
               lowercase     :Union[str, Any]   						=    token_index
             writer.write(" ".join(_lowerCAmelCase					) + "\n"					)
             index += 1

         return src_vocab_file, tgt_vocab_file, merges_file






       def __getstate__( self:       int					):
         lowercase     :Dict   						=    self.__dict__.copy()
         lowercase     :str   						=    None
         return state






       def __setstate__( self:       str							,       _lowerCAmelCase:       Dict					):
         lowercase     :Any   						=    d

         try:
           import sacremoses
         except ImportError:
           raise ImportError(
               "You need to install sacremoses to use XLMTokenizer. "
               "See https://pypi.org/project/sacremoses/ for installation."					)

         lowercase     :List[str]   						=    sacremoses









import decimal



def    UpperCAmelCase__	(  ):
  lowercase     :int   						=    0
  lowercase     :str   						=    decimal.Context(prec=105 )
  for i in range(2,      100 ):
    lowercase     :int   						=    decimal.Decimal(lowerCamelCase )
    lowercase     :str   						=    number.sqrt(lowerCamelCase )
    if len(str(lowerCamelCase ) ) > 1:
      answer += int(str(lowerCamelCase )[0] )
      lowercase     :int   						=    str(lowerCamelCase )[2:101]
      answer += sum(int(lowerCamelCase ) for x in sqrt_number_str )
  return answer


if __name__ == "__main__":
     import doctest

     doctest.testmod()
     print(f'''{solution() = }''')



