


'''simple docstring'''

import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_  :				Union[str, Any]						= logging.get_logger(__name__)

a_  :				List[Any]						= """▁"""

a_  :				List[Any]						= {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

a_  :				Optional[int]						= {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

a_  :				Tuple						= {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

a_  :				Dict						= {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

a_  :				Any						= {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}





class      snake_case    (    lowercase			):





  """simple docstring"""




  _lowerCamelCase		    =  ["input_ids"]

  _lowerCamelCase		    =  VOCAB_FILES_NAMES
  _lowerCamelCase		    =  PRETRAINED_INIT_CONFIGURATION
  _lowerCamelCase		    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase		    =  PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase		    =  RESOURCE_FILES_NAMES






  def __init__(    self			,  UpperCamelCase			,  UpperCamelCase=None			,  UpperCamelCase=False			,  UpperCamelCase="utf8"			,  UpperCamelCase="[UNK]"			,  UpperCamelCase="[SEP]"			,  UpperCamelCase="[PAD]"			,  UpperCamelCase="[CLS]"			,  UpperCamelCase="[MASK]"			,  UpperCamelCase = None			,  **UpperCamelCase			,  ):







   """simple docstring"""
   # Mask token behave like a normal word, i.e. include the space before it and
   # is included in the raw text, there should be a match in a non-normalized sentence.

   lowerCamelCase_      =  {} if sp_model_kwargs is None else sp_model_kwargs
   super().__init__(
       do_lower_case=UpperCamelCase			,  unk_token=UpperCamelCase			,  sep_token=UpperCamelCase			,  pad_token=UpperCamelCase			,  cls_token=UpperCamelCase			,  mask_token=UpperCamelCase			,  vocab_file=UpperCamelCase			,  encoding=UpperCamelCase			,  sp_model_kwargs=self.sp_model_kwargs			,  **UpperCamelCase			,  )
   lowerCamelCase_      =  do_lower_case
   lowerCamelCase_      =  sentencepiece_model_ckpt
   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(UpperCamelCase     )

   # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
   if vocab_file is not None:
    lowerCamelCase_      =  self.load_vocab(filepath=UpperCamelCase     )
   else:
    lowerCamelCase_      =  {self.sp_model.id_to_piece(UpperCamelCase     ): id for id in range(self.sp_model.get_piece_size()     )}
   lowerCamelCase_      =  {v: k for k, v in self.vocab.items()}






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if text is None:
    return None

   lowerCamelCase_      =  self.tokenize(UpperCamelCase     )
   lowerCamelCase_ ,lowerCamelCase_      =  "", []

   for i, ch in enumerate(UpperCamelCase     ):
    if ch in self.SP_CHAR_MAPPING:
     lowerCamelCase_      =  self.SP_CHAR_MAPPING.get(UpperCamelCase     )
    else:
     lowerCamelCase_      =  unicodedata.normalize("NFKC"			,  UpperCamelCase     )
    if self.is_whitespace(UpperCamelCase     ):
     continue
    normalized_text += ch
    char_mapping.extend([i] * len(UpperCamelCase     )     )

   lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_      =  normalized_text, [], 0

   if self.do_lower_case:
    lowerCamelCase_      =  text.lower()

   for token in split_tokens:
    if token[:1] == "▁":
     lowerCamelCase_      =  token[1:]
    lowerCamelCase_      =  text[offset:].index(UpperCamelCase     ) + offset
    lowerCamelCase_      =  start + len(UpperCamelCase     )

    token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)     )
    lowerCamelCase_      =  end
   return token_mapping






  @property
  def   snake_case		(    self     ):







   """simple docstring"""
   return len(self.vocab     )






  def   snake_case		(    self     ):







   """simple docstring"""
   return dict(self.vocab			,  **self.added_tokens_encoder     )






  def __getstate__(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  self.__dict__.copy()
   lowerCamelCase_      =  None
   return state






  def __setstate__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  d

   # for backward compatibility
   if not hasattr(self			,  "sp_model_kwargs"     ):
    lowerCamelCase_      =  {}

   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(self.sentencepiece_model_ckpt     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase			,  UpperCamelCase     ) for c in text)     )






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase=False			,  UpperCamelCase=64			,  UpperCamelCase=0.1     ):







   """simple docstring"""

   if self.sp_model_kwargs.get("enable_sampling"     ) is True:
    lowerCamelCase_      =  True
   if self.sp_model_kwargs.get("alpha"     ) is not None:
    lowerCamelCase_      =  self.sp_model_kwargs.get("alpha"     )
   if self.sp_model_kwargs.get("nbest_size"     ) is not None:
    lowerCamelCase_      =  self.sp_model_kwargs.get("nbest_size"     )

   if not enable_sampling:
    lowerCamelCase_      =  self.sp_model.EncodeAsPieces(UpperCamelCase     )
   else:
    lowerCamelCase_      =  self.sp_model.SampleEncodeAsPieces(UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase     )
   lowerCamelCase_      =  []
   for pi, piece in enumerate(UpperCamelCase     ):
    if piece == SPIECE_UNDERLINE:
     if not pieces[pi + 1].startswith(UpperCamelCase     ) and pi != 0:
      new_pieces.append(UpperCamelCase     )
      continue
     else:
      continue
    lowerCamelCase_      =  0
    for i, chunk in enumerate(UpperCamelCase     ):
     if chunk == SPIECE_UNDERLINE:
      continue
     if self.is_ch_char(UpperCamelCase     ) or self.is_punct(UpperCamelCase     ):
      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
       new_pieces.append(piece[lst_i:i]     )
      new_pieces.append(UpperCamelCase     )
      lowerCamelCase_      =  i + 1
     elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
       new_pieces.append(piece[lst_i:i]     )
      lowerCamelCase_      =  i
     elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
       new_pieces.append(piece[lst_i:i]     )
      lowerCamelCase_      =  i
    if len(UpperCamelCase     ) > lst_i:
     new_pieces.append(piece[lst_i:]     )
   return new_pieces






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  "".join(UpperCamelCase     ).replace(UpperCamelCase			,  " "     ).strip()
   return out_string






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.convert_ids_to_tokens(UpperCamelCase     )
   lowerCamelCase_      =  "".join(UpperCamelCase     ).replace(UpperCamelCase			,  " "     ).strip()
   return out_string






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.vocab.get(UpperCamelCase			,  self.vocab.get(self.unk_token     )     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.reverse_vocab.get(UpperCamelCase			,  self.unk_token     )






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase=None     ):







   """simple docstring"""
   if token_ids_a is None:
    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
   lowerCamelCase_      =  [self.cls_token_id]
   lowerCamelCase_      =  [self.sep_token_id]
   return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase=None     ):







   """simple docstring"""
   if offset_mapping_a is None:
    return [(0, 0)] + offset_mapping_a + [(0, 0)]

   return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase=None			,  UpperCamelCase=False     ):







   """simple docstring"""

   if already_has_special_tokens:
    if token_ids_a is not None:
     raise ValueError(
         "You should not supply a second sequence if the provided sequence of "
         "ids is already formatted with special tokens for the model."     )
    return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

   if token_ids_a is not None:
    return [1] + ([0] * len(UpperCamelCase     )) + [1, 1] + ([0] * len(UpperCamelCase     )) + [1]
   return [1] + ([0] * len(UpperCamelCase     )) + [1]






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
   if token_ids_a is None:
    # [CLS] X [SEP]
    return (len(UpperCamelCase     ) + 2) * [0]

   # [CLS] A [SEP] [SEP] B [SEP]
   return [0] * (len(UpperCamelCase     ) + 1) + [1] * (len(UpperCamelCase     ) + 3)






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if "\u4e00" <= char <= "\u9fff":
    return True
   return False






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if ("a" <= char <= "z") or ("A" <= char <= "Z"):
    return True
   return False






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if char in ",;:.?!~，；：。？！《》【】":
    return True
   return False






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if char == " " or char == "\t" or char == "\n" or char == "\r":
    return True
   if len(UpperCamelCase     ) == 1:
    lowerCamelCase_      =  unicodedata.category(UpperCamelCase     )
    if cat == "Zs":
     return True
   return False






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  {}
   with io.open(UpperCamelCase			,  "r"			,  encoding="utf-8"     ) as f:
    for index, line in enumerate(UpperCamelCase     ):
     lowerCamelCase_      =  line.rstrip("\n"     )
     lowerCamelCase_      =  int(UpperCamelCase     )

   return token_to_idx







  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  0
   if os.path.isdir(UpperCamelCase     ):
    lowerCamelCase_      =  os.path.join(
        UpperCamelCase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )
   else:
    lowerCamelCase_      =  (filename_prefix + "-" if filename_prefix else "") + save_directory
   with open(UpperCamelCase			,  "w"			,  encoding="utf-8"     ) as writer:
    for token, token_index in sorted(self.vocab.items()			,  key=lambda UpperCamelCase     : kv[1]     ):
     if index != token_index:
      logger.warning(
          f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
          " Please check that the vocabulary is not corrupted!"     )
      lowerCamelCase_      =  token_index
     writer.write(token + "\n"     )
     index += 1

   lowerCamelCase_      =  os.path.join(UpperCamelCase			,  "sentencepiece.bpe.model"     )
   with open(UpperCamelCase			,  "wb"     ) as fi:
    lowerCamelCase_      =  self.sp_model.serialized_model_proto()
    fi.write(UpperCamelCase     )

   return (vocab_file,)







'''simple docstring'''

import math





def __snake_case					(      UpperCAmelCase_						:						float					,				UpperCAmelCase_						:						float		):
 return math.pow(UpperCAmelCase_					,				2		) - a





def __snake_case					(      UpperCAmelCase_						:						float		):
 return 2 * x





def __snake_case					(      UpperCAmelCase_						:						float		):
 lowerCamelCase_      =  2.0

 while start <= a:
  lowerCamelCase_      =  math.pow(UpperCAmelCase_					,				2		)

 return start





def __snake_case					(      UpperCAmelCase_						:						float					,				UpperCAmelCase_						:						int = 9999					,				UpperCAmelCase_						:						float = 0.00_0000_0000_0001		):

 if a < 0:
  raise ValueError("math domain error"		)

 lowerCamelCase_      =  get_initial_point(UpperCAmelCase_		)

 for _ in range(UpperCAmelCase_		):
  lowerCamelCase_      =  value
  lowerCamelCase_      =  value - fx(UpperCAmelCase_					,				UpperCAmelCase_		) / fx_derivative(UpperCAmelCase_		)
  if abs(prev_value - value		) < tolerance:
   return value

 return value


if __name__ == "__main__":
     from doctest import testmod

     testmod()



