




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_									=	logging.get_logger(__name__)

UpperCAmelCase_									=	'▁'

UpperCAmelCase_									=	{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

UpperCAmelCase_									=	{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

UpperCAmelCase_									=	{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

UpperCAmelCase_									=	{
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

UpperCAmelCase_									=	{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}





class     lowerCamelCase__(					__lowerCamelCase):
      UpperCAmelCase__      :	List[str]    		=   ["input_ids"]

      UpperCAmelCase__      :	Union[str, Any]    		=   VOCAB_FILES_NAMES
      UpperCAmelCase__      :	Dict    		=   PRETRAINED_INIT_CONFIGURATION
      UpperCAmelCase__      :	int    		=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__      :	List[Any]    		=   PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__      :	int    		=   RESOURCE_FILES_NAMES
      def __init__(				self: Union[str, Any]    ,       UpperCamelCase_: Any    ,       UpperCamelCase_: Union[str, Any]=None    ,       UpperCamelCase_: str=False    ,       UpperCamelCase_: Union[str, Any]="utf8"    ,       UpperCamelCase_: int="[UNK]"    ,       UpperCamelCase_: Union[str, Any]="[SEP]"    ,       UpperCamelCase_: Optional[Any]="[PAD]"    ,       UpperCamelCase_: Dict="[CLS]"    ,       UpperCamelCase_: Any="[MASK]"    ,       UpperCamelCase_: Optional[Dict[str, Any]] = None    ,       **UpperCamelCase_: Union[str, Any]    ,       ):
             # Mask token behave like a normal word, i.e. include the space before it and
             # is included in the raw text, there should be a match in a non-normalized sentence.

             __lowerCamelCase   =						{} if sp_model_kwargs is None else sp_model_kwargs
             super().__init__(
                 do_lower_case=UpperCamelCase_    ,       unk_token=UpperCamelCase_    ,       sep_token=UpperCamelCase_    ,       pad_token=UpperCamelCase_    ,       cls_token=UpperCamelCase_    ,       mask_token=UpperCamelCase_    ,       vocab_file=UpperCamelCase_    ,       encoding=UpperCamelCase_    ,       sp_model_kwargs=self.sp_model_kwargs    ,       **UpperCamelCase_    ,       )
             __lowerCamelCase   =						do_lower_case
             __lowerCamelCase   =						sentencepiece_model_ckpt
             __lowerCamelCase   =						spm.SentencePieceProcessor(**self.sp_model_kwargs   )
             self.sp_model.Load(UpperCamelCase_   )

             # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
             if vocab_file is not None:
                    __lowerCamelCase   =						self.load_vocab(filepath=UpperCamelCase_   )
             else:
                    __lowerCamelCase   =						{self.sp_model.id_to_piece(UpperCamelCase_   ): id for id in range(self.sp_model.get_piece_size()   )}
             __lowerCamelCase   =						{v: k for k, v in self.vocab.items()}
      def 			lowerCAmelCase__						(				self: List[str]    ,       UpperCamelCase_: Optional[int]   ):
             if text is None:
                    return None

             __lowerCamelCase   =						self.tokenize(UpperCamelCase_   )
             __lowerCamelCase,   __lowerCamelCase   =						"""""", []

             for i, ch in enumerate(UpperCamelCase_   ):
                    if ch in self.SP_CHAR_MAPPING:
                           __lowerCamelCase   =						self.SP_CHAR_MAPPING.get(UpperCamelCase_   )
                    else:
                           __lowerCamelCase   =						unicodedata.normalize("""NFKC"""    ,       UpperCamelCase_   )
                    if self.is_whitespace(UpperCamelCase_   ):
                           continue
                    normalized_text += ch
                    char_mapping.extend([i] * len(UpperCamelCase_   )   )

             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						normalized_text, [], 0

             if self.do_lower_case:
                    __lowerCamelCase   =						text.lower()

             for token in split_tokens:
                    if token[:1] == "▁":
                           __lowerCamelCase   =						token[1:]
                    __lowerCamelCase   =						text[offset:].index(UpperCamelCase_   ) + offset
                    __lowerCamelCase   =						start + len(UpperCamelCase_   )

                    token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)   )
                    __lowerCamelCase   =						end
             return token_mapping
      @property
      def 			lowerCAmelCase__						(				self: Optional[int]   ):
             return len(self.vocab   )
      def 			lowerCAmelCase__						(				self: List[str]   ):
             return dict(self.vocab    ,       **self.added_tokens_encoder   )
      def __getstate__(				self: List[Any]   ):
             __lowerCamelCase   =						self.__dict__.copy()
             __lowerCamelCase   =						None
             return state
      def __setstate__(				self: str    ,       UpperCamelCase_: Optional[Any]   ):
             __lowerCamelCase   =						d

             # for backward compatibility
             if not hasattr(self    ,       """sp_model_kwargs"""   ):
                    __lowerCamelCase   =						{}

             __lowerCamelCase   =						spm.SentencePieceProcessor(**self.sp_model_kwargs   )
             self.sp_model.Load(self.sentencepiece_model_ckpt   )
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: List[Any]   ):
             return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase_    ,       UpperCamelCase_   ) for c in text)   )
      def 			lowerCAmelCase__						(				self: Union[str, Any]    ,       UpperCamelCase_: Tuple    ,       UpperCamelCase_: Any=False    ,       UpperCamelCase_: Union[str, Any]=64    ,       UpperCamelCase_: Union[str, Any]=0.1   ):

             if self.sp_model_kwargs.get("""enable_sampling"""   ) is True:
                    __lowerCamelCase   =						True
             if self.sp_model_kwargs.get("""alpha"""   ) is not None:
                    __lowerCamelCase   =						self.sp_model_kwargs.get("""alpha"""   )
             if self.sp_model_kwargs.get("""nbest_size"""   ) is not None:
                    __lowerCamelCase   =						self.sp_model_kwargs.get("""nbest_size"""   )

             if not enable_sampling:
                    __lowerCamelCase   =						self.sp_model.EncodeAsPieces(UpperCamelCase_   )
             else:
                    __lowerCamelCase   =						self.sp_model.SampleEncodeAsPieces(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_   )
             __lowerCamelCase   =						[]
             for pi, piece in enumerate(UpperCamelCase_   ):
                    if piece == SPIECE_UNDERLINE:
                           if not pieces[pi + 1].startswith(UpperCamelCase_   ) and pi != 0:
                                  new_pieces.append(UpperCamelCase_   )
                                  continue
                           else:
                                  continue
                    __lowerCamelCase   =						0
                    for i, chunk in enumerate(UpperCamelCase_   ):
                           if chunk == SPIECE_UNDERLINE:
                                  continue
                           if self.is_ch_char(UpperCamelCase_   ) or self.is_punct(UpperCamelCase_   ):
                                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                         new_pieces.append(piece[lst_i:i]   )
                                  new_pieces.append(UpperCamelCase_   )
                                  __lowerCamelCase   =						i + 1
                           elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                         new_pieces.append(piece[lst_i:i]   )
                                  __lowerCamelCase   =						i
                           elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                         new_pieces.append(piece[lst_i:i]   )
                                  __lowerCamelCase   =						i
                    if len(UpperCamelCase_   ) > lst_i:
                           new_pieces.append(piece[lst_i:]   )
             return new_pieces
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: Any   ):
             __lowerCamelCase   =						"""""".join(UpperCamelCase_   ).replace(UpperCamelCase_    ,       """ """   ).strip()
             return out_string
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: Tuple   ):
             __lowerCamelCase   =						self.convert_ids_to_tokens(UpperCamelCase_   )
             __lowerCamelCase   =						"""""".join(UpperCamelCase_   ).replace(UpperCamelCase_    ,       """ """   ).strip()
             return out_string
      def 			lowerCAmelCase__						(				self: List[str]    ,       UpperCamelCase_: Optional[Any]   ):
             return self.vocab.get(UpperCamelCase_    ,       self.vocab.get(self.unk_token   )   )
      def 			lowerCAmelCase__						(				self: Optional[int]    ,       UpperCamelCase_: str   ):
             return self.reverse_vocab.get(UpperCamelCase_    ,       self.unk_token   )
      def 			lowerCAmelCase__						(				self: str    ,       UpperCamelCase_: Any    ,       UpperCamelCase_: str=None   ):
             if token_ids_a is None:
                    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             __lowerCamelCase   =						[self.cls_token_id]
             __lowerCamelCase   =						[self.sep_token_id]
             return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
      def 			lowerCAmelCase__						(				self: Any    ,       UpperCamelCase_: int    ,       UpperCamelCase_: List[str]=None   ):
             if offset_mapping_a is None:
                    return [(0, 0)] + offset_mapping_a + [(0, 0)]

             return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
      def 			lowerCAmelCase__						(				self: Dict    ,       UpperCamelCase_: List[str]    ,       UpperCamelCase_: Tuple=None    ,       UpperCamelCase_: List[str]=False   ):

             if already_has_special_tokens:
                    if token_ids_a is not None:
                           raise ValueError(
                               """You should not supply a second sequence if the provided sequence of """
                               """ids is already formatted with special tokens for the model."""   )
                    return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

             if token_ids_a is not None:
                    return [1] + ([0] * len(UpperCamelCase_   )) + [1, 1] + ([0] * len(UpperCamelCase_   )) + [1]
             return [1] + ([0] * len(UpperCamelCase_   )) + [1]
      def 			lowerCAmelCase__						(				self: Optional[int]    ,       UpperCamelCase_: List[int]    ,       UpperCamelCase_: Optional[List[int]] = None   ):
             # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
             if token_ids_a is None:
                    # [CLS] X [SEP]
                    return (len(UpperCamelCase_   ) + 2) * [0]

             # [CLS] A [SEP] [SEP] B [SEP]
             return [0] * (len(UpperCamelCase_   ) + 1) + [1] * (len(UpperCamelCase_   ) + 3)
      def 			lowerCAmelCase__						(				self: Union[str, Any]    ,       UpperCamelCase_: List[str]   ):
             if "\u4e00" <= char <= "\u9fff":
                    return True
             return False
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: List[str]   ):
             if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                    return True
             return False
      def 			lowerCAmelCase__						(				self: Dict    ,       UpperCamelCase_: List[str]   ):
             if char in ",;:.?!~，；：。？！《》【】":
                    return True
             return False
      def 			lowerCAmelCase__						(				self: Dict    ,       UpperCamelCase_: Union[str, Any]   ):
             if char == " " or char == "\t" or char == "\n" or char == "\r":
                    return True
             if len(UpperCamelCase_   ) == 1:
                    __lowerCamelCase   =						unicodedata.category(UpperCamelCase_   )
                    if cat == "Zs":
                           return True
             return False
      def 			lowerCAmelCase__						(				self: Optional[int]    ,       UpperCamelCase_: int   ):
             __lowerCamelCase   =						{}
             with io.open(UpperCamelCase_    ,       """r"""    ,       encoding="""utf-8"""   ) as f:
                    for index, line in enumerate(UpperCamelCase_   ):
                           __lowerCamelCase   =						line.rstrip("""\n"""   )
                           __lowerCamelCase   =						int(UpperCamelCase_   )

             return token_to_idx




      def 			lowerCAmelCase__						(				self: List[str]    ,       UpperCamelCase_: str    ,       UpperCamelCase_: Optional[str] = None   ):
             __lowerCamelCase   =						0
             if os.path.isdir(UpperCamelCase_   ):
                    __lowerCamelCase   =						os.path.join(
                        UpperCamelCase_    ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
             else:
                    __lowerCamelCase   =						(filename_prefix + """-""" if filename_prefix else """""") + save_directory
             with open(UpperCamelCase_    ,       """w"""    ,       encoding="""utf-8"""   ) as writer:
                    for token, token_index in sorted(self.vocab.items()    ,       key=lambda UpperCamelCase_   : kv[1]   ):
                           if index != token_index:
                                  logger.warning(
                                      F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                                      """ Please check that the vocabulary is not corrupted!"""   )
                                  __lowerCamelCase   =						token_index
                           writer.write(token + """\n"""   )
                           index += 1

             __lowerCamelCase   =						os.path.join(UpperCamelCase_    ,       """sentencepiece.bpe.model"""   )
             with open(UpperCamelCase_    ,       """wb"""   ) as fi:
                    __lowerCamelCase   =						self.sp_model.serialized_model_proto()
                    fi.write(UpperCamelCase_   )

             return (vocab_file,)


'''simple docstring'''





from transformers import DistilBertTokenizer, DistilBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest




@require_tokenizers
class        __A      ( UpperCamelCase__	):
    a__       :						Optional[Any]          =  DistilBertTokenizer
    a__       :						Any          =  DistilBertTokenizerFast
    a__       :						str          =  True



    @slow
    def 					_lowercase						(self    :    int ):
         UpperCAmelCase_           =   DistilBertTokenizer.from_pretrained("distilbert-base-uncased" )

         UpperCAmelCase_           =   tokenizer.encode("sequence builders"   ,			add_special_tokens=__a )
         UpperCAmelCase_           =   tokenizer.encode("multi-sequence build"   ,			add_special_tokens=__a )

         UpperCAmelCase_           =   tokenizer.build_inputs_with_special_tokens(__a )
         UpperCAmelCase_           =   tokenizer.build_inputs_with_special_tokens(__a   ,			__a )

         assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
         assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
             tokenizer.sep_token_id
         ]




