

'''simple docstring'''
import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase :      Tuple					 =     logging.get_logger(__name__)

lowerCamelCase :      int					 =     "▁"

lowerCamelCase :      Dict					 =     {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

lowerCamelCase :      Optional[int]					 =     {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

lowerCamelCase :      Dict					 =     {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

lowerCamelCase :      Tuple					 =     {
    "ernie-m-base": 5_1_4,
    "ernie-m-large": 5_1_4,
}

lowerCamelCase :      int					 =     {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}







class 							A__    (       A__  ):
  A__		 =				["input_ids"]

  A__		 =				VOCAB_FILES_NAMES
  A__		 =				PRETRAINED_INIT_CONFIGURATION
  A__		 =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A__		 =				PRETRAINED_VOCAB_FILES_MAP
  A__		 =				RESOURCE_FILES_NAMES





  def __init__(		self      :	Tuple							,					_a      :	List[Any]							,					_a      :	Dict=None							,					_a      :	Dict=False							,					_a      :	List[Any]="utf8"							,					_a      :	Dict="[UNK]"							,					_a      :	List[Any]="[SEP]"							,					_a      :	List[str]="[PAD]"							,					_a      :	int="[CLS]"							,					_a      :	Dict="[MASK]"							,					_a      :	Optional[Dict[str, Any]] = None							,					**_a      :	Tuple							,					)	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ={} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=_a							,					unk_token=_a							,					sep_token=_a							,					pad_token=_a							,					cls_token=_a							,					mask_token=_a							,					vocab_file=_a							,					encoding=_a							,					sp_model_kwargs=self.sp_model_kwargs							,					**_a							,					)
       _SCREAMING_SNAKE_CASE							    =do_lower_case
       _SCREAMING_SNAKE_CASE							    =sentencepiece_model_ckpt
       _SCREAMING_SNAKE_CASE							    =spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(_a )

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
            _SCREAMING_SNAKE_CASE							    =self.load_vocab(filepath=_a )
       else:
            _SCREAMING_SNAKE_CASE							    ={self.sp_model.id_to_piece(_a ): id for id in range(self.sp_model.get_piece_size() )}
       _SCREAMING_SNAKE_CASE							    ={v: k for k, v in self.vocab.items()}





  def 						A  (		self      :	Optional[int]							,					_a      :	Any )	->    List[Any]:





       '''simple docstring'''






       if text is None:
            return None

       _SCREAMING_SNAKE_CASE							    =self.tokenize(_a )
       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    ='', []

       for i, ch in enumerate(_a ):
            if ch in self.SP_CHAR_MAPPING:
                 _SCREAMING_SNAKE_CASE							    =self.SP_CHAR_MAPPING.get(_a )
            else:
                 _SCREAMING_SNAKE_CASE							    =unicodedata.normalize('NFKC'							,					_a )
            if self.is_whitespace(_a ):
                 continue
            normalized_text += ch
            char_mapping.extend([i] * len(_a ) )

       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    =normalized_text, [], 0

       if self.do_lower_case:
            _SCREAMING_SNAKE_CASE							    =text.lower()

       for token in split_tokens:
            if token[:1] == "▁":
                 _SCREAMING_SNAKE_CASE							    =token[1:]
            _SCREAMING_SNAKE_CASE							    =text[offset:].index(_a ) + offset
            _SCREAMING_SNAKE_CASE							    =start + len(_a )

            token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1) )
            _SCREAMING_SNAKE_CASE							    =end
       return token_mapping





  @property
  def 						A  (		self      :	List[str] )	->    Union[str, Any]:





       '''simple docstring'''






       return len(self.vocab )





  def 						A  (		self      :	int )	->    Optional[Any]:





       '''simple docstring'''






       return dict(self.vocab							,					**self.added_tokens_encoder )





  def __getstate__(		self      :	Optional[int] )	->    Optional[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.__dict__.copy()
       _SCREAMING_SNAKE_CASE							    =None
       return state





  def __setstate__(		self      :	Any							,					_a      :	List[Any] )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =d

       # for backward compatibility
       if not hasattr(self							,					'sp_model_kwargs' ):
            _SCREAMING_SNAKE_CASE							    ={}

       _SCREAMING_SNAKE_CASE							    =spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(self.sentencepiece_model_ckpt )





  def 						A  (		self      :	Any							,					_a      :	int )	->    Optional[Any]:





       '''simple docstring'''






       return "".join((self.SP_CHAR_MAPPING.get(_a							,					_a ) for c in text) )





  def 						A  (		self      :	List[Any]							,					_a      :	int							,					_a      :	Optional[int]=False							,					_a      :	Any=64							,					_a      :	List[str]=0.1 )	->    Optional[Any]:





       '''simple docstring'''






       if self.sp_model_kwargs.get('enable_sampling' ) is True:
            _SCREAMING_SNAKE_CASE							    =True
       if self.sp_model_kwargs.get('alpha' ) is not None:
            _SCREAMING_SNAKE_CASE							    =self.sp_model_kwargs.get('alpha' )
       if self.sp_model_kwargs.get('nbest_size' ) is not None:
            _SCREAMING_SNAKE_CASE							    =self.sp_model_kwargs.get('nbest_size' )

       if not enable_sampling:
            _SCREAMING_SNAKE_CASE							    =self.sp_model.EncodeAsPieces(_a )
       else:
            _SCREAMING_SNAKE_CASE							    =self.sp_model.SampleEncodeAsPieces(_a							,					_a							,					_a )
       _SCREAMING_SNAKE_CASE							    =[]
       for pi, piece in enumerate(_a ):
            if piece == SPIECE_UNDERLINE:
                 if not pieces[pi + 1].startswith(_a ) and pi != 0:
                      new_pieces.append(_a )
                      continue
                 else:
                      continue
            _SCREAMING_SNAKE_CASE							    =0
            for i, chunk in enumerate(_a ):
                 if chunk == SPIECE_UNDERLINE:
                      continue
                 if self.is_ch_char(_a ) or self.is_punct(_a ):
                      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                           new_pieces.append(piece[lst_i:i] )
                      new_pieces.append(_a )
                      _SCREAMING_SNAKE_CASE							    =i + 1
                 elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                           new_pieces.append(piece[lst_i:i] )
                      _SCREAMING_SNAKE_CASE							    =i
                 elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                           new_pieces.append(piece[lst_i:i] )
                      _SCREAMING_SNAKE_CASE							    =i
            if len(_a ) > lst_i:
                 new_pieces.append(piece[lst_i:] )
       return new_pieces





  def 						A  (		self      :	List[str]							,					_a      :	List[str] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =''.join(_a ).replace(_a							,					' ' ).strip()
       return out_string





  def 						A  (		self      :	str							,					_a      :	int )	->    int:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.convert_ids_to_tokens(_a )
       _SCREAMING_SNAKE_CASE							    =''.join(_a ).replace(_a							,					' ' ).strip()
       return out_string





  def 						A  (		self      :	List[str]							,					_a      :	Optional[int] )	->    Union[str, Any]:





       '''simple docstring'''






       return self.vocab.get(_a							,					self.vocab.get(self.unk_token ) )





  def 						A  (		self      :	List[Any]							,					_a      :	Optional[Any] )	->    Any:





       '''simple docstring'''






       return self.reverse_vocab.get(_a							,					self.unk_token )





  def 						A  (		self      :	Optional[Any]							,					_a      :	Union[str, Any]							,					_a      :	Dict=None )	->    Optional[int]:





       '''simple docstring'''






       if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       _SCREAMING_SNAKE_CASE							    =[self.cls_token_id]
       _SCREAMING_SNAKE_CASE							    =[self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep





  def 						A  (		self      :	List[Any]							,					_a      :	Dict							,					_a      :	Optional[Any]=None )	->    Optional[int]:





       '''simple docstring'''






       if offset_mapping_a is None:
            return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]





  def 						A  (		self      :	int							,					_a      :	Tuple							,					_a      :	Optional[int]=None							,					_a      :	Tuple=False )	->    List[Any]:





       '''simple docstring'''






       if already_has_special_tokens:
            if token_ids_a is not None:
                 raise ValueError(
                     'You should not supply a second sequence if the provided sequence of '
                     'ids is already formatted with special tokens for the model.' )
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
            return [1] + ([0] * len(_a )) + [1, 1] + ([0] * len(_a )) + [1]
       return [1] + ([0] * len(_a )) + [1]





  def 						A  (		self      :	List[str]							,					_a      :	List[int]							,					_a      :	Optional[List[int]] = None )	->    List[int]:





       '''simple docstring'''






       if token_ids_a is None:
            # [CLS] X [SEP]
            return (len(_a ) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(_a ) + 1) + [1] * (len(_a ) + 3)





  def 						A  (		self      :	List[Any]							,					_a      :	str )	->    Optional[int]:





       '''simple docstring'''






       if "\u4e00" <= char <= "\u9fff":
            return True
       return False





  def 						A  (		self      :	Tuple							,					_a      :	Tuple )	->    Optional[Any]:





       '''simple docstring'''






       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
            return True
       return False





  def 						A  (		self      :	Tuple							,					_a      :	Optional[int] )	->    Any:





       '''simple docstring'''






       if char in ",;:.?!~，；：。？！《》【】":
            return True
       return False





  def 						A  (		self      :	str							,					_a      :	int )	->    List[Any]:





       '''simple docstring'''






       if char == " " or char == "\t" or char == "\n" or char == "\r":
            return True
       if len(_a ) == 1:
            _SCREAMING_SNAKE_CASE							    =unicodedata.category(_a )
            if cat == "Zs":
                 return True
       return False





  def 						A  (		self      :	str							,					_a      :	List[Any] )	->    Optional[int]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ={}
       with io.open(_a							,					'r'							,					encoding='utf-8' ) as f:
            for index, line in enumerate(_a ):
                 _SCREAMING_SNAKE_CASE							    =line.rstrip('\n' )
                 _SCREAMING_SNAKE_CASE							    =int(_a )

       return token_to_idx





  def 						A  (		self      :	str							,					_a      :	str							,					_a      :	Optional[str] = None )	->    Tuple[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =0
       if os.path.isdir(_a ):
            _SCREAMING_SNAKE_CASE							    =os.path.join(
                _a							,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
       else:
            _SCREAMING_SNAKE_CASE							    =(filename_prefix + '-' if filename_prefix else '') + save_directory
       with open(_a							,					'w'							,					encoding='utf-8' ) as writer:
            for token, token_index in sorted(self.vocab.items()							,					key=lambda _a : kv[1] ):
                 if index != token_index:
                      logger.warning(
                          f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                          ' Please check that the vocabulary is not corrupted!' )
                      _SCREAMING_SNAKE_CASE							    =token_index
                 writer.write(token + '\n' )
                 index += 1

       _SCREAMING_SNAKE_CASE							    =os.path.join(_a							,					'sentencepiece.bpe.model' )
       with open(_a							,					'wb' ) as fi:
            _SCREAMING_SNAKE_CASE							    =self.sp_model.serialized_model_proto()
            fi.write(_a )

       return (vocab_file,)



'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase :      int					 =     {"configuration_mbart": ["MBART_PRETRAINED_CONFIG_ARCHIVE_MAP", "MBartConfig", "MBartOnnxConfig"]}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCamelCase :      List[Any]					 =     ["MBartTokenizer"]

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCamelCase :      Dict					 =     ["MBartTokenizerFast"]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCamelCase :      int					 =     [
         "MBART_PRETRAINED_MODEL_ARCHIVE_LIST",
         "MBartForCausalLM",
         "MBartForConditionalGeneration",
         "MBartForQuestionAnswering",
         "MBartForSequenceClassification",
         "MBartModel",
         "MBartPreTrainedModel",
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCamelCase :      Union[str, Any]					 =     [
         "TFMBartForConditionalGeneration",
         "TFMBartModel",
         "TFMBartPreTrainedModel",
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCamelCase :      Optional[Any]					 =     [
         "FlaxMBartForConditionalGeneration",
         "FlaxMBartForQuestionAnswering",
         "FlaxMBartForSequenceClassification",
         "FlaxMBartModel",
         "FlaxMBartPreTrainedModel",
     ]


if TYPE_CHECKING:
     from .configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfig, MBartOnnxConfig

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_mbart import MBartTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_mbart_fast import MBartTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_mbart import (
              MBART_PRETRAINED_MODEL_ARCHIVE_LIST,
              MBartForCausalLM,
              MBartForConditionalGeneration,
              MBartForQuestionAnswering,
              MBartForSequenceClassification,
              MBartModel,
              MBartPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_mbart import TFMBartForConditionalGeneration, TFMBartModel, TFMBartPreTrainedModel

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_mbart import (
              FlaxMBartForConditionalGeneration,
              FlaxMBartForQuestionAnswering,
              FlaxMBartForSequenceClassification,
              FlaxMBartModel,
              FlaxMBartPreTrainedModel,
          )

else:
     import sys

     lowerCamelCase :      List[Any]					 =     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
