import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase__											=					logging.get_logger(__name__)

UpperCamelCase__											=					{'vocab_file': 'spiece.model'}

UpperCamelCase__											=					{
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}





class      A							(				UpperCAmelCase_     ):


   def __init__(self     :						Tuple       ,  __UpperCAmelCase     :						Optional[int]       ,  __UpperCAmelCase     :						Any=False       ,  __UpperCAmelCase     :						Dict=True       ,  __UpperCAmelCase     :						List[str]=False       ,  __UpperCAmelCase     :						Dict="<s>"       ,  __UpperCAmelCase     :						int="</s>"       ,  __UpperCAmelCase     :						int="<unk>"       ,  __UpperCAmelCase     :						str="<sep>"       ,  __UpperCAmelCase     :						Tuple="<pad>"       ,  __UpperCAmelCase     :						List[str]="<cls>"       ,  __UpperCAmelCase     :						Dict="<mask>"       ,  __UpperCAmelCase     :						int=["<eop>", "<eod>"]       ,  __UpperCAmelCase     :						Optional[Dict[str, Any]] = None       ,  **__UpperCAmelCase     :						Dict       ,  )						->				None:
         """simple docstring"""



         UpperCAmelCase__								=  AddedToken(__UpperCAmelCase       ,  lstrip=__UpperCAmelCase       ,  rstrip=__UpperCAmelCase		) if isinstance(__UpperCAmelCase       ,  __UpperCAmelCase		) else mask_token

         UpperCAmelCase__								=  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=__UpperCAmelCase       ,  remove_space=__UpperCAmelCase       ,  keep_accents=__UpperCAmelCase       ,  bos_token=__UpperCAmelCase       ,  eos_token=__UpperCAmelCase       ,  unk_token=__UpperCAmelCase       ,  sep_token=__UpperCAmelCase       ,  pad_token=__UpperCAmelCase       ,  cls_token=__UpperCAmelCase       ,  mask_token=__UpperCAmelCase       ,  additional_special_tokens=__UpperCAmelCase       ,  sp_model_kwargs=self.sp_model_kwargs       ,  **__UpperCAmelCase       ,  )

         UpperCAmelCase__								=  3

         UpperCAmelCase__								=  do_lower_case
         UpperCAmelCase__								=  remove_space
         UpperCAmelCase__								=  keep_accents
         UpperCAmelCase__								=  vocab_file

         UpperCAmelCase__								=  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
         self.sp_model.Load(__UpperCAmelCase		)

         try:
               import jieba
         except ModuleNotFoundError as error:
               raise error.__class__(
                   "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                   "See https://pypi.org/project/jieba/ for installation."		)
         UpperCAmelCase__								=  jieba
         UpperCAmelCase__								=  str.maketrans(" \n"       ,  "\u2582\u2583"		)


   @property
   # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
   def 			lowercase_      (self     :						Optional[Any]		)						->				List[Any]:
         """simple docstring"""



         return len(self.sp_model		)


   def 			lowercase_      (self     :						int		)						->				Any:
         """simple docstring"""



         UpperCAmelCase__								=  {self.convert_ids_to_tokens(__UpperCAmelCase		): i for i in range(self.vocab_size		)}
         vocab.update(self.added_tokens_encoder		)
         return vocab


   def __getstate__(self     :						int		)						->				Tuple:
         """simple docstring"""



         UpperCAmelCase__								=  self.__dict__.copy()
         UpperCAmelCase__								=  None
         return state


   def __setstate__(self     :						List[Any]       ,  __UpperCAmelCase     :						Dict		)						->				List[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  d

         # for backward compatibility
         if not hasattr(self       ,  "sp_model_kwargs"		):
               UpperCAmelCase__								=  {}

         UpperCAmelCase__								=  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
         self.sp_model.Load(self.vocab_file		)


   def 			lowercase_      (self     :						Optional[Any]       ,  __UpperCAmelCase     :						List[Any]		)						->				int:
         """simple docstring"""



         if self.remove_space:
               UpperCAmelCase__								=  " ".join(inputs.strip().split()		)
         else:
               UpperCAmelCase__								=  inputs
         UpperCAmelCase__								=  outputs.replace("``"       ,  "\""		).replace("''"       ,  "\""		)

         if not self.keep_accents:
               UpperCAmelCase__								=  unicodedata.normalize("NFKD"       ,  __UpperCAmelCase		)
               UpperCAmelCase__								=  "".join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase		)]		)
         if self.do_lower_case:
               UpperCAmelCase__								=  outputs.lower()

         return outputs


   def 			lowercase_      (self     :						Any       ,  __UpperCAmelCase     :						str		)						->				List[str]:
         """simple docstring"""



         UpperCAmelCase__								=  self.preprocess_text(__UpperCAmelCase		)
         UpperCAmelCase__								=  self.sp_model.encode(__UpperCAmelCase       ,  out_type=__UpperCAmelCase		)
         UpperCAmelCase__								=  []
         for piece in pieces:
               if len(__UpperCAmelCase		) > 1 and piece[-1] == str(","		) and piece[-2].isdigit():
                     UpperCAmelCase__								=  self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase       ,  ""		)		)
                     if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                           if len(cur_pieces[0]		) == 1:
                                 UpperCAmelCase__								=  cur_pieces[1:]
                           else:
                                 UpperCAmelCase__								=  cur_pieces[0][1:]
                     cur_pieces.append(piece[-1]		)
                     new_pieces.extend(__UpperCAmelCase		)
               else:
                     new_pieces.append(__UpperCAmelCase		)

         return new_pieces


   def 			lowercase_      (self     :						Optional[Any]       ,  __UpperCAmelCase     :						int		)						->				Optional[int]:
         """simple docstring"""



         return self.sp_model.PieceToId(__UpperCAmelCase		)


   def 			lowercase_      (self     :						Dict       ,  __UpperCAmelCase     :						List[str]		)						->				Optional[Any]:
         """simple docstring"""



         return self.sp_model.IdToPiece(__UpperCAmelCase		)


   def 			lowercase_      (self     :						str       ,  __UpperCAmelCase     :						Optional[int]		)						->				List[str]:
         """simple docstring"""



         UpperCAmelCase__								=  "".join(__UpperCAmelCase		).replace(__UpperCAmelCase       ,  " "		).strip()
         return out_string


   def 			lowercase_      (self     :						Dict       ,  __UpperCAmelCase     :						List[int]       ,  __UpperCAmelCase     :						Optional[List[int]] = None		)						->				List[int]:
         """simple docstring"""



         UpperCAmelCase__								=  [self.sep_token_id]
         UpperCAmelCase__								=  [self.cls_token_id]
         if token_ids_a is None:
               return token_ids_a + sep + cls
         return token_ids_a + sep + token_ids_a + sep + cls


   def 			lowercase_      (self     :						Tuple       ,  __UpperCAmelCase     :						List[int]       ,  __UpperCAmelCase     :						Optional[List[int]] = None       ,  __UpperCAmelCase     :						bool = False		)						->				List[int]:
         """simple docstring"""



         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__UpperCAmelCase       ,  token_ids_a=__UpperCAmelCase       ,  already_has_special_tokens=__UpperCAmelCase		)

         if token_ids_a is not None:
               return ([0] * len(__UpperCAmelCase		)) + [1] + ([0] * len(__UpperCAmelCase		)) + [1, 1]
         return ([0] * len(__UpperCAmelCase		)) + [1, 1]


   def 			lowercase_      (self     :						Optional[Any]       ,  __UpperCAmelCase     :						List[int]       ,  __UpperCAmelCase     :						Optional[List[int]] = None		)						->				List[int]:
         """simple docstring"""



         UpperCAmelCase__								=  [self.sep_token_id]
         UpperCAmelCase__								=  [2]

         if token_ids_a is None:
               return len(token_ids_a + sep		) * [0] + cls_segment_id
         return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id


   def 			lowercase_      (self     :						Union[str, Any]       ,  __UpperCAmelCase     :						str       ,  __UpperCAmelCase     :						Optional[str] = None		)						->				Tuple[str]:
         """simple docstring"""



         if not os.path.isdir(__UpperCAmelCase		):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
               return
         UpperCAmelCase__								=  os.path.join(
             __UpperCAmelCase       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

         if os.path.abspath(self.vocab_file		) != os.path.abspath(__UpperCAmelCase		) and os.path.isfile(self.vocab_file		):
               copyfile(self.vocab_file       ,  __UpperCAmelCase		)
         elif not os.path.isfile(self.vocab_file		):
               with open(__UpperCAmelCase       ,  "wb"		) as fi:
                     UpperCAmelCase__								=  self.sp_model.serialized_model_proto()
                     fi.write(__UpperCAmelCase		)

         return (out_vocab_file,)





   def 			lowercase_      (self     :						List[Any]       ,  *__UpperCAmelCase     :						Tuple       ,  **__UpperCAmelCase     :						List[Any]		)						->				List[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  super()._decode(*__UpperCAmelCase       ,  **__UpperCAmelCase		)
         UpperCAmelCase__								=  text.replace(" "       ,  ""		).replace("\u2582"       ,  " "		).replace("\u2583"       ,  "\n"		)
         return text




from __future__ import annotations

from collections import deque





class      A							:


   def __init__(self     :						Dict       ,  __UpperCAmelCase     :						list[str]		)						->				Optional[int]:
         """simple docstring"""



         UpperCAmelCase__								=  []
         self.adlist.append(
             {"value": "", "next_states": [], "fail_state": 0, "output": []}		)

         for keyword in keywords:
               self.add_keyword(__UpperCAmelCase		)
         self.set_fail_transitions()


   def 			lowercase_      (self     :						Tuple       ,  __UpperCAmelCase     :						int       ,  __UpperCAmelCase     :						str		)						->				int | None:
         """simple docstring"""



         for state in self.adlist[current_state]["next_states"]:
               if char == self.adlist[state]["value"]:
                     return state
         return None


   def 			lowercase_      (self     :						Dict       ,  __UpperCAmelCase     :						str		)						->				None:
         """simple docstring"""



         UpperCAmelCase__								=  0
         for character in keyword:
               UpperCAmelCase__								=  self.find_next_state(__UpperCAmelCase       ,  __UpperCAmelCase		)
               if next_state is None:
                     self.adlist.append(
                         {
                             "value": character,
                             "next_states": [],
                             "fail_state": 0,
                             "output": [],
                         }		)
                     self.adlist[current_state]["next_states"].append(len(self.adlist		) - 1		)
                     UpperCAmelCase__								=  len(self.adlist		) - 1
               else:
                     UpperCAmelCase__								=  next_state
         self.adlist[current_state]["output"].append(__UpperCAmelCase		)


   def 			lowercase_      (self     :						Optional[int]		)						->				None:
         """simple docstring"""



         UpperCAmelCase__								=  deque()
         for node in self.adlist[0]["next_states"]:
               q.append(__UpperCAmelCase		)
               UpperCAmelCase__								=  0
         while q:
               UpperCAmelCase__								=  q.popleft()
               for child in self.adlist[r]["next_states"]:
                     q.append(__UpperCAmelCase		)
                     UpperCAmelCase__								=  self.adlist[r]["fail_state"]
                     while (
                         self.find_next_state(__UpperCAmelCase       ,  self.adlist[child]["value"]		) is None
                         and state != 0
                     ):
                           UpperCAmelCase__								=  self.adlist[state]["fail_state"]
                     UpperCAmelCase__								=  self.find_next_state(
                         __UpperCAmelCase       ,  self.adlist[child]["value"]		)
                     if self.adlist[child]["fail_state"] is None:
                           UpperCAmelCase__								=  0
                     UpperCAmelCase__								=  (
                         self.adlist[child]["output"]
                         + self.adlist[self.adlist[child]["fail_state"]]["output"]
                     )





   def 			lowercase_      (self     :						Union[str, Any]       ,  __UpperCAmelCase     :						str		)						->				dict[str, list[int]]:
         """simple docstring"""



         UpperCAmelCase__								=  {}  # returns a dict with keywords and list of its occurrences
         UpperCAmelCase__								=  0
         for i in range(len(__UpperCAmelCase		)		):
               while (
                   self.find_next_state(__UpperCAmelCase       ,  string[i]		) is None
                   and current_state != 0
               ):
                     UpperCAmelCase__								=  self.adlist[current_state]["fail_state"]
               UpperCAmelCase__								=  self.find_next_state(__UpperCAmelCase       ,  string[i]		)
               if next_state is None:
                     UpperCAmelCase__								=  0
               else:
                     UpperCAmelCase__								=  next_state
                     for key in self.adlist[current_state]["output"]:
                           if key not in result:
                                 UpperCAmelCase__								=  []
                           result[key].append(i - len(__UpperCAmelCase		) + 1		)
         return result


if __name__ == "__main__":
      import doctest

      doctest.testmod()



