


'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase							:   Optional[int]  						=	logging.get_logger(__name__)

lowerCamelCase							:   Union[str, Any]  						=	{'vocab_file': 'spiece.model'}

lowerCamelCase							:   int  						=	{
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      def __init__(self    :    List[Any]   ,						UpperCamelCase    :    Optional[int]   ,						UpperCamelCase    :    List[str]=False   ,						UpperCamelCase    :    Tuple=True   ,						UpperCamelCase    :    Optional[int]=False   ,						UpperCamelCase    :    str="<s>"   ,						UpperCamelCase    :    Optional[Any]="</s>"   ,						UpperCamelCase    :    Any="<unk>"   ,						UpperCamelCase    :    Tuple="<sep>"   ,						UpperCamelCase    :    Union[str, Any]="<pad>"   ,						UpperCamelCase    :    Any="<cls>"   ,						UpperCamelCase    :    Tuple="<mask>"   ,						UpperCamelCase    :    Any=["<eop>", "<eod>"]   ,						UpperCamelCase    :    Optional[Dict[str, Any]] = None   ,						**UpperCamelCase    :    Optional[int]   ,						):

             '''simple docstring'''







             lowercase__      			= AddedToken(UpperCamelCase   ,						lstrip=UpperCamelCase   ,						rstrip=UpperCamelCase ) if isinstance(UpperCamelCase   ,						UpperCamelCase ) else mask_token

             lowercase__      			= {} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=UpperCamelCase   ,						remove_space=UpperCamelCase   ,						keep_accents=UpperCamelCase   ,						bos_token=UpperCamelCase   ,						eos_token=UpperCamelCase   ,						unk_token=UpperCamelCase   ,						sep_token=UpperCamelCase   ,						pad_token=UpperCamelCase   ,						cls_token=UpperCamelCase   ,						mask_token=UpperCamelCase   ,						additional_special_tokens=UpperCamelCase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**UpperCamelCase   ,						)

             lowercase__      			= 3

             lowercase__      			= do_lower_case
             lowercase__      			= remove_space
             lowercase__      			= keep_accents
             lowercase__      			= vocab_file

             lowercase__      			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
             self.sp_model.Load(UpperCamelCase )

             try:
                    import jieba
             except ModuleNotFoundError as error:
                    raise error.__class__(
                        '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
                        '''See https://pypi.org/project/jieba/ for installation.''' )
             lowercase__      			= jieba
             lowercase__      			= str.maketrans(''' \n'''   ,						'''\u2582\u2583''' )



      @property
      # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             return len(self.sp_model )



      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= {self.convert_ids_to_tokens(UpperCamelCase ): i for i in range(self.vocab_size )}
             vocab.update(self.added_tokens_encoder )
             return vocab



      def __getstate__(self    :    str ):

             '''simple docstring'''







             lowercase__      			= self.__dict__.copy()
             lowercase__      			= None
             return state



      def __setstate__(self    :    Dict   ,						UpperCamelCase    :    int ):

             '''simple docstring'''







             lowercase__      			= d

             # for backward compatibility
             if not hasattr(self   ,						'''sp_model_kwargs''' ):
                    lowercase__      			= {}

             lowercase__      			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
             self.sp_model.Load(self.vocab_file )



      def       UpperCamelCase__		(self    :    Optional[int]   ,						UpperCamelCase    :    Optional[Any] ):

             '''simple docstring'''







             if self.remove_space:
                    lowercase__      			= ''' '''.join(inputs.strip().split() )
             else:
                    lowercase__      			= inputs
             lowercase__      			= outputs.replace('''``'''   ,						'''"''' ).replace('''\'\''''   ,						'''"''' )

             if not self.keep_accents:
                    lowercase__      			= unicodedata.normalize('''NFKD'''   ,						UpperCamelCase )
                    lowercase__      			= ''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase )] )
             if self.do_lower_case:
                    lowercase__      			= outputs.lower()

             return outputs



      def       UpperCamelCase__		(self    :    List[str]   ,						UpperCamelCase    :    str ):

             '''simple docstring'''







             lowercase__      			= self.preprocess_text(UpperCamelCase )
             lowercase__      			= self.sp_model.encode(UpperCamelCase   ,						out_type=UpperCamelCase )
             lowercase__      			= []
             for piece in pieces:
                    if len(UpperCamelCase ) > 1 and piece[-1] == str(''',''' ) and piece[-2].isdigit():
                           lowercase__      			= self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase   ,						'''''' ) )
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0] ) == 1:
                                         lowercase__      			= cur_pieces[1:]
                                  else:
                                         lowercase__      			= cur_pieces[0][1:]
                           cur_pieces.append(piece[-1] )
                           new_pieces.extend(UpperCamelCase )
                    else:
                           new_pieces.append(UpperCamelCase )

             return new_pieces



      def       UpperCamelCase__		(self    :    Dict   ,						UpperCamelCase    :    Optional[int] ):

             '''simple docstring'''







             return self.sp_model.PieceToId(UpperCamelCase )



      def       UpperCamelCase__		(self    :    int   ,						UpperCamelCase    :    Any ):

             '''simple docstring'''







             return self.sp_model.IdToPiece(UpperCamelCase )



      def       UpperCamelCase__		(self    :    List[str]   ,						UpperCamelCase    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= ''''''.join(UpperCamelCase ).replace(UpperCamelCase   ,						''' ''' ).strip()
             return out_string



      def       UpperCamelCase__		(self    :    Optional[int]   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None ):

             '''simple docstring'''







             lowercase__      			= [self.sep_token_id]
             lowercase__      			= [self.cls_token_id]
             if token_ids_a is None:
                    return token_ids_a + sep + cls
             return token_ids_a + sep + token_ids_a + sep + cls



      def       UpperCamelCase__		(self    :    int   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None   ,						UpperCamelCase    :    bool = False ):

             '''simple docstring'''







             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=UpperCamelCase   ,						token_ids_a=UpperCamelCase   ,						already_has_special_tokens=UpperCamelCase )

             if token_ids_a is not None:
                    return ([0] * len(UpperCamelCase )) + [1] + ([0] * len(UpperCamelCase )) + [1, 1]
             return ([0] * len(UpperCamelCase )) + [1, 1]



      def       UpperCamelCase__		(self    :    str   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None ):

             '''simple docstring'''







             lowercase__      			= [self.sep_token_id]
             lowercase__      			= [2]

             if token_ids_a is None:
                    return len(token_ids_a + sep ) * [0] + cls_segment_id
             return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id



      def       UpperCamelCase__		(self    :    Dict   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    Optional[str] = None ):

             '''simple docstring'''







             if not os.path.isdir(UpperCamelCase ):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory" )
                    return
             lowercase__      			= os.path.join(
                 UpperCamelCase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

             if os.path.abspath(self.vocab_file ) != os.path.abspath(UpperCamelCase ) and os.path.isfile(self.vocab_file ):
                    copyfile(self.vocab_file   ,						UpperCamelCase )
             elif not os.path.isfile(self.vocab_file ):
                    with open(UpperCamelCase   ,						'''wb''' ) as fi:
                           lowercase__      			= self.sp_model.serialized_model_proto()
                           fi.write(UpperCamelCase )

             return (out_vocab_file,)






      def       UpperCamelCase__		(self    :    Optional[Any]   ,						*UpperCamelCase    :    int   ,						**UpperCamelCase    :    List[str] ):

             '''simple docstring'''







             lowercase__      			= super()._decode(*UpperCamelCase   ,						**UpperCamelCase )
             lowercase__      			= text.replace(''' '''   ,						'''''' ).replace('''\u2582'''   ,						''' ''' ).replace('''\u2583'''   ,						'''\n''' )
             return text








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCamelCase							:   str  						=	{
    'configuration_rag': ['RagConfig'],
    'retrieval_rag': ['RagRetriever'],
    'tokenization_rag': ['RagTokenizer'],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:   Dict  						=	[
     'RagModel',
     'RagPreTrainedModel',
     'RagSequenceForGeneration',
     'RagTokenForGeneration',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:   Optional[int]  						=	[
     'TFRagModel',
     'TFRagPreTrainedModel',
     'TFRagSequenceForGeneration',
     'TFRagTokenForGeneration',
 ]


if TYPE_CHECKING:
 from .configuration_rag import RagConfig
 from .retrieval_rag import RagRetriever
 from .tokenization_rag import RagTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_rag import RagModel, RagPreTrainedModel, RagSequenceForGeneration, RagTokenForGeneration

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_rag import (
      TFRagModel,
      TFRagPreTrainedModel,
      TFRagSequenceForGeneration,
      TFRagTokenForGeneration,
  )

else:
 import sys

 lowerCamelCase							:   Tuple  						=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




