

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowercase_       		=  logging.get_logger(__name__)

lowercase_       		=  {"vocab_file": "spiece.model"}

lowercase_       		=  {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

lowercase_       		=  {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
lowercase_       		=  0
lowercase_       		=  1
lowercase_       		=  2
lowercase_       		=  3
lowercase_       		=  4


class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    lowerCamelCase			=							VOCAB_FILES_NAMES
    lowerCamelCase			=							PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase			=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase			=							'left'


    def __init__(      self :       Dict,lowercase_ :       List[Any],lowercase_ :       Dict=False,lowercase_ :       List[str]=True,lowercase_ :       Any=False,lowercase_ :       Optional[int]="<s>",lowercase_ :       List[str]="</s>",lowercase_ :       List[str]="<unk>",lowercase_ :       str="<sep>",lowercase_ :       str="<pad>",lowercase_ :       List[str]="<cls>",lowercase_ :       Dict="<mask>",lowercase_ :       Tuple=["<eop>", "<eod>"],lowercase_ :       Optional[Dict[str, Any]] = None,**lowercase_ :       Optional[Any],)->    None:





        '''simple docstring'''







        A__          =   AddedToken(lowercase_,lstrip=lowercase_,rstrip=lowercase_		) if isinstance(lowercase_,lowercase_		) else mask_token

        A__          =   {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            do_lower_case=lowercase_,remove_space=lowercase_,keep_accents=lowercase_,bos_token=lowercase_,eos_token=lowercase_,unk_token=lowercase_,sep_token=lowercase_,pad_token=lowercase_,cls_token=lowercase_,mask_token=lowercase_,additional_special_tokens=lowercase_,sp_model_kwargs=self.sp_model_kwargs,**lowercase_,)

        A__          =   3

        A__          =   do_lower_case
        A__          =   remove_space
        A__          =   keep_accents
        A__          =   vocab_file

        A__          =   spm.SentencePieceProcessor(**self.sp_model_kwargs		)
        self.sp_model.Load(lowercase_		)


    @property
    def 						snake_case__     (      self :       List[str]		)->    Optional[Any]:





        '''simple docstring'''







        return len(self.sp_model		)


    def 						snake_case__     (      self :       Union[str, Any]		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   {self.convert_ids_to_tokens(lowercase_		): i for i in range(self.vocab_size		)}
        vocab.update(self.added_tokens_encoder		)
        return vocab


    def __getstate__(      self :       Union[str, Any]		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   self.__dict__.copy()
        A__          =   None
        return state


    def __setstate__(      self :       Optional[Any],lowercase_ :       Optional[int]		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   d

        # for backward compatibility
        if not hasattr(self,'sp_model_kwargs'		):
            A__          =   {}

        A__          =   spm.SentencePieceProcessor(**self.sp_model_kwargs		)
        self.sp_model.Load(self.vocab_file		)


    def 						snake_case__     (      self :       int,lowercase_ :       List[Any]		)->    List[Any]:





        '''simple docstring'''







        if self.remove_space:
            A__          =   ' '.join(inputs.strip().split()		)
        else:
            A__          =   inputs
        A__          =   outputs.replace('``','"'		).replace('\'\'','"'		)

        if not self.keep_accents:
            A__          =   unicodedata.normalize('NFKD',lowercase_		)
            A__          =   ''.join([c for c in outputs if not unicodedata.combining(lowercase_		)]		)
        if self.do_lower_case:
            A__          =   outputs.lower()

        return outputs


    def 						snake_case__     (      self :       Optional[Any],lowercase_ :       str		)->    List[str]:





        '''simple docstring'''







        A__          =   self.preprocess_text(lowercase_		)
        A__          =   self.sp_model.encode(lowercase_,out_type=lowercase_		)
        A__          =   []
        for piece in pieces:
            if len(lowercase_		) > 1 and piece[-1] == str(','		) and piece[-2].isdigit():
                A__          =   self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase_,''		)		)
                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                    if len(cur_pieces[0]		) == 1:
                        A__          =   cur_pieces[1:]
                    else:
                        A__          =   cur_pieces[0][1:]
                cur_pieces.append(piece[-1]		)
                new_pieces.extend(lowercase_		)
            else:
                new_pieces.append(lowercase_		)

        return new_pieces


    def 						snake_case__     (      self :       Dict,lowercase_ :       Any		)->    Optional[Any]:





        '''simple docstring'''







        return self.sp_model.PieceToId(lowercase_		)


    def 						snake_case__     (      self :       Optional[Any],lowercase_ :       Any		)->    Optional[int]:





        '''simple docstring'''







        return self.sp_model.IdToPiece(lowercase_		)


    def 						snake_case__     (      self :       Optional[Any],lowercase_ :       str		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   ''.join(lowercase_		).replace(lowercase_,' '		).strip()
        return out_string


    def 						snake_case__     (      self :       Union[str, Any],lowercase_ :       List[int],lowercase_ :       bool = False,lowercase_ :       bool = None,lowercase_ :       bool = True,**lowercase_ :       List[str],)->    str:





        '''simple docstring'''







        A__          =   kwargs.pop('use_source_tokenizer',lowercase_		)

        A__          =   self.convert_ids_to_tokens(lowercase_,skip_special_tokens=lowercase_		)

        # To avoid mixing byte-level and unicode for byte-level BPT
        # we need to build string separately for added tokens and byte-level tokens
        # cf. https://github.com/huggingface/transformers/issues/1133
        A__          =   []
        A__          =   []
        for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                continue
            if token in self.added_tokens_encoder:
                if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(lowercase_		)		)
                    A__          =   []
                sub_texts.append(lowercase_		)
            else:
                current_sub_text.append(lowercase_		)
        if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(lowercase_		)		)

        # Mimic the behavior of the Rust tokenizer:
        # By default, there are no spaces between special tokens
        A__          =   ''.join(lowercase_		)

        A__          =   (
            clean_up_tokenization_spaces
            if clean_up_tokenization_spaces is not None
            else self.clean_up_tokenization_spaces
        )
        if clean_up_tokenization_spaces:
            A__          =   self.clean_up_tokenization(lowercase_		)
            return clean_text
        else:
            return text


    def 						snake_case__     (      self :       List[Any],lowercase_ :       List[int],lowercase_ :       Optional[List[int]] = None		)->    List[int]:





        '''simple docstring'''







        A__          =   [self.sep_token_id]
        A__          =   [self.cls_token_id]
        if token_ids_a is None:
            return token_ids_a + sep + cls
        return token_ids_a + sep + token_ids_a + sep + cls


    def 						snake_case__     (      self :       str,lowercase_ :       List[int],lowercase_ :       Optional[List[int]] = None,lowercase_ :       bool = False		)->    List[int]:





        '''simple docstring'''







        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=lowercase_,token_ids_a=lowercase_,already_has_special_tokens=lowercase_		)

        if token_ids_a is not None:
            return ([0] * len(lowercase_		)) + [1] + ([0] * len(lowercase_		)) + [1, 1]
        return ([0] * len(lowercase_		)) + [1, 1]


    def 						snake_case__     (      self :       List[str],lowercase_ :       List[int],lowercase_ :       Optional[List[int]] = None		)->    List[int]:





        '''simple docstring'''







        A__          =   [self.sep_token_id]
        A__          =   [2]

        if token_ids_a is None:
            return len(token_ids_a + sep		) * [0] + cls_segment_id
        return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id


    def 						snake_case__     (      self :       List[Any],lowercase_ :       str,lowercase_ :       Optional[str] = None		)->    Tuple[str]:





        '''simple docstring'''







        if not os.path.isdir(lowercase_		):
            logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
            return
        A__          =   os.path.join(
            lowercase_,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

        if os.path.abspath(self.vocab_file		) != os.path.abspath(lowercase_		) and os.path.isfile(self.vocab_file		):
            copyfile(self.vocab_file,lowercase_		)
        elif not os.path.isfile(self.vocab_file		):
            with open(lowercase_,'wb'		) as fi:
                A__          =   self.sp_model.serialized_model_proto()
                fi.write(lowercase_		)

        return (out_vocab_file,)




"""simple docstring"""





import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin




class 					_lowerCAmelCase  (	unittest.TestCase     ,lowercase   ):







 """simple docstring"""







 def 					_lowercase (			self							:			List[Any]	):
    __lowercase								=						load_tool("text-classification"	)
    self.tool.setup()
    __lowercase								=						load_tool("text-classification",					remote=UpperCAmelCase__	)


 def 					_lowercase (			self							:			str	):
    __lowercase								=						self.tool("That's quite cool",					["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)


 def 					_lowercase (			self							:			str	):
    __lowercase								=						self.remote_tool("That's quite cool",					["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)


 def 					_lowercase (			self							:			List[str]	):
    __lowercase								=						self.tool(text="That's quite cool",					labels=["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)



 def 					_lowercase (			self							:			Tuple	):
    __lowercase								=						self.remote_tool(text="That's quite cool",					labels=["positive", "negative"]	)
    self.assertEqual(UpperCAmelCase__,					"positive"	)




