

"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


__lowerCAmelCase				      =   logging.get_logger(__name__)

__lowerCAmelCase				      =   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCAmelCase				      =   {
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

__lowerCAmelCase				      =   {
    '''microsoft/mpnet-base''': 512,
}

__lowerCAmelCase				      =   {
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}
class 	_lowerCAmelCase    (      __snake_case       ):
							__lowerCAmelCase					:    Union[str, Any]     =  VOCAB_FILES_NAMES
							__lowerCAmelCase					:    Union[str, Any]     =  PRETRAINED_VOCAB_FILES_MAP
							__lowerCAmelCase					:    Optional[Any]     =  PRETRAINED_INIT_CONFIGURATION
							__lowerCAmelCase					:    Tuple     =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowerCAmelCase					:    Dict     =  MPNetTokenizer
							__lowerCAmelCase					:    List[str]     =  ['''input_ids''', '''attention_mask''']

							def __init__( self       :    Optional[int] ,		a       :    Tuple=None ,		a       :    int=None ,		a       :    Any=True ,		a       :    int="<s>" ,		a       :    Any="</s>" ,		a       :    Union[str, Any]="</s>" ,		a       :    List[Any]="<s>" ,		a       :    int="[UNK]" ,		a       :    Dict="<pad>" ,		a       :    List[str]="<mask>" ,		a       :    Any=True ,		a       :    List[Any]=None ,		**a       :    Union[str, Any] ,		)   ->    Tuple:

									"""simple docstring"""
									super().__init__(
									    a ,		tokenizer_file=a ,		do_lower_case=a ,		bos_token=a ,		eos_token=a ,		sep_token=a ,		cls_token=a ,		unk_token=a ,		pad_token=a ,		mask_token=a ,		tokenize_chinese_chars=a ,		strip_accents=a ,		**a ,		)

									lowercase										=							json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
									if (
									    pre_tok_state.get('''lowercase''' ,		a    ) != do_lower_case
									    or pre_tok_state.get('''strip_accents''' ,		a    ) != strip_accents
									):
											lowercase										=							getattr(a ,		pre_tok_state.pop('''type'''    )    )
											lowercase										=							do_lower_case
											lowercase										=							strip_accents
											lowercase										=							pre_tok_class(**a    )

									lowercase										=							do_lower_case

							@property
							def 					_lowerCAmelCase  ( self       :    Optional[Any]    )   ->    str:

									"""simple docstring"""
									if self._mask_token is None:
											if self.verbose:
													logger.error('''Using mask_token, but it is not set yet.'''    )
											return None
									return str(self._mask_token    )

							@mask_token.setter
							def 					_lowerCAmelCase  ( self       :    Any ,		a       :    Any    )   ->    Dict:

									"""simple docstring"""
									lowercase										=							AddedToken(a ,		lstrip=a ,		rstrip=a    ) if isinstance(a ,		a    ) else value
									lowercase										=							value

							def 					_lowerCAmelCase  ( self       :    Any ,		a       :    Optional[int] ,		a       :    Any=None    )   ->    str:

									"""simple docstring"""
									lowercase										=							[self.bos_token_id] + token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
											return output

									return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    List[int] ,		a       :    Optional[List[int]] = None    )   ->    List[int]:

									"""simple docstring"""
									lowercase										=							[self.sep_token_id]
									lowercase										=							[self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep    ) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]

							def 					_lowerCAmelCase  ( self       :    Union[str, Any] ,		a       :    str ,		a       :    Optional[str] = None    )   ->    Tuple[str]:

									"""simple docstring"""
									lowercase										=							self._tokenizer.model.save(a ,		name=a    )
									return tuple(a    )


"""simple docstring"""






from ..utils import DummyObject, requires_backends
class 	_lowerCAmelCase    (      metaclass=__snake_case       ):
							__lowerCAmelCase					:    Optional[Any]     =  ['''tensorflow_text''']

							def __init__( self       :    Optional[int] ,		*a       :    int ,		**a       :    str    )   ->    str:

									"""simple docstring"""
									requires_backends(self ,		['''tensorflow_text''']    )