




'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


a__         =     logging.get_logger(__name__)

a__         =     {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a__         =     {
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

a__         =     {
    '''microsoft/mpnet-base''': 512,
}

a__         =     {
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}





class __magic_name__(						__lowerCAmelCase      ):
						UpperCAmelCase_		:					List[str]    =  VOCAB_FILES_NAMES
						UpperCAmelCase_		:					str    =  PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase_		:					Optional[Any]    =  PRETRAINED_INIT_CONFIGURATION
						UpperCAmelCase_		:					Optional[int]    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase_		:					int    =  MPNetTokenizer
						UpperCAmelCase_		:					Any    =  ["""input_ids""", """attention_mask"""]






						def __init__(   self						:						Tuple					,   __UpperCamelCase						:						Tuple=None					,   __UpperCamelCase						:						str=None					,   __UpperCamelCase						:						str=True					,   __UpperCamelCase						:						Optional[int]="<s>"					,   __UpperCamelCase						:						Optional[int]="</s>"					,   __UpperCamelCase						:						Dict="</s>"					,   __UpperCamelCase						:						str="<s>"					,   __UpperCamelCase						:						Optional[int]="[UNK]"					,   __UpperCamelCase						:						Optional[int]="<pad>"					,   __UpperCamelCase						:						Dict="<mask>"					,   __UpperCamelCase						:						Dict=True					,   __UpperCamelCase						:						Dict=None					,   **__UpperCamelCase						:						Union[str, Any]					,   ):



									'''simple docstring'''






									super().__init__(
									    __UpperCamelCase					,   tokenizer_file=__UpperCamelCase					,   do_lower_case=__UpperCamelCase					,   bos_token=__UpperCamelCase					,   eos_token=__UpperCamelCase					,   sep_token=__UpperCamelCase					,   cls_token=__UpperCamelCase					,   unk_token=__UpperCamelCase					,   pad_token=__UpperCamelCase					,   mask_token=__UpperCamelCase					,   tokenize_chinese_chars=__UpperCamelCase					,   strip_accents=__UpperCamelCase					,   **__UpperCamelCase					,   )

									snake_case__          =						json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
									if (
									    pre_tok_state.get("""lowercase"""					,   __UpperCamelCase       ) != do_lower_case
									    or pre_tok_state.get("""strip_accents"""					,   __UpperCamelCase       ) != strip_accents
									):
												snake_case__          =						getattr(__UpperCamelCase					,   pre_tok_state.pop("""type"""       )       )
												snake_case__          =						do_lower_case
												snake_case__          =						strip_accents
												snake_case__          =						pre_tok_class(**__UpperCamelCase       )

									snake_case__          =						do_lower_case






						@property
						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									if self._mask_token is None:
												if self.verbose:
															logger.error("""Using mask_token, but it is not set yet."""       )
												return None
									return str(self._mask_token       )






						@mask_token.setter
						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						AddedToken(__UpperCamelCase					,   lstrip=__UpperCamelCase					,   rstrip=__UpperCamelCase       ) if isinstance(__UpperCamelCase					,   __UpperCamelCase       ) else value
									snake_case__          =						value






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						Any=None       ):



									'''simple docstring'''






									snake_case__          =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
												return output

									return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						List[int]					,   __UpperCamelCase						:						Optional[List[int]] = None       ):



									'''simple docstring'''






									snake_case__          =						[self.sep_token_id]
									snake_case__          =						[self.cls_token_id]

									if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]







						def       __lowerCAmelCase(   self						:						List[Any]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Optional[str] = None       ):



									'''simple docstring'''






									snake_case__          =						self._tokenizer.model.save(__UpperCamelCase					,   name=__UpperCamelCase       )
									return tuple(__UpperCamelCase       )







"""simple docstring"""





def 	lowerCAmelCase_(			SCREAMING_SNAKE_CASE       )       ->						int:
    """simple docstring"""

    if not isinstance(SCREAMING_SNAKE_CASE			,						SCREAMING_SNAKE_CASE       ):
        raise ValueError('double_factorial() only accepts integral values'       )
    if num < 0:
        raise ValueError('double_factorial() not defined for negative values'       )
    UpperCamelCase__								=     1
    for i in range(SCREAMING_SNAKE_CASE			,						0			,						-2       ):
        value *= i
    return value


if __name__ == "__main__":
  import doctest

  doctest.testmod()




