





"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


A       	=     logging.get_logger(__name__)

A       	=     {'''vocab_file''': '''vocab.txt'''}

A       	=     {
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

A       	=     {
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


A       	=     {
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}




class  __lowercase	(	A_							):







					'''simple docstring'''




					__lowerCAmelCase     =     VOCAB_FILES_NAMES
					__lowerCAmelCase     =     PRETRAINED_VOCAB_FILES_MAP
					__lowerCAmelCase     =     PRETRAINED_INIT_CONFIGURATION
					__lowerCAmelCase     =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__lowerCAmelCase     =     ConvBertTokenizer







					def __init__(  self		,	_UpperCAmelCase=None		,	_UpperCAmelCase=None		,	_UpperCAmelCase=True		,	_UpperCAmelCase="[UNK]"		,	_UpperCAmelCase="[SEP]"		,	_UpperCAmelCase="[PAD]"		,	_UpperCAmelCase="[CLS]"		,	_UpperCAmelCase="[MASK]"		,	_UpperCAmelCase=True		,	_UpperCAmelCase=None		,	**_UpperCAmelCase		,	):
									super().__init__(
									    _snake_case		,	tokenizer_file=_snake_case		,	do_lower_case=_snake_case		,	unk_token=_snake_case		,	sep_token=_snake_case		,	pad_token=_snake_case		,	cls_token=_snake_case		,	mask_token=_snake_case		,	tokenize_chinese_chars=_snake_case		,	strip_accents=_snake_case		,	**_snake_case		,	)

									__a       :  int					=     json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
									if (
									    normalizer_state.get('''lowercase'''		,	_snake_case   ) != do_lower_case
									    or normalizer_state.get('''strip_accents'''		,	_snake_case   ) != strip_accents
									    or normalizer_state.get('''handle_chinese_chars'''		,	_snake_case   ) != tokenize_chinese_chars
									):
													__a       :  int					=     getattr(_snake_case		,	normalizer_state.pop('''type'''   )   )
													__a       :  Optional[Any]					=     do_lower_case
													__a       :  List[str]					=     strip_accents
													__a       :  Optional[Any]					=     tokenize_chinese_chars
													__a       :  List[str]					=     normalizer_class(**_snake_case   )

									__a       :  Optional[int]					=     do_lower_case







					def        _lowerCamelCase						(  self		,	_UpperCAmelCase		,	_UpperCAmelCase=None   ):
									__a       :  List[Any]					=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

									return output







					def        _lowerCamelCase						(  self		,	_UpperCAmelCase		,	_UpperCAmelCase = None   ):
									__a       :  Tuple					=     [self.sep_token_id]
									__a       :  Union[str, Any]					=     [self.cls_token_id]
									if token_ids_a is None:
													return len(cls + token_ids_a + sep   ) * [0]
									return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







					def        _lowerCamelCase						(  self		,	_UpperCAmelCase		,	_UpperCAmelCase = None   ):
									__a       :  str					=     self._tokenizer.model.save(_snake_case		,	name=_snake_case   )
									return tuple(_snake_case   )




"""simple docstring"""







from ..utils import DummyObject, requires_backends
class        __A       (	metaclass=A_    ):







		'''simple docstring'''





		lowerCAmelCase			:					List[str]   	=				["torch", "torchsde"]







		def __init__(	self						:  Tuple			,*_snake_case						:  Union[str, Any]			,**_snake_case						:  Any						)					->		Union[str, Any]:






				"""simple docstring"""



				requires_backends(self			,['''torch''', '''torchsde''']						)







		@classmethod
		def 					UpperCAmelCase    (	cls						:  List[str]			,*_snake_case						:  int			,**_snake_case						:  Union[str, Any]						)					->		str:






				"""simple docstring"""



				requires_backends(cls			,['''torch''', '''torchsde''']						)







		@classmethod
		def 					UpperCAmelCase    (	cls						:  List[Any]			,*_snake_case						:  List[Any]			,**_snake_case						:  List[str]						)					->		List[Any]:






				"""simple docstring"""



				requires_backends(cls			,['''torch''', '''torchsde''']						)
