'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__a: Optional[Any]          =   logging.get_logger(__name__)

__a: Dict          =   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__a: int          =   {
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

__a: Any          =   {
    '''junnyu/roformer_chinese_small''': 15_36,
    '''junnyu/roformer_chinese_base''': 15_36,
    '''junnyu/roformer_chinese_char_small''': 5_12,
    '''junnyu/roformer_chinese_char_base''': 5_12,
    '''junnyu/roformer_small_discriminator''': 1_28,
    '''junnyu/roformer_small_generator''': 1_28,
}


__a: Union[str, Any]          =   {
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}



class 			UpperCAmelCase  (						A__					):





       '''simple docstring'''



       SCREAMING_SNAKE_CASE							=    VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE							=    PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE							=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE							=    PRETRAINED_INIT_CONFIGURATION
       SCREAMING_SNAKE_CASE							=    RoFormerTokenizer

       def __init__(						self	,   __lowerCAmelCase=None	,   __lowerCAmelCase=None	,   __lowerCAmelCase=True	,   __lowerCAmelCase="[UNK]"	,   __lowerCAmelCase="[SEP]"	,   __lowerCAmelCase="[PAD]"	,   __lowerCAmelCase="[CLS]"	,   __lowerCAmelCase="[MASK]"	,   __lowerCAmelCase=True	,   __lowerCAmelCase=None	,   **__lowerCAmelCase	,   )  ->  Optional[Any]:
        super().__init__(
            _A	,   tokenizer_file=_A	,   do_lower_case=_A	,   unk_token=_A	,   sep_token=_A	,   pad_token=_A	,   cls_token=_A	,   mask_token=_A	,   tokenize_chinese_chars=_A	,   strip_accents=_A	,   **_A	,   )

        lowercase__				:    Tuple  =					json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            pre_tok_state.get('''lowercase'''	,   _A ) != do_lower_case
            or pre_tok_state.get('''strip_accents'''	,   _A ) != strip_accents
        ):
         lowercase__				:    Union[str, Any]  =					getattr(_A	,   pre_tok_state.pop('''type''' ) )
         lowercase__				:    Any  =					do_lower_case
         lowercase__				:    List[str]  =					strip_accents
         lowercase__				:    str  =					pre_tok_class(**_A )

        lowercase__				:    List[str]  =					do_lower_case

       def __getstate__(						self )  ->  Optional[int]:
        lowercase__				:    Dict  =					self.__dict__.copy()
        lowercase__				:    Optional[Any]  =					BertPreTokenizer()
        return state

       def __setstate__(						self	,   __lowerCAmelCase )  ->  Optional[int]:
        lowercase__				:    int  =					d
        lowercase__				:    List[str]  =					self.__dict__["_tokenizer"].get_vocab()
        lowercase__				:    Any  =					PreTokenizer.custom(JiebaPreTokenizer(_A ) )

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase=None )  ->  Any:
        lowercase__				:    List[Any]  =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

        return output

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase = None )  ->  Optional[Any]:
        lowercase__				:    Optional[Any]  =					[self.sep_token_id]
        lowercase__				:    List[str]  =					[self.cls_token_id]
        if token_ids_a is None:
         return len(cls + token_ids_a + sep ) * [0]
        return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase = None )  ->  Union[str, Any]:
        lowercase__				:    Union[str, Any]  =					self._tokenizer.model.save(_A	,   name=_A )
        return tuple(_A )


       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase=None	,   __lowerCAmelCase=None	,   __lowerCAmelCase=False	,   **__lowerCAmelCase	,   )  ->  Optional[int]:
        lowercase__				:    Union[str, Any]  =					BertPreTokenizer()
        return super().save_pretrained(_A	,   _A	,   _A	,   _A	,   **_A )


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowerCamelCase					:	Union[str, Any]       =  {
    '''configuration_chinese_clip''': [
        '''CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''ChineseCLIPConfig''',
        '''ChineseCLIPOnnxConfig''',
        '''ChineseCLIPTextConfig''',
        '''ChineseCLIPVisionConfig''',
    ],
    '''processing_chinese_clip''': ['''ChineseCLIPProcessor'''],
}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__lowerCamelCase					:	Any       =  ['''ChineseCLIPFeatureExtractor''']
		__lowerCamelCase					:	Optional[int]       =  ['''ChineseCLIPImageProcessor''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__lowerCamelCase					:	int       =  [
		    '''CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''ChineseCLIPModel''',
		    '''ChineseCLIPPreTrainedModel''',
		    '''ChineseCLIPTextModel''',
		    '''ChineseCLIPVisionModel''',
		]

if TYPE_CHECKING:
		from .configuration_chinese_clip import (
		    CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    ChineseCLIPConfig,
		    ChineseCLIPOnnxConfig,
		    ChineseCLIPTextConfig,
		    ChineseCLIPVisionConfig,
		)
		from .processing_chinese_clip import ChineseCLIPProcessor

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_chinese_clip import (
				    CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ChineseCLIPModel,
				    ChineseCLIPPreTrainedModel,
				    ChineseCLIPTextModel,
				    ChineseCLIPVisionModel,
				)

else:
		import sys

		__lowerCamelCase					:	Any       =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
