
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A     :	Any    = logging.get_logger(__name__)

A     :	Tuple    = {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A     :	List[Any]    = {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

A     :	str    = {
    'junnyu/roformer_chinese_small': 1_5_3_6,
    'junnyu/roformer_chinese_base': 1_5_3_6,
    'junnyu/roformer_chinese_char_small': 5_1_2,
    'junnyu/roformer_chinese_char_base': 5_1_2,
    'junnyu/roformer_small_discriminator': 1_2_8,
    'junnyu/roformer_small_generator': 1_2_8,
}


A     :	Tuple    = {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}







class 			__A( a			):
				snake_case_						   =    VOCAB_FILES_NAMES
				snake_case_						   =    PRETRAINED_VOCAB_FILES_MAP
				snake_case_						   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				snake_case_						   =    PRETRAINED_INIT_CONFIGURATION
				snake_case_						   =    RoFormerTokenizer




				def __init__(     self		,     _snake_case=None		,     _snake_case=None		,     _snake_case=True		,     _snake_case="[UNK]"		,     _snake_case="[SEP]"		,     _snake_case="[PAD]"		,     _snake_case="[CLS]"		,     _snake_case="[MASK]"		,     _snake_case=True		,     _snake_case=None		,     **_snake_case		,     )    ->      List[Any]:


								'''simple docstring'''




								super().__init__(
								    _snake_case		,     tokenizer_file=_snake_case		,     do_lower_case=_snake_case		,     unk_token=_snake_case		,     sep_token=_snake_case		,     pad_token=_snake_case		,     cls_token=_snake_case		,     mask_token=_snake_case		,     tokenize_chinese_chars=_snake_case		,     strip_accents=_snake_case		,     **_snake_case		,     )

								__a										=    json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    pre_tok_state.get('''lowercase'''		,     _snake_case				) != do_lower_case
								    or pre_tok_state.get('''strip_accents'''		,     _snake_case				) != strip_accents
								):
												__a										=    getattr(_snake_case		,     pre_tok_state.pop('''type'''				)				)
												__a										=    do_lower_case
												__a										=    strip_accents
												__a										=    pre_tok_class(**_snake_case				)

								__a										=    do_lower_case




				def __getstate__(     self				)    ->      Tuple:


								'''simple docstring'''




								__a										=    self.__dict__.copy()
								__a										=    BertPreTokenizer()
								return state




				def __setstate__(     self		,     _snake_case				)    ->      Any:


								'''simple docstring'''




								__a										=    d
								__a										=    self.__dict__['''_tokenizer'''].get_vocab()
								__a										=    PreTokenizer.custom(JiebaPreTokenizer(_snake_case				)				)




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case=None				)    ->      List[str]:


								'''simple docstring'''




								__a										=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case = None				)    ->      List[int]:


								'''simple docstring'''




								__a										=    [self.sep_token_id]
								__a										=    [self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep				) * [0]
								return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case = None				)    ->      Tuple[str]:


								'''simple docstring'''




								__a										=    self._tokenizer.model.save(_snake_case		,     name=_snake_case				)
								return tuple(_snake_case				)




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case=None		,     _snake_case=None		,     _snake_case=False		,     **_snake_case		,     )    ->      Any:


								'''simple docstring'''




								__a										=    BertPreTokenizer()
								return super().save_pretrained(_snake_case		,     _snake_case		,     _snake_case		,     _snake_case		,     **_snake_case				)

from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Value
from .base import TaskTemplate







@dataclass(frozen=a			)
class 			__A( a			):
				snake_case_						   =    field(default='''language-modeling'''  ,   metadata={'''include_in_asdict_even_if_is_default''': True}			)

				snake_case_						   =    Features({'''text''': Value('''string'''			)}			)
				snake_case_						   =    Features({}			)
				snake_case_						   =    "text"




				@property
				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Dict[str, str]:


								'''simple docstring'''




								return {self.text_column: "text"}