




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_lowerCamelCase	:		str							       =   logging.get_logger(__name__)

_lowerCamelCase	:		Optional[int]							       =   {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowerCamelCase	:		Optional[int]							       =   {
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

_lowerCamelCase	:		Optional[Any]							       =   {
    """junnyu/roformer_chinese_small""": 1536,
    """junnyu/roformer_chinese_base""": 1536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


_lowerCamelCase	:		List[Any]							       =   {
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}


class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      VOCAB_FILES_NAMES
    UpperCAmelCase__							  =      PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__							  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__							  =      PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase__							  =      RoFormerTokenizer


    def __init__(			self      :	Any					, UpperCAmelCase__      :	int=None					, UpperCAmelCase__      :	Optional[int]=None					, UpperCAmelCase__      :	str=True					, UpperCAmelCase__      :	Optional[int]="[UNK]"					, UpperCAmelCase__      :	Optional[int]="[SEP]"					, UpperCAmelCase__      :	Union[str, Any]="[PAD]"					, UpperCAmelCase__      :	str="[CLS]"					, UpperCAmelCase__      :	Tuple="[MASK]"					, UpperCAmelCase__      :	int=True					, UpperCAmelCase__      :	Tuple=None					, **UpperCAmelCase__      :	Optional[Any]					, )						->List[Any]:





           '''simple docstring'''

           super().__init__(
               UpperCAmelCase__					, tokenizer_file=UpperCAmelCase__					, do_lower_case=UpperCAmelCase__					, unk_token=UpperCAmelCase__					, sep_token=UpperCAmelCase__					, pad_token=UpperCAmelCase__					, cls_token=UpperCAmelCase__					, mask_token=UpperCAmelCase__					, tokenize_chinese_chars=UpperCAmelCase__					, strip_accents=UpperCAmelCase__					, **UpperCAmelCase__					, )

           A__       =    json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               pre_tok_state.get('''lowercase'''					, UpperCAmelCase__) != do_lower_case
               or pre_tok_state.get('''strip_accents'''					, UpperCAmelCase__) != strip_accents
           ):
                  A__       =    getattr(UpperCAmelCase__					, pre_tok_state.pop('''type'''))
                  A__       =    do_lower_case
                  A__       =    strip_accents
                  A__       =    pre_tok_class(**UpperCAmelCase__)

           A__       =    do_lower_case


    def __getstate__(			self      :	Tuple)						->List[str]:





           '''simple docstring'''

           A__       =    self.__dict__.copy()
           A__       =    BertPreTokenizer()
           return state


    def __setstate__(			self      :	Any					, UpperCAmelCase__      :	Tuple)						->List[Any]:





           '''simple docstring'''

           A__       =    d
           A__       =    self.__dict__['''_tokenizer'''].get_vocab()
           A__       =    PreTokenizer.custom(JiebaPreTokenizer(UpperCAmelCase__))


    def        SCREAMING_SNAKE_CASE							(			self      :	Any					, UpperCAmelCase__      :	Optional[int]					, UpperCAmelCase__      :	Union[str, Any]=None)						->Dict:





           '''simple docstring'''

           A__       =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output


    def        SCREAMING_SNAKE_CASE							(			self      :	Dict					, UpperCAmelCase__      :	List[int]					, UpperCAmelCase__      :	Optional[List[int]] = None)						->List[int]:





           '''simple docstring'''

           A__       =    [self.sep_token_id]
           A__       =    [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple					, UpperCAmelCase__      :	str					, UpperCAmelCase__      :	Optional[str] = None)						->Tuple[str]:





           '''simple docstring'''

           A__       =    self._tokenizer.model.save(UpperCAmelCase__					, name=UpperCAmelCase__)
           return tuple(UpperCAmelCase__)





    def        SCREAMING_SNAKE_CASE							(			self      :	Any					, UpperCAmelCase__      :	List[Any]					, UpperCAmelCase__      :	Union[str, Any]=None					, UpperCAmelCase__      :	Dict=None					, UpperCAmelCase__      :	List[Any]=False					, **UpperCAmelCase__      :	str					, )						->List[str]:





           '''simple docstring'''

           A__       =    BertPreTokenizer()
           return super().save_pretrained(UpperCAmelCase__					, UpperCAmelCase__					, UpperCAmelCase__					, UpperCAmelCase__					, **UpperCAmelCase__)








import os
import sys
import unittest


_lowerCamelCase	:		Optional[Any]							       =   os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import get_test_info  # noqa: E402
from get_test_info import (  # noqa: E402
    get_model_to_test_mapping,
    get_model_to_tester_mapping,
    get_test_to_tester_mapping,
)


_lowerCamelCase	:		Any							       =   os.path.join("""tests""", """models""", """bert""", """test_modeling_bert.py""")
_lowerCamelCase	:		str							       =   os.path.join("""tests""", """models""", """blip""", """test_modeling_blip.py""")


class 				UpperCamelCase_    ( unittest.TestCase     ):

    '''simple docstring'''






    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple)						->Tuple:





           '''simple docstring'''

           A__       =    get_test_to_tester_mapping(UpperCAmelCase__)
           A__       =    get_test_to_tester_mapping(UpperCAmelCase__)

           A__       =    {'''BertModelTest''': '''BertModelTester'''}

           A__       =    {
               '''BlipModelTest''': '''BlipModelTester''',
               '''BlipTextImageModelTest''': '''BlipTextImageModelsModelTester''',
               '''BlipTextModelTest''': '''BlipTextModelTester''',
               '''BlipTextRetrievalModelTest''': '''BlipTextRetrievalModelTester''',
               '''BlipVQAModelTest''': '''BlipVQAModelTester''',
               '''BlipVisionModelTest''': '''BlipVisionModelTester''',
           }

           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)
           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple)						->List[Any]:





           '''simple docstring'''

           A__       =    get_model_to_test_mapping(UpperCAmelCase__)
           A__       =    get_model_to_test_mapping(UpperCAmelCase__)

           A__       =    {
               '''BertForMaskedLM''': ['''BertModelTest'''],
               '''BertForMultipleChoice''': ['''BertModelTest'''],
               '''BertForNextSentencePrediction''': ['''BertModelTest'''],
               '''BertForPreTraining''': ['''BertModelTest'''],
               '''BertForQuestionAnswering''': ['''BertModelTest'''],
               '''BertForSequenceClassification''': ['''BertModelTest'''],
               '''BertForTokenClassification''': ['''BertModelTest'''],
               '''BertLMHeadModel''': ['''BertModelTest'''],
               '''BertModel''': ['''BertModelTest'''],
           }

           A__       =    {
               '''BlipForConditionalGeneration''': ['''BlipTextImageModelTest'''],
               '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTest'''],
               '''BlipForQuestionAnswering''': ['''BlipVQAModelTest'''],
               '''BlipModel''': ['''BlipModelTest'''],
               '''BlipTextModel''': ['''BlipTextModelTest'''],
               '''BlipVisionModel''': ['''BlipVisionModelTest'''],
           }

           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)
           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)





    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any])						->str:





           '''simple docstring'''

           A__       =    get_model_to_tester_mapping(UpperCAmelCase__)
           A__       =    get_model_to_tester_mapping(UpperCAmelCase__)

           A__       =    {
               '''BertForMaskedLM''': ['''BertModelTester'''],
               '''BertForMultipleChoice''': ['''BertModelTester'''],
               '''BertForNextSentencePrediction''': ['''BertModelTester'''],
               '''BertForPreTraining''': ['''BertModelTester'''],
               '''BertForQuestionAnswering''': ['''BertModelTester'''],
               '''BertForSequenceClassification''': ['''BertModelTester'''],
               '''BertForTokenClassification''': ['''BertModelTester'''],
               '''BertLMHeadModel''': ['''BertModelTester'''],
               '''BertModel''': ['''BertModelTester'''],
           }

           A__       =    {
               '''BlipForConditionalGeneration''': ['''BlipTextImageModelsModelTester'''],
               '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTester'''],
               '''BlipForQuestionAnswering''': ['''BlipVQAModelTester'''],
               '''BlipModel''': ['''BlipModelTester'''],
               '''BlipTextModel''': ['''BlipTextModelTester'''],
               '''BlipVisionModel''': ['''BlipVisionModelTester'''],
           }

           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)
           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)


