






"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_UpperCamelCase		:  str						=		logging.get_logger(__name__)

_UpperCamelCase		:  Optional[int]						=		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase		:  Optional[int]						=		{
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

_UpperCamelCase		:  Optional[Any]						=		{
    """junnyu/roformer_chinese_small""": 1_5_3_6,
    """junnyu/roformer_chinese_base""": 1_5_3_6,
    """junnyu/roformer_chinese_char_small""": 5_1_2,
    """junnyu/roformer_chinese_char_base""": 5_1_2,
    """junnyu/roformer_small_discriminator""": 1_2_8,
    """junnyu/roformer_small_generator""": 1_2_8,
}


_UpperCamelCase		:  List[Any]						=		{
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}

class        a     (   UpperCAmelCase__	):
    UpperCAmelCase_ :     List[str]	     =VOCAB_FILES_NAMES
    UpperCAmelCase_ :     str	     =PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase_ :     str	     =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase_ :     Dict	     =PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase_ :     Union[str, Any]	     =RoFormerTokenizer







    def __init__(			self							,		_lowerCamelCase=None							,		_lowerCamelCase=None							,		_lowerCamelCase=True							,		_lowerCamelCase="[UNK]"							,		_lowerCamelCase="[SEP]"							,		_lowerCamelCase="[PAD]"							,		_lowerCamelCase="[CLS]"							,		_lowerCamelCase="[MASK]"							,		_lowerCamelCase=True							,		_lowerCamelCase=None							,		**_lowerCamelCase							,		):
      super().__init__(
          UpperCAmelCase__							,		tokenizer_file=UpperCAmelCase__							,		do_lower_case=UpperCAmelCase__							,		unk_token=UpperCAmelCase__							,		sep_token=UpperCAmelCase__							,		pad_token=UpperCAmelCase__							,		cls_token=UpperCAmelCase__							,		mask_token=UpperCAmelCase__							,		tokenize_chinese_chars=UpperCAmelCase__							,		strip_accents=UpperCAmelCase__							,		**UpperCAmelCase__							,		)

      lowercase							    =       json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
      if (
          pre_tok_state.get('lowercase'							,		UpperCAmelCase__       ) != do_lower_case
          or pre_tok_state.get('strip_accents'							,		UpperCAmelCase__       ) != strip_accents
      ):
        lowercase							    =       getattr(UpperCAmelCase__							,		pre_tok_state.pop('type'       )       )
        lowercase							    =       do_lower_case
        lowercase							    =       strip_accents
        lowercase							    =       pre_tok_class(**UpperCAmelCase__       )

      lowercase							    =       do_lower_case







    def __getstate__(			self       ):
      lowercase							    =       self.__dict__.copy()
      lowercase							    =       BertPreTokenizer()
      return state







    def __setstate__(			self							,		_lowerCamelCase       ):
      lowercase							    =       d
      lowercase							    =       self.__dict__['_tokenizer'].get_vocab()
      lowercase							    =       PreTokenizer.custom(JiebaPreTokenizer(UpperCAmelCase__       )       )







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		_lowerCamelCase=None       ):
      lowercase							    =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		_lowerCamelCase = None       ):
      lowercase							    =       [self.sep_token_id]
      lowercase							    =       [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		_lowerCamelCase = None       ):
      lowercase							    =       self._tokenizer.model.save(UpperCAmelCase__							,		name=UpperCAmelCase__       )
      return tuple(UpperCAmelCase__       )







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		_lowerCamelCase=None							,		_lowerCamelCase=None							,		_lowerCamelCase=False							,		**_lowerCamelCase							,		):
      lowercase							    =       BertPreTokenizer()
      return super().save_pretrained(UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__							,		**UpperCAmelCase__       )








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


_lowerCamelCase	:		int							       =   {
    """configuration_blip""": [
        """BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """BlipConfig""",
        """BlipTextConfig""",
        """BlipVisionConfig""",
    ],
    """processing_blip""": ["""BlipProcessor"""],
}

try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCamelCase	:		Tuple							       =   ["""BlipImageProcessor"""]


try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCamelCase	:		List[Any]							       =   [
      """BLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """BlipModel""",
      """BlipPreTrainedModel""",
      """BlipForConditionalGeneration""",
      """BlipForQuestionAnswering""",
      """BlipVisionModel""",
      """BlipTextModel""",
      """BlipForImageTextRetrieval""",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCamelCase	:		Optional[Any]							       =   [
      """TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TFBlipModel""",
      """TFBlipPreTrainedModel""",
      """TFBlipForConditionalGeneration""",
      """TFBlipForQuestionAnswering""",
      """TFBlipVisionModel""",
      """TFBlipTextModel""",
      """TFBlipForImageTextRetrieval""",
  ]

if TYPE_CHECKING:
  from .configuration_blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig
  from .processing_blip import BlipProcessor

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .image_processing_blip import BlipImageProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_blip import (
        BLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
        BlipForConditionalGeneration,
        BlipForImageTextRetrieval,
        BlipForQuestionAnswering,
        BlipModel,
        BlipPreTrainedModel,
        BlipTextModel,
        BlipVisionModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_blip import (
        TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFBlipForConditionalGeneration,
        TFBlipForImageTextRetrieval,
        TFBlipForQuestionAnswering,
        TFBlipModel,
        TFBlipPreTrainedModel,
        TFBlipTextModel,
        TFBlipVisionModel,
    )

else:
  import sys

  _lowerCamelCase	:		List[Any]							       =   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


