






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


A_       :int       =						{
    """configuration_nezha""": ["""NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """NezhaConfig"""],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							A_       :Any       =						[
							    """NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """NezhaForNextSentencePrediction""",
							    """NezhaForMaskedLM""",
							    """NezhaForPreTraining""",
							    """NezhaForMultipleChoice""",
							    """NezhaForQuestionAnswering""",
							    """NezhaForSequenceClassification""",
							    """NezhaForTokenClassification""",
							    """NezhaModel""",
							    """NezhaPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_nezha import (
														    NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
														    NezhaForMaskedLM,
														    NezhaForMultipleChoice,
														    NezhaForNextSentencePrediction,
														    NezhaForPreTraining,
														    NezhaForQuestionAnswering,
														    NezhaForSequenceClassification,
														    NezhaForTokenClassification,
														    NezhaModel,
														    NezhaPreTrainedModel,
														)


else:
							import sys

							A_       :Any       =						_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase				:       Optional[int]	   =				logging.get_logger(__name__)

_UpperCAmelCase				:       Any	   =				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase				:       int	   =				{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase				:       int	   =				{
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

_UpperCAmelCase				:       Any	   =				{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class    lowerCAmelCase							(					__UpperCamelCase			):
 UpperCAmelCase__													=		VOCAB_FILES_NAMES
 UpperCAmelCase__													=		PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__													=		PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__													=		RealmTokenizer
 def __init__(      self  :      Optional[int]			,     UpperCAmelCase  :      Tuple=None			,     UpperCAmelCase  :      Any=None			,     UpperCAmelCase  :      List[Any]=True			,     UpperCAmelCase  :      Optional[Any]="[UNK]"			,     UpperCAmelCase  :      Any="[SEP]"			,     UpperCAmelCase  :      Tuple="[PAD]"			,     UpperCAmelCase  :      List[Any]="[CLS]"			,     UpperCAmelCase  :      Union[str, Any]="[MASK]"			,     UpperCAmelCase  :      Optional[Any]=True			,     UpperCAmelCase  :      Any=None			,     **UpperCAmelCase  :      Optional[int]			,     )			->	str:
     super().__init__(
         UpperCAmelCase			,     tokenizer_file=UpperCAmelCase			,     do_lower_case=UpperCAmelCase			,     unk_token=UpperCAmelCase			,     sep_token=UpperCAmelCase			,     pad_token=UpperCAmelCase			,     cls_token=UpperCAmelCase			,     mask_token=UpperCAmelCase			,     tokenize_chinese_chars=UpperCAmelCase			,     strip_accents=UpperCAmelCase			,     **UpperCAmelCase			,     )

     lowerCamelCase__    :      List[Any]					 =		json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
     if (
         normalizer_state.get('lowercase'			,     UpperCAmelCase	) != do_lower_case
         or normalizer_state.get('strip_accents'			,     UpperCAmelCase	) != strip_accents
         or normalizer_state.get('handle_chinese_chars'			,     UpperCAmelCase	) != tokenize_chinese_chars
     ):
         lowerCamelCase__    :      Optional[int]					 =		getattr(UpperCAmelCase			,     normalizer_state.pop('type'	)	)
         lowerCamelCase__    :      Optional[Any]					 =		do_lower_case
         lowerCamelCase__    :      str					 =		strip_accents
         lowerCamelCase__    :      Optional[Any]					 =		tokenize_chinese_chars
         lowerCamelCase__    :      int					 =		normalizer_class(**UpperCAmelCase	)

     lowerCamelCase__    :      str					 =		do_lower_case
 def      A_	(      self  :      Optional[int]			,     UpperCAmelCase  :      int			,     **UpperCAmelCase  :      int	)			->	List[Any]:
     lowerCamelCase__    :      List[Any]					 =		PaddingStrategy.MAX_LENGTH

     lowerCamelCase__    :      Optional[int]					 =		text
     lowerCamelCase__    :      Dict					 =		kwargs.pop('text_pair'			,     UpperCAmelCase	)
     lowerCamelCase__    :      List[Any]					 =		kwargs.pop('return_tensors'			,     UpperCAmelCase	)

     lowerCamelCase__    :      List[Any]					 =		{
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(UpperCAmelCase	):
         if batch_text_pair is not None:
             lowerCamelCase__    :      Tuple					 =		batch_text_pair[idx]
         else:
             lowerCamelCase__    :      Dict					 =		None

         lowerCamelCase__    :      Optional[int]					 =		super().__call__(UpperCAmelCase			,     UpperCAmelCase			,     return_tensors=UpperCAmelCase			,     **UpperCAmelCase	)

         lowerCamelCase__    :      Any					 =		encoded_candidates.get('input_ids'	)
         lowerCamelCase__    :      Union[str, Any]					 =		encoded_candidates.get('attention_mask'	)
         lowerCamelCase__    :      Tuple					 =		encoded_candidates.get('token_type_ids'	)

         if encoded_input_ids is not None:
             output_data["input_ids"].append(UpperCAmelCase	)
         if encoded_attention_mask is not None:
             output_data["attention_mask"].append(UpperCAmelCase	)
         if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(UpperCAmelCase	)

     lowerCamelCase__    :      int					 =		{key: item for key, item in output_data.items() if len(UpperCAmelCase	) != 0}

     return BatchEncoding(UpperCAmelCase			,     tensor_type=UpperCAmelCase	)
 def      A_	(      self  :      int			,     UpperCAmelCase  :      Optional[Any]			,     UpperCAmelCase  :      Optional[int]=None	)			->	List[str]:
     lowerCamelCase__    :      Tuple					 =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output
 def      A_	(      self  :      Tuple			,     UpperCAmelCase  :      List[int]			,     UpperCAmelCase  :      Optional[List[int]] = None	)			->	List[int]:
     lowerCamelCase__    :      List[Any]					 =		[self.sep_token_id]
     lowerCamelCase__    :      Optional[int]					 =		[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[str] = None	)			->	Tuple[str]:
     lowerCamelCase__    :      int					 =		self._tokenizer.model.save(UpperCAmelCase			,     name=UpperCAmelCase	)
     return tuple(UpperCAmelCase	)





