




UpperCAmelCase_									=	[
    'Audio',
    'Array2D',
    'Array3D',
    'Array4D',
    'Array5D',
    'ClassLabel',
    'Features',
    'Sequence',
    'Value',
    'Image',
    'Translation',
    'TranslationVariableLanguages',
]
from .audio import Audio
from .features import ArrayaD, ArrayaD, ArrayaD, ArrayaD, ClassLabel, Features, Sequence, Value
from .image import Image
from .translation import Translation, TranslationVariableLanguages








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase				:       Optional[int]	   =				logging.get_logger(__name__)

_UpperCAmelCase				:       Any	   =				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase				:       int	   =				{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase				:       int	   =				{
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

_UpperCAmelCase				:       Any	   =				{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class    lowerCAmelCase							(					__UpperCamelCase			):
 UpperCAmelCase__													=		VOCAB_FILES_NAMES
 UpperCAmelCase__													=		PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__													=		PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__													=		RealmTokenizer
 def __init__(      self  :      Optional[int]			,     UpperCAmelCase  :      Tuple=None			,     UpperCAmelCase  :      Any=None			,     UpperCAmelCase  :      List[Any]=True			,     UpperCAmelCase  :      Optional[Any]="[UNK]"			,     UpperCAmelCase  :      Any="[SEP]"			,     UpperCAmelCase  :      Tuple="[PAD]"			,     UpperCAmelCase  :      List[Any]="[CLS]"			,     UpperCAmelCase  :      Union[str, Any]="[MASK]"			,     UpperCAmelCase  :      Optional[Any]=True			,     UpperCAmelCase  :      Any=None			,     **UpperCAmelCase  :      Optional[int]			,     )			->	str:
     super().__init__(
         UpperCAmelCase			,     tokenizer_file=UpperCAmelCase			,     do_lower_case=UpperCAmelCase			,     unk_token=UpperCAmelCase			,     sep_token=UpperCAmelCase			,     pad_token=UpperCAmelCase			,     cls_token=UpperCAmelCase			,     mask_token=UpperCAmelCase			,     tokenize_chinese_chars=UpperCAmelCase			,     strip_accents=UpperCAmelCase			,     **UpperCAmelCase			,     )

     lowerCamelCase__    :      List[Any]					 =		json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
     if (
         normalizer_state.get('lowercase'			,     UpperCAmelCase	) != do_lower_case
         or normalizer_state.get('strip_accents'			,     UpperCAmelCase	) != strip_accents
         or normalizer_state.get('handle_chinese_chars'			,     UpperCAmelCase	) != tokenize_chinese_chars
     ):
         lowerCamelCase__    :      Optional[int]					 =		getattr(UpperCAmelCase			,     normalizer_state.pop('type'	)	)
         lowerCamelCase__    :      Optional[Any]					 =		do_lower_case
         lowerCamelCase__    :      str					 =		strip_accents
         lowerCamelCase__    :      Optional[Any]					 =		tokenize_chinese_chars
         lowerCamelCase__    :      int					 =		normalizer_class(**UpperCAmelCase	)

     lowerCamelCase__    :      str					 =		do_lower_case
 def      A_	(      self  :      Optional[int]			,     UpperCAmelCase  :      int			,     **UpperCAmelCase  :      int	)			->	List[Any]:
     lowerCamelCase__    :      List[Any]					 =		PaddingStrategy.MAX_LENGTH

     lowerCamelCase__    :      Optional[int]					 =		text
     lowerCamelCase__    :      Dict					 =		kwargs.pop('text_pair'			,     UpperCAmelCase	)
     lowerCamelCase__    :      List[Any]					 =		kwargs.pop('return_tensors'			,     UpperCAmelCase	)

     lowerCamelCase__    :      List[Any]					 =		{
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(UpperCAmelCase	):
         if batch_text_pair is not None:
             lowerCamelCase__    :      Tuple					 =		batch_text_pair[idx]
         else:
             lowerCamelCase__    :      Dict					 =		None

         lowerCamelCase__    :      Optional[int]					 =		super().__call__(UpperCAmelCase			,     UpperCAmelCase			,     return_tensors=UpperCAmelCase			,     **UpperCAmelCase	)

         lowerCamelCase__    :      Any					 =		encoded_candidates.get('input_ids'	)
         lowerCamelCase__    :      Union[str, Any]					 =		encoded_candidates.get('attention_mask'	)
         lowerCamelCase__    :      Tuple					 =		encoded_candidates.get('token_type_ids'	)

         if encoded_input_ids is not None:
             output_data["input_ids"].append(UpperCAmelCase	)
         if encoded_attention_mask is not None:
             output_data["attention_mask"].append(UpperCAmelCase	)
         if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(UpperCAmelCase	)

     lowerCamelCase__    :      int					 =		{key: item for key, item in output_data.items() if len(UpperCAmelCase	) != 0}

     return BatchEncoding(UpperCAmelCase			,     tensor_type=UpperCAmelCase	)
 def      A_	(      self  :      int			,     UpperCAmelCase  :      Optional[Any]			,     UpperCAmelCase  :      Optional[int]=None	)			->	List[str]:
     lowerCamelCase__    :      Tuple					 =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output
 def      A_	(      self  :      Tuple			,     UpperCAmelCase  :      List[int]			,     UpperCAmelCase  :      Optional[List[int]] = None	)			->	List[int]:
     lowerCamelCase__    :      List[Any]					 =		[self.sep_token_id]
     lowerCamelCase__    :      Optional[int]					 =		[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[str] = None	)			->	Tuple[str]:
     lowerCamelCase__    :      int					 =		self._tokenizer.model.save(UpperCAmelCase			,     name=UpperCAmelCase	)
     return tuple(UpperCAmelCase	)





