





'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


a__							:							Optional[Any] =logging.get_logger(__name__)

a__							:							str ={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a__							:							Optional[int] ={
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

a__							:							List[Any] ={
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

a__							:							Tuple ={
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class      snake_case     (    __lowerCamelCase       ):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_	:					str					=VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE_	:					Union[str, Any]					=PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE_	:					Tuple					=PRETRAINED_INIT_CONFIGURATION
 SCREAMING_SNAKE_CASE_	:					Optional[int]					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE_	:					List[Any]					=RealmTokenizer






 def __init__(       self       :							Tuple							,	__A       :							Optional[Any]=None							,	__A       :							Optional[Any]=None							,	__A       :							Tuple=True							,	__A       :							Dict="[UNK]"							,	__A       :							Union[str, Any]="[SEP]"							,	__A       :							Union[str, Any]="[PAD]"							,	__A       :							Optional[int]="[CLS]"							,	__A       :							List[Any]="[MASK]"							,	__A       :							Union[str, Any]=True							,	__A       :							Optional[int]=None							,	**__A       :							Any							,	):
        super().__init__(
            __A							,	tokenizer_file=__A							,	do_lower_case=__A							,	unk_token=__A							,	sep_token=__A							,	pad_token=__A							,	cls_token=__A							,	mask_token=__A							,	tokenize_chinese_chars=__A							,	strip_accents=__A							,	**__A							,	)

        __UpperCamelCase       =  json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
        if (
            normalizer_state.get('lowercase'							,	__A	) != do_lower_case
            or normalizer_state.get('strip_accents'							,	__A	) != strip_accents
            or normalizer_state.get('handle_chinese_chars'							,	__A	) != tokenize_chinese_chars
        ):
               __UpperCamelCase       =  getattr(__A							,	normalizer_state.pop('type'	)	)
               __UpperCamelCase       =  do_lower_case
               __UpperCamelCase       =  strip_accents
               __UpperCamelCase       =  tokenize_chinese_chars
               __UpperCamelCase       =  normalizer_class(**__A	)

        __UpperCamelCase       =  do_lower_case






 def        _lowerCamelCase					(       self       :							List[str]							,	__A       :							Union[str, Any]							,	**__A       :							List[str]	):
        __UpperCamelCase       =  PaddingStrategy.MAX_LENGTH

        __UpperCamelCase       =  text
        __UpperCamelCase       =  kwargs.pop('text_pair'							,	__A	)
        __UpperCamelCase       =  kwargs.pop('return_tensors'							,	__A	)

        __UpperCamelCase       =  {
            """input_ids""": [],
            """attention_mask""": [],
            """token_type_ids""": [],
        }

        for idx, candidate_text in enumerate(__A	):
               if batch_text_pair is not None:
                      __UpperCamelCase       =  batch_text_pair[idx]
               else:
                      __UpperCamelCase       =  None

               __UpperCamelCase       =  super().__call__(__A							,	__A							,	return_tensors=__A							,	**__A	)

               __UpperCamelCase       =  encoded_candidates.get('input_ids'	)
               __UpperCamelCase       =  encoded_candidates.get('attention_mask'	)
               __UpperCamelCase       =  encoded_candidates.get('token_type_ids'	)

               if encoded_input_ids is not None:
                      output_data["input_ids"].append(__A	)
               if encoded_attention_mask is not None:
                      output_data["attention_mask"].append(__A	)
               if encoded_token_type_ids is not None:
                      output_data["token_type_ids"].append(__A	)

        __UpperCamelCase       =  {key: item for key, item in output_data.items() if len(__A	) != 0}

        return BatchEncoding(__A							,	tensor_type=__A	)






 def        _lowerCamelCase					(       self       :							Union[str, Any]							,	__A       :							Optional[Any]							,	__A       :							Tuple=None	):
        __UpperCamelCase       =  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

        return output






 def        _lowerCamelCase					(       self       :							Optional[Any]							,	__A       :							Optional[int]							,	__A       :							List[Any] = None	):
        __UpperCamelCase       =  [self.sep_token_id]
        __UpperCamelCase       =  [self.cls_token_id]
        if token_ids_a is None:
               return len(cls + token_ids_a + sep	) * [0]
        return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






 def        _lowerCamelCase					(       self       :							Union[str, Any]							,	__A       :							str							,	__A       :							Any = None	):
        __UpperCamelCase       =  self._tokenizer.model.save(__A							,	name=__A	)
        return tuple(__A	)


def 						_snake_case(  SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				) ->    List[str]:
       if index == r:
              for j in range(SCREAMING_SNAKE_CASE__				):
                     print(data[j]					,	end=""" """				)
              print(""" """				)
              return
       #  When no more elements are there to put in data[]
       if i >= n:
              return
       # current is included, put next at next location
       lowercase							:     Tuple       	=      arr[i]
       combination_util(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	index + 1					,	SCREAMING_SNAKE_CASE__					,	i + 1				)
       # current is excluded, replace it with
       # next (Note that i+1 is passed, but
       # index is not changed)
       combination_util(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	i + 1				)
       # The main function that prints all combinations
       # of size r in arr[] of size n. This function
       # mainly uses combinationUtil()







def 						_snake_case(  SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__				) ->    List[str]:
       # A temporary array to store all combination one by one
       lowercase							:     Optional[int]       	=      [0] * r
       # Print all combination using temporary array 'data[]'
       combination_util(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	0					,	SCREAMING_SNAKE_CASE__					,	0				)


if __name__ == "__main__":
  # Driver code to check the function above
  lowercase  :					int						    =							[10, 20, 30, 40, 50]
  print_combination(arr, len(arr), 3)
  # This code is contributed by Ambuj sahu
