


'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


a_  :				Optional[int]						= logging.get_logger(__name__)

a_  :				Dict						= {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a_  :				Any						= {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

a_  :				int						= {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

a_  :				int						= {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}





class      snake_case    (    lowercase			):





  """simple docstring"""




  _lowerCamelCase		    =  VOCAB_FILES_NAMES
  _lowerCamelCase		    =  PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase		    =  PRETRAINED_INIT_CONFIGURATION
  _lowerCamelCase		    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase		    =  RealmTokenizer






  def __init__(    self			,  UpperCamelCase=None			,  UpperCamelCase=None			,  UpperCamelCase=True			,  UpperCamelCase="[UNK]"			,  UpperCamelCase="[SEP]"			,  UpperCamelCase="[PAD]"			,  UpperCamelCase="[CLS]"			,  UpperCamelCase="[MASK]"			,  UpperCamelCase=True			,  UpperCamelCase=None			,  **UpperCamelCase			,  ):







   """simple docstring"""
   super().__init__(
       UpperCamelCase			,  tokenizer_file=UpperCamelCase			,  do_lower_case=UpperCamelCase			,  unk_token=UpperCamelCase			,  sep_token=UpperCamelCase			,  pad_token=UpperCamelCase			,  cls_token=UpperCamelCase			,  mask_token=UpperCamelCase			,  tokenize_chinese_chars=UpperCamelCase			,  strip_accents=UpperCamelCase			,  **UpperCamelCase			,  )

   lowerCamelCase_      =  json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
   if (
       normalizer_state.get("lowercase"			,  UpperCamelCase     ) != do_lower_case
       or normalizer_state.get("strip_accents"			,  UpperCamelCase     ) != strip_accents
       or normalizer_state.get("handle_chinese_chars"			,  UpperCamelCase     ) != tokenize_chinese_chars
   ):
    lowerCamelCase_      =  getattr(UpperCamelCase			,  normalizer_state.pop("type"     )     )
    lowerCamelCase_      =  do_lower_case
    lowerCamelCase_      =  strip_accents
    lowerCamelCase_      =  tokenize_chinese_chars
    lowerCamelCase_      =  normalizer_class(**UpperCamelCase     )

   lowerCamelCase_      =  do_lower_case






  def   snake_case		(    self			,  UpperCamelCase			,  **UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  PaddingStrategy.MAX_LENGTH

   lowerCamelCase_      =  text
   lowerCamelCase_      =  kwargs.pop("text_pair"			,  UpperCamelCase     )
   lowerCamelCase_      =  kwargs.pop("return_tensors"			,  UpperCamelCase     )

   lowerCamelCase_      =  {
       "input_ids": [],
       "attention_mask": [],
       "token_type_ids": [],
   }

   for idx, candidate_text in enumerate(UpperCamelCase     ):
    if batch_text_pair is not None:
     lowerCamelCase_      =  batch_text_pair[idx]
    else:
     lowerCamelCase_      =  None

    lowerCamelCase_      =  super().__call__(UpperCamelCase			,  UpperCamelCase			,  return_tensors=UpperCamelCase			,  **UpperCamelCase     )

    lowerCamelCase_      =  encoded_candidates.get("input_ids"     )
    lowerCamelCase_      =  encoded_candidates.get("attention_mask"     )
    lowerCamelCase_      =  encoded_candidates.get("token_type_ids"     )

    if encoded_input_ids is not None:
     output_data["input_ids"].append(UpperCamelCase     )
    if encoded_attention_mask is not None:
     output_data["attention_mask"].append(UpperCamelCase     )
    if encoded_token_type_ids is not None:
     output_data["token_type_ids"].append(UpperCamelCase     )

   lowerCamelCase_      =  {key: item for key, item in output_data.items() if len(UpperCamelCase     ) != 0}

   return BatchEncoding(UpperCamelCase			,  tensor_type=UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase=None     ):







   """simple docstring"""
   lowerCamelCase_      =  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
    output += token_ids_a + [self.sep_token_id]

   return output






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  [self.sep_token_id]
   lowerCamelCase_      =  [self.cls_token_id]
   if token_ids_a is None:
    return len(cls + token_ids_a + sep     ) * [0]
   return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  self._tokenizer.model.save(UpperCamelCase			,  name=UpperCamelCase     )
   return tuple(UpperCamelCase     )







'''simple docstring'''

from __future__ import annotations

from typing import Generic, TypeVar

a_  :				List[str]						= TypeVar("""T""")





class      snake_case    (    Generic[T]			):





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  data
   lowerCamelCase_      =  self
   lowerCamelCase_      =  0





class      snake_case    (    Generic[T]			):





  """simple docstring"""




  def __init__(    self     ):







   """simple docstring"""
   # map from node name to the node object
   lowerCamelCase_      =  {}






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   # create a new set with x as its member
   lowerCamelCase_      =  DisjointSetTreeNode(UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   # find the set x belongs to (with path-compression)
   lowerCamelCase_      =  self.map[data]
   if elem_ref != elem_ref.parent:
    lowerCamelCase_      =  self.find_set(elem_ref.parent.data     )
   return elem_ref.parent






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase     ):







   """simple docstring"""
   # helper function for union operation
   if nodea.rank > nodea.rank:
    lowerCamelCase_      =  nodea
   else:
    lowerCamelCase_      =  nodea
    if nodea.rank == nodea.rank:
     nodea.rank += 1







  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase     ):







   """simple docstring"""
   # merge 2 disjoint sets
   self.link(self.find_set(UpperCamelCase     )			,  self.find_set(UpperCamelCase     )     )





class      snake_case    (    Generic[T]			):





  """simple docstring"""




  def __init__(    self     ):







   """simple docstring"""
   # connections: map from the node to the neighbouring nodes (with weights)
   lowerCamelCase_      =  {}






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   # add a node ONLY if its not present in the graph
   if node not in self.connections:
    lowerCamelCase_      =  {}






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase     ):







   """simple docstring"""
   # add an edge with the given weight
   self.add_node(UpperCamelCase     )
   self.add_node(UpperCamelCase     )
   lowerCamelCase_      =  weight
   lowerCamelCase_      =  weight







  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  []
   lowerCamelCase_      =  set()
   for start in self.connections:
    for end in self.connections[start]:
     if (start, end) not in seen:
      seen.add((end, start)     )
      edges.append((start, end, self.connections[start][end])     )
   edges.sort(key=lambda UpperCamelCase     : x[2]     )

   # creating the disjoint set
   lowerCamelCase_      =  DisjointSetTree[T]()
   for node in self.connections:
    disjoint_set.make_set(UpperCamelCase     )

   # MST generation
   lowerCamelCase_      =  0
   lowerCamelCase_      =  0
   lowerCamelCase_      =  GraphUndirectedWeighted[T]()
   while num_edges < len(self.connections     ) - 1:
    lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_      =  edges[index]
    index += 1
    lowerCamelCase_      =  disjoint_set.find_set(UpperCamelCase     )
    lowerCamelCase_      =  disjoint_set.find_set(UpperCamelCase     )
    if parent_u != parent_v:
     num_edges += 1
     graph.add_edge(UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase     )
     disjoint_set.union(UpperCamelCase			,  UpperCamelCase     )
   return graph



