





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


UpperCAmelCase_       : Any							=    logging.get_logger(__name__)

UpperCAmelCase_       : Tuple							=    {"""vocab_file""": """vocab.txt"""}

UpperCAmelCase_       : str							=    {
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

UpperCAmelCase_       : Optional[int]							=    {
    """tau/splinter-base""": 5_12,
    """tau/splinter-base-qass""": 5_12,
    """tau/splinter-large""": 5_12,
    """tau/splinter-large-qass""": 5_12,
}

UpperCAmelCase_       : Dict							=    {
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}





class   lowercase__     (				__A	):
  __UpperCamelCase						  =	VOCAB_FILES_NAMES
  __UpperCamelCase						  =	PRETRAINED_VOCAB_FILES_MAP
  __UpperCamelCase						  =	PRETRAINED_INIT_CONFIGURATION
  __UpperCamelCase						  =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __UpperCamelCase						  =	SplinterTokenizer







  def __init__(						self     ,    _lowercase=None     ,    _lowercase=None     ,    _lowercase=True     ,    _lowercase="[UNK]"     ,    _lowercase="[SEP]"     ,    _lowercase="[PAD]"     ,    _lowercase="[CLS]"     ,    _lowercase="[MASK]"     ,    _lowercase="[QUESTION]"     ,    _lowercase=True     ,    _lowercase=None     ,    **_lowercase     ,    ):
       super().__init__(
           _lowercase     ,    tokenizer_file=_lowercase     ,    do_lower_case=_lowercase     ,    unk_token=_lowercase     ,    sep_token=_lowercase     ,    pad_token=_lowercase     ,    cls_token=_lowercase     ,    mask_token=_lowercase     ,    tokenize_chinese_chars=_lowercase     ,    strip_accents=_lowercase     ,    additional_special_tokens=(question_token,)     ,    **_lowercase     ,    )

       lowerCAmelCase_       :							List[Any]			 =		json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
       if (
           pre_tok_state.get("""lowercase"""     ,    _lowercase    ) != do_lower_case
           or pre_tok_state.get("""strip_accents"""     ,    _lowercase    ) != strip_accents
       ):
            lowerCAmelCase_       :							Dict			 =		getattr(_lowercase     ,    pre_tok_state.pop("""type"""    )    )
            lowerCAmelCase_       :							List[Any]			 =		do_lower_case
            lowerCAmelCase_       :							List[Any]			 =		strip_accents
            lowerCAmelCase_       :							Union[str, Any]			 =		pre_tok_class(**_lowercase    )

       lowerCAmelCase_       :							Optional[int]			 =		do_lower_case







  @property
  def 						UpperCAmelCase__		(						self    ):
       return self.convert_tokens_to_ids(self.question_token    )







  def 						UpperCAmelCase__		(						self     ,    _lowercase     ,    _lowercase = None    ):
       if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       lowerCAmelCase_       :							Any			 =		[self.cls_token_id]
       lowerCAmelCase_       :							Optional[Any]			 =		[self.sep_token_id]
       lowerCAmelCase_       :							List[Any]			 =		[self.question_token_id] + [self.convert_tokens_to_ids("""."""    )]
       if self.padding_side == "right":
            # Input is question-then-context
            return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
       else:
            # Input is context-then-question
            return cls + token_ids_a + sep + token_ids_a + question_suffix + sep







  def 						UpperCAmelCase__		(						self     ,    _lowercase     ,    _lowercase = None    ):
       lowerCAmelCase_       :							Union[str, Any]			 =		[self.sep_token_id]
       lowerCAmelCase_       :							Any			 =		[self.cls_token_id]
       lowerCAmelCase_       :							int			 =		[self.question_token_id] + [self.convert_tokens_to_ids("""."""    )]
       if token_ids_a is None:
            return len(cls + token_ids_a + sep    ) * [0]

       if self.padding_side == "right":
            # Input is question-then-context
            return len(cls + token_ids_a + question_suffix + sep    ) * [0] + len(token_ids_a + sep    ) * [1]
       else:
            # Input is context-then-question
            return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + question_suffix + sep    ) * [1]







  def 						UpperCAmelCase__		(						self     ,    _lowercase     ,    _lowercase = None    ):
       lowerCAmelCase_       :							Optional[int]			 =		self._tokenizer.model.save(_lowercase     ,    name=_lowercase    )
       return tuple(_lowercase    )







from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCAmelCase_       : List[Any]							=    {
    """configuration_retribert""": ["""RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RetriBertConfig"""],
    """tokenization_retribert""": ["""RetriBertTokenizer"""],
}

try:
      if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_       : str							=    ["""RetriBertTokenizerFast"""]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_       : Optional[Any]							=    [
          """RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """RetriBertModel""",
          """RetriBertPreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
      from .tokenization_retribert import RetriBertTokenizer

      try:
            if not is_tokenizers_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_retribert_fast import RetriBertTokenizerFast

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_retribert import (
                RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                RetriBertModel,
                RetriBertPreTrainedModel,
            )

else:
      import sys

      UpperCAmelCase_       : Tuple							=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
