




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__A						:		Any          =						logging.get_logger(__name__)

__A						:		Optional[Any]          =						{"vocab_file": "vocab.txt"}

__A						:		str          =						{
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

__A						:		Tuple          =						{
    "tau/splinter-base": 512,
    "tau/splinter-base-qass": 512,
    "tau/splinter-large": 512,
    "tau/splinter-large-qass": 512,
}

__A						:		Dict          =						{
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}

class 		lowercase_       (	lowerCAmelCase__				):
					__UpperCamelCase     	= VOCAB_FILES_NAMES
					__UpperCamelCase     	= PRETRAINED_VOCAB_FILES_MAP
					__UpperCamelCase     	= PRETRAINED_INIT_CONFIGURATION
					__UpperCamelCase     	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__UpperCamelCase     	= SplinterTokenizer




					def __init__(   self:		List[Any],     _lowercase:		List[str]=None,     _lowercase:		int=None,     _lowercase:		List[str]=True,     _lowercase:		Dict="[UNK]",     _lowercase:		Tuple="[SEP]",     _lowercase:		Union[str, Any]="[PAD]",     _lowercase:		Optional[Any]="[CLS]",     _lowercase:		Optional[int]="[MASK]",     _lowercase:		str="[QUESTION]",     _lowercase:		int=True,     _lowercase:		str=None,     **_lowercase:		List[Any],     ):


									'''simple docstring'''


									super().__init__(
									    _lowercase,     tokenizer_file=_lowercase,     do_lower_case=_lowercase,     unk_token=_lowercase,     sep_token=_lowercase,     pad_token=_lowercase,     cls_token=_lowercase,     mask_token=_lowercase,     tokenize_chinese_chars=_lowercase,     strip_accents=_lowercase,     additional_special_tokens=(question_token,),     **_lowercase,     )

									__lowerCAmelCase					=  json.loads(self.backend_tokenizer.normalizer.__getstate__())
									if (
									    pre_tok_state.get("""lowercase""",     _lowercase) != do_lower_case
									    or pre_tok_state.get("""strip_accents""",     _lowercase) != strip_accents
									):
													__lowerCAmelCase					=  getattr(_lowercase,     pre_tok_state.pop("""type"""))
													__lowerCAmelCase					=  do_lower_case
													__lowerCAmelCase					=  strip_accents
													__lowerCAmelCase					=  pre_tok_class(**_lowercase)

									__lowerCAmelCase					=  do_lower_case




					@property
					def     _lowercase						(   self:		List[str]):


									'''simple docstring'''


									return self.convert_tokens_to_ids(self.question_token)




					def     _lowercase						(   self:		Optional[int],     _lowercase:		List[int],     _lowercase:		Optional[List[int]] = None):


									'''simple docstring'''


									if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									__lowerCAmelCase					=  [self.cls_token_id]
									__lowerCAmelCase					=  [self.sep_token_id]
									__lowerCAmelCase					=  [self.question_token_id] + [self.convert_tokens_to_ids(""".""")]
									if self.padding_side == "right":
													# Input is question-then-context
													return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
									else:
													# Input is context-then-question
													return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




					def     _lowercase						(   self:		str,     _lowercase:		List[int],     _lowercase:		Optional[List[int]] = None):


									'''simple docstring'''


									__lowerCAmelCase					=  [self.sep_token_id]
									__lowerCAmelCase					=  [self.cls_token_id]
									__lowerCAmelCase					=  [self.question_token_id] + [self.convert_tokens_to_ids(""".""")]
									if token_ids_a is None:
													return len(cls + token_ids_a + sep) * [0]

									if self.padding_side == "right":
													# Input is question-then-context
													return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
									else:
													# Input is context-then-question
													return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]







					def     _lowercase						(   self:		Optional[int],     _lowercase:		str,     _lowercase:		Optional[str] = None):


									'''simple docstring'''


									__lowerCAmelCase					=  self._tokenizer.model.save(_lowercase,     name=_lowercase)
									return tuple(_lowercase)













from __future__ import annotations

from typing import Any
class 							__snake_case:



 def __init__(						self : str			,		UpperCAmelCase__ : Any			,		UpperCAmelCase__ : Any						):  # noqa: A002

    """simple docstring"""






    lowercase__    :						Optional[int]       =			item
    lowercase__    :						Union[str, Any]       =			next
class 							__snake_case:



 def __init__(						self : List[Any]						):

    """simple docstring"""






    lowercase__    :						Node | None       =			None
    lowercase__    :						Optional[Any]       =			0



 def     UpperCAmelCase     (						self : List[Any]			,		UpperCAmelCase__ : Any						):

    """simple docstring"""






    lowercase__    :						Any       =			Node(UpperCAmelCase__			,		self.head						)
    self.size += 1



 def     UpperCAmelCase     (						self : Dict						):

    """simple docstring"""






    if self.head is None:
       return None
    else:
       lowercase__    :						List[Any]       =			self.head.item
       lowercase__    :						Optional[Any]       =			self.head.next
       self.size -= 1
       return item



 def     UpperCAmelCase     (						self : Union[str, Any]						):

    """simple docstring"""






    return self.head is None



 def __str__(						self : List[str]						):

    """simple docstring"""






    if self.is_empty():
       return ""
    else:
       lowercase__    :						Optional[Any]       =			self.head
       lowercase__    :						str       =			""""""
       lowercase__    :						list[str]       =			[]
       while iterate:
          item_list.append(str(iterate.item						)						)
          lowercase__    :						List[str]       =			iterate.next

       lowercase__    :						List[str]       =			""" --> """.join(UpperCAmelCase__						)

       return item_str







 def __len__(						self : List[Any]						):

    """simple docstring"""






    return self.size


