






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


UpperCamelCase					=     logging.get_logger(__name__)

UpperCamelCase					=     {'vocab_file': 'vocab.txt'}

UpperCamelCase					=     {
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

UpperCamelCase					=     {
    'tau/splinter-base': 5_12,
    'tau/splinter-base-qass': 5_12,
    'tau/splinter-large': 5_12,
    'tau/splinter-large-qass': 5_12,
}

UpperCamelCase					=     {
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}







class 						_a  (		__UpperCamelCase       ):





   '''simple docstring'''






   UpperCamelCase__     	=							VOCAB_FILES_NAMES
   UpperCamelCase__     	=							PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase__     	=							PRETRAINED_INIT_CONFIGURATION
   UpperCamelCase__     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase__     	=							SplinterTokenizer





   def __init__(       self					, UpperCAmelCase_=None					, UpperCAmelCase_=None					, UpperCAmelCase_=True					, UpperCAmelCase_="[UNK]"					, UpperCAmelCase_="[SEP]"					, UpperCAmelCase_="[PAD]"					, UpperCAmelCase_="[CLS]"					, UpperCAmelCase_="[MASK]"					, UpperCAmelCase_="[QUESTION]"					, UpperCAmelCase_=True					, UpperCAmelCase_=None					, **UpperCAmelCase_					, )			->   Any:




          '''simple docstring'''




          super().__init__(
              __UpperCamelCase					, tokenizer_file=__UpperCamelCase					, do_lower_case=__UpperCamelCase					, unk_token=__UpperCamelCase					, sep_token=__UpperCamelCase					, pad_token=__UpperCamelCase					, cls_token=__UpperCamelCase					, mask_token=__UpperCamelCase					, tokenize_chinese_chars=__UpperCamelCase					, strip_accents=__UpperCamelCase					, additional_special_tokens=(question_token,)					, **__UpperCamelCase					, )

          lowercase__:							List[str]         =     json.loads(self.backend_tokenizer.normalizer.__getstate__())
          if (
              pre_tok_state.get("lowercase"					, __UpperCamelCase) != do_lower_case
              or pre_tok_state.get("strip_accents"					, __UpperCamelCase) != strip_accents
          ):
                 lowercase__:							Dict         =     getattr(__UpperCamelCase					, pre_tok_state.pop("type"))
                 lowercase__:							int         =     do_lower_case
                 lowercase__:							Dict         =     strip_accents
                 lowercase__:							Union[str, Any]         =     pre_tok_class(**__UpperCamelCase)

          lowercase__:							Optional[int]         =     do_lower_case





   @property
   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          return self.convert_tokens_to_ids(self.question_token)





   def __lowercase							(       self					, UpperCAmelCase_					, UpperCAmelCase_ = None)			->   Optional[int]:




          '''simple docstring'''




          if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          lowercase__:							Optional[Any]         =     [self.cls_token_id]
          lowercase__:							int         =     [self.sep_token_id]
          lowercase__:							Dict         =     [self.question_token_id] + [self.convert_tokens_to_ids(".")]
          if self.padding_side == "right":
                 # Input is question-then-context
                 return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
          else:
                 # Input is context-then-question
                 return cls + token_ids_a + sep + token_ids_a + question_suffix + sep





   def __lowercase							(       self					, UpperCAmelCase_					, UpperCAmelCase_ = None)			->   Tuple:




          '''simple docstring'''




          lowercase__:							List[str]         =     [self.sep_token_id]
          lowercase__:							List[Any]         =     [self.cls_token_id]
          lowercase__:							List[Any]         =     [self.question_token_id] + [self.convert_tokens_to_ids(".")]
          if token_ids_a is None:
                 return len(cls + token_ids_a + sep) * [0]

          if self.padding_side == "right":
                 # Input is question-then-context
                 return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
          else:
                 # Input is context-then-question
                 return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]





   def __lowercase							(       self					, UpperCAmelCase_					, UpperCAmelCase_ = None)			->   Dict:




          '''simple docstring'''




          lowercase__:							Optional[Any]         =     self._tokenizer.model.save(__UpperCamelCase					, name=__UpperCamelCase)
          return tuple(__UpperCamelCase)








'''simple docstring'''







def 				_snake_case (		lowercase						)       ->       int:
				if not isinstance(lowercase			,    lowercase						):
								__a       :     Union[str, Any]								=	F"""Input value of [number={number}] must be an integer"""
								raise TypeError(lowercase						)
				if number < 1:
								raise ValueError("""Input must be a positive integer"""						)
				return number * (2 * number - 1)


if __name__ == "__main__":
					import doctest

					doctest.testmod()