import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__lowerCamelCase     :					List[str]				  =						logging.get_logger(__name__)

__lowerCamelCase     :					List[Any]				  =						{'''vocab_file''': '''vocab.txt'''}

__lowerCamelCase     :					List[str]				  =						{
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

__lowerCamelCase     :					List[str]				  =						{
    '''tau/splinter-base''': 512,
    '''tau/splinter-base-qass''': 512,
    '''tau/splinter-large''': 512,
    '''tau/splinter-large-qass''': 512,
}

__lowerCamelCase     :					Tuple				  =						{
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}







class __snake_case							(		lowerCamelCase_						):
   lowerCAmelCase_								=				VOCAB_FILES_NAMES
   lowerCAmelCase_								=				PRETRAINED_VOCAB_FILES_MAP
   lowerCAmelCase_								=				PRETRAINED_INIT_CONFIGURATION
   lowerCAmelCase_								=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCAmelCase_								=				SplinterTokenizer




   def __init__(       self     :							List[Any]			,      _lowercase     :							Optional[int]=None			,      _lowercase     :							List[Any]=None			,      _lowercase     :							List[Any]=True			,      _lowercase     :							Tuple="[UNK]"			,      _lowercase     :							Optional[Any]="[SEP]"			,      _lowercase     :							int="[PAD]"			,      _lowercase     :							Union[str, Any]="[CLS]"			,      _lowercase     :							Dict="[MASK]"			,      _lowercase     :							Optional[Any]="[QUESTION]"			,      _lowercase     :							List[Any]=True			,      _lowercase     :							Optional[int]=None			,      **_lowercase     :							Union[str, Any]			,      ):



     """simple docstring"""






     super().__init__(
         _lowercase			,      tokenizer_file=_lowercase			,      do_lower_case=_lowercase			,      unk_token=_lowercase			,      sep_token=_lowercase			,      pad_token=_lowercase			,      cls_token=_lowercase			,      mask_token=_lowercase			,      tokenize_chinese_chars=_lowercase			,      strip_accents=_lowercase			,      additional_special_tokens=(question_token,)			,      **_lowercase			,      )

     SCREAMING_SNAKE_CASE__     				= json.loads(self.backend_tokenizer.normalizer.__getstate__() )
     if (
         pre_tok_state.get("""lowercase"""			,      _lowercase ) != do_lower_case
         or pre_tok_state.get("""strip_accents"""			,      _lowercase ) != strip_accents
     ):
       SCREAMING_SNAKE_CASE__     				= getattr(_lowercase			,      pre_tok_state.pop("""type""" ) )
       SCREAMING_SNAKE_CASE__     				= do_lower_case
       SCREAMING_SNAKE_CASE__     				= strip_accents
       SCREAMING_SNAKE_CASE__     				= pre_tok_class(**_lowercase )

     SCREAMING_SNAKE_CASE__     				= do_lower_case




   @property
   def       __a		(       self     :							Tuple ):



     """simple docstring"""






     return self.convert_tokens_to_ids(self.question_token )




   def       __a		(       self     :							Optional[int]			,      _lowercase     :							List[int]			,      _lowercase     :							Optional[List[int]] = None ):



     """simple docstring"""






     if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     SCREAMING_SNAKE_CASE__     				= [self.cls_token_id]
     SCREAMING_SNAKE_CASE__     				= [self.sep_token_id]
     SCREAMING_SNAKE_CASE__     				= [self.question_token_id] + [self.convert_tokens_to_ids(""".""" )]
     if self.padding_side == "right":
       # Input is question-then-context
       return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
     else:
       # Input is context-then-question
       return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




   def       __a		(       self     :							Optional[Any]			,      _lowercase     :							List[int]			,      _lowercase     :							Optional[List[int]] = None ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= [self.sep_token_id]
     SCREAMING_SNAKE_CASE__     				= [self.cls_token_id]
     SCREAMING_SNAKE_CASE__     				= [self.question_token_id] + [self.convert_tokens_to_ids(""".""" )]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep ) * [0]

     if self.padding_side == "right":
       # Input is question-then-context
       return len(cls + token_ids_a + question_suffix + sep ) * [0] + len(token_ids_a + sep ) * [1]
     else:
       # Input is context-then-question
       return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + question_suffix + sep ) * [1]




   def       __a		(       self     :							List[Any]			,      _lowercase     :							str			,      _lowercase     :							Optional[str] = None ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= self._tokenizer.model.save(_lowercase			,      name=_lowercase )
     return tuple(_lowercase )

import unittest

from transformers import is_torch_available, load_tool

from .test_tools_common import ToolTesterMixin


if is_torch_available():
   import torch







class __snake_case							(		unittest.TestCase  ,						lowerCamelCase_						):




   def       __a		(       self     :							Dict ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= load_tool("""speech-to-text""" )
     self.tool.setup()




   def       __a		(       self     :							Tuple ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= self.tool(torch.ones(30_00 ) )
     self.assertEqual(_lowercase			,      """ you""" )




   def       __a		(       self     :							Any ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= self.tool(audio=torch.ones(30_00 ) )
     self.assertEqual(_lowercase			,      """ you""" )
