



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__magic_name__       =     logging.get_logger(__name__)

__magic_name__       =     {'''vocab_file''': '''vocab.txt'''}

__magic_name__       =     {
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

__magic_name__       =     {
    '''tau/splinter-base''': 512,
    '''tau/splinter-base-qass''': 512,
    '''tau/splinter-large''': 512,
    '''tau/splinter-large-qass''': 512,
}

__magic_name__       =     {
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}


class 	a__       (       _snake_case       ):


					"""simple docstring"""



					A__  :				str					 =				VOCAB_FILES_NAMES
					A__  :				List[str]					 =				PRETRAINED_VOCAB_FILES_MAP
					A__  :				Union[str, Any]					 =				PRETRAINED_INIT_CONFIGURATION
					A__  :				List[str]					 =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__  :				List[Any]					 =				SplinterTokenizer

					def __init__(		self     :Any      ,  lowercase__     :Tuple=None      ,  lowercase__     :Optional[Any]=None      ,  lowercase__     :int=True      ,  lowercase__     :Any="[UNK]"      ,  lowercase__     :Optional[int]="[SEP]"      ,  lowercase__     :Tuple="[PAD]"      ,  lowercase__     :int="[CLS]"      ,  lowercase__     :str="[MASK]"      ,  lowercase__     :Optional[int]="[QUESTION]"      ,  lowercase__     :str=True      ,  lowercase__     :str=None      ,  **lowercase__     :List[str]      ,  ):
										super().__init__(
										    lowercase__      ,  tokenizer_file=lowercase__      ,  do_lower_case=lowercase__      ,  unk_token=lowercase__      ,  sep_token=lowercase__      ,  pad_token=lowercase__      ,  cls_token=lowercase__      ,  mask_token=lowercase__      ,  tokenize_chinese_chars=lowercase__      ,  strip_accents=lowercase__      ,  additional_special_tokens=(question_token,)      ,  **lowercase__      ,  )

										lowercase						   =       json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
										if (
										    pre_tok_state.get('lowercase'      ,  lowercase__    ) != do_lower_case
										    or pre_tok_state.get('strip_accents'      ,  lowercase__    ) != strip_accents
										):
															lowercase						   =       getattr(lowercase__      ,  pre_tok_state.pop('type'    )    )
															lowercase						   =       do_lower_case
															lowercase						   =       strip_accents
															lowercase						   =       pre_tok_class(**lowercase__    )

										lowercase						   =       do_lower_case

					@property
					def 				__UpperCAmelCase		(		self     :str    ):
										return self.convert_tokens_to_ids(self.question_token    )

					def 				__UpperCAmelCase		(		self     :List[Any]      ,  lowercase__     :List[int]      ,  lowercase__     :Optional[List[int]] = None    ):
										if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										lowercase						   =       [self.cls_token_id]
										lowercase						   =       [self.sep_token_id]
										lowercase						   =       [self.question_token_id] + [self.convert_tokens_to_ids('.'    )]
										if self.padding_side == "right":
															# Input is question-then-context
															return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
										else:
															# Input is context-then-question
															return cls + token_ids_a + sep + token_ids_a + question_suffix + sep

					def 				__UpperCAmelCase		(		self     :Optional[Any]      ,  lowercase__     :List[int]      ,  lowercase__     :Optional[List[int]] = None    ):
										lowercase						   =       [self.sep_token_id]
										lowercase						   =       [self.cls_token_id]
										lowercase						   =       [self.question_token_id] + [self.convert_tokens_to_ids('.'    )]
										if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]

										if self.padding_side == "right":
															# Input is question-then-context
															return len(cls + token_ids_a + question_suffix + sep    ) * [0] + len(token_ids_a + sep    ) * [1]
										else:
															# Input is context-then-question
															return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + question_suffix + sep    ) * [1]



					def 				__UpperCAmelCase		(		self     :str      ,  lowercase__     :str      ,  lowercase__     :Optional[str] = None    ):
										lowercase						   =       self._tokenizer.model.save(lowercase__      ,  name=lowercase__    )
										return tuple(lowercase__    )











import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
		from .testing_utils import (
		    floats_tensor,
		    load_hf_numpy,
		    load_image,
		    load_numpy,
		    load_pt,
		    nightly,
		    parse_flag_from_env,
		    print_tensor_test,
		    require_torch_a,
		    require_torch_gpu,
		    skip_mps,
		    slow,
		    torch_all_close,
		    torch_device,
		)
		from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


__magic_name__       =     get_logger(__name__)





def 		__snake_case						(    _UpperCAmelCase				):
					"""simple docstring"""






					if version.parse(_UpperCAmelCase				) < version.parse(_UpperCAmelCase				):
										if "dev" in min_version:
															lowercase						   =       (
															    'This example requires a source install from HuggingFace diffusers (see '
															    '`https://huggingface.co/docs/diffusers/installation#install-from-source`),'
															)
										else:
															lowercase						   =       f"""This example requires a minimum version of {min_version},"""
										error_message += f""" but the version found is {__version__}.\n"""
										raise ImportError(_UpperCAmelCase				)






