



"""simple docstring"""


import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


snake_case_								=			"""block_records.npy"""


snake_case_								=			logging.get_logger(__name__)

def 					_lowerCAmelCase (  lowercase_						,      lowercase_		):
						import tensorflow.compat.va as tf

						UpperCAmelCase			 =					tf.data.TFRecordDataset(lowercase_						,      buffer_size=512 * 1024 * 1024		)
						UpperCAmelCase			 =					blocks_dataset.batch(lowercase_						,      drop_remainder=lowercase_		)
						UpperCAmelCase			 =					next(blocks_dataset.take(1		).as_numpy_iterator()		)

						return np_record







class  A_     :



						"""simple docstring"""






						def __init__(   self					:Optional[Any]  ,    lowercase_					:List[Any]  ,    lowercase_					:Dict  ,    lowercase_					:List[str]=2  ,    lowercase_					:Optional[int]=10_00  ,    lowercase_					:List[str]=1_00  ,    lowercase_					:Tuple=10_00_00  ,    )							->     Any:
												from scann.scann_ops.py.scann_ops_pybind import builder as Builder

												UpperCAmelCase			 =					Builder(db=lowercase_  ,    num_neighbors=lowercase_  ,    distance_measure='dot_product'   )
												UpperCAmelCase			 =					builder.tree(
												    num_leaves=lowercase_  ,    num_leaves_to_search=lowercase_  ,    training_sample_size=lowercase_   )
												UpperCAmelCase			 =					builder.score_ah(dimensions_per_block=lowercase_   )

												UpperCAmelCase			 =					builder.build()






						def       UpperCAmelCase__	(   self					:Dict  ,    lowercase_					:Union[str, Any]   )							->     Dict:
												UpperCAmelCase ,       UpperCAmelCase			 =					self.searcher.search_batched(question_projection.detach().cpu()   )
												return retrieved_block_ids.astype('int64'   )







class  A_     :



						"""simple docstring"""






						def __init__(   self					:Optional[int]  ,    lowercase_					:Optional[Any]  ,    lowercase_					:int   )							->     List[Any]:
												super().__init__()
												UpperCAmelCase			 =					block_records
												UpperCAmelCase			 =					tokenizer






						def __call__(   self					:Any  ,    lowercase_					:Union[str, Any]  ,    lowercase_					:Any  ,    lowercase_					:str  ,    lowercase_					:List[str]=None  ,    lowercase_					:List[str]="pt"   )							->     List[Any]:
												UpperCAmelCase			 =					np.take(self.block_records  ,    indices=lowercase_  ,    axis=0   )

												UpperCAmelCase			 =					self.tokenizer.decode(question_input_ids[0]  ,    skip_special_tokens=lowercase_   )

												UpperCAmelCase			 =					[]
												UpperCAmelCase			 =					[]
												for retrieved_block in retrieved_blocks:
																		text.append(lowercase_   )
																		text_pair.append(retrieved_block.decode()   )

												UpperCAmelCase			 =					self.tokenizer(
												    lowercase_  ,    lowercase_  ,    padding=lowercase_  ,    truncation=lowercase_  ,    return_special_tokens_mask=lowercase_  ,    max_length=lowercase_   )
												UpperCAmelCase			 =					concat_inputs.convert_to_tensors(lowercase_   )

												if answer_ids is not None:
																		return self.block_has_answer(lowercase_  ,    lowercase_   ) + (concat_inputs_tensors,)
												else:
																		return (None, None, None, concat_inputs_tensors)






						@classmethod
						def       UpperCAmelCase__	(   cls					:Any  ,    lowercase_					:Optional[Union[str, os.PathLike]]  ,    *lowercase_					:Optional[int]  ,    **lowercase_					:Optional[Any]   )							->     int:
												if os.path.isdir(lowercase_   ):
																		UpperCAmelCase			 =					os.path.join(lowercase_  ,    _REALM_BLOCK_RECORDS_FILENAME   )
												else:
																		UpperCAmelCase			 =					hf_hub_download(
																		    repo_id=lowercase_  ,    filename=_REALM_BLOCK_RECORDS_FILENAME  ,    **lowercase_   )
												UpperCAmelCase			 =					np.load(lowercase_  ,    allow_pickle=lowercase_   )

												UpperCAmelCase			 =					AutoTokenizer.from_pretrained(lowercase_  ,    *lowercase_  ,    **lowercase_   )

												return cls(lowercase_  ,    lowercase_   )






						def       UpperCAmelCase__	(   self					:Any  ,    lowercase_					:Tuple   )							->     Tuple:
												# save block records
												np.save(os.path.join(lowercase_  ,    _REALM_BLOCK_RECORDS_FILENAME   )  ,    self.block_records   )
												# save tokenizer
												self.tokenizer.save_pretrained(lowercase_   )






						def       UpperCAmelCase__	(   self					:Union[str, Any]  ,    lowercase_					:List[str]  ,    lowercase_					:Optional[Any]   )							->     Union[str, Any]:
												UpperCAmelCase			 =					[]
												UpperCAmelCase			 =					[]
												UpperCAmelCase			 =					[]
												UpperCAmelCase			 =					0

												for input_id in concat_inputs.input_ids:
																		UpperCAmelCase			 =					input_id.tolist()
																		# Check answers between two [SEP] tokens
																		UpperCAmelCase			 =					input_id_list.index(self.tokenizer.sep_token_id   )
																		UpperCAmelCase			 =					first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id   )

																		start_pos.append([]   )
																		end_pos.append([]   )
																		for answer in answer_ids:
																								for idx in range(first_sep_idx + 1  ,    lowercase_   ):
																														if answer[0] == input_id_list[idx]:
																																				if input_id_list[idx : idx + len(lowercase_   )] == answer:
																																										start_pos[-1].append(lowercase_   )
																																										end_pos[-1].append(idx + len(lowercase_   ) - 1   )

																		if len(start_pos[-1]   ) == 0:
																								has_answers.append(lowercase_   )
																		else:
																								has_answers.append(lowercase_   )
																								if len(start_pos[-1]   ) > max_answers:
																														UpperCAmelCase			 =					len(start_pos[-1]   )

        # Pad -1 to max_answers
												for start_pos_, end_pos_ in zip(lowercase_  ,    lowercase_   ):
																		if len(start_pos_   ) < max_answers:
																								UpperCAmelCase			 =					[-1] * (max_answers - len(start_pos_   ))
																								start_pos_ += padded
																								end_pos_ += padded
												return has_answers, start_pos, end_pos










"""simple docstring"""


import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


snake_case_								=			logging.get_logger(__name__)

snake_case_								=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case_								=			{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

snake_case_								=			{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

snake_case_								=			{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}







class  A_     (	SCREAMING_SNAKE_CASE_							):



						"""simple docstring"""
						__UpperCamelCase													=				VOCAB_FILES_NAMES
						__UpperCamelCase													=				PRETRAINED_VOCAB_FILES_MAP
						__UpperCamelCase													=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__UpperCamelCase													=				["""input_ids""", """attention_mask""", """token_type_ids"""]
						__UpperCamelCase													=				DebertaTokenizer






						def __init__(   self					:Optional[Any]  ,    lowercase_					:List[str]=None  ,    lowercase_					:List[str]=None  ,    lowercase_					:List[Any]=None  ,    lowercase_					:Optional[Any]="replace"  ,    lowercase_					:int="[CLS]"  ,    lowercase_					:Union[str, Any]="[SEP]"  ,    lowercase_					:int="[SEP]"  ,    lowercase_					:int="[CLS]"  ,    lowercase_					:Optional[int]="[UNK]"  ,    lowercase_					:int="[PAD]"  ,    lowercase_					:List[Any]="[MASK]"  ,    lowercase_					:Union[str, Any]=False  ,    **lowercase_					:int  ,    )							->     Tuple:
												super().__init__(
												    lowercase_  ,    lowercase_  ,    tokenizer_file=lowercase_  ,    errors=lowercase_  ,    bos_token=lowercase_  ,    eos_token=lowercase_  ,    unk_token=lowercase_  ,    sep_token=lowercase_  ,    cls_token=lowercase_  ,    pad_token=lowercase_  ,    mask_token=lowercase_  ,    add_prefix_space=lowercase_  ,    **lowercase_  ,    )
												UpperCAmelCase			 =					kwargs.pop('add_bos_token'  ,    lowercase_   )

												UpperCAmelCase			 =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
												if pre_tok_state.get('add_prefix_space'  ,    lowercase_   ) != add_prefix_space:
																		UpperCAmelCase			 =					getattr(lowercase_  ,    pre_tok_state.pop('type'   )   )
																		UpperCAmelCase			 =					add_prefix_space
																		UpperCAmelCase			 =					pre_tok_class(**lowercase_   )

												UpperCAmelCase			 =					add_prefix_space






						@property
						def       UpperCAmelCase__	(   self					:str   )							->     str:
												if self._mask_token is None:
																		if self.verbose:
																								logger.error('Using mask_token, but it is not set yet.'   )
																		return None
												return str(self._mask_token   )






						@mask_token.setter
						def       UpperCAmelCase__	(   self					:Any  ,    lowercase_					:Any   )							->     Any:
												UpperCAmelCase			 =					AddedToken(lowercase_  ,    lstrip=lowercase_  ,    rstrip=lowercase_   ) if isinstance(lowercase_  ,    lowercase_   ) else value
												UpperCAmelCase			 =					value






						def       UpperCAmelCase__	(   self					:Optional[Any]  ,    lowercase_					:List[int]  ,    lowercase_					:Optional[List[int]] = None   )							->     List[int]:
												if token_ids_a is None:
																		return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												UpperCAmelCase			 =					[self.cls_token_id]
												UpperCAmelCase			 =					[self.sep_token_id]
												return cls + token_ids_a + sep + token_ids_a + sep






						def       UpperCAmelCase__	(   self					:Dict  ,    lowercase_					:List[int]  ,    lowercase_					:Optional[List[int]] = None   )							->     List[int]:
												UpperCAmelCase			 =					[self.sep_token_id]
												UpperCAmelCase			 =					[self.cls_token_id]

												if token_ids_a is None:
																		return len(cls + token_ids_a + sep   ) * [0]
												return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






						def       UpperCAmelCase__	(   self					:int  ,    *lowercase_					:str  ,    **lowercase_					:Optional[Any]   )							->     BatchEncoding:
												UpperCAmelCase			 =					kwargs.get('is_split_into_words'  ,    lowercase_   )
												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*lowercase_  ,    **lowercase_   )






						def       UpperCAmelCase__	(   self					:Optional[Any]  ,    *lowercase_					:Dict  ,    **lowercase_					:Any   )							->     BatchEncoding:
												UpperCAmelCase			 =					kwargs.get('is_split_into_words'  ,    lowercase_   )

												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*lowercase_  ,    **lowercase_   )






						def       UpperCAmelCase__	(   self					:Union[str, Any]  ,    lowercase_					:str  ,    lowercase_					:Optional[str] = None   )							->     Tuple[str]:
												UpperCAmelCase			 =					self._tokenizer.model.save(lowercase_  ,    name=lowercase_   )
												return tuple(lowercase_   )






						def       UpperCAmelCase__	(   self					:int  ,    lowercase_					:"Conversation"   )							->     List[int]:
												UpperCAmelCase			 =					[]
												for is_user, text in conversation.iter_texts():
																		input_ids.extend(self.encode(lowercase_  ,    add_special_tokens=lowercase_   ) + [self.eos_token_id]   )

												if len(lowercase_   ) > self.model_max_length:
																		UpperCAmelCase			 =					input_ids[-self.model_max_length :]
												return input_ids





