



import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


_lowercase:					Optional[int]      =						"block_records.npy"


_lowercase:					List[Any]      =						logging.get_logger(__name__)






def        a( A			:       str					,	A			:       int			)						->			np.ndarray:
					"""simple docstring"""





					import tensorflow.compat.va as tf

					a   	=  tf.data.TFRecordDataset(A					,	buffer_size=512 * 1024 * 1024			)
					a   	=  blocks_dataset.batch(A					,	drop_remainder=A			)
					a   	=  next(blocks_dataset.take(1			).as_numpy_iterator()			)

					return np_record



class       _lowercase				:





	"""simple docstring"""




	def __init__(self						,			lowerCamelCase_						,			lowerCamelCase_						,			lowerCamelCase_=2						,			lowerCamelCase_=1000						,			lowerCamelCase_=100						,			lowerCamelCase_=100000						,			):
						"""simple docstring"""

						from scann.scann_ops.py.scann_ops_pybind import builder as Builder

						a   	=  Builder(db=lowerCamelCase_						,			num_neighbors=lowerCamelCase_						,			distance_measure="dot_product"						)
						a   	=  builder.tree(
						    num_leaves=lowerCamelCase_						,			num_leaves_to_search=lowerCamelCase_						,			training_sample_size=lowerCamelCase_						)
						a   	=  builder.score_ah(dimensions_per_block=lowerCamelCase_						)

						a   	=  builder.build()




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a	,							a   	=  self.searcher.search_batched(question_projection.detach().cpu()						)
						return retrieved_block_ids.astype("int64"						)




class       _lowercase				:





	"""simple docstring"""




	def __init__(self						,			lowerCamelCase_						,			lowerCamelCase_						):
						"""simple docstring"""

						super().__init__()
						a   	=  block_records
						a   	=  tokenizer




	def __call__(self						,			lowerCamelCase_						,			lowerCamelCase_						,			lowerCamelCase_						,			lowerCamelCase_=None						,			lowerCamelCase_="pt"						):
						"""simple docstring"""

						a   	=  np.take(self.block_records						,			indices=lowerCamelCase_						,			axis=0						)

						a   	=  self.tokenizer.decode(question_input_ids[0]						,			skip_special_tokens=lowerCamelCase_						)

						a   	=  []
						a   	=  []
						for retrieved_block in retrieved_blocks:
											text.append(lowerCamelCase_						)
											text_pair.append(retrieved_block.decode()						)

						a   	=  self.tokenizer(
						    lowerCamelCase_						,			lowerCamelCase_						,			padding=lowerCamelCase_						,			truncation=lowerCamelCase_						,			return_special_tokens_mask=lowerCamelCase_						,			max_length=lowerCamelCase_						)
						a   	=  concat_inputs.convert_to_tensors(lowerCamelCase_						)

						if answer_ids is not None:
											return self.block_has_answer(lowerCamelCase_						,			lowerCamelCase_						) + (concat_inputs_tensors,)
						else:
											return (None, None, None, concat_inputs_tensors)




	@classmethod
	def 	UpperCamelCase_			(cls						,			lowerCamelCase_						,			*lowerCamelCase_						,			**lowerCamelCase_						):
						"""simple docstring"""

						if os.path.isdir(lowerCamelCase_						):
											a   	=  os.path.join(lowerCamelCase_						,			_REALM_BLOCK_RECORDS_FILENAME						)
						else:
											a   	=  hf_hub_download(
											    repo_id=lowerCamelCase_						,			filename=_REALM_BLOCK_RECORDS_FILENAME						,			**lowerCamelCase_						)
						a   	=  np.load(lowerCamelCase_						,			allow_pickle=lowerCamelCase_						)

						a   	=  AutoTokenizer.from_pretrained(lowerCamelCase_						,			*lowerCamelCase_						,			**lowerCamelCase_						)

						return cls(lowerCamelCase_						,			lowerCamelCase_						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						np.save(os.path.join(lowerCamelCase_						,			_REALM_BLOCK_RECORDS_FILENAME						)						,			self.block_records						)
						# save tokenizer
						self.tokenizer.save_pretrained(lowerCamelCase_						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  []
						a   	=  []
						a   	=  []
						a   	=  0

						for input_id in concat_inputs.input_ids:
											a   	=  input_id.tolist()
											# Check answers between two [SEP] tokens
											a   	=  input_id_list.index(self.tokenizer.sep_token_id						)
											a   	=  first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id						)

											start_pos.append([]						)
											end_pos.append([]						)
											for answer in answer_ids:
																for idx in range(first_sep_idx + 1						,			lowerCamelCase_						):
																					if answer[0] == input_id_list[idx]:
																										if input_id_list[idx : idx + len(lowerCamelCase_						)] == answer:
																															start_pos[-1].append(lowerCamelCase_						)
																															end_pos[-1].append(idx + len(lowerCamelCase_						) - 1						)

											if len(start_pos[-1]						) == 0:
																has_answers.append(lowerCamelCase_						)
											else:
																has_answers.append(lowerCamelCase_						)
																if len(start_pos[-1]						) > max_answers:
																					a   	=  len(start_pos[-1]						)

        # Pad -1 to max_answers
						for start_pos_, end_pos_ in zip(lowerCamelCase_						,			lowerCamelCase_						):
											if len(start_pos_						) < max_answers:
																a   	=  [-1] * (max_answers - len(start_pos_						))
																start_pos_ += padded
																end_pos_ += padded
						return has_answers, start_pos, end_pos





from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowercase:					int      =						logging.get_logger(__name__)



class       _lowercase				(  lowerCAmelCase      ):





	"""simple docstring"""
	__A		      =				["input_ids", "attention_mask"]




	def __init__(self						,			lowerCamelCase_="[PAD]"						,			lowerCamelCase_="[BOS]"						,			lowerCamelCase_="[EOS]"						,			lowerCamelCase_="[MASK]"						,			lowerCamelCase_="[CLS]"						,			lowerCamelCase_="[SEP]"						,			lowerCamelCase_=2048						,			**lowerCamelCase_						,			):
						"""simple docstring"""

						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else pad_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else bos_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else eos_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else mask_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else cls_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else sep_token

						super().__init__(
						    pad_token=lowerCamelCase_						,			bos_token=lowerCamelCase_						,			eos_token=lowerCamelCase_						,			mask_token=lowerCamelCase_						,			cls_token=lowerCamelCase_						,			sep_token=lowerCamelCase_						,			model_max_length=lowerCamelCase_						,			**lowerCamelCase_						,			)

						a   	=  2**8  # utf is 8 bits

						# define special tokens dict
						a   	=  {
						    self.pad_token: 0,
						    self.bos_token: 1,
						    self.eos_token: 2,
						    self.mask_token: 3,
						    self.cls_token: 4,
						    self.sep_token: 5,
						}
						a   	=  len(self.special_tokens_encoder						)
						a   	=  {v: k for k, v in self.special_tokens_encoder.items()}




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						a   	=  self.special_tokens_encoder.copy()
						vocab.update(self.added_tokens_encoder						)
						for i in range(self._utf_vocab_size						):
											a   	=  chr(lowerCamelCase_						)
											a   	=  i + len(self.special_tokens_encoder						)
						return vocab




	@property
	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						return self._utf_vocab_size + self._num_special_tokens




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						,			lowerCamelCase_ = False						):
						"""simple docstring"""

						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCamelCase_						,			token_ids_a=lowerCamelCase_						,			already_has_special_tokens=lowerCamelCase_						)

						# normal case: some special tokens
						if token_ids_a is None:
											return [1] + [0] * len(lowerCamelCase_						) + [1]
						return [1] + ([0] * len(lowerCamelCase_						)) + [1] + ([0] * len(lowerCamelCase_						)) + [1]




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						):
						"""simple docstring"""

						if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						else:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  [chr(lowerCamelCase_						) for i in text.encode("utf-8"						)]
						return tokens




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						if token in self.special_tokens_encoder:
											a   	=  self.special_tokens_encoder[token]
						elif token in self.added_tokens_encoder:
											a   	=  self.added_tokens_encoder[token]
						elif len(lowerCamelCase_						) != 1:
											a   	=  self.unk_token_id
						else:
											a   	=  ord(lowerCamelCase_						) + self._num_special_tokens
						return token_id




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						if index in self.special_tokens_decoder:
											a   	=  self.special_tokens_decoder[index]
						elif index in self.added_tokens_decoder:
											a   	=  self.added_tokens_decoder[index]
						else:
											a   	=  chr(index - self._num_special_tokens						)
						return token




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  B""
						for token in tokens:
											if token in self.special_tokens_decoder:
																a   	=  self.special_tokens_decoder[token].encode("utf-8"						)
											elif token in self.added_tokens_decoder:
																a   	=  self.special_tokens_decoder[token].encode("utf-8"						)
											elif token in self.special_tokens_encoder:
																a   	=  token.encode("utf-8"						)
											elif token in self.added_tokens_encoder:
																a   	=  token.encode("utf-8"						)
											else:
																a   	=  bytes([ord(lowerCamelCase_						)]						)
											bstring += tok_string
						a   	=  bstring.decode("utf-8"						,			errors="replace"						)
						return string




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						):
						"""simple docstring"""

						return ()
