
'''simple docstring'''
import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


a_           =    "block_records.npy"


a_           =    logging.get_logger(__name__)

def     UpperCamelCase_  (						__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							):
    """simple docstring"""






    import tensorflow.compat.va as tf

    snake_case_			:     Union[str, Any]        =    tf.data.TFRecordDataset(__SCREAMING_SNAKE_CASE,		buffer_size=5_1_2 * 1_0_2_4 * 1_0_2_4							)
    snake_case_			:     Tuple        =    blocks_dataset.batch(__SCREAMING_SNAKE_CASE,		drop_remainder=__SCREAMING_SNAKE_CASE							)
    snake_case_			:     Any        =    next(blocks_dataset.take(1							).as_numpy_iterator()							)

    return np_record






class  UpperCAmelCase_					:





 def __init__(     self			,							lowercase_			,							lowercase_			,							lowercase_=2			,							lowercase_=10_00			,							lowercase_=1_00			,							lowercase_=10_00_00			,							):
     from scann.scann_ops.py.scann_ops_pybind import builder as Builder

     snake_case_			:     Tuple        =    Builder(db=lowercase_			,							num_neighbors=lowercase_			,							distance_measure="dot_product")
     snake_case_			:     str        =    builder.tree(
         num_leaves=lowercase_			,							num_leaves_to_search=lowercase_			,							training_sample_size=lowercase_)
     snake_case_			:     Tuple        =    builder.score_ah(dimensions_per_block=lowercase_)

     snake_case_			:     Any        =    builder.build()





 def 							snake_case__		(     self			,							lowercase_):
     snake_case_      ,			snake_case_			:     List[str]        =    self.searcher.search_batched(question_projection.detach().cpu())
     return retrieved_block_ids.astype("int64")






class  UpperCAmelCase_					:





 def __init__(     self			,							lowercase_			,							lowercase_):
     super().__init__()
     snake_case_			:     Union[str, Any]        =    block_records
     snake_case_			:     int        =    tokenizer





 def __call__(     self			,							lowercase_			,							lowercase_			,							lowercase_			,							lowercase_=None			,							lowercase_="pt"):
     snake_case_			:     List[Any]        =    np.take(self.block_records			,							indices=lowercase_			,							axis=0)

     snake_case_			:     Union[str, Any]        =    self.tokenizer.decode(question_input_ids[0]			,							skip_special_tokens=lowercase_)

     snake_case_			:     Any        =    []
     snake_case_			:     int        =    []
     for retrieved_block in retrieved_blocks:
         text.append(lowercase_)
         text_pair.append(retrieved_block.decode())

     snake_case_			:     Any        =    self.tokenizer(
         lowercase_			,							lowercase_			,							padding=lowercase_			,							truncation=lowercase_			,							return_special_tokens_mask=lowercase_			,							max_length=lowercase_)
     snake_case_			:     List[str]        =    concat_inputs.convert_to_tensors(lowercase_)

     if answer_ids is not None:
         return self.block_has_answer(lowercase_			,							lowercase_) + (concat_inputs_tensors,)
     else:
         return (None, None, None, concat_inputs_tensors)





 @classmethod
 def 							snake_case__		(     cls			,							lowercase_			,							*lowercase_			,							**lowercase_):
     if os.path.isdir(lowercase_):
         snake_case_			:     List[Any]        =    os.path.join(lowercase_			,							_REALM_BLOCK_RECORDS_FILENAME)
     else:
         snake_case_			:     Optional[int]        =    hf_hub_download(
             repo_id=lowercase_			,							filename=_REALM_BLOCK_RECORDS_FILENAME			,							**lowercase_)
     snake_case_			:     List[Any]        =    np.load(lowercase_			,							allow_pickle=lowercase_)

     snake_case_			:     List[str]        =    AutoTokenizer.from_pretrained(lowercase_			,							*lowercase_			,							**lowercase_)

     return cls(lowercase_			,							lowercase_)





 def 							snake_case__		(     self			,							lowercase_):
     # save block records
     np.save(os.path.join(lowercase_			,							_REALM_BLOCK_RECORDS_FILENAME)			,							self.block_records)
     # save tokenizer
     self.tokenizer.save_pretrained(lowercase_)





 def 							snake_case__		(     self			,							lowercase_			,							lowercase_):
     snake_case_			:     int        =    []
     snake_case_			:     Union[str, Any]        =    []
     snake_case_			:     Union[str, Any]        =    []
     snake_case_			:     List[Any]        =    0

     for input_id in concat_inputs.input_ids:
         snake_case_			:     Any        =    input_id.tolist()
         # Check answers between two [SEP] tokens
         snake_case_			:     Dict        =    input_id_list.index(self.tokenizer.sep_token_id)
         snake_case_			:     Union[str, Any]        =    first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id)

         start_pos.append([])
         end_pos.append([])
         for answer in answer_ids:
             for idx in range(first_sep_idx + 1			,							lowercase_):
                 if answer[0] == input_id_list[idx]:
                     if input_id_list[idx : idx + len(lowercase_)] == answer:
                         start_pos[-1].append(lowercase_)
                         end_pos[-1].append(idx + len(lowercase_) - 1)

         if len(start_pos[-1]) == 0:
             has_answers.append(lowercase_)
         else:
             has_answers.append(lowercase_)
             if len(start_pos[-1]) > max_answers:
                 snake_case_			:     int        =    len(start_pos[-1])

        # Pad -1 to max_answers
     for start_pos_, end_pos_ in zip(lowercase_			,							lowercase_):
         if len(start_pos_) < max_answers:
             snake_case_			:     Any        =    [-1] * (max_answers - len(start_pos_))
             start_pos_ += padded
             end_pos_ += padded
     return has_answers, start_pos, end_pos






'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


a_           =    {
    "configuration_sam": [
        "SAM_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "SamConfig",
        "SamMaskDecoderConfig",
        "SamPromptEncoderConfig",
        "SamVisionConfig",
    ],
    "processing_sam": ["SamProcessor"],
}


try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a_           =    [
      "SAM_PRETRAINED_MODEL_ARCHIVE_LIST",
      "SamModel",
      "SamPreTrainedModel",
  ]
try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a_           =    [
      "TF_SAM_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TFSamModel",
      "TFSamPreTrainedModel",
  ]
try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a_           =    ["SamImageProcessor"]


if TYPE_CHECKING:
  from .configuration_sam import (
      SAM_PRETRAINED_CONFIG_ARCHIVE_MAP,
      SamConfig,
      SamMaskDecoderConfig,
      SamPromptEncoderConfig,
      SamVisionConfig,
  )
  from .processing_sam import SamProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_sam import SAM_PRETRAINED_MODEL_ARCHIVE_LIST, SamModel, SamPreTrainedModel

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_sam import TF_SAM_PRETRAINED_MODEL_ARCHIVE_LIST, TFSamModel, TFSamPreTrainedModel

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .image_processing_sam import SamImageProcessor

else:
  import sys

  a_           =    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




