


import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__UpperCamelCase							:							int    							=					"block_records.npy"


__UpperCamelCase							:							Union[str, Any]    							=					logging.get_logger(__name__)

def       _a	(					SCREAMING_SNAKE_CASE						:					str     ,			SCREAMING_SNAKE_CASE						:					int  ):







 """simple docstring"""



 import tensorflow.compat.va as tf

 UpperCamelCase__       :     str 							= tf.data.TFRecordDataset(SCREAMING_SNAKE_CASE     ,			buffer_size=512 * 1024 * 1024  )
 UpperCamelCase__       :     Any 							= blocks_dataset.batch(SCREAMING_SNAKE_CASE     ,			drop_remainder=SCREAMING_SNAKE_CASE  )
 UpperCamelCase__       :     Union[str, Any] 							= next(blocks_dataset.take(1  ).as_numpy_iterator()  )

 return np_record

class    __magic_name__						:


     def __init__(   self				:  Union[str, Any]				,						lowerCamelCase__				:  List[Any]				,						lowerCamelCase__				:  Union[str, Any]				,						lowerCamelCase__				:  Optional[int]=2				,						lowerCamelCase__				:  Tuple=1000				,						lowerCamelCase__				:  Dict=100				,						lowerCamelCase__				:  Optional[int]=100000				,						)	->   Tuple:





      '''simple docstring'''






      from scann.scann_ops.py.scann_ops_pybind import builder as Builder

      UpperCamelCase__       :     Optional[Any] 							= Builder(db=lowerCamelCase__				,						num_neighbors=lowerCamelCase__				,						distance_measure='''dot_product'''      )
      UpperCamelCase__       :     Optional[Any] 							= builder.tree(
          num_leaves=lowerCamelCase__				,						num_leaves_to_search=lowerCamelCase__				,						training_sample_size=lowerCamelCase__      )
      UpperCamelCase__       :     Tuple 							= builder.score_ah(dimensions_per_block=lowerCamelCase__      )

      UpperCamelCase__       :     List[Any] 							= builder.build()


     def   UpperCAmelCase__						(   self				:  Tuple				,						lowerCamelCase__				:  Any      )	->   str:





      '''simple docstring'''






      UpperCamelCase__							,					UpperCamelCase__       :     List[Any] 							= self.searcher.search_batched(question_projection.detach().cpu()      )
      return retrieved_block_ids.astype('''int64'''      )



class    __magic_name__						:


     def __init__(   self				:  Dict				,						lowerCamelCase__				:  Dict				,						lowerCamelCase__				:  List[str]      )	->   Optional[Any]:





      '''simple docstring'''






      super().__init__()
      UpperCamelCase__       :     Optional[int] 							= block_records
      UpperCamelCase__       :     Optional[int] 							= tokenizer


     def __call__(   self				:  List[str]				,						lowerCamelCase__				:  str				,						lowerCamelCase__				:  Union[str, Any]				,						lowerCamelCase__				:  int				,						lowerCamelCase__				:  Any=None				,						lowerCamelCase__				:  List[str]="pt"      )	->   Any:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[Any] 							= np.take(self.block_records				,						indices=lowerCamelCase__				,						axis=0      )

      UpperCamelCase__       :     int 							= self.tokenizer.decode(question_input_ids[0]				,						skip_special_tokens=lowerCamelCase__      )

      UpperCamelCase__       :     Optional[Any] 							= []
      UpperCamelCase__       :     Any 							= []
      for retrieved_block in retrieved_blocks:
       text.append(lowerCamelCase__      )
       text_pair.append(retrieved_block.decode()      )

      UpperCamelCase__       :     Any 							= self.tokenizer(
          lowerCamelCase__				,						lowerCamelCase__				,						padding=lowerCamelCase__				,						truncation=lowerCamelCase__				,						return_special_tokens_mask=lowerCamelCase__				,						max_length=lowerCamelCase__      )
      UpperCamelCase__       :     Optional[int] 							= concat_inputs.convert_to_tensors(lowerCamelCase__      )

      if answer_ids is not None:
       return self.block_has_answer(lowerCamelCase__				,						lowerCamelCase__      ) + (concat_inputs_tensors,)
      else:
       return (None, None, None, concat_inputs_tensors)


     @classmethod
     def   UpperCAmelCase__						(   cls				:  List[str]				,						lowerCamelCase__				:  Optional[Union[str, os.PathLike]]				,						*lowerCamelCase__				:  int				,						**lowerCamelCase__				:  Optional[int]      )	->   Optional[int]:





      '''simple docstring'''






      if os.path.isdir(lowerCamelCase__      ):
       UpperCamelCase__       :     List[Any] 							= os.path.join(lowerCamelCase__				,						_REALM_BLOCK_RECORDS_FILENAME      )
      else:
       UpperCamelCase__       :     List[str] 							= hf_hub_download(
           repo_id=lowerCamelCase__				,						filename=_REALM_BLOCK_RECORDS_FILENAME				,						**lowerCamelCase__      )
      UpperCamelCase__       :     Any 							= np.load(lowerCamelCase__				,						allow_pickle=lowerCamelCase__      )

      UpperCamelCase__       :     int 							= AutoTokenizer.from_pretrained(lowerCamelCase__				,						*lowerCamelCase__				,						**lowerCamelCase__      )

      return cls(lowerCamelCase__				,						lowerCamelCase__      )


     def   UpperCAmelCase__						(   self				:  Optional[Any]				,						lowerCamelCase__				:  Optional[int]      )	->   Optional[Any]:





      '''simple docstring'''






      np.save(os.path.join(lowerCamelCase__				,						_REALM_BLOCK_RECORDS_FILENAME      )				,						self.block_records      )
      # save tokenizer
      self.tokenizer.save_pretrained(lowerCamelCase__      )


     def   UpperCAmelCase__						(   self				:  List[str]				,						lowerCamelCase__				:  List[str]				,						lowerCamelCase__				:  int      )	->   Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     List[Any] 							= []
      UpperCamelCase__       :     Optional[int] 							= []
      UpperCamelCase__       :     int 							= []
      UpperCamelCase__       :     Dict 							= 0

      for input_id in concat_inputs.input_ids:
       UpperCamelCase__       :     List[str] 							= input_id.tolist()
       # Check answers between two [SEP] tokens
       UpperCamelCase__       :     Any 							= input_id_list.index(self.tokenizer.sep_token_id      )
       UpperCamelCase__       :     Dict 							= first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id      )

       start_pos.append([]      )
       end_pos.append([]      )
       for answer in answer_ids:
        for idx in range(first_sep_idx + 1				,						lowerCamelCase__      ):
         if answer[0] == input_id_list[idx]:
          if input_id_list[idx : idx + len(lowerCamelCase__      )] == answer:
           start_pos[-1].append(lowerCamelCase__      )
           end_pos[-1].append(idx + len(lowerCamelCase__      ) - 1      )

       if len(start_pos[-1]      ) == 0:
        has_answers.append(lowerCamelCase__      )
       else:
        has_answers.append(lowerCamelCase__      )
        if len(start_pos[-1]      ) > max_answers:
         UpperCamelCase__       :     Dict 							= len(start_pos[-1]      )

        # Pad -1 to max_answers
      for start_pos_, end_pos_ in zip(lowerCamelCase__				,						lowerCamelCase__      ):
       if len(start_pos_      ) < max_answers:
        UpperCamelCase__       :     Union[str, Any] 							= [-1] * (max_answers - len(start_pos_      ))
        start_pos_ += padded
        end_pos_ += padded
      return has_answers, start_pos, end_pos






import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir

@dataclass
class    __magic_name__						:
     A:		str



class    __magic_name__						(  unittest.TestCase):


     def   UpperCAmelCase__						(   self				:  Any      )	->   List[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[Any] 							= get_tests_dir('''fixtures/test_sentencepiece.model'''      )
      UpperCamelCase__       :     Any 							= get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model'''      )

      UpperCamelCase__       :     List[str] 							= FakeOriginalTokenizer(vocab_file=lowerCamelCase__      )

      with warnings.catch_warnings(record=lowerCamelCase__      ) as w:
       UpperCamelCase__       :     Optional[int] 							= SpmConverter(lowerCamelCase__      )
      self.assertEqual(len(lowerCamelCase__      )				,						0      )

      UpperCamelCase__       :     Tuple 							= FakeOriginalTokenizer(vocab_file=lowerCamelCase__      )

      with warnings.catch_warnings(record=lowerCamelCase__      ) as w:
       UpperCamelCase__       :     str 							= SpmConverter(lowerCamelCase__      )
      self.assertEqual(len(lowerCamelCase__      )				,						1      )

      self.assertIn(
          '''The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'''
          ''' which is not implemented in the fast tokenizers.'''				,						str(w[0].message      )				,						)


