
'''simple docstring'''

import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__UpperCAmelCase												=   "block_records.npy"


__UpperCAmelCase												=   logging.get_logger(__name__)



def        SCREAMING_SNAKE_CASE_  (	snake_case_				:	str   ,     snake_case_				:	int							)       ->	List[Any]:
 import tensorflow.compat.va as tf

 SCREAMING_SNAKE_CASE  :  str								=		tf.data.TFRecordDataset(snake_case_   ,     buffer_size=512 * 1024 * 1024							)
 SCREAMING_SNAKE_CASE  :  Any								=		blocks_dataset.batch(snake_case_   ,     drop_remainder=snake_case_							)
 SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		next(blocks_dataset.take(1							).as_numpy_iterator()							)

 return np_record


class 		_a     :







   """simple docstring"""




   def __init__(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE=2    ,__SCREAMING_SNAKE_CASE=1000    ,__SCREAMING_SNAKE_CASE=100    ,__SCREAMING_SNAKE_CASE=100000    ,):
    from scann.scann_ops.py.scann_ops_pybind import builder as Builder

    SCREAMING_SNAKE_CASE  :  Optional[Any]								=		Builder(db=lowerCamelCase__    ,num_neighbors=lowerCamelCase__    ,distance_measure='dot_product'      )
    SCREAMING_SNAKE_CASE  :  Optional[Any]								=		builder.tree(
        num_leaves=lowerCamelCase__    ,num_leaves_to_search=lowerCamelCase__    ,training_sample_size=lowerCamelCase__      )
    SCREAMING_SNAKE_CASE  :  Tuple								=		builder.score_ah(dimensions_per_block=lowerCamelCase__      )

    SCREAMING_SNAKE_CASE  :  List[Any]								=		builder.build()






   def 					__a			(       self    ,__SCREAMING_SNAKE_CASE      ):
    SCREAMING_SNAKE_CASE  :  List[Any]								=		self.searcher.search_batched(question_projection.detach().cpu()      )
    return retrieved_block_ids.astype('int64'      )



class 		_a     :







   """simple docstring"""




   def __init__(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE      ):
    super().__init__()
    SCREAMING_SNAKE_CASE  :  Optional[int]								=		block_records
    SCREAMING_SNAKE_CASE  :  Optional[int]								=		tokenizer






   def __call__(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE=None    ,__SCREAMING_SNAKE_CASE="pt"      ):
    SCREAMING_SNAKE_CASE  :  Optional[Any]								=		np.take(self.block_records    ,indices=lowerCamelCase__    ,axis=0      )

    SCREAMING_SNAKE_CASE  :  int								=		self.tokenizer.decode(question_input_ids[0]    ,skip_special_tokens=lowerCamelCase__      )

    SCREAMING_SNAKE_CASE  :  Optional[Any]								=		[]
    SCREAMING_SNAKE_CASE  :  Any								=		[]
    for retrieved_block in retrieved_blocks:
     text.append(lowerCamelCase__      )
     text_pair.append(retrieved_block.decode()      )

    SCREAMING_SNAKE_CASE  :  Any								=		self.tokenizer(
        lowerCamelCase__    ,lowerCamelCase__    ,padding=lowerCamelCase__    ,truncation=lowerCamelCase__    ,return_special_tokens_mask=lowerCamelCase__    ,max_length=lowerCamelCase__      )
    SCREAMING_SNAKE_CASE  :  Optional[int]								=		concat_inputs.convert_to_tensors(lowerCamelCase__      )

    if answer_ids is not None:
     return self.block_has_answer(lowerCamelCase__    ,lowerCamelCase__      ) + (concat_inputs_tensors,)
    else:
     return (None, None, None, concat_inputs_tensors)






   @classmethod
   def 					__a			(       cls    ,__SCREAMING_SNAKE_CASE    ,*__SCREAMING_SNAKE_CASE    ,**__SCREAMING_SNAKE_CASE      ):






    if os.path.isdir(lowerCamelCase__      ):
     SCREAMING_SNAKE_CASE  :  List[Any]								=		os.path.join(lowerCamelCase__    ,_REALM_BLOCK_RECORDS_FILENAME      )
    else:
     SCREAMING_SNAKE_CASE  :  List[str]								=		hf_hub_download(
         repo_id=lowerCamelCase__    ,filename=_REALM_BLOCK_RECORDS_FILENAME    ,**lowerCamelCase__      )
    SCREAMING_SNAKE_CASE  :  Any								=		np.load(lowerCamelCase__    ,allow_pickle=lowerCamelCase__      )

    SCREAMING_SNAKE_CASE  :  int								=		AutoTokenizer.from_pretrained(lowerCamelCase__    ,*lowerCamelCase__    ,**lowerCamelCase__      )

    return cls(lowerCamelCase__    ,lowerCamelCase__      )






   def 					__a			(       self    ,__SCREAMING_SNAKE_CASE      ):
    np.save(os.path.join(lowerCamelCase__    ,_REALM_BLOCK_RECORDS_FILENAME      )    ,self.block_records      )
    # save tokenizer
    self.tokenizer.save_pretrained(lowerCamelCase__      )






   def 					__a			(       self    ,__SCREAMING_SNAKE_CASE    ,__SCREAMING_SNAKE_CASE      ):
    SCREAMING_SNAKE_CASE  :  List[Any]								=		[]
    SCREAMING_SNAKE_CASE  :  Optional[int]								=		[]
    SCREAMING_SNAKE_CASE  :  int								=		[]
    SCREAMING_SNAKE_CASE  :  Dict								=		0

    for input_id in concat_inputs.input_ids:
     SCREAMING_SNAKE_CASE  :  List[str]								=		input_id.tolist()
     # Check answers between two [SEP] tokens
     SCREAMING_SNAKE_CASE  :  Any								=		input_id_list.index(self.tokenizer.sep_token_id      )
     SCREAMING_SNAKE_CASE  :  Dict								=		first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id      )

     start_pos.append([]      )
     end_pos.append([]      )
     for answer in answer_ids:
      for idx in range(first_sep_idx + 1    ,lowerCamelCase__      ):
       if answer[0] == input_id_list[idx]:
        if input_id_list[idx : idx + len(lowerCamelCase__      )] == answer:
         start_pos[-1].append(lowerCamelCase__      )
         end_pos[-1].append(idx + len(lowerCamelCase__      ) - 1      )

     if len(start_pos[-1]      ) == 0:
      has_answers.append(lowerCamelCase__      )
     else:
      has_answers.append(lowerCamelCase__      )
      if len(start_pos[-1]      ) > max_answers:
       SCREAMING_SNAKE_CASE  :  Dict								=		len(start_pos[-1]      )

      # Pad -1 to max_answers
    for start_pos_, end_pos_ in zip(lowerCamelCase__    ,lowerCamelCase__      ):
     if len(start_pos_      ) < max_answers:
      SCREAMING_SNAKE_CASE  :  Union[str, Any]								=		[-1] * (max_answers - len(start_pos_      ))
      start_pos_ += padded
      end_pos_ += padded
    return has_answers, start_pos, end_pos






import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir

@dataclass
class    __magic_name__						:
     A:		str



class    __magic_name__						(  unittest.TestCase):


     def   UpperCAmelCase__						(   self				:  Any      )	->   List[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[Any] 							= get_tests_dir('''fixtures/test_sentencepiece.model'''      )
      UpperCamelCase__       :     Any 							= get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model'''      )

      UpperCamelCase__       :     List[str] 							= FakeOriginalTokenizer(vocab_file=lowerCamelCase__      )

      with warnings.catch_warnings(record=lowerCamelCase__      ) as w:
       UpperCamelCase__       :     Optional[int] 							= SpmConverter(lowerCamelCase__      )
      self.assertEqual(len(lowerCamelCase__      )				,						0      )

      UpperCamelCase__       :     Tuple 							= FakeOriginalTokenizer(vocab_file=lowerCamelCase__      )

      with warnings.catch_warnings(record=lowerCamelCase__      ) as w:
       UpperCamelCase__       :     str 							= SpmConverter(lowerCamelCase__      )
      self.assertEqual(len(lowerCamelCase__      )				,						1      )

      self.assertIn(
          '''The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'''
          ''' which is not implemented in the fast tokenizers.'''				,						str(w[0].message      )				,						)


