






'''simple docstring'''





import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__A				:				List[Any]					=		"block_records.npy"


__A				:				Dict					=		logging.get_logger(__name__)






def  lowerCamelCase_  (     lowercase__					,  lowercase__):
 import tensorflow.compat.va as tf

 lowerCamelCase__	=  tf.data.TFRecordDataset(SCREAMING_SNAKE_CASE_					,  buffer_size=512 * 1024 * 1024)
 lowerCamelCase__	=  blocks_dataset.batch(SCREAMING_SNAKE_CASE_					,  drop_remainder=SCREAMING_SNAKE_CASE_)
 lowerCamelCase__	=  next(blocks_dataset.take(1).as_numpy_iterator())

 return np_record
class 					lowercase							:


    '''simple docstring'''

    def __init__(       self  :  Union[str, Any]					,  __lowerCamelCase  :  Optional[Any]					,  __lowerCamelCase  :  Any					,  __lowerCamelCase  :  Any=2					,  __lowerCamelCase  :  str=1000					,  __lowerCamelCase  :  Any=100					,  __lowerCamelCase  :  str=100000					,  )				->				Dict:

     '''simple docstring'''




     from scann.scann_ops.py.scann_ops_pybind import builder as Builder

     lowerCamelCase__	=  Builder(db=__SCREAMING_SNAKE_CASE					,  num_neighbors=__SCREAMING_SNAKE_CASE					,  distance_measure="dot_product"  )
     lowerCamelCase__	=  builder.tree(
         num_leaves=__SCREAMING_SNAKE_CASE					,  num_leaves_to_search=__SCREAMING_SNAKE_CASE					,  training_sample_size=__SCREAMING_SNAKE_CASE  )
     lowerCamelCase__	=  builder.score_ah(dimensions_per_block=__SCREAMING_SNAKE_CASE  )

     lowerCamelCase__	=  builder.build()




    def  a__					(       self  :  Optional[int]					,  __lowerCamelCase  :  List[Any]  )				->				List[Any]:

     '''simple docstring'''




     lowerCamelCase__	=  self.searcher.search_batched(question_projection.detach().cpu()  )
     return retrieved_block_ids.astype("int64"  )
class 					lowercase							:


    '''simple docstring'''

    def __init__(       self  :  Any					,  __lowerCamelCase  :  List[Any]					,  __lowerCamelCase  :  Optional[Any]  )				->				List[str]:

     '''simple docstring'''




     super().__init__()
     lowerCamelCase__	=  block_records
     lowerCamelCase__	=  tokenizer

    def __call__(       self  :  Union[str, Any]					,  __lowerCamelCase  :  Tuple					,  __lowerCamelCase  :  List[Any]					,  __lowerCamelCase  :  Union[str, Any]					,  __lowerCamelCase  :  int=None					,  __lowerCamelCase  :  int="pt"  )				->				Any:

     '''simple docstring'''




     lowerCamelCase__	=  np.take(self.block_records					,  indices=__SCREAMING_SNAKE_CASE					,  axis=0  )

     lowerCamelCase__	=  self.tokenizer.decode(question_input_ids[0]					,  skip_special_tokens=__SCREAMING_SNAKE_CASE  )

     lowerCamelCase__	=  []
     lowerCamelCase__	=  []
     for retrieved_block in retrieved_blocks:
      text.append(__SCREAMING_SNAKE_CASE  )
      text_pair.append(retrieved_block.decode()  )

     lowerCamelCase__	=  self.tokenizer(
         __SCREAMING_SNAKE_CASE					,  __SCREAMING_SNAKE_CASE					,  padding=__SCREAMING_SNAKE_CASE					,  truncation=__SCREAMING_SNAKE_CASE					,  return_special_tokens_mask=__SCREAMING_SNAKE_CASE					,  max_length=__SCREAMING_SNAKE_CASE  )
     lowerCamelCase__	=  concat_inputs.convert_to_tensors(__SCREAMING_SNAKE_CASE  )

     if answer_ids is not None:
      return self.block_has_answer(__SCREAMING_SNAKE_CASE					,  __SCREAMING_SNAKE_CASE  ) + (concat_inputs_tensors,)
     else:
      return (None, None, None, concat_inputs_tensors)

    @classmethod
    def  a__					(       cls  :  Optional[int]					,  __lowerCamelCase  :  List[str]					,  *__lowerCamelCase  :  Dict					,  **__lowerCamelCase  :  int  )				->				int:

     '''simple docstring'''




     if os.path.isdir(__SCREAMING_SNAKE_CASE  ):
      lowerCamelCase__	=  os.path.join(__SCREAMING_SNAKE_CASE					,  _REALM_BLOCK_RECORDS_FILENAME  )
     else:
      lowerCamelCase__	=  hf_hub_download(
          repo_id=__SCREAMING_SNAKE_CASE					,  filename=_REALM_BLOCK_RECORDS_FILENAME					,  **__SCREAMING_SNAKE_CASE  )
     lowerCamelCase__	=  np.load(__SCREAMING_SNAKE_CASE					,  allow_pickle=__SCREAMING_SNAKE_CASE  )

     lowerCamelCase__	=  AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,  *__SCREAMING_SNAKE_CASE					,  **__SCREAMING_SNAKE_CASE  )

     return cls(__SCREAMING_SNAKE_CASE					,  __SCREAMING_SNAKE_CASE  )

    def  a__					(       self  :  Union[str, Any]					,  __lowerCamelCase  :  List[str]  )				->				List[Any]:

     '''simple docstring'''




     np.save(os.path.join(__SCREAMING_SNAKE_CASE					,  _REALM_BLOCK_RECORDS_FILENAME  )					,  self.block_records  )
     # save tokenizer
     self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE  )




    def  a__					(       self  :  Any					,  __lowerCamelCase  :  Dict					,  __lowerCamelCase  :  str  )				->				int:

     '''simple docstring'''




     lowerCamelCase__	=  []
     lowerCamelCase__	=  []
     lowerCamelCase__	=  []
     lowerCamelCase__	=  0

     for input_id in concat_inputs.input_ids:
      lowerCamelCase__	=  input_id.tolist()
      # Check answers between two [SEP] tokens
      lowerCamelCase__	=  input_id_list.index(self.tokenizer.sep_token_id  )
      lowerCamelCase__	=  first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id  )

      start_pos.append([]  )
      end_pos.append([]  )
      for answer in answer_ids:
       for idx in range(first_sep_idx + 1					,  __SCREAMING_SNAKE_CASE  ):
        if answer[0] == input_id_list[idx]:
         if input_id_list[idx : idx + len(__SCREAMING_SNAKE_CASE  )] == answer:
          start_pos[-1].append(__SCREAMING_SNAKE_CASE  )
          end_pos[-1].append(idx + len(__SCREAMING_SNAKE_CASE  ) - 1  )

      if len(start_pos[-1]  ) == 0:
       has_answers.append(__SCREAMING_SNAKE_CASE  )
      else:
       has_answers.append(__SCREAMING_SNAKE_CASE  )
       if len(start_pos[-1]  ) > max_answers:
        lowerCamelCase__	=  len(start_pos[-1]  )

        # Pad -1 to max_answers
     for start_pos_, end_pos_ in zip(__SCREAMING_SNAKE_CASE					,  __SCREAMING_SNAKE_CASE  ):
      if len(start_pos_  ) < max_answers:
       lowerCamelCase__	=  [-1] * (max_answers - len(start_pos_  ))
       start_pos_ += padded
       end_pos_ += padded
     return has_answers, start_pos, end_pos





import random
import shutil
import tempfile
import unittest

import numpy as np

from transformers import TaTokenizer, TaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torch
from transformers.utils.import_utils import is_speech_available, is_torch_available


if is_torch_available():
  pass

if is_speech_available():
  from transformers import EncodecFeatureExtractor, MusicgenProcessor


__UpperCAmelCase			:			Union[str, Any]			      =					random.Random()


def   a						(				SCREAMING_SNAKE_CASE_   :		Optional[Any]				,      SCREAMING_SNAKE_CASE_   :		Optional[Any]=1.0				,      SCREAMING_SNAKE_CASE_   :		Optional[Any]=None				,      SCREAMING_SNAKE_CASE_   :		Union[str, Any]=None							):






       """simple docstring"""





       if rng is None:
              UpperCamelCase     :						str 						=		global_rng

       UpperCamelCase     :						Union[str, Any] 						=		[]
       for batch_idx in range(shape[0]							):
              values.append([]							)
              for _ in range(shape[1]							):
                     values[-1].append(rng.random() * scale							)

       return values



@require_torch
@require_sentencepiece
class  UpperCAmelCase_						(				unittest.TestCase):
       '''simple docstring'''






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						int 						=		'''facebook/musicgen-small'''
              UpperCamelCase     :						str 						=		tempfile.mkdtemp()






       def 			_lowercase (  self			,       **__SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              return TaTokenizer.from_pretrained(self.checkpoint			,       **__SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self			,       **__SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              return EncodecFeatureExtractor.from_pretrained(self.checkpoint			,       **__SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              shutil.rmtree(self.tmpdirname    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						Union[str, Any] 						=		self.get_tokenizer()
              UpperCamelCase     :						Any 						=		self.get_feature_extractor()

              UpperCamelCase     :						Optional[int] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              processor.save_pretrained(self.tmpdirname    )
              UpperCamelCase     :						int 						=		MusicgenProcessor.from_pretrained(self.tmpdirname    )

              self.assertEqual(processor.tokenizer.get_vocab()			,       tokenizer.get_vocab()    )
              self.assertIsInstance(processor.tokenizer			,       __SCREAMING_SNAKE_CASE    )

              self.assertEqual(processor.feature_extractor.to_json_string()			,       feature_extractor.to_json_string()    )
              self.assertIsInstance(processor.feature_extractor			,       __SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						str 						=		MusicgenProcessor(tokenizer=self.get_tokenizer()			,       feature_extractor=self.get_feature_extractor()    )
              processor.save_pretrained(self.tmpdirname    )

              UpperCamelCase     :						Any 						=		self.get_tokenizer(bos_token='''(BOS)'''			,       eos_token='''(EOS)'''    )
              UpperCamelCase     :						Union[str, Any] 						=		self.get_feature_extractor(do_normalize=__SCREAMING_SNAKE_CASE			,       padding_value=1.0    )

              UpperCamelCase     :						Union[str, Any] 						=		MusicgenProcessor.from_pretrained(
                  self.tmpdirname			,       bos_token='''(BOS)'''			,       eos_token='''(EOS)'''			,       do_normalize=__SCREAMING_SNAKE_CASE			,       padding_value=1.0    )

              self.assertEqual(processor.tokenizer.get_vocab()			,       tokenizer_add_kwargs.get_vocab()    )
              self.assertIsInstance(processor.tokenizer			,       __SCREAMING_SNAKE_CASE    )

              self.assertEqual(processor.feature_extractor.to_json_string()			,       feature_extractor_add_kwargs.to_json_string()    )
              self.assertIsInstance(processor.feature_extractor			,       __SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						int 						=		self.get_feature_extractor()
              UpperCamelCase     :						Tuple 						=		self.get_tokenizer()

              UpperCamelCase     :						Any 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Union[str, Any] 						=		floats_list((3, 1_000)    )

              UpperCamelCase     :						Tuple 						=		feature_extractor(__SCREAMING_SNAKE_CASE			,       return_tensors='''np'''    )
              UpperCamelCase     :						Optional[Any] 						=		processor(__SCREAMING_SNAKE_CASE			,       return_tensors='''np'''    )

              for key in input_feat_extract.keys():
                     self.assertAlmostEqual(input_feat_extract[key].sum()			,       input_processor[key].sum()			,       delta=1e-2    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						int 						=		self.get_feature_extractor()
              UpperCamelCase     :						List[str] 						=		self.get_tokenizer()

              UpperCamelCase     :						Optional[int] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Tuple 						=		'''This is a test string'''

              UpperCamelCase     :						Dict 						=		processor(text=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Union[str, Any] 						=		tokenizer(__SCREAMING_SNAKE_CASE    )

              for key in encoded_tok.keys():
                     self.assertListEqual(encoded_tok[key]			,       encoded_processor[key]    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						Any 						=		self.get_feature_extractor()
              UpperCamelCase     :						int 						=		self.get_tokenizer()

              UpperCamelCase     :						Optional[int] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						int 						=		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

              UpperCamelCase     :						int 						=		processor.batch_decode(sequences=__SCREAMING_SNAKE_CASE    )
              UpperCamelCase     :						Union[str, Any] 						=		tokenizer.batch_decode(__SCREAMING_SNAKE_CASE    )

              self.assertListEqual(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						Dict 						=		self.get_feature_extractor()
              UpperCamelCase     :						Union[str, Any] 						=		self.get_tokenizer()

              UpperCamelCase     :						str 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              self.assertListEqual(
                  processor.model_input_names			,       feature_extractor.model_input_names			,       msg='''`processor` and `feature_extractor` model input names do not match'''			,       )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						str 						=		self.get_feature_extractor(padding_side='''left'''    )
              UpperCamelCase     :						Optional[Any] 						=		self.get_tokenizer()

              UpperCamelCase     :						List[Any] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						List[str] 						=		[floats_list((1, x)    )[0] for x in range(5			,       20			,       5    )]
              UpperCamelCase     :						Tuple 						=		processor(__SCREAMING_SNAKE_CASE    ).padding_mask

              UpperCamelCase     :						List[str] 						=		np.asarray(floats_list((3, 20)    )    )[:, None, :]
              UpperCamelCase     :						Optional[int] 						=		processor.batch_decode(__SCREAMING_SNAKE_CASE			,       padding_mask=__SCREAMING_SNAKE_CASE    )

              self.assertIsInstance(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    )

              for audio in decoded_audios:
                     self.assertIsInstance(__SCREAMING_SNAKE_CASE			,       np.ndarray    )

              self.assertTrue(decoded_audios[0].shape == (1, 10)    )
              self.assertTrue(decoded_audios[1].shape == (1, 15)    )
              self.assertTrue(decoded_audios[2].shape == (1, 20)    )

