


import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__UpperCAmelCase			:			List[Any]			      =					"block_records.npy"


__UpperCAmelCase			:			Dict			      =					logging.get_logger(__name__)


def   a						(				SCREAMING_SNAKE_CASE_   :		str				,      SCREAMING_SNAKE_CASE_   :		int							):






       """simple docstring"""





       import tensorflow.compat.va as tf

       UpperCamelCase     :						Tuple 						=		tf.data.TFRecordDataset(SCREAMING_SNAKE_CASE_				,      buffer_size=5_1_2 * 1_0_2_4 * 1_0_2_4							)
       UpperCamelCase     :						Dict 						=		blocks_dataset.batch(SCREAMING_SNAKE_CASE_				,      drop_remainder=SCREAMING_SNAKE_CASE_							)
       UpperCamelCase     :						Any 						=		next(blocks_dataset.take(1							).as_numpy_iterator()							)

       return np_record



class  UpperCAmelCase_						:
       '''simple docstring'''






       def __init__(  self			,       __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE=2			,       __SCREAMING_SNAKE_CASE=1_000			,       __SCREAMING_SNAKE_CASE=100			,       __SCREAMING_SNAKE_CASE=100_000			,       ):




              """simple docstring"""




              from scann.scann_ops.py.scann_ops_pybind import builder as Builder

              UpperCamelCase     :						List[str] 						=		Builder(db=__SCREAMING_SNAKE_CASE			,       num_neighbors=__SCREAMING_SNAKE_CASE			,       distance_measure='''dot_product'''    )
              UpperCamelCase     :						Optional[Any] 						=		builder.tree(
                  num_leaves=__SCREAMING_SNAKE_CASE			,       num_leaves_to_search=__SCREAMING_SNAKE_CASE			,       training_sample_size=__SCREAMING_SNAKE_CASE    )
              UpperCamelCase     :						Any 						=		builder.score_ah(dimensions_per_block=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Optional[int] 						=		builder.build()






       def 			_lowercase (  self			,       __SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              UpperCamelCase						,			UpperCamelCase     :						Optional[Any] 						=		self.searcher.search_batched(question_projection.detach().cpu()    )
              return retrieved_block_ids.astype('''int64'''    )




class  UpperCAmelCase_						:
       '''simple docstring'''






       def __init__(  self			,       __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              super().__init__()
              UpperCamelCase     :						str 						=		block_records
              UpperCamelCase     :						Optional[int] 						=		tokenizer






       def __call__(  self			,       __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE=None			,       __SCREAMING_SNAKE_CASE="pt"    ):




              """simple docstring"""




              UpperCamelCase     :						Dict 						=		np.take(self.block_records			,       indices=__SCREAMING_SNAKE_CASE			,       axis=0    )

              UpperCamelCase     :						Optional[int] 						=		self.tokenizer.decode(question_input_ids[0]			,       skip_special_tokens=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Union[str, Any] 						=		[]
              UpperCamelCase     :						Optional[Any] 						=		[]
              for retrieved_block in retrieved_blocks:
                     text.append(__SCREAMING_SNAKE_CASE    )
                     text_pair.append(retrieved_block.decode()    )

              UpperCamelCase     :						Optional[Any] 						=		self.tokenizer(
                  __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE			,       padding=__SCREAMING_SNAKE_CASE			,       truncation=__SCREAMING_SNAKE_CASE			,       return_special_tokens_mask=__SCREAMING_SNAKE_CASE			,       max_length=__SCREAMING_SNAKE_CASE    )
              UpperCamelCase     :						str 						=		concat_inputs.convert_to_tensors(__SCREAMING_SNAKE_CASE    )

              if answer_ids is not None:
                     return self.block_has_answer(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    ) + (concat_inputs_tensors,)
              else:
                     return (None, None, None, concat_inputs_tensors)






       @classmethod
       def 			_lowercase (  cls			,       __SCREAMING_SNAKE_CASE			,       *__SCREAMING_SNAKE_CASE			,       **__SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              if os.path.isdir(__SCREAMING_SNAKE_CASE    ):
                     UpperCamelCase     :						int 						=		os.path.join(__SCREAMING_SNAKE_CASE			,       _REALM_BLOCK_RECORDS_FILENAME    )
              else:
                     UpperCamelCase     :						str 						=		hf_hub_download(
                         repo_id=__SCREAMING_SNAKE_CASE			,       filename=_REALM_BLOCK_RECORDS_FILENAME			,       **__SCREAMING_SNAKE_CASE    )
              UpperCamelCase     :						str 						=		np.load(__SCREAMING_SNAKE_CASE			,       allow_pickle=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						List[Any] 						=		AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			,       *__SCREAMING_SNAKE_CASE			,       **__SCREAMING_SNAKE_CASE    )

              return cls(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self			,       __SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              np.save(os.path.join(__SCREAMING_SNAKE_CASE			,       _REALM_BLOCK_RECORDS_FILENAME    )			,       self.block_records    )
              # save tokenizer
              self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self			,       __SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              UpperCamelCase     :						Optional[Any] 						=		[]
              UpperCamelCase     :						str 						=		[]
              UpperCamelCase     :						Union[str, Any] 						=		[]
              UpperCamelCase     :						str 						=		0

              for input_id in concat_inputs.input_ids:
                     UpperCamelCase     :						Tuple 						=		input_id.tolist()
                     # Check answers between two [SEP] tokens
                     UpperCamelCase     :						Union[str, Any] 						=		input_id_list.index(self.tokenizer.sep_token_id    )
                     UpperCamelCase     :						Dict 						=		first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id    )

                     start_pos.append([]    )
                     end_pos.append([]    )
                     for answer in answer_ids:
                            for idx in range(first_sep_idx + 1			,       __SCREAMING_SNAKE_CASE    ):
                                   if answer[0] == input_id_list[idx]:
                                          if input_id_list[idx : idx + len(__SCREAMING_SNAKE_CASE    )] == answer:
                                                 start_pos[-1].append(__SCREAMING_SNAKE_CASE    )
                                                 end_pos[-1].append(idx + len(__SCREAMING_SNAKE_CASE    ) - 1    )

                     if len(start_pos[-1]    ) == 0:
                            has_answers.append(__SCREAMING_SNAKE_CASE    )
                     else:
                            has_answers.append(__SCREAMING_SNAKE_CASE    )
                            if len(start_pos[-1]    ) > max_answers:
                                   UpperCamelCase     :						Optional[int] 						=		len(start_pos[-1]    )

        # Pad -1 to max_answers
              for start_pos_, end_pos_ in zip(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    ):
                     if len(start_pos_    ) < max_answers:
                            UpperCamelCase     :						int 						=		[-1] * (max_answers - len(start_pos_    ))
                            start_pos_ += padded
                            end_pos_ += padded
              return has_answers, start_pos, end_pos





import random
import shutil
import tempfile
import unittest

import numpy as np

from transformers import TaTokenizer, TaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torch
from transformers.utils.import_utils import is_speech_available, is_torch_available


if is_torch_available():
  pass

if is_speech_available():
  from transformers import EncodecFeatureExtractor, MusicgenProcessor


__UpperCAmelCase			:			Union[str, Any]			      =					random.Random()


def   a						(				SCREAMING_SNAKE_CASE_   :		Optional[Any]				,      SCREAMING_SNAKE_CASE_   :		Optional[Any]=1.0				,      SCREAMING_SNAKE_CASE_   :		Optional[Any]=None				,      SCREAMING_SNAKE_CASE_   :		Union[str, Any]=None							):






       """simple docstring"""





       if rng is None:
              UpperCamelCase     :						str 						=		global_rng

       UpperCamelCase     :						Union[str, Any] 						=		[]
       for batch_idx in range(shape[0]							):
              values.append([]							)
              for _ in range(shape[1]							):
                     values[-1].append(rng.random() * scale							)

       return values



@require_torch
@require_sentencepiece
class  UpperCAmelCase_						(				unittest.TestCase):
       '''simple docstring'''






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						int 						=		'''facebook/musicgen-small'''
              UpperCamelCase     :						str 						=		tempfile.mkdtemp()






       def 			_lowercase (  self			,       **__SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              return TaTokenizer.from_pretrained(self.checkpoint			,       **__SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self			,       **__SCREAMING_SNAKE_CASE    ):




              """simple docstring"""




              return EncodecFeatureExtractor.from_pretrained(self.checkpoint			,       **__SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              shutil.rmtree(self.tmpdirname    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						Union[str, Any] 						=		self.get_tokenizer()
              UpperCamelCase     :						Any 						=		self.get_feature_extractor()

              UpperCamelCase     :						Optional[int] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              processor.save_pretrained(self.tmpdirname    )
              UpperCamelCase     :						int 						=		MusicgenProcessor.from_pretrained(self.tmpdirname    )

              self.assertEqual(processor.tokenizer.get_vocab()			,       tokenizer.get_vocab()    )
              self.assertIsInstance(processor.tokenizer			,       __SCREAMING_SNAKE_CASE    )

              self.assertEqual(processor.feature_extractor.to_json_string()			,       feature_extractor.to_json_string()    )
              self.assertIsInstance(processor.feature_extractor			,       __SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						str 						=		MusicgenProcessor(tokenizer=self.get_tokenizer()			,       feature_extractor=self.get_feature_extractor()    )
              processor.save_pretrained(self.tmpdirname    )

              UpperCamelCase     :						Any 						=		self.get_tokenizer(bos_token='''(BOS)'''			,       eos_token='''(EOS)'''    )
              UpperCamelCase     :						Union[str, Any] 						=		self.get_feature_extractor(do_normalize=__SCREAMING_SNAKE_CASE			,       padding_value=1.0    )

              UpperCamelCase     :						Union[str, Any] 						=		MusicgenProcessor.from_pretrained(
                  self.tmpdirname			,       bos_token='''(BOS)'''			,       eos_token='''(EOS)'''			,       do_normalize=__SCREAMING_SNAKE_CASE			,       padding_value=1.0    )

              self.assertEqual(processor.tokenizer.get_vocab()			,       tokenizer_add_kwargs.get_vocab()    )
              self.assertIsInstance(processor.tokenizer			,       __SCREAMING_SNAKE_CASE    )

              self.assertEqual(processor.feature_extractor.to_json_string()			,       feature_extractor_add_kwargs.to_json_string()    )
              self.assertIsInstance(processor.feature_extractor			,       __SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						int 						=		self.get_feature_extractor()
              UpperCamelCase     :						Tuple 						=		self.get_tokenizer()

              UpperCamelCase     :						Any 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Union[str, Any] 						=		floats_list((3, 1_000)    )

              UpperCamelCase     :						Tuple 						=		feature_extractor(__SCREAMING_SNAKE_CASE			,       return_tensors='''np'''    )
              UpperCamelCase     :						Optional[Any] 						=		processor(__SCREAMING_SNAKE_CASE			,       return_tensors='''np'''    )

              for key in input_feat_extract.keys():
                     self.assertAlmostEqual(input_feat_extract[key].sum()			,       input_processor[key].sum()			,       delta=1e-2    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						int 						=		self.get_feature_extractor()
              UpperCamelCase     :						List[str] 						=		self.get_tokenizer()

              UpperCamelCase     :						Optional[int] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Tuple 						=		'''This is a test string'''

              UpperCamelCase     :						Dict 						=		processor(text=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						Union[str, Any] 						=		tokenizer(__SCREAMING_SNAKE_CASE    )

              for key in encoded_tok.keys():
                     self.assertListEqual(encoded_tok[key]			,       encoded_processor[key]    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						Any 						=		self.get_feature_extractor()
              UpperCamelCase     :						int 						=		self.get_tokenizer()

              UpperCamelCase     :						Optional[int] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						int 						=		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

              UpperCamelCase     :						int 						=		processor.batch_decode(sequences=__SCREAMING_SNAKE_CASE    )
              UpperCamelCase     :						Union[str, Any] 						=		tokenizer.batch_decode(__SCREAMING_SNAKE_CASE    )

              self.assertListEqual(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						Dict 						=		self.get_feature_extractor()
              UpperCamelCase     :						Union[str, Any] 						=		self.get_tokenizer()

              UpperCamelCase     :						str 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              self.assertListEqual(
                  processor.model_input_names			,       feature_extractor.model_input_names			,       msg='''`processor` and `feature_extractor` model input names do not match'''			,       )






       def 			_lowercase (  self    ):




              """simple docstring"""




              UpperCamelCase     :						str 						=		self.get_feature_extractor(padding_side='''left'''    )
              UpperCamelCase     :						Optional[Any] 						=		self.get_tokenizer()

              UpperCamelCase     :						List[Any] 						=		MusicgenProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE    )

              UpperCamelCase     :						List[str] 						=		[floats_list((1, x)    )[0] for x in range(5			,       20			,       5    )]
              UpperCamelCase     :						Tuple 						=		processor(__SCREAMING_SNAKE_CASE    ).padding_mask

              UpperCamelCase     :						List[str] 						=		np.asarray(floats_list((3, 20)    )    )[:, None, :]
              UpperCamelCase     :						Optional[int] 						=		processor.batch_decode(__SCREAMING_SNAKE_CASE			,       padding_mask=__SCREAMING_SNAKE_CASE    )

              self.assertIsInstance(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE    )

              for audio in decoded_audios:
                     self.assertIsInstance(__SCREAMING_SNAKE_CASE			,       np.ndarray    )

              self.assertTrue(decoded_audios[0].shape == (1, 10)    )
              self.assertTrue(decoded_audios[1].shape == (1, 15)    )
              self.assertTrue(decoded_audios[2].shape == (1, 20)    )

