




'''simple docstring'''

import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    HubertConfig,
    HubertForCTC,
    HubertModel,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
a  :				str									=  logging.get_logger(__name__)

a  :				Tuple									=  {
    'post_extract_proj': 'feature_projection.projection',
    'encoder.pos_conv.0': 'encoder.pos_conv_embed.conv',
    'self_attn.k_proj': 'encoder.layers.*.attention.k_proj',
    'self_attn.v_proj': 'encoder.layers.*.attention.v_proj',
    'self_attn.q_proj': 'encoder.layers.*.attention.q_proj',
    'self_attn.out_proj': 'encoder.layers.*.attention.out_proj',
    'self_attn_layer_norm': 'encoder.layers.*.layer_norm',
    'fc1': 'encoder.layers.*.feed_forward.intermediate_dense',
    'fc2': 'encoder.layers.*.feed_forward.output_dense',
    'final_layer_norm': 'encoder.layers.*.final_layer_norm',
    'encoder.layer_norm': 'encoder.layer_norm',
    'w2v_model.layer_norm': 'feature_projection.layer_norm',
    'w2v_encoder.proj': 'lm_head',
    'mask_emb': 'masked_spec_embed',
}


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)		->					Any:

					'''simple docstring'''

					for attribute in key.split('''.'''	):
										snake_case_									=				getattr(__UpperCAmelCase,      __UpperCAmelCase	)

					if weight_type is not None:
										snake_case_									=				getattr(__UpperCAmelCase,      __UpperCAmelCase	).shape
					else:
										snake_case_									=				hf_pointer.shape

					assert hf_shape == value.shape, (
					    F"Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be"
					    F" {value.shape} for {full_name}"
					)

					if weight_type == "weight":
										snake_case_									=				value
					elif weight_type == "weight_g":
										snake_case_									=				value
					elif weight_type == "weight_v":
										snake_case_									=				value
					elif weight_type == "bias":
										snake_case_									=				value
					else:
										snake_case_									=				value

					logger.info(F"{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}."	)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)		->					List[Any]:

					'''simple docstring'''

					snake_case_									=				[]
					snake_case_									=				fairseq_model.state_dict()

					snake_case_									=				hf_model.hubert.feature_extractor if is_finetuned else hf_model.feature_extractor

					for name, value in fairseq_dict.items():
										snake_case_									=				False
										if "conv_layers" in name:
															load_conv_layer(
															    __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      hf_model.config.feat_extract_norm == '''group''',      )
															snake_case_									=				True
										else:
															for key, mapped_key in MAPPING.items():
																				snake_case_									=				'''hubert.''' + mapped_key if (is_finetuned and mapped_key != '''lm_head''') else mapped_key

																				if key in name or (key.split('''w2v_model.'''	)[-1] == name.split('''.'''	)[0] and not is_finetuned):
																									snake_case_									=				True
																									if "*" in mapped_key:
																														snake_case_									=				name.split(__UpperCAmelCase	)[0].split('''.'''	)[-2]
																														snake_case_									=				mapped_key.replace('''*''',      __UpperCAmelCase	)
																									if "weight_g" in name:
																														snake_case_									=				'''weight_g'''
																									elif "weight_v" in name:
																														snake_case_									=				'''weight_v'''
																									elif "weight" in name:
																														snake_case_									=				'''weight'''
																									elif "bias" in name:
																														snake_case_									=				'''bias'''
																									else:
																														snake_case_									=				None
																									set_recursively(__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)
																				continue
										if not is_used:
															unused_weights.append(__UpperCAmelCase	)

					logger.warning(F"Unused weights: {unused_weights}"	)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)		->					int:

					'''simple docstring'''

					snake_case_									=				full_name.split('''conv_layers.'''	)[-1]
					snake_case_									=				name.split('''.'''	)
					snake_case_									=				int(items[0]	)
					snake_case_									=				int(items[1]	)

					if type_id == 0:
										if "bias" in name:
															assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
															    F"{full_name} has size {value.shape}, but"
															    F" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."
															)
															snake_case_									=				value
															logger.info(F"Feat extract conv layer {layer_id} was initialized from {full_name}."	)
										elif "weight" in name:
															assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
															    F"{full_name} has size {value.shape}, but"
															    F" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."
															)
															snake_case_									=				value
															logger.info(F"Feat extract conv layer {layer_id} was initialized from {full_name}."	)
					elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
										if "bias" in name:
															assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
															    F"{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was"
															    " found."
															)
															snake_case_									=				value
															logger.info(F"Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."	)
										elif "weight" in name:
															assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
															    F"{full_name} has size {value.shape}, but"
															    F" {feature_extractor[layer_id].layer_norm.weight.data.shape} was found."
															)
															snake_case_									=				value
															logger.info(F"Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."	)
					else:
										unused_weights.append(__UpperCAmelCase	)


@torch.no_grad()
def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase=None,      __UpperCAmelCase=None,      __UpperCAmelCase=True	)		->					List[Any]:

					'''simple docstring'''

					if config_path is not None:
										snake_case_									=				HubertConfig.from_pretrained(__UpperCAmelCase	)
					else:
										snake_case_									=				HubertConfig()

					if is_finetuned:
										if dict_path:
															snake_case_									=				Dictionary.load(__UpperCAmelCase	)

															# important change bos & pad token id since CTC symbol is <pad> and
															# not <s> as in fairseq
															snake_case_									=				target_dict.pad_index
															snake_case_									=				target_dict.bos_index
															snake_case_									=				target_dict.eos_index
															snake_case_									=				len(target_dict.symbols	)
															snake_case_									=				os.path.join(__UpperCAmelCase,      '''vocab.json'''	)
															if not os.path.isdir(__UpperCAmelCase	):
																				logger.error('''--pytorch_dump_folder_path ({}) should be a directory'''.format(__UpperCAmelCase	)	)
																				return
															os.makedirs(__UpperCAmelCase,      exist_ok=__UpperCAmelCase	)
															with open(__UpperCAmelCase,      '''w''',      encoding='''utf-8'''	) as vocab_handle:
																				json.dump(target_dict.indices,      __UpperCAmelCase	)
															snake_case_									=				WavaVecaCTCTokenizer(
															    __UpperCAmelCase,      unk_token=target_dict.unk_word,      pad_token=target_dict.pad_word,      bos_token=target_dict.bos_word,      eos_token=target_dict.eos_word,      word_delimiter_token='''|''',      do_lower_case=__UpperCAmelCase,      )
															snake_case_									=				True if config.feat_extract_norm == '''layer''' else False
															snake_case_									=				WavaVecaFeatureExtractor(
															    feature_size=1,      sampling_rate=1_6000,      padding_value=0,      do_normalize=__UpperCAmelCase,      return_attention_mask=__UpperCAmelCase,      )
															snake_case_									=				WavaVecaProcessor(feature_extractor=__UpperCAmelCase,      tokenizer=__UpperCAmelCase	)
															processor.save_pretrained(__UpperCAmelCase	)

										snake_case_									=				HubertForCTC(__UpperCAmelCase	)
					else:
										snake_case_									=				HubertModel(__UpperCAmelCase	)

					if is_finetuned:
										snake_case_			,snake_case_			,snake_case_									=				fairseq.checkpoint_utils.load_model_ensemble_and_task(
										    [checkpoint_path],      arg_overrides={'''data''': '''/'''.join(dict_path.split('''/'''	)[:-1]	)}	)
					else:
										snake_case_			,snake_case_			,snake_case_									=				fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]	)

					snake_case_									=				model[0].eval()

					recursively_load_weights(__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)

					hf_wavavec.save_pretrained(__UpperCAmelCase	)


if __name__ == "__main__":
							a  :				Tuple									=  argparse.ArgumentParser()
							parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
							parser.add_argument('--checkpoint_path', default=None, type=str, help='Path to fairseq checkpoint')
							parser.add_argument('--dict_path', default=None, type=str, help='Path to dict of fine-tuned model')
							parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
							parser.add_argument(
							    '--not_finetuned', action='store_true', help='Whether the model to convert is a fine-tuned model or not'
							)
							a  :				str									=  parser.parse_args()
							convert_hubert_checkpoint(
							    args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
							)







'''simple docstring'''

import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
							import faiss






@require_faiss
class 						a				(   _lowerCamelCase     ):


				def   A_ (		self    :					str					):
									snake_case_									=				tempfile.mkdtemp()
									snake_case_									=				8

									# DPR tok
									snake_case_									=				[
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									snake_case_									=				os.path.join(self.tmpdirname       ,				'''dpr_tokenizer'''					)
									os.makedirs(lowercase_       ,				exist_ok=lowercase_					)
									snake_case_									=				os.path.join(lowercase_       ,				DPR_VOCAB_FILES_NAMES['''vocab_file''']					)
									with open(self.vocab_file       ,				'''w'''       ,				encoding='''utf-8'''					) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)

									# BART tok
									snake_case_									=				[
									    '''l''',
									    '''o''',
									    '''w''',
									    '''e''',
									    '''r''',
									    '''s''',
									    '''t''',
									    '''i''',
									    '''d''',
									    '''n''',
									    '''\u0120''',
									    '''\u0120l''',
									    '''\u0120n''',
									    '''\u0120lo''',
									    '''\u0120low''',
									    '''er''',
									    '''\u0120lowest''',
									    '''\u0120newer''',
									    '''\u0120wider''',
									    '''<unk>''',
									]
									snake_case_									=				dict(zip(lowercase_       ,				range(len(lowercase_					)					)					)					)
									snake_case_									=				['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
									snake_case_									=				{'''unk_token''': '''<unk>'''}

									snake_case_									=				os.path.join(self.tmpdirname       ,				'''bart_tokenizer'''					)
									os.makedirs(lowercase_       ,				exist_ok=lowercase_					)
									snake_case_									=				os.path.join(lowercase_       ,				BART_VOCAB_FILES_NAMES['''vocab_file''']					)
									snake_case_									=				os.path.join(lowercase_       ,				BART_VOCAB_FILES_NAMES['''merges_file''']					)
									with open(self.vocab_file       ,				'''w'''       ,				encoding='''utf-8'''					) as fp:
														fp.write(json.dumps(lowercase_					) + '''\n'''					)
									with open(self.merges_file       ,				'''w'''       ,				encoding='''utf-8'''					) as fp:
														fp.write('''\n'''.join(lowercase_					)					)


				def   A_ (		self    :					Union[str, Any]					):
									return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,				'''dpr_tokenizer'''					)					)


				def   A_ (		self    :					Union[str, Any]					):
									return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,				'''dpr_tokenizer'''					)					)


				def   A_ (		self    :					int					):
									return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,				'''bart_tokenizer'''					)					)


				def   A_ (		self    :					str					):
									shutil.rmtree(self.tmpdirname					)


				def   A_ (		self    :					str					):
									snake_case_									=				Dataset.from_dict(
									    {
									        '''id''': ['''0''', '''1'''],
									        '''text''': ['''foo''', '''bar'''],
									        '''title''': ['''Foo''', '''Bar'''],
									        '''embeddings''': [np.ones(self.retrieval_vector_size					), 2 * np.ones(self.retrieval_vector_size					)],
									    }					)
									dataset.add_faiss_index('''embeddings'''       ,				string_factory='''Flat'''       ,				metric_type=faiss.METRIC_INNER_PRODUCT					)
									return dataset


				def   A_ (		self    :					str					):
									snake_case_									=				self.get_dummy_dataset()
									snake_case_									=				RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size       ,				question_encoder=DPRConfig().to_dict()       ,				generator=BartConfig().to_dict()       ,				)
									with patch('''transformers.models.rag.retrieval_rag.load_dataset'''					) as mock_load_dataset:
														snake_case_									=				dataset
														snake_case_									=				RagRetriever(
														    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()       ,				)
									return retriever


				def   A_ (		self    :					str       ,				lowercase_    :					bool					):
									snake_case_									=				self.get_dummy_dataset()
									snake_case_									=				RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size       ,				question_encoder=DPRConfig().to_dict()       ,				generator=BartConfig().to_dict()       ,				index_name='''custom'''       ,				)
									if from_disk:
														snake_case_									=				os.path.join(self.tmpdirname       ,				'''dataset'''					)
														snake_case_									=				os.path.join(self.tmpdirname       ,				'''index.faiss'''					)
														dataset.get_index('''embeddings'''					).save(os.path.join(self.tmpdirname       ,				'''index.faiss'''					)					)
														dataset.drop_index('''embeddings'''					)
														dataset.save_to_disk(os.path.join(self.tmpdirname       ,				'''dataset'''					)					)
														del dataset
														snake_case_									=				RagRetriever(
														    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()       ,				)
									else:
														snake_case_									=				RagRetriever(
														    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()       ,				index=CustomHFIndex(config.retrieval_vector_size       ,				lowercase_					)       ,				)
									return retriever


				def   A_ (		self    :					Tuple					):
									snake_case_									=				Dataset.from_dict(
									    {
									        '''id''': ['''0''', '''1'''],
									        '''text''': ['''foo''', '''bar'''],
									        '''title''': ['''Foo''', '''Bar'''],
									        '''embeddings''': [np.ones(self.retrieval_vector_size + 1					), 2 * np.ones(self.retrieval_vector_size + 1					)],
									    }					)
									dataset.add_faiss_index('''embeddings'''       ,				string_factory='''Flat'''       ,				metric_type=faiss.METRIC_INNER_PRODUCT					)

									snake_case_									=				os.path.join(self.tmpdirname       ,				'''hf_bert_base.hnswSQ8_correct_phi_128.c_index'''					)
									dataset.save_faiss_index('''embeddings'''       ,				index_file_name + '''.index.dpr'''					)
									pickle.dump(dataset['''id''']       ,				open(index_file_name + '''.index_meta.dpr'''       ,				'''wb'''					)					)

									snake_case_									=				os.path.join(self.tmpdirname       ,				'''psgs_w100.tsv.pkl'''					)
									snake_case_									=				{sample['''id''']: [sample['''text'''], sample['''title''']] for sample in dataset}
									pickle.dump(lowercase_       ,				open(lowercase_       ,				'''wb'''					)					)

									snake_case_									=				RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size       ,				question_encoder=DPRConfig().to_dict()       ,				generator=BartConfig().to_dict()       ,				index_name='''legacy'''       ,				index_path=self.tmpdirname       ,				)
									snake_case_									=				RagRetriever(
									    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()					)
									return retriever


				def   A_ (		self    :					Optional[Any]					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_canonical_hf_index_retriever()
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''embeddings''', '''id''', '''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''id''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''id'''][0]       ,				'''1'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]       ,				'''0'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					str					):
									snake_case_									=				self.get_dummy_canonical_hf_index_retriever()
									with tempfile.TemporaryDirectory() as tmp_dirname:
														with patch('''transformers.models.rag.retrieval_rag.load_dataset'''					) as mock_load_dataset:
																			snake_case_									=				self.get_dummy_dataset()
																			retriever.save_pretrained(lowercase_					)
																			snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
																			self.assertIsInstance(lowercase_       ,				lowercase_					)
																			snake_case_									=				np.array(
																			    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
																			snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
																			self.assertTrue(out is not None					)


				def   A_ (		self    :					int					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''embeddings''', '''id''', '''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''id''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''id'''][0]       ,				'''1'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]       ,				'''0'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					int					):
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									with tempfile.TemporaryDirectory() as tmp_dirname:
														retriever.save_pretrained(lowercase_					)
														snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
														self.assertIsInstance(lowercase_       ,				lowercase_					)
														snake_case_									=				np.array(
														    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
														snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
														self.assertTrue(out is not None					)


				def   A_ (		self    :					str					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''embeddings''', '''id''', '''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''id''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''id'''][0]       ,				'''1'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]       ,				'''0'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					Any					):
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									with tempfile.TemporaryDirectory() as tmp_dirname:
														retriever.save_pretrained(lowercase_					)
														snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
														self.assertIsInstance(lowercase_       ,				lowercase_					)
														snake_case_									=				np.array(
														    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
														snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
														self.assertTrue(out is not None					)


				def   A_ (		self    :					Any					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_legacy_index_retriever()
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''text''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''text'''][0]       ,				'''bar'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''text'''][0]       ,				'''foo'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					int					):
									snake_case_									=				self.get_dummy_legacy_index_retriever()
									with tempfile.TemporaryDirectory() as tmp_dirname:
														retriever.save_pretrained(lowercase_					)
														snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
														self.assertIsInstance(lowercase_       ,				lowercase_					)
														snake_case_									=				np.array(
														    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
														snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
														self.assertTrue(out is not None					)


				@require_torch
				@require_tokenizers
				@require_sentencepiece
				def   A_ (		self    :					List[str]					):
									import torch

									snake_case_									=				1
									snake_case_									=				self.get_dummy_canonical_hf_index_retriever()
									snake_case_									=				[[5, 7], [10, 11]]
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_									=				retriever(lowercase_       ,				lowercase_       ,				prefix=retriever.config.generator.prefix       ,				n_docs=lowercase_					)
									snake_case_			,snake_case_			,snake_case_									=				(
									    out['''context_input_ids'''],
									    out['''context_attention_mask'''],
									    out['''retrieved_doc_embeds'''],
									)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertIsInstance(lowercase_       ,				lowercase_					)
									self.assertIsInstance(lowercase_       ,				lowercase_					)
									self.assertIsInstance(lowercase_       ,				np.ndarray					)

									snake_case_									=				retriever(
									    lowercase_       ,				lowercase_       ,				prefix=retriever.config.generator.prefix       ,				n_docs=lowercase_       ,				return_tensors='''pt'''       ,				)
									snake_case_			,snake_case_			,snake_case_			,snake_case_									=				(  # noqa: F841
									    out['''context_input_ids'''],
									    out['''context_attention_mask'''],
									    out['''retrieved_doc_embeds'''],
									    out['''doc_ids'''],
									)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertIsInstance(lowercase_       ,				torch.Tensor					)
									self.assertIsInstance(lowercase_       ,				torch.Tensor					)
									self.assertIsInstance(lowercase_       ,				torch.Tensor					)


				@require_torch
				@require_tokenizers
				@require_sentencepiece
				def   A_ (		self    :					Tuple					):
									snake_case_									=				self.get_dpr_ctx_encoder_tokenizer()
									snake_case_									=				1
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									retriever.set_ctx_encoder_tokenizer(lowercase_					)

									snake_case_									=				[[5, 7], [10, 11]]
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_									=				retriever(lowercase_       ,				lowercase_       ,				prefix=retriever.config.generator.prefix       ,				n_docs=lowercase_					)

									self.assertEqual(
									    len(lowercase_					)       ,				6					)  # check whether the retriever output consist of 6 attributes including tokenized docs
									self.assertEqual(
									    all(k in out for k in ('''tokenized_doc_ids''', '''tokenized_doc_attention_mask''')					)       ,				lowercase_					)  # check for doc token related keys in dictionary.

