


"""simple docstring"""




import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_ta import TaTokenizer
else:
				lowercase_			     =  None


lowercase_			     =  logging.get_logger(__name__)

lowercase_			     =  {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowercase_			     =  {
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
lowercase_			     =  {
    't5-small': 5_1_2,
    't5-base': 5_1_2,
    't5-large': 5_1_2,
    't5-3b': 5_1_2,
    't5-11b': 5_1_2,
}
class 					__lowerCAmelCase					(   _lowerCamelCase       ):


		'''simple docstring'''



		__UpperCAmelCase		:			List[str]										=				VOCAB_FILES_NAMES
		__UpperCAmelCase		:			int										=				PRETRAINED_VOCAB_FILES_MAP
		__UpperCAmelCase		:			Optional[Any]										=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__UpperCAmelCase		:			Optional[int]										=				['input_ids', 'attention_mask']
		__UpperCAmelCase		:			int										=				TaTokenizer

		__UpperCAmelCase		:			Union[str, Any]										=				[]

		def __init__(  self							, _a=None							, _a=None							, _a="</s>"							, _a="<unk>"							, _a="<pad>"							, _a=100							, _a=None							, **_a							, ):
				# Add extra_ids to the special token list
				if extra_ids > 0 and additional_special_tokens is None:
						__a           =			[f'''<extra_id_{i}>''' for i in range(lowercase_      )]
				elif extra_ids > 0 and additional_special_tokens is not None:
						# Check that we have the right number of extra special tokens
						__a           =			len(set(filter(lambda _a      : bool('''extra_id_''' in str(lowercase_      )      )							, lowercase_      )      )      )
						if extra_tokens != extra_ids:
								raise ValueError(
								    f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
								    ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
								    ''' tokens'''      )

				super().__init__(
				    lowercase_							, tokenizer_file=lowercase_							, eos_token=lowercase_							, unk_token=lowercase_							, pad_token=lowercase_							, extra_ids=lowercase_							, additional_special_tokens=lowercase_							, **lowercase_							, )

				__a           =			vocab_file
				__a           =			False if not self.vocab_file else True
				__a           =			extra_ids

		@staticmethod
		def 			__UpperCAmelCase       (  _a							, _a							, _a      ):
				if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
						__a           =			TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
						if init_max_model_length is not None and init_max_model_length != max_model_length:
								return init_max_model_length
						elif init_max_model_length is None:
								warnings.warn(
								    '''This tokenizer was incorrectly instantiated with a model max length of'''
								    f''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
								    ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
								    ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
								    f''' {pretrained_model_name_or_path} automatically truncating your input to'''
								    f''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
								    f''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
								    ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
								    ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''							, lowercase_							, )

				return max_model_length

		def 			__UpperCAmelCase       (  self							, _a							, _a = None      ):
				if not self.can_save_slow_tokenizer:
						raise ValueError(
						    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
						    '''tokenizer.'''      )

				if not os.path.isdir(lowercase_      ):
						logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
						return
				__a           =			os.path.join(
				    lowercase_							, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

				if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowercase_      ):
						copyfile(self.vocab_file							, lowercase_      )
						logger.info(f'''Copy vocab file to {out_vocab_file}'''      )

				return (out_vocab_file,)

		def 			__UpperCAmelCase       (  self							, _a							, _a = None      ):
				__a           =			token_ids_a + [self.eos_token_id]
				if token_ids_a is None:
						return self.prefix_tokens + token_ids_a
				else:
						__a           =			token_ids_a + [self.eos_token_id]
						return self.prefix_tokens + token_ids_a + token_ids_a

		def 			__UpperCAmelCase       (  self							, _a							, _a = None      ):
				__a           =			[self.eos_token_id]

				if token_ids_a is None:
						return len(token_ids_a + eos      ) * [0]
				return len(token_ids_a + eos + token_ids_a + eos      ) * [0]

		def 			__UpperCAmelCase       (  self      ):
				return list(
				    set(filter(lambda _a      : bool(re.search(R'''<extra_id_\d+>'''							, lowercase_      )      ) is not None							, self.additional_special_tokens      )      )      )





		def 			__UpperCAmelCase       (  self      ):
				return [self.convert_tokens_to_ids(lowercase_      ) for token in self.get_sentinel_tokens()]












'''simple docstring'''

import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
							import faiss






@require_faiss
class 						a				(   _lowerCamelCase     ):


				def   A_ (		self    :					str					):
									snake_case_									=				tempfile.mkdtemp()
									snake_case_									=				8

									# DPR tok
									snake_case_									=				[
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									snake_case_									=				os.path.join(self.tmpdirname       ,				'''dpr_tokenizer'''					)
									os.makedirs(lowercase_       ,				exist_ok=lowercase_					)
									snake_case_									=				os.path.join(lowercase_       ,				DPR_VOCAB_FILES_NAMES['''vocab_file''']					)
									with open(self.vocab_file       ,				'''w'''       ,				encoding='''utf-8'''					) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)

									# BART tok
									snake_case_									=				[
									    '''l''',
									    '''o''',
									    '''w''',
									    '''e''',
									    '''r''',
									    '''s''',
									    '''t''',
									    '''i''',
									    '''d''',
									    '''n''',
									    '''\u0120''',
									    '''\u0120l''',
									    '''\u0120n''',
									    '''\u0120lo''',
									    '''\u0120low''',
									    '''er''',
									    '''\u0120lowest''',
									    '''\u0120newer''',
									    '''\u0120wider''',
									    '''<unk>''',
									]
									snake_case_									=				dict(zip(lowercase_       ,				range(len(lowercase_					)					)					)					)
									snake_case_									=				['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
									snake_case_									=				{'''unk_token''': '''<unk>'''}

									snake_case_									=				os.path.join(self.tmpdirname       ,				'''bart_tokenizer'''					)
									os.makedirs(lowercase_       ,				exist_ok=lowercase_					)
									snake_case_									=				os.path.join(lowercase_       ,				BART_VOCAB_FILES_NAMES['''vocab_file''']					)
									snake_case_									=				os.path.join(lowercase_       ,				BART_VOCAB_FILES_NAMES['''merges_file''']					)
									with open(self.vocab_file       ,				'''w'''       ,				encoding='''utf-8'''					) as fp:
														fp.write(json.dumps(lowercase_					) + '''\n'''					)
									with open(self.merges_file       ,				'''w'''       ,				encoding='''utf-8'''					) as fp:
														fp.write('''\n'''.join(lowercase_					)					)


				def   A_ (		self    :					Union[str, Any]					):
									return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,				'''dpr_tokenizer'''					)					)


				def   A_ (		self    :					Union[str, Any]					):
									return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,				'''dpr_tokenizer'''					)					)


				def   A_ (		self    :					int					):
									return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,				'''bart_tokenizer'''					)					)


				def   A_ (		self    :					str					):
									shutil.rmtree(self.tmpdirname					)


				def   A_ (		self    :					str					):
									snake_case_									=				Dataset.from_dict(
									    {
									        '''id''': ['''0''', '''1'''],
									        '''text''': ['''foo''', '''bar'''],
									        '''title''': ['''Foo''', '''Bar'''],
									        '''embeddings''': [np.ones(self.retrieval_vector_size					), 2 * np.ones(self.retrieval_vector_size					)],
									    }					)
									dataset.add_faiss_index('''embeddings'''       ,				string_factory='''Flat'''       ,				metric_type=faiss.METRIC_INNER_PRODUCT					)
									return dataset


				def   A_ (		self    :					str					):
									snake_case_									=				self.get_dummy_dataset()
									snake_case_									=				RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size       ,				question_encoder=DPRConfig().to_dict()       ,				generator=BartConfig().to_dict()       ,				)
									with patch('''transformers.models.rag.retrieval_rag.load_dataset'''					) as mock_load_dataset:
														snake_case_									=				dataset
														snake_case_									=				RagRetriever(
														    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()       ,				)
									return retriever


				def   A_ (		self    :					str       ,				lowercase_    :					bool					):
									snake_case_									=				self.get_dummy_dataset()
									snake_case_									=				RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size       ,				question_encoder=DPRConfig().to_dict()       ,				generator=BartConfig().to_dict()       ,				index_name='''custom'''       ,				)
									if from_disk:
														snake_case_									=				os.path.join(self.tmpdirname       ,				'''dataset'''					)
														snake_case_									=				os.path.join(self.tmpdirname       ,				'''index.faiss'''					)
														dataset.get_index('''embeddings'''					).save(os.path.join(self.tmpdirname       ,				'''index.faiss'''					)					)
														dataset.drop_index('''embeddings'''					)
														dataset.save_to_disk(os.path.join(self.tmpdirname       ,				'''dataset'''					)					)
														del dataset
														snake_case_									=				RagRetriever(
														    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()       ,				)
									else:
														snake_case_									=				RagRetriever(
														    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()       ,				index=CustomHFIndex(config.retrieval_vector_size       ,				lowercase_					)       ,				)
									return retriever


				def   A_ (		self    :					Tuple					):
									snake_case_									=				Dataset.from_dict(
									    {
									        '''id''': ['''0''', '''1'''],
									        '''text''': ['''foo''', '''bar'''],
									        '''title''': ['''Foo''', '''Bar'''],
									        '''embeddings''': [np.ones(self.retrieval_vector_size + 1					), 2 * np.ones(self.retrieval_vector_size + 1					)],
									    }					)
									dataset.add_faiss_index('''embeddings'''       ,				string_factory='''Flat'''       ,				metric_type=faiss.METRIC_INNER_PRODUCT					)

									snake_case_									=				os.path.join(self.tmpdirname       ,				'''hf_bert_base.hnswSQ8_correct_phi_128.c_index'''					)
									dataset.save_faiss_index('''embeddings'''       ,				index_file_name + '''.index.dpr'''					)
									pickle.dump(dataset['''id''']       ,				open(index_file_name + '''.index_meta.dpr'''       ,				'''wb'''					)					)

									snake_case_									=				os.path.join(self.tmpdirname       ,				'''psgs_w100.tsv.pkl'''					)
									snake_case_									=				{sample['''id''']: [sample['''text'''], sample['''title''']] for sample in dataset}
									pickle.dump(lowercase_       ,				open(lowercase_       ,				'''wb'''					)					)

									snake_case_									=				RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size       ,				question_encoder=DPRConfig().to_dict()       ,				generator=BartConfig().to_dict()       ,				index_name='''legacy'''       ,				index_path=self.tmpdirname       ,				)
									snake_case_									=				RagRetriever(
									    lowercase_       ,				question_encoder_tokenizer=self.get_dpr_tokenizer()       ,				generator_tokenizer=self.get_bart_tokenizer()					)
									return retriever


				def   A_ (		self    :					Optional[Any]					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_canonical_hf_index_retriever()
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''embeddings''', '''id''', '''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''id''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''id'''][0]       ,				'''1'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]       ,				'''0'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					str					):
									snake_case_									=				self.get_dummy_canonical_hf_index_retriever()
									with tempfile.TemporaryDirectory() as tmp_dirname:
														with patch('''transformers.models.rag.retrieval_rag.load_dataset'''					) as mock_load_dataset:
																			snake_case_									=				self.get_dummy_dataset()
																			retriever.save_pretrained(lowercase_					)
																			snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
																			self.assertIsInstance(lowercase_       ,				lowercase_					)
																			snake_case_									=				np.array(
																			    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
																			snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
																			self.assertTrue(out is not None					)


				def   A_ (		self    :					int					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''embeddings''', '''id''', '''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''id''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''id'''][0]       ,				'''1'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]       ,				'''0'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					int					):
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									with tempfile.TemporaryDirectory() as tmp_dirname:
														retriever.save_pretrained(lowercase_					)
														snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
														self.assertIsInstance(lowercase_       ,				lowercase_					)
														snake_case_									=				np.array(
														    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
														snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
														self.assertTrue(out is not None					)


				def   A_ (		self    :					str					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''embeddings''', '''id''', '''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''id''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''id'''][0]       ,				'''1'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]       ,				'''0'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					Any					):
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									with tempfile.TemporaryDirectory() as tmp_dirname:
														retriever.save_pretrained(lowercase_					)
														snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
														self.assertIsInstance(lowercase_       ,				lowercase_					)
														snake_case_									=				np.array(
														    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
														snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
														self.assertTrue(out is not None					)


				def   A_ (		self    :					Any					):
									snake_case_									=				1
									snake_case_									=				self.get_dummy_legacy_index_retriever()
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_			,snake_case_			,snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=lowercase_					)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertEqual(len(lowercase_					)       ,				2					)
									self.assertEqual(sorted(doc_dicts[0]					)       ,				['''text''', '''title''']					)
									self.assertEqual(len(doc_dicts[0]['''text''']					)       ,				lowercase_					)
									self.assertEqual(doc_dicts[0]['''text'''][0]       ,				'''bar'''					)  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''text'''][0]       ,				'''foo'''					)  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()       ,				[[1], [0]]					)


				def   A_ (		self    :					int					):
									snake_case_									=				self.get_dummy_legacy_index_retriever()
									with tempfile.TemporaryDirectory() as tmp_dirname:
														retriever.save_pretrained(lowercase_					)
														snake_case_									=				RagRetriever.from_pretrained(lowercase_					)
														self.assertIsInstance(lowercase_       ,				lowercase_					)
														snake_case_									=				np.array(
														    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
														snake_case_									=				retriever.retrieve(lowercase_       ,				n_docs=1					)
														self.assertTrue(out is not None					)


				@require_torch
				@require_tokenizers
				@require_sentencepiece
				def   A_ (		self    :					List[str]					):
									import torch

									snake_case_									=				1
									snake_case_									=				self.get_dummy_canonical_hf_index_retriever()
									snake_case_									=				[[5, 7], [10, 11]]
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_									=				retriever(lowercase_       ,				lowercase_       ,				prefix=retriever.config.generator.prefix       ,				n_docs=lowercase_					)
									snake_case_			,snake_case_			,snake_case_									=				(
									    out['''context_input_ids'''],
									    out['''context_attention_mask'''],
									    out['''retrieved_doc_embeds'''],
									)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertIsInstance(lowercase_       ,				lowercase_					)
									self.assertIsInstance(lowercase_       ,				lowercase_					)
									self.assertIsInstance(lowercase_       ,				np.ndarray					)

									snake_case_									=				retriever(
									    lowercase_       ,				lowercase_       ,				prefix=retriever.config.generator.prefix       ,				n_docs=lowercase_       ,				return_tensors='''pt'''       ,				)
									snake_case_			,snake_case_			,snake_case_			,snake_case_									=				(  # noqa: F841
									    out['''context_input_ids'''],
									    out['''context_attention_mask'''],
									    out['''retrieved_doc_embeds'''],
									    out['''doc_ids'''],
									)
									self.assertEqual(retrieved_doc_embeds.shape       ,				(2, n_docs, self.retrieval_vector_size)					)
									self.assertIsInstance(lowercase_       ,				torch.Tensor					)
									self.assertIsInstance(lowercase_       ,				torch.Tensor					)
									self.assertIsInstance(lowercase_       ,				torch.Tensor					)


				@require_torch
				@require_tokenizers
				@require_sentencepiece
				def   A_ (		self    :					Tuple					):
									snake_case_									=				self.get_dpr_ctx_encoder_tokenizer()
									snake_case_									=				1
									snake_case_									=				self.get_dummy_custom_hf_index_retriever(from_disk=lowercase_					)
									retriever.set_ctx_encoder_tokenizer(lowercase_					)

									snake_case_									=				[[5, 7], [10, 11]]
									snake_case_									=				np.array(
									    [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]       ,				dtype=np.floataa					)
									snake_case_									=				retriever(lowercase_       ,				lowercase_       ,				prefix=retriever.config.generator.prefix       ,				n_docs=lowercase_					)

									self.assertEqual(
									    len(lowercase_					)       ,				6					)  # check whether the retriever output consist of 6 attributes including tokenized docs
									self.assertEqual(
									    all(k in out for k in ('''tokenized_doc_ids''', '''tokenized_doc_attention_mask''')					)       ,				lowercase_					)  # check for doc token related keys in dictionary.

