



import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__SCREAMING_SNAKE_CASE     =	logging.getLogger(__name__)

class 						lowerCamelCase_     :






						'''simple docstring'''







						def __init__(						self   :      str			)   ->					int:
									A				:       Optional[Any]														=    False







						def       SCREAMING_SNAKE_CASE__		(						self   :      List[Any]						,						__lowerCamelCase   :      Dict						,						__lowerCamelCase   :      Dict						,						__lowerCamelCase   :      str						,						__lowerCamelCase   :      int			)   ->					Tuple:
									if not self.initialized:
												A				:       int														=    RagRetriever(
												    __lowerCamelCase						,						question_encoder_tokenizer=__lowerCamelCase						,						generator_tokenizer=__lowerCamelCase						,						index=__lowerCamelCase						,						init_retrieval=__lowerCamelCase						,						)
												A				:       Dict														=    True







						def       SCREAMING_SNAKE_CASE__		(						self   :      Tuple			)   ->					List[Any]:
									self.retriever.index.init_index()







						def       SCREAMING_SNAKE_CASE__		(						self   :      int			)   ->					str:
									# delete the old self.retriever object before assigning the new index
									del self.retriever
									A				:       Any														=    False







						def       SCREAMING_SNAKE_CASE__		(						self   :      List[str]						,						__lowerCamelCase   :      str						,						__lowerCamelCase   :      Optional[int]			)   ->					List[str]:
									A							,			A				:       List[str]														=    self.retriever._main_retrieve(__lowerCamelCase						,						__lowerCamelCase			)
									A				:       List[str]														=    self.retriever.index.get_doc_dicts(__lowerCamelCase			)
									return doc_ids, retrieved_doc_embeds, doc_dicts







class 						lowerCamelCase_     (			_A    ):






						'''simple docstring'''







						def __init__(						self   :      Union[str, Any]						,						__lowerCamelCase   :      int						,						__lowerCamelCase   :      Optional[int]						,						__lowerCamelCase   :      Any						,						__lowerCamelCase   :      Tuple						,						__lowerCamelCase   :      Optional[int]=None			)   ->					Tuple:
									if index is not None and index.is_initialized() and len(__lowerCamelCase			) > 0:
												raise ValueError(
												    "When using Ray for distributed fine-tuning, "
												    "you'll need to provide the paths instead, "
												    "as the dataset and the index are loaded "
												    "separately. More info in examples/rag/use_own_knowledge_dataset.py "			)

									super().__init__(
									    __lowerCamelCase						,						question_encoder_tokenizer=__lowerCamelCase						,						generator_tokenizer=__lowerCamelCase						,						index=__lowerCamelCase						,						init_retrieval=__lowerCamelCase						,						)

									A				:       List[str]														=    retrieval_workers
									A				:       Any														=    question_encoder_tokenizer
									A				:       Dict														=    generator_tokenizer
									if len(self.retrieval_workers			) > 0:
												ray.get(
												    [
												        worker.create_rag_retriever.remote(__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase			)
												        for worker in self.retrieval_workers
												    ]			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      List[Any]			)   ->					Union[str, Any]:
									logger.info("initializing retrieval"			)

									if len(self.retrieval_workers			) > 0:
												ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]			)
									else:
												# Non-distributed training. Load index into this same process.
												self.index.init_index()







						def       SCREAMING_SNAKE_CASE__		(						self   :      Optional[int]						,						__lowerCamelCase   :      Optional[int]						,						__lowerCamelCase   :      Optional[Any]			)   ->					Dict:
									if len(self.retrieval_workers			) > 0:
												# Select a random retrieval actor.
												A				:       Any														=    self.retrieval_workers[random.randint(0						,						len(self.retrieval_workers			) - 1			)]
												A							,			A							,			A				:       Optional[int]														=    ray.get(
												    random_worker.retrieve.remote(__lowerCamelCase						,						__lowerCamelCase			)			)
									else:
												A							,			A				:       str														=    self._main_retrieve(__lowerCamelCase						,						__lowerCamelCase			)
												A				:       Optional[int]														=    self.index.get_doc_dicts(__lowerCamelCase			)
									return retrieved_doc_embeds, doc_ids, doc_dicts







						@classmethod
						def       SCREAMING_SNAKE_CASE__		(						cls   :      Optional[Any]						,						__lowerCamelCase   :      Optional[int]						,						__lowerCamelCase   :      Dict=None						,						**__lowerCamelCase   :      List[Any]			)   ->					str:
									return super(__lowerCamelCase						,						cls			).get_tokenizers(__lowerCamelCase						,						__lowerCamelCase						,						**__lowerCamelCase			)







						@classmethod
						def       SCREAMING_SNAKE_CASE__		(						cls   :      Optional[int]						,						__lowerCamelCase   :      Union[str, Any]						,						__lowerCamelCase   :      Optional[int]						,						__lowerCamelCase   :      Tuple=None						,						**__lowerCamelCase   :      str			)   ->					List[str]:
									A				:       Dict														=    kwargs.pop("config"						,						__lowerCamelCase			) or RagConfig.from_pretrained(__lowerCamelCase						,						**__lowerCamelCase			)
									A				:       List[str]														=    RagTokenizer.from_pretrained(__lowerCamelCase						,						config=__lowerCamelCase			)
									A				:       Tuple														=    rag_tokenizer.question_encoder
									A				:       List[Any]														=    rag_tokenizer.generator

									if indexed_dataset is not None:
												A				:       List[Any]														=    "custom"
												A				:       List[Any]														=    CustomHFIndex(config.retrieval_vector_size						,						__lowerCamelCase			)
									else:
												A				:       str														=    cls._build_index(__lowerCamelCase			)

									return cls(
									    __lowerCamelCase						,						question_encoder_tokenizer=__lowerCamelCase						,						generator_tokenizer=__lowerCamelCase						,						retrieval_workers=__lowerCamelCase						,						index=__lowerCamelCase						,						)







						def       SCREAMING_SNAKE_CASE__		(						self   :      str			)   ->					List[Any]:
									logger.info("re-loading the new dataset with embeddings"			)
									# access from the training loop

									ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]			)

									# build the index object again
									A				:       Any														=    self._build_index(self.config			)

									ray.get(
									    [
									        worker.create_rag_retriever.remote(
									            self.config						,						self.question_encoder_tokenizer						,						self.generator_tokenizer						,						__lowerCamelCase			)
									        for worker in self.retrieval_workers
									    ]			)




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__SCREAMING_SNAKE_CASE     =	{"""configuration_yoso""": ["""YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP""", """YosoConfig"""]}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__SCREAMING_SNAKE_CASE     =	[
							    """YOSO_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """YosoForMaskedLM""",
							    """YosoForMultipleChoice""",
							    """YosoForQuestionAnswering""",
							    """YosoForSequenceClassification""",
							    """YosoForTokenClassification""",
							    """YosoLayer""",
							    """YosoModel""",
							    """YosoPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_yoso import YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP, YosoConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_yoso import (
														    YOSO_PRETRAINED_MODEL_ARCHIVE_LIST,
														    YosoForMaskedLM,
														    YosoForMultipleChoice,
														    YosoForQuestionAnswering,
														    YosoForSequenceClassification,
														    YosoForTokenClassification,
														    YosoLayer,
														    YosoModel,
														    YosoPreTrainedModel,
														)


else:
							import sys

							__SCREAMING_SNAKE_CASE     =	_LazyModule(__name__, globals()["""__file__"""], _import_structure)