






import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__UpperCAmelCase      =				logging.getLogger(__name__)
class 	a_:



     """simple docstring"""



     def __init__(		self					:	Any)							->	Optional[int]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		False





     def     __UpperCamelCase						(		self					:	Dict  ,  lowerCAmelCase__					:	Union[str, Any]  ,  lowerCAmelCase__					:	List[Any]  ,  lowerCAmelCase__					:	Optional[Any]  ,  lowerCAmelCase__					:	List[Any])							->	Optional[Any]:

      """simple docstring"""
      if not self.initialized:
       SCREAMING_SNAKE_CASE				    =		RagRetriever(
           lowerCAmelCase__  ,  question_encoder_tokenizer=lowerCAmelCase__  ,  generator_tokenizer=lowerCAmelCase__  ,  index=lowerCAmelCase__  ,  init_retrieval=lowerCAmelCase__  ,  )
       SCREAMING_SNAKE_CASE				    =		True





     def     __UpperCamelCase						(		self					:	List[str])							->	Any:

      """simple docstring"""
      self.retriever.index.init_index()





     def     __UpperCamelCase						(		self					:	int)							->	Dict:

      """simple docstring"""
      # delete the old self.retriever object before assigning the new index
      del self.retriever
      SCREAMING_SNAKE_CASE				    =		False





     def     __UpperCamelCase						(		self					:	List[str]  ,  lowerCAmelCase__					:	Any  ,  lowerCAmelCase__					:	Optional[int])							->	Union[str, Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.retriever._main_retrieve(lowerCAmelCase__  ,  lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		self.retriever.index.get_doc_dicts(lowerCAmelCase__)
      return doc_ids, retrieved_doc_embeds, doc_dicts

class 	a_(		lowercase__	):



     """simple docstring"""



     def __init__(		self					:	List[str]  ,  lowerCAmelCase__					:	Tuple  ,  lowerCAmelCase__					:	Any  ,  lowerCAmelCase__					:	Union[str, Any]  ,  lowerCAmelCase__					:	Optional[Any]  ,  lowerCAmelCase__					:	Optional[Any]=None)							->	Tuple:

      """simple docstring"""
      if index is not None and index.is_initialized() and len(lowerCAmelCase__) > 0:
       raise ValueError(
           'When using Ray for distributed fine-tuning, '
           'you\'ll need to provide the paths instead, '
           'as the dataset and the index are loaded '
           'separately. More info in examples/rag/use_own_knowledge_dataset.py ')

      super().__init__(
          lowerCAmelCase__  ,  question_encoder_tokenizer=lowerCAmelCase__  ,  generator_tokenizer=lowerCAmelCase__  ,  index=lowerCAmelCase__  ,  init_retrieval=lowerCAmelCase__  ,  )

      SCREAMING_SNAKE_CASE				    =		retrieval_workers
      SCREAMING_SNAKE_CASE				    =		question_encoder_tokenizer
      SCREAMING_SNAKE_CASE				    =		generator_tokenizer
      if len(self.retrieval_workers) > 0:
       ray.get(
           [
               worker.create_rag_retriever.remote(lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__  ,  lowerCAmelCase__)
               for worker in self.retrieval_workers
           ])





     def     __UpperCamelCase						(		self					:	Any)							->	Dict:

      """simple docstring"""
      logger.info('initializing retrieval')

      if len(self.retrieval_workers) > 0:
       ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers])
      else:
       # Non-distributed training. Load index into this same process.
       self.index.init_index()





     def     __UpperCamelCase						(		self					:	Optional[int]  ,  lowerCAmelCase__					:	List[str]  ,  lowerCAmelCase__					:	Union[str, Any])							->	List[str]:

      """simple docstring"""
      if len(self.retrieval_workers) > 0:
       # Select a random retrieval actor.
       SCREAMING_SNAKE_CASE				    =		self.retrieval_workers[random.randint(0  ,  len(self.retrieval_workers) - 1)]
       SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		ray.get(
           random_worker.retrieve.remote(lowerCAmelCase__  ,  lowerCAmelCase__))
      else:
       SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self._main_retrieve(lowerCAmelCase__  ,  lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		self.index.get_doc_dicts(lowerCAmelCase__)
      return retrieved_doc_embeds, doc_ids, doc_dicts





     @classmethod
     def     __UpperCamelCase						(		cls					:	List[str]  ,  lowerCAmelCase__					:	Optional[int]  ,  lowerCAmelCase__					:	Any=None  ,  **lowerCAmelCase__					:	Optional[Any])							->	Union[str, Any]:

      """simple docstring"""
      return super(lowerCAmelCase__  ,  cls).get_tokenizers(lowerCAmelCase__  ,  lowerCAmelCase__  ,  **lowerCAmelCase__)





     @classmethod
     def     __UpperCamelCase						(		cls					:	Any  ,  lowerCAmelCase__					:	Dict  ,  lowerCAmelCase__					:	List[Any]  ,  lowerCAmelCase__					:	Optional[Any]=None  ,  **lowerCAmelCase__					:	str)							->	Dict:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		kwargs.pop('config'  ,  lowerCAmelCase__) or RagConfig.from_pretrained(lowerCAmelCase__  ,  **lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		RagTokenizer.from_pretrained(lowerCAmelCase__  ,  config=lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		rag_tokenizer.question_encoder
      SCREAMING_SNAKE_CASE				    =		rag_tokenizer.generator

      if indexed_dataset is not None:
       SCREAMING_SNAKE_CASE				    =		'custom'
       SCREAMING_SNAKE_CASE				    =		CustomHFIndex(config.retrieval_vector_size  ,  lowerCAmelCase__)
      else:
       SCREAMING_SNAKE_CASE				    =		cls._build_index(lowerCAmelCase__)

      return cls(
          lowerCAmelCase__  ,  question_encoder_tokenizer=lowerCAmelCase__  ,  generator_tokenizer=lowerCAmelCase__  ,  retrieval_workers=lowerCAmelCase__  ,  index=lowerCAmelCase__  ,  )





     def     __UpperCamelCase						(		self					:	List[Any])							->	Union[str, Any]:

      """simple docstring"""
      logger.info('re-loading the new dataset with embeddings')
      # access from the training loop

      ray.get([worker.clear_object.remote() for worker in self.retrieval_workers])

      # build the index object again
      SCREAMING_SNAKE_CASE				    =		self._build_index(self.config)

      ray.get(
          [
              worker.create_rag_retriever.remote(
                  self.config  ,  self.question_encoder_tokenizer  ,  self.generator_tokenizer  ,  lowerCAmelCase__)
              for worker in self.retrieval_workers
          ])










import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
							UpperCamelCase__						=		None


UpperCamelCase__						=		logging.get_logger(__name__)

UpperCamelCase__						=		{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__						=		{
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase__						=		{
    'xlm-roberta-base': 5_1_2,
    'xlm-roberta-large': 5_1_2,
    'xlm-roberta-large-finetuned-conll02-dutch': 5_1_2,
    'xlm-roberta-large-finetuned-conll02-spanish': 5_1_2,
    'xlm-roberta-large-finetuned-conll03-english': 5_1_2,
    'xlm-roberta-large-finetuned-conll03-german': 5_1_2,
}


class 							__SCREAMING_SNAKE_CASE				(						lowerCamelCase    ):
					_lowerCamelCase						    =					VOCAB_FILES_NAMES
					_lowerCamelCase						    =					PRETRAINED_VOCAB_FILES_MAP
					_lowerCamelCase						    =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_lowerCamelCase						    =					["""input_ids""", """attention_mask"""]
					_lowerCamelCase						    =					XLMRobertaTokenizer

					def __init__(							self					,			__lowercase=None					,			__lowercase=None					,			__lowercase="<s>"					,			__lowercase="</s>"					,			__lowercase="</s>"					,			__lowercase="<s>"					,			__lowercase="<unk>"					,			__lowercase="<pad>"					,			__lowercase="<mask>"					,			**__lowercase					,			):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[int]				  =				AddedToken(__lowercase					,			lstrip=__lowercase					,			rstrip=__lowercase   ) if isinstance(__lowercase					,			__lowercase   ) else mask_token

						super().__init__(
						    __lowercase					,			tokenizer_file=__lowercase					,			bos_token=__lowercase					,			eos_token=__lowercase					,			sep_token=__lowercase					,			cls_token=__lowercase					,			unk_token=__lowercase					,			pad_token=__lowercase					,			mask_token=__lowercase					,			**__lowercase					,			)

						lowerCamelCase_				:						str				  =				vocab_file
						lowerCamelCase_				:						str				  =				False if not self.vocab_file else True

					def       lowercase_				(							self					,			__lowercase					,			__lowercase = None   ):







						'''simple docstring'''




						if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						lowerCamelCase_				:						Any				  =				[self.cls_token_id]
						lowerCamelCase_				:						List[Any]				  =				[self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep

					def       lowercase_				(							self					,			__lowercase					,			__lowercase = None   ):







						'''simple docstring'''




						lowerCamelCase_				:						str				  =				[self.sep_token_id]
						lowerCamelCase_				:						Dict				  =				[self.cls_token_id]

						if token_ids_a is None:
							return len(cls + token_ids_a + sep   ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]





					def       lowercase_				(							self					,			__lowercase					,			__lowercase = None   ):







						'''simple docstring'''




						if not self.can_save_slow_tokenizer:
							raise ValueError(
							    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
							    'tokenizer.'   )

						if not os.path.isdir(__lowercase   ):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory."   )
							return
						lowerCamelCase_				:						List[Any]				  =				os.path.join(
						    __lowercase					,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

						if os.path.abspath(self.vocab_file   ) != os.path.abspath(__lowercase   ):
							copyfile(self.vocab_file					,			__lowercase   )

						return (out_vocab_file,)
