'''simple docstring'''




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


__snake_case  : List[str]		       =    logging.getLogger(__name__)





def 	_UpperCAmelCase       (				)      -> int:
    A_      =  argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''  )
    parser.add_argument(
        '''--dataset_name''',  type=_UpperCamelCase,  default='''wikitext''',  help='''Name of the training. Explore datasets at: hf.co/datasets.''',  )
    parser.add_argument(
        '''--dataset_config''',  type=_UpperCamelCase,  default='''wikitext-103-raw-v1''',  help='''Configuration name of the dataset.'''  )
    parser.add_argument(
        '''--batch_size''',  type=_UpperCamelCase,  default=10_00,  help='''Batch size during training.''',  )
    parser.add_argument(
        '''--vocab_size''',  type=_UpperCamelCase,  default=1_00_48,  help='''Size of the desired vocabulary.''',  )
    parser.add_argument(
        '''--limit''',  default=_UpperCamelCase,  type=_UpperCamelCase,  help='''Limit the number of shards (used for debugging).''',  )
    parser.add_argument(
        '''--export_to_hub''',  action='''store_true''',  )

    A_      =  parser.parse_args()
    return args





def 	_UpperCAmelCase       (				_UpperCamelCase       :      Optional[int]  )      -> str:
    A_      =  datasets.load_dataset(args.dataset_name,  args.dataset_config,  split='''train'''  )

    if args.limit is not None:
        A_      =  min(len(_UpperCamelCase  ),  args.limit  )
        A_      =  dataset.select(range(_UpperCamelCase  )  )
        logger.info(F'''Limiting the dataset to {args.limit} entries.'''  )

    def batch_iterator():
        for i in range(0,  len(_UpperCamelCase  ),  args.batch_size  ):
            yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
    A_      =  Tokenizer(Unigram()  )
    A_      =  normalizers.Sequence([normalizers.Replace('''``''',  '''"'''  ), normalizers.Replace('''\'\'''',  '''"'''  )]  )
    A_      =  pre_tokenizers.Metaspace()

    # Prepare the trainer.
    A_      =  UnigramTrainer(
        unk_token='''<unk>''',  special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]'''],  vocab_size=args.vocab_size,  )

    logger.info('''Training the tokenizer.'''  )
    tokenizer.train_from_iterator(batch_iterator(),  trainer=_UpperCamelCase  )
    logger.info('''Tokenizer training complete!'''  )

    A_      =  tokenizer.token_to_id('''[CLS]'''  )
    A_      =  tokenizer.token_to_id('''[SEP]'''  )
    A_      =  processors.TemplateProcessing(
        single='''[CLS]:0 $A:0 [SEP]:0''',  pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1''',  special_tokens=[
            ('''[CLS]''', cls_token_id),
            ('''[SEP]''', sep_token_id),
        ],  )
    A_      =  decoders.Metaspace()

    if args.export_to_hub:
        logger.info('''Exporting the trained tokenzier to Hub.'''  )
        A_      =  AlbertTokenizerFast(tokenizer_object=_UpperCamelCase  )
        new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''  )


if __name__ == "__main__":
  __snake_case  : List[Any]		       =    parse_args()
  main(args)







'''simple docstring'''




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__snake_case  : Optional[int]		       =    logging.getLogger(__name__)




class       __UpperCAmelCase							:





       '''simple docstring'''



       def __init__( self					)						->	List[str]:
           A_      =  False

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	str:
           if not self.initialized:
               A_      =  RagRetriever(
                   _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	init_retrieval=_SCREAMING_SNAKE_CASE       ,	)
               A_      =  True

       def    __A			( self					)						->	List[Any]:
           self.retriever.index.init_index()

       def    __A			( self					)						->	Dict:
           # delete the old self.retriever object before assigning the new index
           del self.retriever
           A_      =  False

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	Optional[int]:
           A_  ,A_      =  self.retriever._main_retrieve(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
           A_      =  self.retriever.index.get_doc_dicts(_SCREAMING_SNAKE_CASE					)
           return doc_ids, retrieved_doc_embeds, doc_dicts




class       __UpperCAmelCase							(							_UpperCamelCase	):





       '''simple docstring'''



       def __init__( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None					)						->	List[Any]:
           if index is not None and index.is_initialized() and len(_SCREAMING_SNAKE_CASE					) > 0:
               raise ValueError(
                   '''When using Ray for distributed fine-tuning, '''
                   '''you\'ll need to provide the paths instead, '''
                   '''as the dataset and the index are loaded '''
                   '''separately. More info in examples/rag/use_own_knowledge_dataset.py '''					)

           super().__init__(
               _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	init_retrieval=_SCREAMING_SNAKE_CASE       ,	)

           A_      =  retrieval_workers
           A_      =  question_encoder_tokenizer
           A_      =  generator_tokenizer
           if len(self.retrieval_workers					) > 0:
               ray.get(
                   [
                       worker.create_rag_retriever.remote(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
                       for worker in self.retrieval_workers
                   ]					)

       def    __A			( self					)						->	List[Any]:
           logger.info('''initializing retrieval'''					)

           if len(self.retrieval_workers					) > 0:
               ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]					)
           else:
               # Non-distributed training. Load index into this same process.
               self.index.init_index()

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	Optional[int]:
           if len(self.retrieval_workers					) > 0:
               # Select a random retrieval actor.
               A_      =  self.retrieval_workers[random.randint(0       ,	len(self.retrieval_workers					) - 1					)]
               A_  ,A_  ,A_      =  ray.get(
                   random_worker.retrieve.remote(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)					)
           else:
               A_  ,A_      =  self._main_retrieve(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
               A_      =  self.index.get_doc_dicts(_SCREAMING_SNAKE_CASE					)
           return retrieved_doc_embeds, doc_ids, doc_dicts

       @classmethod
       def    __A			( cls       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None       ,	**_SCREAMING_SNAKE_CASE					)						->	List[Any]:
           return super(_SCREAMING_SNAKE_CASE       ,	cls					).get_tokenizers(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)

       @classmethod
       def    __A			( cls       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None       ,	**_SCREAMING_SNAKE_CASE					)						->	int:
           A_      =  kwargs.pop('''config'''       ,	_SCREAMING_SNAKE_CASE					) or RagConfig.from_pretrained(_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)
           A_      =  RagTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE       ,	config=_SCREAMING_SNAKE_CASE					)
           A_      =  rag_tokenizer.question_encoder
           A_      =  rag_tokenizer.generator

           if indexed_dataset is not None:
               A_      =  '''custom'''
               A_      =  CustomHFIndex(config.retrieval_vector_size       ,	_SCREAMING_SNAKE_CASE					)
           else:
               A_      =  cls._build_index(_SCREAMING_SNAKE_CASE					)

           return cls(
               _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	retrieval_workers=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	)

       def    __A			( self					)						->	Optional[int]:
           logger.info('''re-loading the new dataset with embeddings'''					)
           # access from the training loop

           ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]					)

           # build the index object again
           A_      =  self._build_index(self.config					)

           ray.get(
               [
                   worker.create_rag_retriever.remote(
                       self.config       ,	self.question_encoder_tokenizer       ,	self.generator_tokenizer       ,	_SCREAMING_SNAKE_CASE					)
                   for worker in self.retrieval_workers
               ]					)






