



from collections.abc import Generator
from math import sin





def      __snake_case	(			_UpperCAmelCase				):
    if len(_UpperCAmelCase				) != 32:
        raise ValueError('''Input must be of length 32'''				)

    __a						       =				b''''''
    for i in [3, 2, 1, 0]:
        little_endian += string_aa[8 * i : 8 * i + 8]
    return little_endian





def      __snake_case	(			_UpperCAmelCase				):
    if i < 0:
        raise ValueError('''Input must be non-negative'''				)

    __a						       =				format(_UpperCAmelCase   , '''08x'''				)[-8:]
    __a						       =				b''''''
    for i in [3, 2, 1, 0]:
        little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode('''utf-8'''				)
    return little_endian_hex





def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				b''''''
    for char in message:
        bit_string += format(_UpperCAmelCase   , '''08b'''				).encode('''utf-8'''				)
    __a						       =				format(len(_UpperCAmelCase				)   , '''064b'''				).encode('''utf-8'''				)

    # Pad bit_string to a multiple of 512 chars
    bit_string += b"1"
    while len(_UpperCAmelCase				) % 512 != 448:
        bit_string += b"0"
    bit_string += to_little_endian(start_len[32:]				) + to_little_endian(start_len[:32]				)

    return bit_string





def      __snake_case	(			_UpperCAmelCase				):
    if len(_UpperCAmelCase				) % 512 != 0:
        raise ValueError('''Input must have length that\'s a multiple of 512'''				)

    for pos in range(0   , len(_UpperCAmelCase				)   , 512				):
        __a						       =				bit_string[pos : pos + 512]
        __a						       =				[]
        for i in range(0   , 512   , 32				):
            block_words.append(int(to_little_endian(block[i : i + 32]				)   , 2				)				)
        yield block_words





def      __snake_case	(			_UpperCAmelCase				):
    if i < 0:
        raise ValueError('''Input must be non-negative'''				)

    __a						       =				format(_UpperCAmelCase   , '''032b'''				)
    __a						       =				''''''
    for c in i_str:
        new_str += "1" if c == "0" else "0"
    return int(_UpperCAmelCase   , 2				)





def      __snake_case	(			_UpperCAmelCase   , _UpperCAmelCase				):
    return (a + b) % 2**32





def      __snake_case	(			_UpperCAmelCase   , _UpperCAmelCase				):
    if i < 0:
        raise ValueError('''Input must be non-negative'''				)
    if shift < 0:
        raise ValueError('''Shift must be non-negative'''				)
    return ((i << shift) ^ (i >> (32 - shift))) % 2**32





def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				preprocess(_UpperCAmelCase				)

    __a						       =				[int(2**32 * abs(sin(i + 1				)				)				) for i in range(64				)]

    # Starting states
    __a						       =				0X67_452_301
    __a						       =				0Xef_cda_b89
    __a						       =				0X98_bad_cfe
    __a						       =				0X10_325_476

    __a						       =				[
        7,
        12,
        17,
        22,
        7,
        12,
        17,
        22,
        7,
        12,
        17,
        22,
        7,
        12,
        17,
        22,
        5,
        9,
        14,
        20,
        5,
        9,
        14,
        20,
        5,
        9,
        14,
        20,
        5,
        9,
        14,
        20,
        4,
        11,
        16,
        23,
        4,
        11,
        16,
        23,
        4,
        11,
        16,
        23,
        4,
        11,
        16,
        23,
        6,
        10,
        15,
        21,
        6,
        10,
        15,
        21,
        6,
        10,
        15,
        21,
        6,
        10,
        15,
        21,
    ]

    # Process bit string in chunks, each with 16 32-char words
    for block_words in get_block_words(_UpperCAmelCase				):
        __a						       =				aa
        __a						       =				ba
        __a						       =				ca
        __a						       =				da

        # Hash current chunk
        for i in range(64				):
            if i <= 15:
                # f = (b & c) | (not_32(b) & d)     # Alternate definition for f
                __a						       =				d ^ (b & (c ^ d))
                __a						       =				i
            elif i <= 31:
                # f = (d & b) | (not_32(d) & c)     # Alternate definition for f
                __a						       =				c ^ (d & (b ^ c))
                __a						       =				(5 * i + 1) % 16
            elif i <= 47:
                __a						       =				b ^ c ^ d
                __a						       =				(3 * i + 5) % 16
            else:
                __a						       =				c ^ (b | not_aa(_UpperCAmelCase				))
                __a						       =				(7 * i) % 16
            __a						       =				(f + a + added_consts[i] + block_words[g]) % 2**32
            __a						       =				d
            __a						       =				c
            __a						       =				b
            __a						       =				sum_aa(_UpperCAmelCase   , left_rotate_aa(_UpperCAmelCase   , shift_amounts[i]				)				)

        # Add hashed chunk to running total
        __a						       =				sum_aa(_UpperCAmelCase   , _UpperCAmelCase				)
        __a						       =				sum_aa(_UpperCAmelCase   , _UpperCAmelCase				)
        __a						       =				sum_aa(_UpperCAmelCase   , _UpperCAmelCase				)
        __a						       =				sum_aa(_UpperCAmelCase   , _UpperCAmelCase				)

    __a						       =				reformat_hex(_UpperCAmelCase				) + reformat_hex(_UpperCAmelCase				) + reformat_hex(_UpperCAmelCase				) + reformat_hex(_UpperCAmelCase				)
    return digest


if __name__ == "__main__":
 import doctest

 doctest.testmod()










import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__snake_case						:List[Any]      		=     logging.getLogger(__name__)

class     _A       :



       def __init__(		self							:     List[str]):


           '''simple docstring'''

           __a						       =				False



       def 		_lowerCamelCase			(		self							:     Any     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     List[str]     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]):


           '''simple docstring'''

           if not self.initialized:
               __a						       =				RagRetriever(
                   __SCREAMING_SNAKE_CASE     ,		question_encoder_tokenizer=__SCREAMING_SNAKE_CASE     ,		generator_tokenizer=__SCREAMING_SNAKE_CASE     ,		index=__SCREAMING_SNAKE_CASE     ,		init_retrieval=__SCREAMING_SNAKE_CASE     ,		)
               __a						       =				True



       def 		_lowerCamelCase			(		self							:     List[str]):


           '''simple docstring'''

           self.retriever.index.init_index()





       def 		_lowerCamelCase			(		self							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]):


           '''simple docstring'''

           __a       ,   __a						       =				self.retriever._main_retrieve(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)
           return doc_ids, retrieved_doc_embeds


class     _A       (	__UpperCAmelCase    ):



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     List[Any]     ,		__SCREAMING_SNAKE_CASE							:     List[str]     ,		__SCREAMING_SNAKE_CASE							:     List[str]     ,		__SCREAMING_SNAKE_CASE							:     int     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None):


           '''simple docstring'''

           if index is not None and index.is_initialized() and len(__SCREAMING_SNAKE_CASE) > 0:
               raise ValueError(
                   '''When using Ray for distributed fine-tuning, '''
                   '''you\'ll need to provide the paths instead, '''
                   '''as the dataset and the index are loaded '''
                   '''separately. More info in examples/rag/use_own_knowledge_dataset.py ''')
           super().__init__(
               __SCREAMING_SNAKE_CASE     ,		question_encoder_tokenizer=__SCREAMING_SNAKE_CASE     ,		generator_tokenizer=__SCREAMING_SNAKE_CASE     ,		index=__SCREAMING_SNAKE_CASE     ,		init_retrieval=__SCREAMING_SNAKE_CASE     ,		)
           __a						       =				retrieval_workers
           if len(self.retrieval_workers) > 0:
               ray.get(
                   [
                       worker.create_rag_retriever.remote(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)
                       for worker in self.retrieval_workers
                   ])



       def 		_lowerCamelCase			(		self							:     List[Any]):


           '''simple docstring'''

           logger.info('''initializing retrieval''')

           if len(self.retrieval_workers) > 0:
               ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers])
           else:
               # Non-distributed training. Load index into this same process.
               self.index.init_index()



       def 		_lowerCamelCase			(		self							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]     ,		__SCREAMING_SNAKE_CASE							:     List[Any]):


           '''simple docstring'''

           if len(self.retrieval_workers) > 0:
               # Select a random retrieval actor.
               __a						       =				self.retrieval_workers[random.randint(0     ,		len(self.retrieval_workers) - 1)]
               __a       ,   __a						       =				ray.get(random_worker.retrieve.remote(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE))
           else:
               __a       ,   __a						       =				self._main_retrieve(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)
           return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(__SCREAMING_SNAKE_CASE)



       @classmethod
       def 		_lowerCamelCase			(		cls							:     Any     ,		__SCREAMING_SNAKE_CASE							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Tuple=None     ,		**__SCREAMING_SNAKE_CASE							:     Optional[int]):


           '''simple docstring'''

           return super(__SCREAMING_SNAKE_CASE     ,		cls).get_tokenizers(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)





       @classmethod
       def 		_lowerCamelCase			(		cls							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     str=None     ,		**__SCREAMING_SNAKE_CASE							:     List[Any]):


           '''simple docstring'''

           __a						       =				kwargs.pop('''config'''     ,		__SCREAMING_SNAKE_CASE) or RagConfig.from_pretrained(__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
           __a						       =				RagTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE     ,		config=__SCREAMING_SNAKE_CASE)
           __a						       =				rag_tokenizer.question_encoder
           __a						       =				rag_tokenizer.generator
           if indexed_dataset is not None:
               __a						       =				'''custom'''
               __a						       =				CustomHFIndex(config.retrieval_vector_size     ,		__SCREAMING_SNAKE_CASE)
           else:
               __a						       =				cls._build_index(__SCREAMING_SNAKE_CASE)
           return cls(
               __SCREAMING_SNAKE_CASE     ,		question_encoder_tokenizer=__SCREAMING_SNAKE_CASE     ,		generator_tokenizer=__SCREAMING_SNAKE_CASE     ,		retrieval_workers=__SCREAMING_SNAKE_CASE     ,		index=__SCREAMING_SNAKE_CASE     ,		)





