

import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer
class        _A						(     A__):

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     Any							=   tempfile.mkdtemp()
        SCREAMING_SNAKE_CASE_ :     Dict							=   5

        # Realm tok
        SCREAMING_SNAKE_CASE_ :     str							=   [
            '[UNK]',
            '[CLS]',
            '[SEP]',
            '[PAD]',
            '[MASK]',
            'test',
            'question',
            'this',
            'is',
            'the',
            'first',
            'second',
            'third',
            'fourth',
            'fifth',
            'record',
            'want',
            '##want',
            '##ed',
            'wa',
            'un',
            'runn',
            '##ing',
            ',',
            'low',
            'lowest',
        ]
        SCREAMING_SNAKE_CASE_ :     Optional[int]							=   os.path.join(self.tmpdirname       ,    'realm_tokenizer'    )
        os.makedirs(_a       ,    exist_ok=_a    )
        SCREAMING_SNAKE_CASE_ :     Union[str, Any]							=   os.path.join(_a       ,    VOCAB_FILES_NAMES['vocab_file']    )
        with open(self.vocab_file       ,    'w'       ,    encoding='utf-8'    ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )

        SCREAMING_SNAKE_CASE_ :     List[str]							=   os.path.join(self.tmpdirname       ,    'realm_block_records'    )
        os.makedirs(_a       ,    exist_ok=_a    )

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,    'realm_tokenizer'    )    )

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        shutil.rmtree(self.tmpdirname    )

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     Any							=   RealmConfig(num_block_records=self.num_block_records    )
        return config

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     Any							=   Dataset.from_dict(
            {
                'id': ['0', '1'],
                'question': ['foo', 'bar'],
                'answers': [['Foo', 'Bar'], ['Bar']],
            }    )
        return dataset

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     Tuple							=   np.array(
            [
                b'This is the first record',
                b'This is the second record',
                b'This is the third record',
                b'This is the fourth record',
                b'This is the fifth record',
                b'This is a longer longer longer record',
            ]       ,    dtype=_a       ,    )
        return block_records

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     str							=   RealmRetriever(
            block_records=self.get_dummy_block_records()       ,    tokenizer=self.get_tokenizer()       ,    )
        return retriever

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     Any							=   self.get_config()
        SCREAMING_SNAKE_CASE_ :     List[Any]							=   self.get_dummy_retriever()
        SCREAMING_SNAKE_CASE_ :     List[Any]							=   retriever.tokenizer

        SCREAMING_SNAKE_CASE_ :     Optional[Any]							=   np.array([0, 3]       ,    dtype='long'    )
        SCREAMING_SNAKE_CASE_ :     Dict							=   tokenizer(['Test question']    ).input_ids
        SCREAMING_SNAKE_CASE_ :     Dict							=   tokenizer(
            ['the fourth']       ,    add_special_tokens=_a       ,    return_token_type_ids=_a       ,    return_attention_mask=_a       ,    ).input_ids
        SCREAMING_SNAKE_CASE_ :     int							=   config.reader_seq_len

        SCREAMING_SNAKE_CASE_				,  SCREAMING_SNAKE_CASE_				,  SCREAMING_SNAKE_CASE_				,  SCREAMING_SNAKE_CASE_ :     Tuple							=   retriever(
            _a       ,    _a       ,    answer_ids=_a       ,    max_length=_a       ,    return_tensors='np'    )

        self.assertEqual(len(_a    )       ,    2    )
        self.assertEqual(len(_a    )       ,    2    )
        self.assertEqual(len(_a    )       ,    2    )
        self.assertEqual(concat_inputs.input_ids.shape       ,    (2, 10)    )
        self.assertEqual(concat_inputs.attention_mask.shape       ,    (2, 10)    )
        self.assertEqual(concat_inputs.token_type_ids.shape       ,    (2, 10)    )
        self.assertEqual(concat_inputs.special_tokens_mask.shape       ,    (2, 10)    )
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]    )       ,    ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']       ,    )
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]    )       ,    ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']       ,    )

  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     Tuple							=   self.get_config()
        SCREAMING_SNAKE_CASE_ :     Dict							=   self.get_dummy_retriever()
        SCREAMING_SNAKE_CASE_ :     Dict							=   retriever.tokenizer

        SCREAMING_SNAKE_CASE_ :     Union[str, Any]							=   np.array([0, 3, 5]       ,    dtype='long'    )
        SCREAMING_SNAKE_CASE_ :     int							=   tokenizer(['Test question']    ).input_ids
        SCREAMING_SNAKE_CASE_ :     Tuple							=   tokenizer(
            ['the fourth', 'longer longer']       ,    add_special_tokens=_a       ,    return_token_type_ids=_a       ,    return_attention_mask=_a       ,    ).input_ids
        SCREAMING_SNAKE_CASE_ :     List[Any]							=   config.reader_seq_len

        SCREAMING_SNAKE_CASE_				,  SCREAMING_SNAKE_CASE_				,  SCREAMING_SNAKE_CASE_				,  SCREAMING_SNAKE_CASE_ :     List[Any]							=   retriever(
            _a       ,    _a       ,    answer_ids=_a       ,    max_length=_a       ,    return_tensors='np'    )

        self.assertEqual([False, True, True]       ,    _a    )
        self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]       ,    _a    )
        self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]       ,    _a    )



  def        UpperCAmelCase			( self    ):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_ :     Dict							=   self.get_dummy_retriever()
        retriever.save_pretrained(os.path.join(self.tmpdirname       ,    'realm_block_records'    )    )

        # Test local path
        SCREAMING_SNAKE_CASE_ :     Optional[int]							=   retriever.from_pretrained(os.path.join(self.tmpdirname       ,    'realm_block_records'    )    )
        self.assertEqual(retriever.block_records[0]       ,    b'This is the first record'    )

        # Test mocked remote path
        with patch('transformers.models.realm.retrieval_realm.hf_hub_download'    ) as mock_hf_hub_download:
              SCREAMING_SNAKE_CASE_ :     Dict							=   os.path.join(
                  os.path.join(self.tmpdirname       ,    'realm_block_records'    )       ,    _REALM_BLOCK_RECORDS_FILENAME    )
              SCREAMING_SNAKE_CASE_ :     List[str]							=   RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa'    )

        self.assertEqual(retriever.block_records[0]       ,    b'This is the first record'    )









'''simple docstring'''
import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp


def        _lowerCAmelCase					(						)      ->			Any:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							    =ArgumentParser(
         description=(
             'PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'
         )				)

     # Optional arguments for the launch helper
     parser.add_argument('--num_cores'							, type=_UpperCamelCase							, default=1							, help='Number of TPU cores to use (1 or 8).'				)

     # positional
     parser.add_argument(
         'training_script'							, type=_UpperCamelCase							, help=(
             'The full path to the single TPU training '
             'program/script to be launched in parallel, '
             'followed by all the arguments for the '
             'training script'
         )							, )

     # rest from the training program
     parser.add_argument('training_script_args'							, nargs=_UpperCamelCase				)

     return parser.parse_args()


def        _lowerCAmelCase					(						)      ->			Optional[int]:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							    =parse_args()

     # Import training_script as a module.
     _SCREAMING_SNAKE_CASE							    =Path(args.training_script				)
     sys.path.append(str(script_fpath.parent.resolve()				)				)
     _SCREAMING_SNAKE_CASE							    =script_fpath.stem
     _SCREAMING_SNAKE_CASE							    =importlib.import_module(_UpperCamelCase				)

     # Patch sys.argv
     _SCREAMING_SNAKE_CASE							    =[args.training_script] + args.training_script_args + ['--tpu_num_cores', str(args.num_cores				)]

     xmp.spawn(mod._mp_fn							, args=()							, nprocs=args.num_cores				)


if __name__ == "__main__":
     main()
