


import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
A      :    Tuple		=					logging.getLogger()

A      :    Optional[int]		=					logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       def 						lowerCAmelCase     (   self							:			Optional[Any]					,      __lowerCamelCase							:			Union[str, Any]       ):


             '''simple docstring'''



             os.makedirs(__SCREAMING_SNAKE_CASE					,      exist_ok=__SCREAMING_SNAKE_CASE       )
             lowerCamelCase__		:				List[str]    =  {"source": "What is love ?", "target": "life"}
             lowerCamelCase__		:				Any    =  {"train": 12, "val": 2, "test": 2}
             for split in ["train", "test", "val"]:
                   for field in ["source", "target"]:
                         lowerCamelCase__		:				List[Any]    =  "\n".join([contents[field]] * n_lines[split]       )
                         with open(os.path.join(__SCREAMING_SNAKE_CASE					,      f"{split}.{field}"       )					,      "w"       ) as f:
                               f.write(__SCREAMING_SNAKE_CASE       )





       def 						lowerCAmelCase     (   self							:			Optional[int]					,      __lowerCamelCase							:			int					,      __lowerCamelCase							:			str = "pytorch"       ):


             '''simple docstring'''



             lowerCamelCase__		:				int    =  self.get_auto_remove_tmp_dir()
             lowerCamelCase__		:				List[str]    =  os.path.join(__SCREAMING_SNAKE_CASE					,      "output"       )
             lowerCamelCase__		:				Tuple    =  os.path.join(__SCREAMING_SNAKE_CASE					,      "data"       )
             self._create_dummy_data(data_dir=__SCREAMING_SNAKE_CASE       )

             lowerCamelCase__		:				List[Any]    =  f"\n                --data_dir {data_dir} \\n                --output_dir {output_dir} \\n                --model_name_or_path facebook/rag-sequence-base \\n                --model_type rag_sequence \\n                --do_train \\n                --do_predict \\n                --n_val -1 \\n                --val_check_interval 1.0 \\n                --train_batch_size 2 \\n                --eval_batch_size 1 \\n                --max_source_length 25 \\n                --max_target_length 25 \\n                --val_max_target_length 25 \\n                --test_max_target_length 25 \\n                --label_smoothing 0.1 \\n                --dropout 0.1 \\n                --attention_dropout 0.1 \\n                --weight_decay 0.001 \\n                --adam_epsilon 1e-08 \\n                --max_grad_norm 0.1 \\n                --lr_scheduler polynomial \\n                --learning_rate 3e-04 \\n                --num_train_epochs 1 \\n                --warmup_steps 4 \\n                --gradient_accumulation_steps 1 \\n                --distributed-port 8787 \\n                --use_dummy_dataset 1 \\n                --distributed_retriever {distributed_retriever} \\n            ".split()

             if gpus > 0:
                   testargs.append(f"--gpus={gpus}"       )
                   if is_apex_available():
                         testargs.append("--fp16"       )
             else:
                   testargs.append("--gpus=0"       )
                   testargs.append("--distributed_backend=ddp_cpu"       )
                   testargs.append("--num_processes=2"       )

             lowerCamelCase__		:				Optional[Any]    =  [sys.executable, str(Path(finetune_rag.__file__       ).resolve()       )] + testargs
             execute_subprocess_async(__SCREAMING_SNAKE_CASE					,      env=self.get_env()       )

             lowerCamelCase__		:				str    =  os.path.join(__SCREAMING_SNAKE_CASE					,      "metrics.json"       )
             with open(__SCREAMING_SNAKE_CASE       ) as f:
                   lowerCamelCase__		:				str    =  json.load(__SCREAMING_SNAKE_CASE       )
             return result





       @require_torch_gpu
       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  self._run_finetune(gpus=1       )
             self.assertGreaterEqual(result["test"][0]["test_avg_em"]					,      0.2       )





       @require_torch_multi_gpu
       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  self._run_finetune(gpus=2       )
             self.assertGreaterEqual(result["test"][0]["test_avg_em"]					,      0.2       )





       @require_torch_gpu
       @require_ray
       def 						lowerCAmelCase     (   self							:			Optional[int]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Dict    =  self._run_finetune(gpus=1					,      distributed_retriever="ray"       )
             self.assertGreaterEqual(result["test"][0]["test_avg_em"]					,      0.2       )





       @require_torch_multi_gpu
       @require_ray
       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  self._run_finetune(gpus=1					,      distributed_retriever="ray"       )
             self.assertGreaterEqual(result["test"][0]["test_avg_em"]					,      0.2       )







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__snake_case						:str      		=     logging.get_logger(__name__)

__snake_case						:int      		=     {'''vocab_file''': '''vocab.txt'''}

__snake_case						:List[Any]      		=     {
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

__snake_case						:List[str]      		=     {
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


__snake_case						:Optional[int]      		=     {
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[int]											=						VOCAB_FILES_NAMES
       UpperCamelCase__  :    str											=						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :    Optional[Any]											=						PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase__  :    Optional[int]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__  :    int											=						ConvBertTokenizer



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=None     ,		__SCREAMING_SNAKE_CASE							:     Dict=None     ,		__SCREAMING_SNAKE_CASE							:     Tuple=True     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[UNK]"     ,		__SCREAMING_SNAKE_CASE							:     int="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     List[Any]="[PAD]"     ,		__SCREAMING_SNAKE_CASE							:     int="[CLS]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[MASK]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=True     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None     ,		**__SCREAMING_SNAKE_CASE							:     Dict     ,		):


           '''simple docstring'''

           super().__init__(
               __SCREAMING_SNAKE_CASE     ,		tokenizer_file=__SCREAMING_SNAKE_CASE     ,		do_lower_case=__SCREAMING_SNAKE_CASE     ,		unk_token=__SCREAMING_SNAKE_CASE     ,		sep_token=__SCREAMING_SNAKE_CASE     ,		pad_token=__SCREAMING_SNAKE_CASE     ,		cls_token=__SCREAMING_SNAKE_CASE     ,		mask_token=__SCREAMING_SNAKE_CASE     ,		tokenize_chinese_chars=__SCREAMING_SNAKE_CASE     ,		strip_accents=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)

           __a						       =				json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               normalizer_state.get('''lowercase'''     ,		__SCREAMING_SNAKE_CASE) != do_lower_case
               or normalizer_state.get('''strip_accents'''     ,		__SCREAMING_SNAKE_CASE) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''     ,		__SCREAMING_SNAKE_CASE) != tokenize_chinese_chars
           ):
               __a						       =				getattr(__SCREAMING_SNAKE_CASE     ,		normalizer_state.pop('''type'''))
               __a						       =				do_lower_case
               __a						       =				strip_accents
               __a						       =				tokenize_chinese_chars
               __a						       =				normalizer_class(**__SCREAMING_SNAKE_CASE)

           __a						       =				do_lower_case



       def 		_lowerCamelCase			(		self							:     int     ,		__SCREAMING_SNAKE_CASE							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Any=None):


           '''simple docstring'''

           __a						       =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

           return output



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]
           __a						       =				[self.cls_token_id]
           if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[str] = None):


           '''simple docstring'''

           __a						       =				self._tokenizer.model.save(__SCREAMING_SNAKE_CASE     ,		name=__SCREAMING_SNAKE_CASE)
           return tuple(__SCREAMING_SNAKE_CASE)





