


import unittest

from cer import CER


__UpperCamelCase							:							Optional[int]    							=					CER()

class    __magic_name__						(  unittest.TestCase):


     def   UpperCAmelCase__						(   self				:  str      )	->   Optional[int]:





      '''simple docstring'''






      UpperCamelCase__       :     Dict 							= ['''White House''']
      UpperCamelCase__       :     Dict 							= ['''white house''']
      # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
      UpperCamelCase__       :     Optional[Any] 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.18_1818_1818      ) < 1E-6      )


     def   UpperCAmelCase__						(   self				:  str      )	->   Dict:





      '''simple docstring'''






      UpperCamelCase__       :     Union[str, Any] 							= ['''were wolf''']
      UpperCamelCase__       :     str 							= ['''werewolf''']
      # S = 0, D = 0, I = 1, N = 9, CER = 1 / 9
      UpperCamelCase__       :     int 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.111_1111      ) < 1E-6      )

      UpperCamelCase__       :     List[str] 							= ['''werewolf''']
      UpperCamelCase__       :     List[Any] 							= ['''weae     wolf''']
      # S = 1, D = 1, I = 0, N = 8, CER = 0.25
      UpperCamelCase__       :     Optional[int] 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.25      ) < 1E-6      )

      # consecutive whitespaces case 1
      UpperCamelCase__       :     Optional[int] 							= ['''were wolf''']
      UpperCamelCase__       :     List[str] 							= ['''were               wolf''']
      # S = 0, D = 0, I = 0, N = 9, CER = 0
      UpperCamelCase__       :     str 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.0      ) < 1E-6      )

      # consecutive whitespaces case 2
      UpperCamelCase__       :     Any 							= ['''were   wolf''']
      UpperCamelCase__       :     Tuple 							= ['''were               wolf''']
      # S = 0, D = 0, I = 0, N = 9, CER = 0
      UpperCamelCase__       :     str 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.0      ) < 1E-6      )


     def   UpperCAmelCase__						(   self				:  Optional[Any]      )	->   int:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[Any] 							= ['''werewolf''']
      UpperCamelCase__       :     str 							= ['''weaewolf''']
      # S = 1, D = 0, I = 0, N = 8, CER = 0.125
      UpperCamelCase__       :     Tuple 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.125      ) < 1E-6      )


     def   UpperCAmelCase__						(   self				:  List[Any]      )	->   str:





      '''simple docstring'''






      UpperCamelCase__       :     Any 							= ['''werewolf''']
      UpperCamelCase__       :     int 							= ['''wereawolf''']
      # S = 0, D = 1, I = 0, N = 8, CER = 0.125
      UpperCamelCase__       :     Tuple 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.125      ) < 1E-6      )


     def   UpperCAmelCase__						(   self				:  str      )	->   Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[Any] 							= ['''werewolf''']
      UpperCamelCase__       :     Dict 							= ['''wereolf''']
      # S = 0, D = 0, I = 1, N = 8, CER = 0.125
      UpperCamelCase__       :     Optional[Any] 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.125      ) < 1E-6      )


     def   UpperCAmelCase__						(   self				:  Tuple      )	->   List[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[int] 							= ['''werewolf''']
      UpperCamelCase__       :     List[Any] 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertEqual(lowerCamelCase__				,						0.0      )


     def   UpperCAmelCase__						(   self				:  Optional[int]      )	->   Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__       :     int 							= ['''werewolf''', '''I am your father''']
      UpperCamelCase__       :     Optional[int] 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertEqual(lowerCamelCase__				,						0.0      )

      UpperCamelCase__       :     List[str] 							= ['''werewolf''', '''I am your father''', '''doge''']
      UpperCamelCase__       :     Any 							= ['''werxwolf''', '''I       am your father''', '''doge''']
      # S = 1, D = 0, I = 0, N = 28, CER = 1 / 28
      UpperCamelCase__       :     str 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.0357_1428      ) < 1E-6      )


     def   UpperCAmelCase__						(   self				:  str      )	->   Any:





      '''simple docstring'''






      UpperCamelCase__       :     str 							= ['''My hovercraft''', '''is full of eels''']
      UpperCamelCase__       :     Any 							= ['''My hovercraft is full''', ''' of eels''']
      # S = 0, D = 0, I = 2, N = 28, CER = 2 / 28
      # whitespace at the front of " of eels" will be strip during preporcessing
      # so need to insert 2 whitespaces
      UpperCamelCase__       :     Dict 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__				,						concatenate_texts=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.07_1428      ) < 1E-6      )


     def   UpperCAmelCase__						(   self				:  str      )	->   Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     List[str] 							= ['''我能吞下玻璃而不伤身体''']
      UpperCamelCase__       :     Optional[int] 							= [''' 能吞虾玻璃而 不霜身体啦''']
      # S = 3, D = 2, I = 0, N = 11, CER = 5 / 11
      UpperCamelCase__       :     List[str] 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.45_4545_4545      ) < 1E-6      )

      UpperCamelCase__       :     int 							= ['''我能吞下玻璃''', '''而不伤身体''']
      UpperCamelCase__       :     List[Any] 							= ['''我    能 吞 下 玻 璃''', '''而不伤身体''']
      # S = 0, D = 5, I = 0, N = 11, CER = 5 / 11
      UpperCamelCase__       :     Dict 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertTrue(abs(char_error_rate - 0.4545_4545_4545      ) < 1E-6      )

      UpperCamelCase__       :     List[Any] 							= ['''我能吞下玻璃而不伤身体''']
      UpperCamelCase__       :     Union[str, Any] 							= cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )
      self.assertFalse(lowerCamelCase__				,						0.0      )


     def   UpperCAmelCase__						(   self				:  str      )	->   Any:





      '''simple docstring'''






      UpperCamelCase__       :     str 							= ['''''']
      UpperCamelCase__       :     Union[str, Any] 							= ['''Hypothesis''']
      with self.assertRaises(lowerCamelCase__      ):
       cer.compute(predictions=lowerCamelCase__				,						references=lowerCamelCase__      )


if __name__ == "__main__":
       unittest.main()






import argparse
import collections
from pathlib import Path

import torch
from torch.serialization import default_restore_location

from transformers import BertConfig, DPRConfig, DPRContextEncoder, DPRQuestionEncoder, DPRReader


__UpperCamelCase							:							Tuple    							=					collections.namedtuple(
    "CheckpointState", ["model_dict", "optimizer_dict", "scheduler_dict", "offset", "epoch", "encoder_params"]
)

def       _a	(					SCREAMING_SNAKE_CASE						:					str  ):







 """simple docstring"""



 print(F"Reading saved model from {model_file}"  )
 UpperCamelCase__       :     int 							= torch.load(SCREAMING_SNAKE_CASE     ,			map_location=lambda SCREAMING_SNAKE_CASE     ,			SCREAMING_SNAKE_CASE  : default_restore_location(SCREAMING_SNAKE_CASE     ,			'''cpu'''  )  )
 return CheckpointState(**SCREAMING_SNAKE_CASE  )

class    __magic_name__						:


     def __init__(   self				:  str				,						lowerCamelCase__				:  Path      )	->   Tuple:





      '''simple docstring'''






      UpperCamelCase__       :     List[str] 							= src_file


     def   UpperCAmelCase__						(   self				:  Dict      )	->   int:





      '''simple docstring'''






      raise NotImplementedError


     @staticmethod
     def   UpperCAmelCase__						(   lowerCamelCase__				:  str				,						*lowerCamelCase__				:  Union[str, Any]				,						**lowerCamelCase__				:  Optional[int]      )	->   "DPRState":





      '''simple docstring'''






      if comp_type.startswith('''c'''      ):
       return DPRContextEncoderState(*lowerCamelCase__				,						**lowerCamelCase__      )
      if comp_type.startswith('''q'''      ):
       return DPRQuestionEncoderState(*lowerCamelCase__				,						**lowerCamelCase__      )
      if comp_type.startswith('''r'''      ):
       return DPRReaderState(*lowerCamelCase__				,						**lowerCamelCase__      )
      else:
       raise ValueError('''Component type must be either \'ctx_encoder\', \'question_encoder\' or \'reader\'.'''      )



class    __magic_name__						(  __lowerCAmelCase):


     def   UpperCAmelCase__						(   self				:  List[str]      )	->   Dict:





      '''simple docstring'''






      UpperCamelCase__       :     Union[str, Any] 							= DPRContextEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''      )[0]      )      )
      print(F"Loading DPR biencoder from {self.src_file}"      )
      UpperCamelCase__       :     int 							= load_states_from_checkpoint(self.src_file      )
      UpperCamelCase__							,					UpperCamelCase__       :     Any 							= model.ctx_encoder, '''ctx_model.'''
      # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
      UpperCamelCase__       :     Union[str, Any] 							= {'''bert_model.embeddings.position_ids''': model.ctx_encoder.bert_model.embeddings.position_ids}
      for key, value in saved_state.model_dict.items():
       if key.startswith(lowerCamelCase__      ):
        UpperCamelCase__       :     int 							= key[len(lowerCamelCase__      ) :]
        if not key.startswith('''encode_proj.'''      ):
         UpperCamelCase__       :     Union[str, Any] 							= '''bert_model.''' + key
        UpperCamelCase__       :     Dict 							= value
      encoder.load_state_dict(lowerCamelCase__      )
      return model



class    __magic_name__						(  __lowerCAmelCase):


     def   UpperCAmelCase__						(   self				:  Any      )	->   Dict:





      '''simple docstring'''






      UpperCamelCase__       :     List[Any] 							= DPRQuestionEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''      )[0]      )      )
      print(F"Loading DPR biencoder from {self.src_file}"      )
      UpperCamelCase__       :     Optional[int] 							= load_states_from_checkpoint(self.src_file      )
      UpperCamelCase__							,					UpperCamelCase__       :     int 							= model.question_encoder, '''question_model.'''
      # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
      UpperCamelCase__       :     Optional[int] 							= {'''bert_model.embeddings.position_ids''': model.question_encoder.bert_model.embeddings.position_ids}
      for key, value in saved_state.model_dict.items():
       if key.startswith(lowerCamelCase__      ):
        UpperCamelCase__       :     int 							= key[len(lowerCamelCase__      ) :]
        if not key.startswith('''encode_proj.'''      ):
         UpperCamelCase__       :     Union[str, Any] 							= '''bert_model.''' + key
        UpperCamelCase__       :     str 							= value
      encoder.load_state_dict(lowerCamelCase__      )
      return model



class    __magic_name__						(  __lowerCAmelCase):


     def   UpperCAmelCase__						(   self				:  int      )	->   Any:





      '''simple docstring'''






      UpperCamelCase__       :     int 							= DPRReader(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased'''      )[0]      )      )
      print(F"Loading DPR reader from {self.src_file}"      )
      UpperCamelCase__       :     List[str] 							= load_states_from_checkpoint(self.src_file      )
      # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
      UpperCamelCase__       :     Tuple 							= {
          '''encoder.bert_model.embeddings.position_ids''': model.span_predictor.encoder.bert_model.embeddings.position_ids
      }
      for key, value in saved_state.model_dict.items():
       if key.startswith('''encoder.'''      ) and not key.startswith('''encoder.encode_proj'''      ):
        UpperCamelCase__       :     List[str] 							= '''encoder.bert_model.''' + key[len('''encoder.'''      ) :]
       UpperCamelCase__       :     List[Any] 							= value
      model.span_predictor.load_state_dict(lowerCamelCase__      )
      return model



def       _a	(					SCREAMING_SNAKE_CASE						:					str     ,			SCREAMING_SNAKE_CASE						:					Path     ,			SCREAMING_SNAKE_CASE						:					Path  ):







 """simple docstring"""



 UpperCamelCase__       :     Any 							= Path(SCREAMING_SNAKE_CASE  )
 dest_dir.mkdir(exist_ok=SCREAMING_SNAKE_CASE  )

 UpperCamelCase__       :     str 							= DPRState.from_type(SCREAMING_SNAKE_CASE     ,			src_file=SCREAMING_SNAKE_CASE  )
 UpperCamelCase__       :     str 							= dpr_state.load_dpr_model()
 model.save_pretrained(SCREAMING_SNAKE_CASE  )
 model.from_pretrained(SCREAMING_SNAKE_CASE  )  # sanity check


if __name__ == "__main__":
       __UpperCamelCase							:							Optional[Any]    							=					argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           "--type", type=str, help="Type of the component to convert: 'ctx_encoder', 'question_encoder' or 'reader'."
       )
       parser.add_argument(
           "--src",
           type=str,
           help=(
               "Path to the dpr checkpoint file. They can be downloaded from the official DPR repo"
               " https://github.com/facebookresearch/DPR. Note that in the official repo, both encoders are stored in the"
               " 'retriever' checkpoints."
           ),
       )
       parser.add_argument("--dest", type=str, default=None, help="Path to the output PyTorch model directory.")
       __UpperCamelCase							:							Union[str, Any]    							=					parser.parse_args()

       __UpperCamelCase							:							Optional[int]    							=					Path(args.src)
       __UpperCamelCase							:							Dict    							=					f"converted-{src_file.name}" if args.dest is None else args.dest
       __UpperCamelCase							:							str    							=					Path(dest_dir)
       assert src_file.exists()
       assert (
           args.type is not None
       ), "Please specify the component type of the DPR model to convert: 'ctx_encoder', 'question_encoder' or 'reader'."
       convert(args.type, src_file, dest_dir)


