




import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm
  import torch_xla.debug.metrics as met


class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    def __init__(			self      :	Optional[int]					, *UpperCAmelCase__      :	int					, UpperCAmelCase__      :	Optional[int]=None					, UpperCAmelCase__      :	List[Any]=None					, **UpperCAmelCase__      :	Optional[Any])						->Any:





           '''simple docstring'''

           super().__init__(*UpperCAmelCase__					, **UpperCAmelCase__)
           A__       =    eval_examples
           A__       =    post_process_function


    def        SCREAMING_SNAKE_CASE							(			self      :	Any					, UpperCAmelCase__      :	Optional[Dataset] = None					, UpperCAmelCase__      :	Dict=None					, UpperCAmelCase__      :	Optional[List[str]] = None					, UpperCAmelCase__      :	str = "eval"					, **UpperCAmelCase__      :	List[str]					, )						->Dict[str, float]:





           '''simple docstring'''

           A__       =    gen_kwargs.copy()
           A__       =    (
               gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length''') is not None else self.args.generation_max_length
           )
           A__       =    (
               gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.args.generation_num_beams
           )
           A__       =    gen_kwargs

           A__       =    self.eval_dataset if eval_dataset is None else eval_dataset
           A__       =    self.get_eval_dataloader(UpperCAmelCase__)
           A__       =    self.eval_examples if eval_examples is None else eval_examples

           # Temporarily disable metric computation, we will do it in the loop here.
           A__       =    self.compute_metrics
           A__       =    None
           A__       =    time.time()
           A__       =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
           try:
                  A__       =    eval_loop(
                      UpperCAmelCase__					, description='''Evaluation'''					, prediction_loss_only=True if compute_metrics is None else None					, ignore_keys=UpperCAmelCase__					, metric_key_prefix=UpperCAmelCase__					, )
           finally:
                  A__       =    compute_metrics
           A__       =    self.args.eval_batch_size * self.args.world_size
           if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
                  start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
           output.metrics.update(
               speed_metrics(
                   UpperCAmelCase__					, UpperCAmelCase__					, num_samples=output.num_samples					, num_steps=math.ceil(output.num_samples / total_batch_size)					, ))

           if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
                  # Only the main node write the results by default
                  A__       =    self.post_process_function(UpperCAmelCase__					, UpperCAmelCase__					, UpperCAmelCase__)
                  A__       =    self.compute_metrics(UpperCAmelCase__)

                  # Prefix all keys with metric_key_prefix + '_'
                  for key in list(metrics.keys()):
                         if not key.startswith(f"""{metric_key_prefix}_"""):
                                A__       =    metrics.pop(UpperCAmelCase__)

                  metrics.update(output.metrics)
           else:
                  A__       =    output.metrics

           if self.args.should_log:
                  # Only the main node log the results by default
                  self.log(UpperCAmelCase__)

           if self.args.tpu_metrics_debug or self.args.debug:
                  # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
                  xm.master_print(met.metrics_report())

           A__       =    self.callback_handler.on_evaluate(self.args					, self.state					, self.control					, UpperCAmelCase__)
           return metrics





    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any]					, UpperCAmelCase__      :	int					, UpperCAmelCase__      :	Dict					, UpperCAmelCase__      :	Dict=None					, UpperCAmelCase__      :	str = "test"					, **UpperCAmelCase__      :	int)						->int:





           '''simple docstring'''

           A__       =    gen_kwargs.copy()

           A__       =    self.get_test_dataloader(UpperCAmelCase__)

           # Temporarily disable metric computation, we will do it in the loop here.
           A__       =    self.compute_metrics
           A__       =    None
           A__       =    time.time()
           A__       =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
           try:
                  A__       =    eval_loop(
                      UpperCAmelCase__					, description='''Prediction'''					, prediction_loss_only=True if compute_metrics is None else None					, ignore_keys=UpperCAmelCase__					, metric_key_prefix=UpperCAmelCase__					, )
           finally:
                  A__       =    compute_metrics

           A__       =    self.args.eval_batch_size * self.args.world_size
           if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
                  start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
           output.metrics.update(
               speed_metrics(
                   UpperCAmelCase__					, UpperCAmelCase__					, num_samples=output.num_samples					, num_steps=math.ceil(output.num_samples / total_batch_size)					, ))
           if self.post_process_function is None or self.compute_metrics is None:
                  return output

           A__       =    self.post_process_function(UpperCAmelCase__					, UpperCAmelCase__					, UpperCAmelCase__					, '''predict''')
           A__       =    self.compute_metrics(UpperCAmelCase__)

           # Prefix all keys with metric_key_prefix + '_'
           for key in list(metrics.keys()):
                  if not key.startswith(f"""{metric_key_prefix}_"""):
                         A__       =    metrics.pop(UpperCAmelCase__)
           metrics.update(output.metrics)
           return PredictionOutput(predictions=predictions.predictions					, label_ids=predictions.label_ids					, metrics=UpperCAmelCase__)




'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_rembert import RemBertTokenizer
else:
      lowercase			:				Optional[Any]    		=			None

lowercase			:				Tuple    		=			logging.get_logger(__name__)
lowercase			:				Dict    		=			{'vocab_file': 'sentencepiece.model', 'tokenizer_file': 'tokenizer.json'}

lowercase			:				Tuple    		=			{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
    'tokenizer_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/tokenizer.json',
    },
}

lowercase			:				List[str]    		=			{
    'google/rembert': 2_56,
}

lowercase			:				Dict    		=			'▁'





class  A				(							__snake_case						):
     __magic_name__						=		VOCAB_FILES_NAMES
     __magic_name__						=		PRETRAINED_VOCAB_FILES_MAP
     __magic_name__						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__						=		RemBertTokenizer





     def __init__(							self						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=False						,			SCREAMING_SNAKE_CASE="[CLS]"						,			SCREAMING_SNAKE_CASE="[SEP]"						,			SCREAMING_SNAKE_CASE="<unk>"						,			SCREAMING_SNAKE_CASE="[SEP]"						,			SCREAMING_SNAKE_CASE="<pad>"						,			SCREAMING_SNAKE_CASE="[CLS]"						,			SCREAMING_SNAKE_CASE="[MASK]"						,			**SCREAMING_SNAKE_CASE						,			) ->				List[str]:






         """simple docstring"""






         A       :   Optional[Any]					=						AddedToken(SCREAMING_SNAKE_CASE						,			lstrip=SCREAMING_SNAKE_CASE						,			rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	) else mask_token

         super().__init__(
             SCREAMING_SNAKE_CASE						,			tokenizer_file=SCREAMING_SNAKE_CASE						,			do_lower_case=SCREAMING_SNAKE_CASE						,			remove_space=SCREAMING_SNAKE_CASE						,			keep_accents=SCREAMING_SNAKE_CASE						,			bos_token=SCREAMING_SNAKE_CASE						,			eos_token=SCREAMING_SNAKE_CASE						,			unk_token=SCREAMING_SNAKE_CASE						,			sep_token=SCREAMING_SNAKE_CASE						,			pad_token=SCREAMING_SNAKE_CASE						,			cls_token=SCREAMING_SNAKE_CASE						,			mask_token=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE						,			)

         A       :   List[Any]					=						do_lower_case
         A       :   str					=						remove_space
         A       :   int					=						keep_accents
         A       :   Union[str, Any]					=						vocab_file
         A       :   List[Any]					=						False if not self.vocab_file else True





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   List[Any]					=						[self.sep_token_id]
         A       :   Tuple					=						[self.cls_token_id]
         if token_ids_a is None:
             return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None						,			SCREAMING_SNAKE_CASE = False	) ->				List[int]:






         """simple docstring"""






         if already_has_special_tokens:
             if token_ids_a is not None:
                 raise ValueError(
                     '''You should not supply a second sequence if the provided sequence of '''
                     '''ids is already formatted with special tokens for the model.'''	)
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
             return [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1]
         return [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   Tuple					=						[self.sep_token_id]
         A       :   Dict					=						[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep	) * [0]
         return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				Tuple[str]:






         """simple docstring"""






         if not os.path.isdir(SCREAMING_SNAKE_CASE	):
             logger.error('''Vocabulary path ({}) should be a directory'''.format(SCREAMING_SNAKE_CASE	)	)
             return
         A       :   Any					=						os.path.join(
             SCREAMING_SNAKE_CASE						,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE	):
             copyfile(self.vocab_file						,			SCREAMING_SNAKE_CASE	)

         return (out_vocab_file,)




