



import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case						:int      		=     logging.get_logger(__name__)

__snake_case						:Union[str, Any]      		=     '''▁'''

__snake_case						:List[str]      		=     {'''vocab_file''': '''prophetnet.tokenizer'''}

__snake_case						:List[str]      		=     {
    '''vocab_file''': {
        '''microsoft/xprophetnet-large-wiki100-cased''': (
            '''https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'''
        ),
    }
}

__snake_case						:Dict      		=     {
    '''microsoft/xprophetnet-large-wiki100-cased''': {'''do_lower_case''': False},
}

__snake_case						:str      		=     {
    '''microsoft/xprophetnet-large-wiki100-cased''': 512,
}





def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				collections.OrderedDict()
    with open(_UpperCAmelCase   , '''r'''   , encoding='''utf-8'''				) as reader:
        __a						       =				reader.readlines()
    for index, token in enumerate(_UpperCAmelCase				):
        __a						       =				token.rstrip('''\n'''				)
        __a						       =				index
    return vocab

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[Any]											=						VOCAB_FILES_NAMES
       UpperCamelCase__  :    List[str]											=						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :    Optional[Any]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__  :    Dict											=						['''input_ids''', '''attention_mask''']



       def __init__(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     List[str]="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     str="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[UNK]"     ,		__SCREAMING_SNAKE_CASE							:     str="[PAD]"     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]="[CLS]"     ,		__SCREAMING_SNAKE_CASE							:     Tuple="[MASK]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[Dict[str, Any]] = None     ,		**__SCREAMING_SNAKE_CASE							:     Union[str, Any]     ,		):


           '''simple docstring'''

           __a						       =				{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               bos_token=__SCREAMING_SNAKE_CASE     ,		eos_token=__SCREAMING_SNAKE_CASE     ,		sep_token=__SCREAMING_SNAKE_CASE     ,		unk_token=__SCREAMING_SNAKE_CASE     ,		pad_token=__SCREAMING_SNAKE_CASE     ,		cls_token=__SCREAMING_SNAKE_CASE     ,		mask_token=__SCREAMING_SNAKE_CASE     ,		sp_model_kwargs=self.sp_model_kwargs     ,		**__SCREAMING_SNAKE_CASE     ,		)

           try:
               import sentencepiece as spm
           except ImportError:
               logger.warning(
                   '''You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'''
                   ''' pip install sentencepiece''')
               raise

           __a						       =				spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(str(__SCREAMING_SNAKE_CASE))
           __a						       =				vocab_file

           # Original fairseq vocab and spm vocab must be "aligned":
           # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
           # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
           # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
           # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

           # put special tokens and [unused] tokens into the vocab
           __a						       =				{'''[PAD]''': 0, '''[CLS]''': 1, '''[SEP]''': 2, '''[UNK]''': 3, '''[MASK]''': 4}

           for i in range(10):
               __a						       =				F'[unused{i}]'
               __a						       =				5 + i

           # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
           __a						       =				12
           __a						       =				{v: k for k, v in self.fairseq_tokens_to_ids.items()}
           for k in self.fairseq_tokens_to_ids.keys():
               self.unique_no_split_tokens.append(__SCREAMING_SNAKE_CASE)



       def __getstate__(		self							:     Dict):


           '''simple docstring'''

           __a						       =				self.__dict__.copy()
           __a						       =				None
           return state



       def __setstate__(		self							:     Optional[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]):


           '''simple docstring'''

           __a						       =				d
           try:
               import sentencepiece as spm
           except ImportError:
               logger.warning(
                   '''You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'''
                   ''' pip install sentencepiece''')
               raise

           # for backward compatibility
           if not hasattr(self     ,		'''sp_model_kwargs'''):
               __a						       =				{}

           __a						       =				spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(self.vocab_file)



       def 		_lowerCamelCase			(		self							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None     ,		__SCREAMING_SNAKE_CASE							:     bool = False):


           '''simple docstring'''

           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__SCREAMING_SNAKE_CASE     ,		token_ids_a=__SCREAMING_SNAKE_CASE     ,		already_has_special_tokens=__SCREAMING_SNAKE_CASE)

           if token_ids_a is None:
               return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]
           return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]



       def 		_lowerCamelCase			(		self							:     List[Any]     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]

           if token_ids_a is None:
               return len(token_ids_a + sep) * [0]
           return len(token_ids_a + sep + sep + token_ids_a + sep) * [0]



       @property
       def 		_lowerCamelCase			(		self							:     List[str]):


           '''simple docstring'''

           return len(self.sp_model) + self.fairseq_offset



       def 		_lowerCamelCase			(		self							:     int):


           '''simple docstring'''

           __a						       =				{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE): i for i in range(self.vocab_size)}
           vocab.update(self.added_tokens_encoder)
           return vocab



       def 		_lowerCamelCase			(		self							:     List[str]     ,		__SCREAMING_SNAKE_CASE							:     str):


           '''simple docstring'''

           return self.sp_model.encode(__SCREAMING_SNAKE_CASE     ,		out_type=__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Dict):


           '''simple docstring'''

           if token in self.fairseq_tokens_to_ids:
               return self.fairseq_tokens_to_ids[token]
           __a						       =				self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE)

           # Need to return unknown token if the SP model returned 0
           return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



       def 		_lowerCamelCase			(		self							:     Union[str, Any]     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]):


           '''simple docstring'''

           if index in self.fairseq_ids_to_tokens:
               return self.fairseq_ids_to_tokens[index]
           return self.sp_model.IdToPiece(index - self.fairseq_offset)



       def 		_lowerCamelCase			(		self							:     Union[str, Any]     ,		__SCREAMING_SNAKE_CASE							:     Tuple):


           '''simple docstring'''

           __a						       =				''''''.join(__SCREAMING_SNAKE_CASE).replace(__SCREAMING_SNAKE_CASE     ,		''' ''').strip()
           return out_string



       def 		_lowerCamelCase			(		self							:     List[Any]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[str] = None):


           '''simple docstring'''

           if not os.path.isdir(__SCREAMING_SNAKE_CASE):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory')
               return
           __a						       =				os.path.join(
               __SCREAMING_SNAKE_CASE     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

           if os.path.abspath(self.vocab_file) != os.path.abspath(__SCREAMING_SNAKE_CASE) and os.path.isfile(self.vocab_file):
               copyfile(self.vocab_file     ,		__SCREAMING_SNAKE_CASE)
           elif not os.path.isfile(self.vocab_file):
               with open(__SCREAMING_SNAKE_CASE     ,		'''wb''') as fi:
                   __a						       =				self.sp_model.serialized_model_proto()
                   fi.write(__SCREAMING_SNAKE_CASE)

           return (out_vocab_file,)





       def 		_lowerCamelCase			(		self							:     int     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           if token_ids_a is None:
               return token_ids_a + [self.sep_token_id]
           __a						       =				[self.sep_token_id]
           return token_ids_a + sep + token_ids_a + sep










import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm
 import torch_xla.debug.metrics as met

class     _A       (	__UpperCAmelCase    ):



       def __init__(		self							:     List[Any]     ,		*__SCREAMING_SNAKE_CASE							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Any=None     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None     ,		**__SCREAMING_SNAKE_CASE							:     str):


           '''simple docstring'''

           super().__init__(*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
           __a						       =				eval_examples
           __a						       =				post_process_function



       def 		_lowerCamelCase			(		self							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Optional[Dataset] = None     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=None     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[str]] = None     ,		__SCREAMING_SNAKE_CASE							:     str = "eval"     ,		**__SCREAMING_SNAKE_CASE							:     Any     ,		):


           '''simple docstring'''

           __a						       =				gen_kwargs.copy()
           __a						       =				(
               gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length''') is not None else self.args.generation_max_length
           )
           __a						       =				(
               gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.args.generation_num_beams
           )
           __a						       =				gen_kwargs

           __a						       =				self.eval_dataset if eval_dataset is None else eval_dataset
           __a						       =				self.get_eval_dataloader(__SCREAMING_SNAKE_CASE)
           __a						       =				self.eval_examples if eval_examples is None else eval_examples

           # Temporarily disable metric computation, we will do it in the loop here.
           __a						       =				self.compute_metrics
           __a						       =				None
           __a						       =				time.time()
           __a						       =				self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
           try:
               __a						       =				eval_loop(
                   __SCREAMING_SNAKE_CASE     ,		description='''Evaluation'''     ,		prediction_loss_only=True if compute_metrics is None else None     ,		ignore_keys=__SCREAMING_SNAKE_CASE     ,		metric_key_prefix=__SCREAMING_SNAKE_CASE     ,		)
           finally:
               __a						       =				compute_metrics
           __a						       =				self.args.eval_batch_size * self.args.world_size
           if F'{metric_key_prefix}_jit_compilation_time' in output.metrics:
               start_time += output.metrics[F'{metric_key_prefix}_jit_compilation_time']
           output.metrics.update(
               speed_metrics(
                   __SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		num_samples=output.num_samples     ,		num_steps=math.ceil(output.num_samples / total_batch_size)     ,		))

           if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
               # Only the main node write the results by default
               __a						       =				self.post_process_function(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)
               __a						       =				self.compute_metrics(__SCREAMING_SNAKE_CASE)

               # Prefix all keys with metric_key_prefix + '_'
               for key in list(metrics.keys()):
                   if not key.startswith(F'{metric_key_prefix}_'):
                       __a						       =				metrics.pop(__SCREAMING_SNAKE_CASE)

               metrics.update(output.metrics)
           else:
               __a						       =				output.metrics

           if self.args.should_log:
               # Only the main node log the results by default
               self.log(__SCREAMING_SNAKE_CASE)

           if self.args.tpu_metrics_debug or self.args.debug:
               # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
               xm.master_print(met.metrics_report())

           __a						       =				self.callback_handler.on_evaluate(self.args     ,		self.state     ,		self.control     ,		__SCREAMING_SNAKE_CASE)
           return metrics





       def 		_lowerCamelCase			(		self							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Any     ,		__SCREAMING_SNAKE_CASE							:     int     ,		__SCREAMING_SNAKE_CASE							:     Tuple=None     ,		__SCREAMING_SNAKE_CASE							:     str = "test"     ,		**__SCREAMING_SNAKE_CASE							:     Dict):


           '''simple docstring'''

           __a						       =				gen_kwargs.copy()

           __a						       =				self.get_test_dataloader(__SCREAMING_SNAKE_CASE)

           # Temporarily disable metric computation, we will do it in the loop here.
           __a						       =				self.compute_metrics
           __a						       =				None
           __a						       =				time.time()
           __a						       =				self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
           try:
               __a						       =				eval_loop(
                   __SCREAMING_SNAKE_CASE     ,		description='''Prediction'''     ,		prediction_loss_only=True if compute_metrics is None else None     ,		ignore_keys=__SCREAMING_SNAKE_CASE     ,		metric_key_prefix=__SCREAMING_SNAKE_CASE     ,		)
           finally:
               __a						       =				compute_metrics

           __a						       =				self.args.eval_batch_size * self.args.world_size
           if F'{metric_key_prefix}_jit_compilation_time' in output.metrics:
               start_time += output.metrics[F'{metric_key_prefix}_jit_compilation_time']
           output.metrics.update(
               speed_metrics(
                   __SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		num_samples=output.num_samples     ,		num_steps=math.ceil(output.num_samples / total_batch_size)     ,		))
           if self.post_process_function is None or self.compute_metrics is None:
               return output

           __a						       =				self.post_process_function(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		'''predict''')
           __a						       =				self.compute_metrics(__SCREAMING_SNAKE_CASE)

           # Prefix all keys with metric_key_prefix + '_'
           for key in list(metrics.keys()):
               if not key.startswith(F'{metric_key_prefix}_'):
                   __a						       =				metrics.pop(__SCREAMING_SNAKE_CASE)
           metrics.update(output.metrics)
           return PredictionOutput(predictions=predictions.predictions     ,		label_ids=predictions.label_ids     ,		metrics=__SCREAMING_SNAKE_CASE)





