




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_big_bird import BigBirdTokenizer
else:
    snake_case_ : Union[str, Any]    		=       None

snake_case_ : Union[str, Any]    		=       logging.get_logger(__name__)
snake_case_ : List[str]    		=       {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

snake_case_ : Tuple    		=       {
    "vocab_file": {
        "google/bigbird-roberta-base": "https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model",
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"
        ),
    },
    "tokenizer_file": {
        "google/bigbird-roberta-base": (
            "https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json"
        ),
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json"
        ),
    },
}

snake_case_ : Tuple    		=       {
    "google/bigbird-roberta-base": 4096,
    "google/bigbird-roberta-large": 4096,
    "google/bigbird-base-trivia-itc": 4096,
}


snake_case_ : Dict    		=       "▁"







class __snake_case			(			a		):
   UpperCAmelCase__  :    Optional[Any]          =  VOCAB_FILES_NAMES
   UpperCAmelCase__  :    Tuple          =  PRETRAINED_VOCAB_FILES_MAP
   UpperCAmelCase__  :    List[Any]          =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCAmelCase__  :    List[str]          =  BigBirdTokenizer
   UpperCAmelCase__  :    Any          =  ['''input_ids''', '''attention_mask''']
   UpperCAmelCase__  :    List[int]          =  []







   def __init__(      self		:						List[str]					,						_snake_case		:						Dict=None					,						_snake_case		:						List[Any]=None					,						_snake_case		:						str="<unk>"					,						_snake_case		:						Optional[int]="<s>"					,						_snake_case		:						Any="</s>"					,						_snake_case		:						List[Any]="<pad>"					,						_snake_case		:						Tuple="[SEP]"					,						_snake_case		:						List[Any]="[MASK]"					,						_snake_case		:						str="[CLS]"					,						**_snake_case		:						List[Any]					,						):

        """simple docstring"""


        UpperCAmelCase_          =							AddedToken(_snake_case					,						lstrip=_snake_case					,						rstrip=_snake_case) if isinstance(_snake_case					,						_snake_case) else bos_token
        UpperCAmelCase_          =							AddedToken(_snake_case					,						lstrip=_snake_case					,						rstrip=_snake_case) if isinstance(_snake_case					,						_snake_case) else eos_token
        UpperCAmelCase_          =							AddedToken(_snake_case					,						lstrip=_snake_case					,						rstrip=_snake_case) if isinstance(_snake_case					,						_snake_case) else unk_token
        UpperCAmelCase_          =							AddedToken(_snake_case					,						lstrip=_snake_case					,						rstrip=_snake_case) if isinstance(_snake_case					,						_snake_case) else pad_token
        UpperCAmelCase_          =							AddedToken(_snake_case					,						lstrip=_snake_case					,						rstrip=_snake_case) if isinstance(_snake_case					,						_snake_case) else cls_token
        UpperCAmelCase_          =							AddedToken(_snake_case					,						lstrip=_snake_case					,						rstrip=_snake_case) if isinstance(_snake_case					,						_snake_case) else sep_token

        # Mask token behave like a normal word, i.e. include the space before it
        UpperCAmelCase_          =							AddedToken(_snake_case					,						lstrip=_snake_case					,						rstrip=_snake_case) if isinstance(_snake_case					,						_snake_case) else mask_token

        super().__init__(
            _snake_case					,						tokenizer_file=_snake_case					,						bos_token=_snake_case					,						eos_token=_snake_case					,						unk_token=_snake_case					,						sep_token=_snake_case					,						pad_token=_snake_case					,						cls_token=_snake_case					,						mask_token=_snake_case					,						**_snake_case					,						)

        UpperCAmelCase_          =							vocab_file
        UpperCAmelCase_          =							False if not self.vocab_file else True







   def lowerCamelCase						(      self		:						Any					,						_snake_case		:						List[int]					,						_snake_case		:						Optional[List[int]] = None):

        """simple docstring"""


        UpperCAmelCase_          =							[self.sep_token_id]
        UpperCAmelCase_          =							[self.cls_token_id]
        if token_ids_a is None:
             return cls + token_ids_a + sep
        return cls + token_ids_a + sep + token_ids_a + sep







   def lowerCamelCase						(      self		:						Optional[Any]					,						_snake_case		:						List[int]					,						_snake_case		:						Optional[List[int]] = None					,						_snake_case		:						bool = False):

        """simple docstring"""


        if already_has_special_tokens:
             if token_ids_a is not None:
                  raise ValueError(
                      '''You should not supply a second sequence if the provided sequence of '''
                      '''ids is already formatted with special tokens for the model.''')
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

        if token_ids_a is None:
             return [1] + ([0] * len(_snake_case)) + [1]
        return [1] + ([0] * len(_snake_case)) + [1] + ([0] * len(_snake_case)) + [1]







   def lowerCamelCase						(      self		:						str					,						_snake_case		:						List[int]					,						_snake_case		:						Optional[List[int]] = None):

        """simple docstring"""


        UpperCAmelCase_          =							[self.sep_token_id]
        UpperCAmelCase_          =							[self.cls_token_id]

        if token_ids_a is None:
             return len(cls + token_ids_a + sep) * [0]
        return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







   def lowerCamelCase						(      self		:						Tuple					,						_snake_case		:						str					,						_snake_case		:						Optional[str] = None):

        """simple docstring"""


        if not self.can_save_slow_tokenizer:
             raise ValueError(
                 '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                 '''tokenizer.''')

        if not os.path.isdir(_snake_case):
             logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
             return
        UpperCAmelCase_          =							os.path.join(
            _snake_case					,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

        if os.path.abspath(self.vocab_file) != os.path.abspath(_snake_case):
             copyfile(self.vocab_file					,						_snake_case)

        return (out_vocab_file,)











import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm
 import torch_xla.debug.metrics as met

class     _A       (	__UpperCAmelCase    ):



       def __init__(		self							:     List[Any]     ,		*__SCREAMING_SNAKE_CASE							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Any=None     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None     ,		**__SCREAMING_SNAKE_CASE							:     str):


           '''simple docstring'''

           super().__init__(*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
           __a						       =				eval_examples
           __a						       =				post_process_function



       def 		_lowerCamelCase			(		self							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Optional[Dataset] = None     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=None     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[str]] = None     ,		__SCREAMING_SNAKE_CASE							:     str = "eval"     ,		**__SCREAMING_SNAKE_CASE							:     Any     ,		):


           '''simple docstring'''

           __a						       =				gen_kwargs.copy()
           __a						       =				(
               gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length''') is not None else self.args.generation_max_length
           )
           __a						       =				(
               gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.args.generation_num_beams
           )
           __a						       =				gen_kwargs

           __a						       =				self.eval_dataset if eval_dataset is None else eval_dataset
           __a						       =				self.get_eval_dataloader(__SCREAMING_SNAKE_CASE)
           __a						       =				self.eval_examples if eval_examples is None else eval_examples

           # Temporarily disable metric computation, we will do it in the loop here.
           __a						       =				self.compute_metrics
           __a						       =				None
           __a						       =				time.time()
           __a						       =				self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
           try:
               __a						       =				eval_loop(
                   __SCREAMING_SNAKE_CASE     ,		description='''Evaluation'''     ,		prediction_loss_only=True if compute_metrics is None else None     ,		ignore_keys=__SCREAMING_SNAKE_CASE     ,		metric_key_prefix=__SCREAMING_SNAKE_CASE     ,		)
           finally:
               __a						       =				compute_metrics
           __a						       =				self.args.eval_batch_size * self.args.world_size
           if F'{metric_key_prefix}_jit_compilation_time' in output.metrics:
               start_time += output.metrics[F'{metric_key_prefix}_jit_compilation_time']
           output.metrics.update(
               speed_metrics(
                   __SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		num_samples=output.num_samples     ,		num_steps=math.ceil(output.num_samples / total_batch_size)     ,		))

           if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
               # Only the main node write the results by default
               __a						       =				self.post_process_function(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)
               __a						       =				self.compute_metrics(__SCREAMING_SNAKE_CASE)

               # Prefix all keys with metric_key_prefix + '_'
               for key in list(metrics.keys()):
                   if not key.startswith(F'{metric_key_prefix}_'):
                       __a						       =				metrics.pop(__SCREAMING_SNAKE_CASE)

               metrics.update(output.metrics)
           else:
               __a						       =				output.metrics

           if self.args.should_log:
               # Only the main node log the results by default
               self.log(__SCREAMING_SNAKE_CASE)

           if self.args.tpu_metrics_debug or self.args.debug:
               # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
               xm.master_print(met.metrics_report())

           __a						       =				self.callback_handler.on_evaluate(self.args     ,		self.state     ,		self.control     ,		__SCREAMING_SNAKE_CASE)
           return metrics





       def 		_lowerCamelCase			(		self							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Any     ,		__SCREAMING_SNAKE_CASE							:     int     ,		__SCREAMING_SNAKE_CASE							:     Tuple=None     ,		__SCREAMING_SNAKE_CASE							:     str = "test"     ,		**__SCREAMING_SNAKE_CASE							:     Dict):


           '''simple docstring'''

           __a						       =				gen_kwargs.copy()

           __a						       =				self.get_test_dataloader(__SCREAMING_SNAKE_CASE)

           # Temporarily disable metric computation, we will do it in the loop here.
           __a						       =				self.compute_metrics
           __a						       =				None
           __a						       =				time.time()
           __a						       =				self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
           try:
               __a						       =				eval_loop(
                   __SCREAMING_SNAKE_CASE     ,		description='''Prediction'''     ,		prediction_loss_only=True if compute_metrics is None else None     ,		ignore_keys=__SCREAMING_SNAKE_CASE     ,		metric_key_prefix=__SCREAMING_SNAKE_CASE     ,		)
           finally:
               __a						       =				compute_metrics

           __a						       =				self.args.eval_batch_size * self.args.world_size
           if F'{metric_key_prefix}_jit_compilation_time' in output.metrics:
               start_time += output.metrics[F'{metric_key_prefix}_jit_compilation_time']
           output.metrics.update(
               speed_metrics(
                   __SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		num_samples=output.num_samples     ,		num_steps=math.ceil(output.num_samples / total_batch_size)     ,		))
           if self.post_process_function is None or self.compute_metrics is None:
               return output

           __a						       =				self.post_process_function(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		'''predict''')
           __a						       =				self.compute_metrics(__SCREAMING_SNAKE_CASE)

           # Prefix all keys with metric_key_prefix + '_'
           for key in list(metrics.keys()):
               if not key.startswith(F'{metric_key_prefix}_'):
                   __a						       =				metrics.pop(__SCREAMING_SNAKE_CASE)
           metrics.update(output.metrics)
           return PredictionOutput(predictions=predictions.predictions     ,		label_ids=predictions.label_ids     ,		metrics=__SCREAMING_SNAKE_CASE)





