
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


__A				=logging.get_logger(__name__)

__A				={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A				={
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

__A				={
    '''yjernite/retribert-base-uncased''': 5_1_2,
}


__A				={
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}




class     _SCREAMING_SNAKE_CASE     (      snake_case_						):
	lowerCAmelCase__         = VOCAB_FILES_NAMES
	lowerCAmelCase__         = PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__         = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__         = PRETRAINED_INIT_CONFIGURATION
	lowerCAmelCase__         = RetriBertTokenizer
	lowerCAmelCase__         = ['input_ids', 'attention_mask']



	def __init__(    self						,  lowercase=None						,  lowercase=None						,  lowercase=True						,  lowercase="[UNK]"						,  lowercase="[SEP]"						,  lowercase="[PAD]"						,  lowercase="[CLS]"						,  lowercase="[MASK]"						,  lowercase=True						,  lowercase=None						,  **lowercase						,  )				->       List[Any]:
					super().__init__(
					    lowercase						,  tokenizer_file=lowercase						,  do_lower_case=lowercase						,  unk_token=lowercase						,  sep_token=lowercase						,  pad_token=lowercase						,  cls_token=lowercase						,  mask_token=lowercase						,  tokenize_chinese_chars=lowercase						,  strip_accents=lowercase						,  **lowercase						,  )

					lowerCamelCase_						=			json.loads(self.backend_tokenizer.normalizer.__getstate__() )
					if (
					    normalizer_state.get("lowercase"						,  lowercase ) != do_lower_case
					    or normalizer_state.get("strip_accents"						,  lowercase ) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"						,  lowercase ) != tokenize_chinese_chars
					):
									lowerCamelCase_						=			getattr(lowercase						,  normalizer_state.pop("type" ) )
									lowerCamelCase_						=			do_lower_case
									lowerCamelCase_						=			strip_accents
									lowerCamelCase_						=			tokenize_chinese_chars
									lowerCamelCase_						=			normalizer_class(**lowercase )

					lowerCamelCase_						=			do_lower_case



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase=None )				->       int:
					lowerCamelCase_						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

					return output



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase = None )				->       List[int]:
					lowerCamelCase_						=			[self.sep_token_id]
					lowerCamelCase_						=			[self.cls_token_id]
					if token_ids_a is None:
									return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase = None )				->       Tuple[str]:
					lowerCamelCase_						=			self._tokenizer.model.save(lowercase						,  name=lowercase )
					return tuple(lowercase )




import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met




class     _SCREAMING_SNAKE_CASE     (      snake_case_						):



	def __init__(    self						,  *lowercase						,  lowercase=None						,  lowercase=None						,  **lowercase )				->       List[str]:
					super().__init__(*lowercase						,  **lowercase )
					lowerCamelCase_						=			eval_examples
					lowerCamelCase_						=			post_process_function



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase = None						,  lowercase=None						,  lowercase = None						,  lowercase = "eval"						,  **lowercase						,  )				->       Dict[str, float]:
					lowerCamelCase_						=			gen_kwargs.copy()
					lowerCamelCase_						=			(
					    gen_kwargs["max_length"] if gen_kwargs.get("max_length" ) is not None else self.args.generation_max_length
					)
					lowerCamelCase_						=			(
					    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams" ) is not None else self.args.generation_num_beams
					)
					lowerCamelCase_						=			gen_kwargs

					lowerCamelCase_						=			self.eval_dataset if eval_dataset is None else eval_dataset
					lowerCamelCase_						=			self.get_eval_dataloader(lowercase )
					lowerCamelCase_						=			self.eval_examples if eval_examples is None else eval_examples

					# Temporarily disable metric computation, we will do it in the loop here.
					lowerCamelCase_						=			self.compute_metrics
					lowerCamelCase_						=			None
					lowerCamelCase_						=			time.time()
					lowerCamelCase_						=			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
					try:
									lowerCamelCase_						=			eval_loop(
									    lowercase						,  description="Evaluation"						,  prediction_loss_only=True if compute_metrics is None else None						,  ignore_keys=lowercase						,  metric_key_prefix=lowercase						,  )
					finally:
									lowerCamelCase_						=			compute_metrics
					lowerCamelCase_						=			self.args.eval_batch_size * self.args.world_size
					if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
									start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
					output.metrics.update(
					    speed_metrics(
					        lowercase						,  lowercase						,  num_samples=output.num_samples						,  num_steps=math.ceil(output.num_samples / total_batch_size )						,  ) )

					if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
									# Only the main node write the results by default
									lowerCamelCase_						=			self.post_process_function(lowercase						,  lowercase						,  lowercase )
									lowerCamelCase_						=			self.compute_metrics(lowercase )

									# Prefix all keys with metric_key_prefix + '_'
									for key in list(metrics.keys() ):
													if not key.startswith(f'{metric_key_prefix}_' ):
																	lowerCamelCase_						=			metrics.pop(lowercase )

									metrics.update(output.metrics )
					else:
									lowerCamelCase_						=			output.metrics

					if self.args.should_log:
									# Only the main node log the results by default
									self.log(lowercase )

					if self.args.tpu_metrics_debug or self.args.debug:
									# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
									xm.master_print(met.metrics_report() )

					lowerCamelCase_						=			self.callback_handler.on_evaluate(self.args						,  self.state						,  self.control						,  lowercase )
					return metrics



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase=None						,  lowercase = "test"						,  **lowercase )				->       Union[str, Any]:
					lowerCamelCase_						=			gen_kwargs.copy()

					lowerCamelCase_						=			self.get_test_dataloader(lowercase )

					# Temporarily disable metric computation, we will do it in the loop here.
					lowerCamelCase_						=			self.compute_metrics
					lowerCamelCase_						=			None
					lowerCamelCase_						=			time.time()
					lowerCamelCase_						=			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
					try:
									lowerCamelCase_						=			eval_loop(
									    lowercase						,  description="Prediction"						,  prediction_loss_only=True if compute_metrics is None else None						,  ignore_keys=lowercase						,  metric_key_prefix=lowercase						,  )
					finally:
									lowerCamelCase_						=			compute_metrics

					lowerCamelCase_						=			self.args.eval_batch_size * self.args.world_size
					if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
									start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
					output.metrics.update(
					    speed_metrics(
					        lowercase						,  lowercase						,  num_samples=output.num_samples						,  num_steps=math.ceil(output.num_samples / total_batch_size )						,  ) )
					if self.post_process_function is None or self.compute_metrics is None:
									return output

					lowerCamelCase_						=			self.post_process_function(lowercase						,  lowercase						,  lowercase						,  "predict" )
					lowerCamelCase_						=			self.compute_metrics(lowercase )

					# Prefix all keys with metric_key_prefix + '_'
					for key in list(metrics.keys() ):
									if not key.startswith(f'{metric_key_prefix}_' ):
													lowerCamelCase_						=			metrics.pop(lowercase )
					metrics.update(output.metrics )
					return PredictionOutput(predictions=predictions.predictions						,  label_ids=predictions.label_ids						,  metrics=lowercase )


