





'''simple docstring'''







import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
					import torch

					from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
					import pyanvml.pyanvml as nvml


snake_case_  :							List[Any]            =	logging.get_logger(__name__)


class        __a   (lowerCamelCase  ):
							__a       : PyTorchBenchmarkArguments
							__a       : PretrainedConfig
							__a       : str          =							"PyTorch"

							@property
							def      UpperCAmelCase__						(				self				:  List[str] )       ->							Union[str, Any]:







													"""simple docstring"""
													return torch.__version__

							def      UpperCAmelCase__						(				self				:  Optional[Any]							,  __magic_name__				:  str							,  __magic_name__				:  int							,  __magic_name__				:  int )       ->							float:







													"""simple docstring"""
													UpperCAmelCase_      :   List[str]      =	self._prepare_inference_func(__magic_name__							,  __magic_name__							,  __magic_name__ )
													return self._measure_speed(_inference )

							def      UpperCAmelCase__						(				self				:  int							,  __magic_name__				:  str							,  __magic_name__				:  int							,  __magic_name__				:  int )       ->							[Memory, Optional[MemorySummary]]:







													"""simple docstring"""
													UpperCAmelCase_      :   Optional[Any]      =	self._prepare_inference_func(__magic_name__							,  __magic_name__							,  __magic_name__ )
													return self._measure_memory(_inference )

							def      UpperCAmelCase__						(				self				:  Optional[Any]							,  __magic_name__				:  str							,  __magic_name__				:  int							,  __magic_name__				:  int )       ->							float:







													"""simple docstring"""
													UpperCAmelCase_      :   str      =	self._prepare_train_func(__magic_name__							,  __magic_name__							,  __magic_name__ )
													return self._measure_speed(_train )

							def      UpperCAmelCase__						(				self				:  Any							,  __magic_name__				:  str							,  __magic_name__				:  int							,  __magic_name__				:  int )       ->							[Memory, Optional[MemorySummary]]:







													"""simple docstring"""
													UpperCAmelCase_      :   List[str]      =	self._prepare_train_func(__magic_name__							,  __magic_name__							,  __magic_name__ )
													return self._measure_memory(_train )

							def      UpperCAmelCase__						(				self				:  Optional[Any]							,  __magic_name__				:  str							,  __magic_name__				:  int							,  __magic_name__				:  int )       ->							Callable[[], None]:







													"""simple docstring"""
													UpperCAmelCase_      :   Dict      =	self.config_dict[model_name]

													if self.args.torchscript:
																			UpperCAmelCase_      :   int      =	True

													UpperCAmelCase_      :   Any      =	(
													    hasattr(__magic_name__							,  '''architectures''' )
													    and isinstance(config.architectures							,  __magic_name__ )
													    and len(config.architectures ) > 0
													)
													if not self.args.only_pretrain_model and has_model_class_in_config:
																			try:
																									UpperCAmelCase_      :   List[Any]      =	config.architectures[0]
																									UpperCAmelCase_      :   List[Any]      =	__import__('''transformers'''							,  fromlist=[model_class] )
																									UpperCAmelCase_      :   Dict      =	getattr(__magic_name__							,  __magic_name__ )
																									UpperCAmelCase_      :   Union[str, Any]      =	model_cls(__magic_name__ )
																			except ImportError:
																									raise ImportError(
																									    F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
																									    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.''' )
													else:
																			UpperCAmelCase_      :   int      =	MODEL_MAPPING[config.__class__](__magic_name__ )

													model.eval()
													model.to(self.args.device )

													# encoder-decoder has vocab size saved differently
													UpperCAmelCase_      :   Dict      =	config.vocab_size if hasattr(__magic_name__							,  '''vocab_size''' ) else config.encoder.vocab_size
													UpperCAmelCase_      :   Optional[Any]      =	torch.randint(__magic_name__							,  (batch_size, sequence_length)							,  dtype=torch.long							,  device=self.args.device )

													if self.args.fpaa:
																			logger.info('''Running training in Mixed Precision...''' )
																			if not self.args.is_gpu:
																									raise ValueError('''Mixed precision is possible only for GPU.''' )
																			# amp seems to have memory leaks so that memory usage
																			# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
																			model.half()

													if self.args.torchscript:
																			with torch.no_grad():
																									UpperCAmelCase_      :   int      =	torch.jit.trace(__magic_name__							,  __magic_name__ )
													else:
																			UpperCAmelCase_      :   Any      =	model

													def encoder_decoder_forward():
																			with torch.no_grad():
																									UpperCAmelCase_      :   Any      =	inference_model(__magic_name__							,  decoder_input_ids=__magic_name__ )
																			return outputs

													def encoder_forward():
																			with torch.no_grad():
																									UpperCAmelCase_      :   int      =	inference_model(__magic_name__ )
																			return outputs

													UpperCAmelCase_      :   Optional[Any]      =	encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
													return _forward

							def      UpperCAmelCase__						(				self				:  Optional[Any]							,  __magic_name__				:  str							,  __magic_name__				:  int							,  __magic_name__				:  int )       ->							Callable[[], None]:







													"""simple docstring"""
													UpperCAmelCase_      :   str      =	self.config_dict[model_name]

													UpperCAmelCase_      :   int      =	(
													    hasattr(__magic_name__							,  '''architectures''' )
													    and isinstance(config.architectures							,  __magic_name__ )
													    and len(config.architectures ) > 0
													)
													if not self.args.only_pretrain_model and has_model_class_in_config:
																			try:
																									UpperCAmelCase_      :   Tuple      =	config.architectures[0]
																									UpperCAmelCase_      :   Optional[int]      =	__import__('''transformers'''							,  fromlist=[model_class] )
																									UpperCAmelCase_      :   List[Any]      =	getattr(__magic_name__							,  __magic_name__ )
																									UpperCAmelCase_      :   Dict      =	model_cls(__magic_name__ )
																			except ImportError:
																									raise ImportError(
																									    F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
																									    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.''' )
													else:
																			UpperCAmelCase_      :   Optional[int]      =	MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__magic_name__ )

													if self.args.torchscript:
																			raise NotImplementedError('''Training for torchscript is currently not implemented''' )
													else:
																			UpperCAmelCase_      :   Tuple      =	model

													model.train()
													model.to(self.args.device )

													# encoder-decoder has vocab size saved differently
													UpperCAmelCase_      :   Tuple      =	config.vocab_size if hasattr(__magic_name__							,  '''vocab_size''' ) else config.encoder.vocab_size
													UpperCAmelCase_      :   Tuple      =	torch.randint(__magic_name__							,  (batch_size, sequence_length)							,  dtype=torch.long							,  device=self.args.device )

													if self.args.fpaa:
																			logger.info('''Running training in Mixed Precision...''' )
																			if not self.args.is_gpu:
																									raise ValueError('''Mixed precision is possible only for GPU.''' )

																			# amp seems to have memory leaks so that memory usage
																			# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
																			model.half()

													def compute_loss_and_backprob_encoder():
																			UpperCAmelCase_      :   List[str]      =	train_model(__magic_name__							,  labels=__magic_name__ )[0]
																			loss.backward()
																			return loss

													def compute_loss_and_backprob_encoder_decoder():
																			UpperCAmelCase_      :   Dict      =	train_model(__magic_name__							,  decoder_input_ids=__magic_name__							,  labels=__magic_name__ )[0]
																			loss.backward()
																			return loss

													UpperCAmelCase_      :   List[str]      =	(
													    compute_loss_and_backprob_encoder_decoder
													    if config.is_encoder_decoder
													    else compute_loss_and_backprob_encoder
													)
													return _train

							def      UpperCAmelCase__						(				self				:  Union[str, Any]							,  __magic_name__				:  Union[str, Any] )       ->							float:







													"""simple docstring"""
													try:
																			if self.args.is_tpu or self.args.torchscript:
																									# run additional 10 times to stabilize compilation for tpu and torchscript
																									logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation''' )
																									timeit.repeat(
																									    __magic_name__							,  repeat=1							,  number=5							,  )

																			# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
																			UpperCAmelCase_      :   str      =	timeit.repeat(
																			    __magic_name__							,  repeat=self.args.repeat							,  number=10							,  )

																			if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
																									import torch_xla.debug.metrics as met

																									self.print_fn(met.metrics_report() )

																			return min(__magic_name__ ) / 1_0.0
													except RuntimeError as e:
																			self.print_fn(F"""Doesn't fit on GPU. {e}""" )
																			return "N/A"



							def      UpperCAmelCase__						(				self				:  List[str]							,  __magic_name__				:  Callable[[], None] )       ->							[Memory, MemorySummary]:







													"""simple docstring"""
													try:
																			if self.args.trace_memory_line_by_line:
																									UpperCAmelCase_      :   int      =	start_memory_tracing('''transformers''' )

																			if self.args.is_tpu:
																									# tpu
																									raise NotImplementedError(
																									    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
																									    ''' `--no-memory` or `args.memory=False`''' )
																			elif self.args.is_gpu:
																									if not is_pyanvml_available():
																															logger.warning(
																															    '''py3nvml not installed, we won\'t log GPU memory usage. '''
																															    '''Install py3nvml (pip install py3nvml) to log information about GPU.''' )
																															UpperCAmelCase_      :   int      =	'''N/A'''
																									else:
																															logger.info(
																															    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
																															    ''' on the same GPU.''' )
																															# init nvml
																															nvml.nvmlInit()
																															func()
																															UpperCAmelCase_      :   Dict      =	nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx )
																															UpperCAmelCase_      :   Any      =	nvml.nvmlDeviceGetMemoryInfo(__magic_name__ )
																															UpperCAmelCase_      :   int      =	meminfo.used
																															UpperCAmelCase_      :   Any      =	Memory(__magic_name__ )
																															# shutdown nvml
																															nvml.nvmlShutdown()
																			else:
																									# cpu
																									UpperCAmelCase_      :   Optional[Any]      =	measure_peak_memory_cpu(__magic_name__ )
																									UpperCAmelCase_      :   Dict      =	Memory(__magic_name__ ) if isinstance(__magic_name__							,  __magic_name__ ) else memory_bytes

																			if self.args.trace_memory_line_by_line:
																									UpperCAmelCase_      :   Dict      =	stop_memory_tracing(__magic_name__ )
																			else:
																									UpperCAmelCase_      :   str      =	None

																			return memory, summary
													except RuntimeError as e:
																			self.print_fn(F"""Doesn't fit on GPU. {e}""" )
																			return "N/A", None












'''simple docstring'''







import argparse
import json
import re
import string
import sys
from collections import Counter





def 							lowerCamelCase_   (	SCREAMING_SNAKE_CASE__   :      Optional[Any] )			->						str:

						def remove_articles(SCREAMING_SNAKE_CASE__   :      Union[str, Any] ):
												return re.sub(R'''\b(a|an|the)\b''',      ''' ''',      SCREAMING_SNAKE_CASE__ )

						def white_space_fix(SCREAMING_SNAKE_CASE__   :      Dict ):
												return " ".join(text.split() )

						def remove_punc(SCREAMING_SNAKE_CASE__   :      Union[str, Any] ):
												UpperCAmelCase_      :   int      =	set(string.punctuation )
												return "".join(ch for ch in text if ch not in exclude )

						def lower(SCREAMING_SNAKE_CASE__   :      str ):
												return text.lower()

						return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE__ ) ) ) )





def 							lowerCamelCase_   (	SCREAMING_SNAKE_CASE__   :      Union[str, Any],      SCREAMING_SNAKE_CASE__   :      Union[str, Any] )			->						Tuple:
						UpperCAmelCase_      :   List[Any]      =	normalize_answer(SCREAMING_SNAKE_CASE__ ).split()
						UpperCAmelCase_      :   Optional[Any]      =	normalize_answer(SCREAMING_SNAKE_CASE__ ).split()
						UpperCAmelCase_      :   Any      =	Counter(SCREAMING_SNAKE_CASE__ ) & Counter(SCREAMING_SNAKE_CASE__ )
						UpperCAmelCase_      :   Tuple      =	sum(common.values() )
						if num_same == 0:
												return 0
						UpperCAmelCase_      :   Union[str, Any]      =	1.0 * num_same / len(SCREAMING_SNAKE_CASE__ )
						UpperCAmelCase_      :   Dict      =	1.0 * num_same / len(SCREAMING_SNAKE_CASE__ )
						UpperCAmelCase_      :   str      =	(2 * precision * recall) / (precision + recall)
						return fa





def 							lowerCamelCase_   (	SCREAMING_SNAKE_CASE__   :      int,      SCREAMING_SNAKE_CASE__   :      int )			->						Any:
						return normalize_answer(SCREAMING_SNAKE_CASE__ ) == normalize_answer(SCREAMING_SNAKE_CASE__ )





def 							lowerCamelCase_   (	SCREAMING_SNAKE_CASE__   :      Union[str, Any],      SCREAMING_SNAKE_CASE__   :      Any,      SCREAMING_SNAKE_CASE__   :      Dict )			->						Optional[int]:
						UpperCAmelCase_      :   Dict      =	[]
						for ground_truth in ground_truths:
												UpperCAmelCase_      :   str      =	metric_fn(SCREAMING_SNAKE_CASE__,      SCREAMING_SNAKE_CASE__ )
												scores_for_ground_truths.append(SCREAMING_SNAKE_CASE__ )
						return max(SCREAMING_SNAKE_CASE__ )





def 							lowerCamelCase_   (	SCREAMING_SNAKE_CASE__   :      List[str],      SCREAMING_SNAKE_CASE__   :      Optional[Any] )			->						Any:
						UpperCAmelCase_      :   str      =	0
						for article in dataset:
												for paragraph in article["paragraphs"]:
																		for qa in paragraph["qas"]:
																								total += 1
																								if qa["id"] not in predictions:
																														UpperCAmelCase_      :   Union[str, Any]      =	'''Unanswered question ''' + qa['''id'''] + ''' will receive score 0.'''
																														print(SCREAMING_SNAKE_CASE__,      file=sys.stderr )
																														continue
																								UpperCAmelCase_      :   Optional[Any]      =	[x['''text'''] for x in qa['''answers''']]
																								UpperCAmelCase_      :   int      =	predictions[qa['''id''']]
																								exact_match += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE__,      SCREAMING_SNAKE_CASE__,      SCREAMING_SNAKE_CASE__ )
																								fa += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE__,      SCREAMING_SNAKE_CASE__,      SCREAMING_SNAKE_CASE__ )

						UpperCAmelCase_      :   Any      =	1_00.0 * exact_match / total
						UpperCAmelCase_      :   Optional[int]      =	1_00.0 * fa / total

						return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
					snake_case_  :							Any            =	"1.1"
					snake_case_  :							Any            =	argparse.ArgumentParser(description="Evaluation for SQuAD " + expected_version)
					parser.add_argument("dataset_file", help="Dataset file")
					parser.add_argument("prediction_file", help="Prediction File")
					snake_case_  :							int            =	parser.parse_args()
					with open(args.dataset_file) as dataset_file:
										snake_case_  :							List[Any]            =	json.load(dataset_file)
										if dataset_json["version"] != expected_version:
															print(
															    "Evaluation expects v-" + expected_version + ", but got dataset with v-" + dataset_json["version"],
															    file=sys.stderr,
															)
										snake_case_  :							Tuple            =	dataset_json["data"]
					with open(args.prediction_file) as prediction_file:
										snake_case_  :							Tuple            =	json.load(prediction_file)
					print(json.dumps(evaluate(dataset, predictions)))





