



import datasets

from .evaluate import evaluate


_snake_case		=  "\\n@article{hendrycks2021cuad,\n      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},\n      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},\n      journal={arXiv preprint arXiv:2103.06268},\n      year={2021}\n}\n"

_snake_case		=  "\nThis metric wrap the official scoring script for version 1 of the Contract\nUnderstanding Atticus Dataset (CUAD).\nContract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510\ncommercial legal contracts that have been manually labeled to identify 41 categories of important\nclauses that lawyers look for when reviewing contracts in connection with corporate transactions.\n"

_snake_case		=  "\nComputes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair as given in the references (see below)\n        - 'prediction_text': list of possible texts for the answer, as a list of strings\n        depending on a threshold on the confidence probability of each prediction.\n    references: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair (see above),\n        - 'answers': a Dict in the CUAD dataset format\n            {\n                'text': list of possible texts for the answer, as a list of strings\n                'answer_start': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    'exact_match': Exact match (the normalized answer exactly match the gold answer)\n    'f1': The F-score of predicted tokens versus the gold answer\n    'aupr': Area Under the Precision-Recall curve\n    'prec_at_80_recall': Precision at 80% recall\n    'prec_at_90_recall': Precision at 90% recall\nExamples:\n    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]\n    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]\n    >>> cuad_metric = datasets.load_metric(\"cuad\")\n    >>> results = cuad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}\n"



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,					_KWARGS_DESCRIPTION)
class   UpperCAmelCase_    (       datasets.Metric):







							def    snake_case__      (   self):

														'''simple docstring'''

														return datasets.MetricInfo(
														    description=_DESCRIPTION,			citation=_CITATION,			inputs_description=_KWARGS_DESCRIPTION,			features=datasets.Features(
														        {
														            "predictions": {
														                "id": datasets.Value("string"),
														                "prediction_text": datasets.features.Sequence(datasets.Value("string")),
														            },
														            "references": {
														                "id": datasets.Value("string"),
														                "answers": datasets.features.Sequence(
														                    {
														                        "text": datasets.Value("string"),
														                        "answer_start": datasets.Value("int32"),
														                    }),
														            },
														        }),			codebase_urls=["https://www.atticusprojectai.org/cuad"],			reference_urls=["https://www.atticusprojectai.org/cuad"],			)







							def    snake_case__      (   self,			__a,			__a):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
														_lowerCAmelCase     :							Optional[Any]    =    [
														    {
														        "paragraphs": [
														            {
														                "qas": [
														                    {
														                        "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
														                        "id": ref["id"],
														                    }
														                    for ref in references
														                ]
														            }
														        ]
														    }
														]
														_lowerCAmelCase     :							int    =    evaluate(dataset=__a,			predictions=__a)
														return score





from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_torch_dist_unique_port,
    require_torch_multi_gpu,
    require_torch_neuroncore,
)
from transformers.training_args import ParallelMode
from transformers.utils import logging


_snake_case		=  logging.get_logger(__name__)


if is_torch_available():
						import torch
						from torch import nn
						from torch.utils.data import Dataset

						from transformers import Trainer



						class   UpperCAmelCase_    (       a):







													def __init__(   self,			__a = 101):

																				'''simple docstring'''

																				_lowerCAmelCase     :							str    =    length







													def __len__(   self):

																				'''simple docstring'''

																				return self.length







													def __getitem__(   self,			__a):

																				'''simple docstring'''

																				return i



						class   UpperCAmelCase_    :







													def __call__(   self,			__a):

																				'''simple docstring'''

																				return {"input_ids": torch.tensor(__a), "labels": torch.tensor(__a)}



						class   UpperCAmelCase_    (       nn.Module):







													def __init__(   self):

																				'''simple docstring'''

																				super().__init__()
																				# Add some (unused) params otherwise DDP will complain.
																				_lowerCAmelCase     :							str    =    nn.Linear(120,			80)







													def    snake_case__      (   self,			__a,			__a=None):

																				'''simple docstring'''

																				if labels is not None:
																											return torch.tensor(0.0,			device=input_ids.device), input_ids
																				else:
																											return input_ids



class   UpperCAmelCase_    (       a):







							@require_torch_neuroncore
							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    f"--nproc_per_node=2\n            --master_port={get_torch_dist_unique_port()}\n            {self.test_file_dir}/test_trainer_distributed.py\n        ".split()
														_lowerCAmelCase     :							Tuple    =    self.get_auto_remove_tmp_dir()
														_lowerCAmelCase     :							Optional[int]    =    f"--output_dir {output_dir}".split()
														_lowerCAmelCase     :							List[Any]    =    ["torchrun"] + distributed_args + args
														execute_subprocess_async(__a,			env=self.get_env())
														# successful return here == success - any errors would have caused an error in the sub-call



class   UpperCAmelCase_    (       a):







							@require_torch_multi_gpu
							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    f"--nproc_per_node={torch.cuda.device_count()}\n            --master_port={get_torch_dist_unique_port()}\n            {self.test_file_dir}/test_trainer_distributed.py\n        ".split()
														_lowerCAmelCase     :							Any    =    self.get_auto_remove_tmp_dir()
														_lowerCAmelCase     :							Optional[int]    =    f"--output_dir {output_dir}".split()
														_lowerCAmelCase     :							Any    =    ["torchrun"] + distributed_args + args
														execute_subprocess_async(__a,			env=self.get_env())
														# successful return here == success - any errors would have caused an error in the sub-call


if __name__ == "__main__":
						# The script below is meant to be run under torch.distributed, on a machine with multiple GPUs:
						#
						# PYTHONPATH="src" python -m torch.distributed.run --nproc_per_node 2 --output_dir output_dir ./tests/test_trainer_distributed.py

						_snake_case		=  HfArgumentParser((TrainingArguments,))
						_snake_case		=  parser.parse_args_into_dataclasses()[0]

						logger.warning(
						    f'''Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, '''
						    f'''distributed training: {training_args.parallel_mode != ParallelMode.NOT_DISTRIBUTED}'''
						)

						# Essentially, what we want to verify in the distributed case is that we get all samples back,
						# in the right order. (this is crucial for prediction for instance)
						for dataset_length in [101, 40, 7]:
												_snake_case		=  DummyDataset(dataset_length)






												def 				A				(					_lowerCamelCase  ):





																			'''simple docstring'''




																			_lowerCAmelCase     :							Dict    =    list(range(len(_lowerCamelCase  )  )  )
																			_lowerCAmelCase     :							Union[str, Any]    =    p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
																			if not success and training_args.local_rank == 0:
																										logger.warning(
																										    "Predictions and/or labels do not match expected results:\n  - predictions: "
																										    F"{p.predictions.tolist()}\n  - labels: {p.label_ids.tolist()}\n  - expected: {sequential}"  )
																			return {"success": success}

												_snake_case		=  Trainer(
												    model=DummyModel(),
												    args=training_args,
												    data_collator=DummyDataCollator(),
												    eval_dataset=dataset,
												    compute_metrics=compute_metrics,
												)
												_snake_case		=  trainer.evaluate()
												logger.info(metrics)
												if metrics["eval_success"] is not True:
																		logger.error(metrics)
																		exit(1)

												_snake_case		=  trainer.predict(dataset)
												logger.info(p.metrics)
												if p.metrics["test_success"] is not True:
																		logger.error(p.metrics)
																		exit(1)

												_snake_case		=  2

												_snake_case		=  trainer.evaluate()
												logger.info(metrics)
												if metrics["eval_success"] is not True:
																		logger.error(metrics)
																		exit(1)

												_snake_case		=  trainer.predict(dataset)
												logger.info(p.metrics)
												if p.metrics["test_success"] is not True:
																		logger.error(p.metrics)
																		exit(1)



												_snake_case		=  None
