




import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

UpperCAmelCase_   							=      32




def    lowerCamelCase__			(		A__			:   Accelerator    ,     A__			:   int = 16      ):





       '''simple docstring'''



       __lowerCamelCase  =		AutoTokenizer.from_pretrained("""bert-base-cased"""      )
       __lowerCamelCase  =		load_dataset("""glue"""    ,     """mrpc"""      )

       def tokenize_function(A__			:   List[str]      ):
              # max_length=None => use the model max length (it's actually the default)
              __lowerCamelCase  =		tokenizer(examples["""sentence1"""]    ,     examples["""sentence2"""]    ,     truncation=A__    ,     max_length=A__      )
              return outputs

       # Apply the method we just defined to all the examples in all the splits of the dataset
       # starting with the main process first:
       with accelerator.main_process_first():
              __lowerCamelCase  =		datasets.map(
                  A__    ,     batched=A__    ,     remove_columns=["""idx""", """sentence1""", """sentence2"""]    ,     )

       # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
       # transformers library
       __lowerCamelCase  =		tokenized_datasets.rename_column("""label"""    ,     """labels"""      )

       def collate_fn(A__			:   Any      ):
              # When using mixed precision we want round multiples of 8/16
              if accelerator.mixed_precision == "fp8":
                     __lowerCamelCase  =		16
              elif accelerator.mixed_precision != "no":
                     __lowerCamelCase  =		8
              else:
                     __lowerCamelCase  =		None

              return tokenizer.pad(
                  A__    ,     padding="""longest"""    ,     pad_to_multiple_of=A__    ,     return_tensors="""pt"""    ,     )

       # Instantiate dataloaders.
       __lowerCamelCase  =		DataLoader(
           tokenized_datasets["""train"""]    ,     shuffle=A__    ,     collate_fn=A__    ,     batch_size=A__      )
       __lowerCamelCase  =		DataLoader(
           tokenized_datasets["""validation"""]    ,     shuffle=A__    ,     collate_fn=A__    ,     batch_size=A__      )

       return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
       from accelerate.test_utils.training import mocked_dataloaders

       UpperCAmelCase_   							=      mocked_dataloaders  # noqa: F811




def    lowerCamelCase__			(		A__			:   str    ,     A__			:   List[Any]      ):





       '''simple docstring'''



       if os.environ.get("""TESTING_MOCKED_DATALOADERS"""    ,     A__      ) == "1":
              __lowerCamelCase  =		2
       # Initialize accelerator
       __lowerCamelCase  =		Accelerator(cpu=args.cpu    ,     mixed_precision=args.mixed_precision      )
       # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
       __lowerCamelCase  =		config["""lr"""]
       __lowerCamelCase  =		int(config["""num_epochs"""]      )
       __lowerCamelCase  =		int(config["""seed"""]      )
       __lowerCamelCase  =		int(config["""batch_size"""]      )

       __lowerCamelCase  =		evaluate.load("""glue"""    ,     """mrpc"""      )

       # New Code #
       # We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
       # to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
       # half each time. From this, we can calculate the number of gradient accumulation steps needed
       # and modify the Accelerator object as a result
       @find_executable_batch_size(starting_batch_size=int(A__      )      )
       def inner_training_loop(A__			:   List[str]      ):
              # Since we need to modify the outside accelerator object, we need to bring it
              # to the local scope
              nonlocal accelerator

              # We can calculate the number of gradient accumulation steps based on the current
              # batch size vs the starting batch size
              __lowerCamelCase  =		observed_batch_size // batch_size

              # And then set it in the Accelerator directly:
              __lowerCamelCase  =		num_gradient_accumulation_steps

              # Next we need to free all of the stored model references in the Accelerator each time
              accelerator.free_memory()

              # And set the seed so our results are reproducable each reset
              set_seed(A__      )

              # Instantiate the model (we build the model here so that the seed also control new weights initialization)
              __lowerCamelCase  =		AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""    ,     return_dict=A__      )

              # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
              # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
              # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
              __lowerCamelCase  =		model.to(accelerator.device      )

              # Instantiate optimizer
              __lowerCamelCase  =		AdamW(params=model.parameters()    ,     lr=A__      )
              __lowerCamelCase,  __lowerCamelCase  =		get_dataloaders(A__    ,     A__      )

              # Instantiate scheduler
              __lowerCamelCase  =		get_linear_schedule_with_warmup(
                  optimizer=A__    ,     num_warmup_steps=100    ,     num_training_steps=(len(A__      ) * num_epochs)    ,     )

              # Prepare everything
              # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
              # prepare method.
              __lowerCamelCase,  __lowerCamelCase,  __lowerCamelCase,  __lowerCamelCase,  __lowerCamelCase  =		accelerator.prepare(
                  A__    ,     A__    ,     A__    ,     A__    ,     A__      )

              # Now we train the model
              for epoch in range(A__      ):
                     model.train()
                     for step, batch in enumerate(A__      ):
                            # And perform gradient accumulation
                            with accelerator.accumulate(A__      ):
                                   # We could avoid this line since we set the accelerator with `device_placement=True`.
                                   batch.to(accelerator.device      )
                                   __lowerCamelCase  =		model(**A__      )
                                   __lowerCamelCase  =		outputs.loss
                                   accelerator.backward(A__      )
                                   optimizer.step()
                                   lr_scheduler.step()
                                   optimizer.zero_grad()

                     model.eval()
                     for step, batch in enumerate(A__      ):
                            # We could avoid this line since we set the accelerator with `device_placement=True`.
                            batch.to(accelerator.device      )
                            with torch.no_grad():
                                   __lowerCamelCase  =		model(**A__      )
                            __lowerCamelCase  =		outputs.logits.argmax(dim=-1      )
                            __lowerCamelCase,  __lowerCamelCase  =		accelerator.gather_for_metrics((predictions, batch["""labels"""])      )
                            metric.add_batch(
                                predictions=A__    ,     references=A__    ,     )

                     __lowerCamelCase  =		metric.compute()
                     # Use accelerator.print to print only on the main process.
                     accelerator.print(f'epoch {epoch}:'    ,     A__      )

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
       inner_training_loop()




def    lowerCamelCase__			(		):





       '''simple docstring'''



       __lowerCamelCase  =		argparse.ArgumentParser(description="""Simple example of training script."""      )
       parser.add_argument(
           """--mixed_precision"""    ,     type=A__    ,     default=A__    ,     choices=["""no""", """fp16""", """bf16""", """fp8"""]    ,     help="""Whether to use mixed precision. Choose"""
           """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
           """and an Nvidia Ampere GPU."""    ,     )
       parser.add_argument("""--cpu"""    ,     action="""store_true"""    ,     help="""If passed, will train on the CPU."""      )
       __lowerCamelCase  =		parser.parse_args()
       # New Code #
       # We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
       __lowerCamelCase  =		{"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 256}
       training_function(A__    ,     A__      )


if __name__ == "__main__":
       main()







import argparse
import json
import re
import string
import sys

import numpy as np


UpperCAmelCase_   							=      0.5




def    lowerCamelCase__			(		A__			:   Optional[Any]    ,     A__			:   Union[str, Any]      ):





       '''simple docstring'''



       __lowerCamelCase  =		[""".""", """,""", """;""", """:"""]

       for token in remove_tokens:
              __lowerCamelCase  =		ground_truth.replace(A__    ,     """"""      )
              __lowerCamelCase  =		prediction.replace(A__    ,     """"""      )

       __lowerCamelCase,  __lowerCamelCase  =		ground_truth.lower(), prediction.lower()
       __lowerCamelCase,  __lowerCamelCase  =		ground_truth.replace("""/"""    ,     """ """      ), prediction.replace("""/"""    ,     """ """      )
       __lowerCamelCase,  __lowerCamelCase  =		set(ground_truth.split(""" """      )      ), set(prediction.split(""" """      )      )

       __lowerCamelCase  =		ground_truth.intersection(A__      )
       __lowerCamelCase  =		ground_truth.union(A__      )
       __lowerCamelCase  =		len(A__      ) / len(A__      )
       return jaccard




def    lowerCamelCase__			(		A__			:   Optional[Any]      ):





       '''simple docstring'''



       def remove_articles(A__			:   Tuple      ):
              return re.sub(R"""\b(a|an|the)\b"""    ,     """ """    ,     A__      )

       def white_space_fix(A__			:   Optional[Any]      ):
              return " ".join(text.split()      )

       def remove_punc(A__			:   Optional[Any]      ):
              __lowerCamelCase  =		set(string.punctuation      )
              return "".join(ch for ch in text if ch not in exclude      )

       def lower(A__			:   Tuple      ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(A__      )      )      )      )




def    lowerCamelCase__			(		A__			:   Any    ,     A__			:   Optional[Any]    ,     A__			:   Optional[Any]      ):





       '''simple docstring'''



       __lowerCamelCase,  __lowerCamelCase,  __lowerCamelCase  =		0, 0, 0

       __lowerCamelCase  =		"""Parties""" in qa_id

       # first check if ground truth is empty
       if len(A__      ) == 0:
              if len(A__      ) > 0:
                     fp += len(A__      )  # false positive for each one
       else:
              for ground_truth in ground_truths:
                     assert len(A__      ) > 0
                     # check if there is a match
                     __lowerCamelCase  =		False
                     for pred in predictions:
                            if substr_ok:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH or ground_truth in pred
                            else:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH
                            if is_match:
                                   __lowerCamelCase  =		True

                     if match_found:
                            tp += 1
                     else:
                            fn += 1

        # now also get any fps by looping through preds
              for pred in predictions:
                     # Check if there's a match. if so, don't count (don't want to double count based on the above)
                     # but if there's no match, then this is a false positive.
                     # (Note: we get the true positives in the above loop instead of this loop so that we don't double count
                     # multiple predictions that are matched with the same answer.)
                     __lowerCamelCase  =		False
                     for ground_truth in ground_truths:
                            assert len(A__      ) > 0
                            if substr_ok:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH or ground_truth in pred
                            else:
                                   __lowerCamelCase  =		get_jaccard(A__    ,     A__      ) >= IOU_THRESH
                            if is_match:
                                   __lowerCamelCase  =		True

                     if not match_found:
                            fp += 1

       __lowerCamelCase  =		tp / (tp + fp) if tp + fp > 0 else np.nan
       __lowerCamelCase  =		tp / (tp + fn) if tp + fn > 0 else np.nan

       return precision, recall




def    lowerCamelCase__			(		A__			:   int      ):





       '''simple docstring'''



       __lowerCamelCase  =		precisions[::-1]
       for i in range(1    ,     len(A__      )      ):
              __lowerCamelCase  =		max(precision_best[i - 1]    ,     precision_best[i]      )
       __lowerCamelCase  =		precision_best[::-1]
       return precisions




def    lowerCamelCase__			(		A__			:   str    ,     A__			:   Union[str, Any]      ):





       '''simple docstring'''



       __lowerCamelCase  =		process_precisions(A__      )
       __lowerCamelCase  =		np.trapz(A__    ,     A__      )
       if np.isnan(A__      ):
              return 0
       return aupr




def    lowerCamelCase__			(		A__			:   Optional[Any]    ,     A__			:   List[str]    ,     A__			:   int      ):





       '''simple docstring'''



       __lowerCamelCase  =		process_precisions(A__      )
       __lowerCamelCase  =		0
       for prec, recall in zip(A__    ,     A__      ):
              if recall >= recall_thresh:
                     __lowerCamelCase  =		prec
                     break
       return prec_at_recall




def    lowerCamelCase__			(		A__			:   Optional[Any]    ,     A__			:   List[Any]      ):





       '''simple docstring'''



       return normalize_answer(A__      ) == normalize_answer(A__      )




def    lowerCamelCase__			(		A__			:   Tuple    ,     A__			:   int    ,     A__			:   Dict      ):





       '''simple docstring'''



       __lowerCamelCase  =		0
       for pred in predictions:
              for ground_truth in ground_truths:
                     __lowerCamelCase  =		metric_fn(A__    ,     A__      )
                     if score == 1:  # break the loop when one prediction matches the ground truth
                            break
              if score == 1:
                     break
       return score




def    lowerCamelCase__			(		A__			:   Union[str, Any]    ,     A__			:   Optional[Any]      ):





       '''simple docstring'''



       __lowerCamelCase  =		__lowerCamelCase  =		__lowerCamelCase  =		0
       __lowerCamelCase  =		[]
       __lowerCamelCase  =		[]
       for article in dataset:
              for paragraph in article["paragraphs"]:
                     for qa in paragraph["qas"]:
                            total += 1
                            if qa["id"] not in predictions:
                                   __lowerCamelCase  =		"""Unanswered question """ + qa["""id"""] + """ will receive score 0."""
                                   print(A__    ,     file=sys.stderr      )
                                   continue
                            __lowerCamelCase  =		[x["""text"""] for x in qa["""answers"""]]
                            __lowerCamelCase  =		predictions[qa["""id"""]]
                            __lowerCamelCase,  __lowerCamelCase  =		compute_precision_recall(A__    ,     A__    ,     qa["""id"""]      )

                            precisions.append(A__      )
                            recalls.append(A__      )

                            if precision == 0 and recall == 0:
                                   fa += 0
                            else:
                                   fa += 2 * (precision * recall) / (precision + recall)

                            exact_match += metric_max_over_ground_truths(A__    ,     A__    ,     A__      )

       __lowerCamelCase  =		[x for _, x in sorted(zip(A__    ,     A__      )      )]
       recalls.sort()

       __lowerCamelCase  =		100.0 * fa / total
       __lowerCamelCase  =		100.0 * exact_match / total
       __lowerCamelCase  =		get_aupr(A__    ,     A__      )

       __lowerCamelCase  =		get_prec_at_recall(A__    ,     A__    ,     recall_thresh=0.9      )
       __lowerCamelCase  =		get_prec_at_recall(A__    ,     A__    ,     recall_thresh=0.8      )

       return {
           "exact_match": exact_match,
           "f1": fa,
           "aupr": aupr,
           "prec_at_80_recall": prec_at_aa_recall,
           "prec_at_90_recall": prec_at_aa_recall,
       }


if __name__ == "__main__":
       UpperCAmelCase_   							=      argparse.ArgumentParser(description='Evaluation for CUAD')
       parser.add_argument('dataset_file', help='Dataset file')
       parser.add_argument('prediction_file', help='Prediction File')
       UpperCAmelCase_   							=      parser.parse_args()
       with open(args.dataset_file) as dataset_file:
              UpperCAmelCase_   							=      json.load(dataset_file)
              UpperCAmelCase_   							=      dataset_json['data']
       with open(args.prediction_file) as prediction_file:
              UpperCAmelCase_   							=      json.load(prediction_file)
       print(json.dumps(evaluate(dataset, predictions)))

