






import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

a 			=							32
def    a__    (	A		,					A = 16       ):
   lowercase__    :						List[Any]       =			AutoTokenizer.from_pretrained("""bert-base-cased"""       )
   lowercase__    :						Union[str, Any]       =			load_dataset("""glue"""		,					"""mrpc"""       )

   def tokenize_function(A       ):
      # max_length=None => use the model max length (it's actually the default)
      lowercase__    :						Any       =			tokenizer(examples["""sentence1"""]		,					examples["""sentence2"""]		,					truncation=A		,					max_length=A       )
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      lowercase__    :						str       =			datasets.map(
          A		,					batched=A		,					remove_columns=["""idx""", """sentence1""", """sentence2"""]		,					)

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   lowercase__    :						List[Any]       =			tokenized_datasets.rename_column("""label"""		,					"""labels"""       )

   def collate_fn(A       ):
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         lowercase__    :						Optional[int]       =			16
      elif accelerator.mixed_precision != "no":
         lowercase__    :						Any       =			8
      else:
         lowercase__    :						Any       =			None

      return tokenizer.pad(
          A		,					padding="""longest"""		,					pad_to_multiple_of=A		,					return_tensors="""pt"""		,					)

   # Instantiate dataloaders.
   lowercase__    :						Dict       =			DataLoader(
       tokenized_datasets["""train"""]		,					shuffle=A		,					collate_fn=A		,					batch_size=A       )
   lowercase__    :						Union[str, Any]       =			DataLoader(
       tokenized_datasets["""validation"""]		,					shuffle=A		,					collate_fn=A		,					batch_size=A       )

   return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
 from accelerate.test_utils.training import mocked_dataloaders

 a 			=							mocked_dataloaders  # noqa: F811
def    a__    (	A		,					A       ):
   # For testing only
   if os.environ.get("""TESTING_MOCKED_DATALOADERS"""		,					A       ) == "1":
      lowercase__    :						Union[str, Any]       =			2
   # Initialize accelerator
   lowercase__    :						int       =			Accelerator(cpu=args.cpu		,					mixed_precision=args.mixed_precision       )
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   lowercase__    :						int       =			config["""lr"""]
   lowercase__    :						Optional[Any]       =			int(config["""num_epochs"""]       )
   lowercase__    :						str       =			int(config["""seed"""]       )
   lowercase__    :						str       =			int(config["""batch_size"""]       )

   lowercase__    :						Union[str, Any]       =			evaluate.load("""glue"""		,					"""mrpc"""       )

   # New Code #
   # We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
   # to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
   # half each time. From this, we can calculate the number of gradient accumulation steps needed
   # and modify the Accelerator object as a result
   @find_executable_batch_size(starting_batch_size=int(A       )       )
   def inner_training_loop(A       ):
      # Since we need to modify the outside accelerator object, we need to bring it
      # to the local scope
      nonlocal accelerator

      # We can calculate the number of gradient accumulation steps based on the current
      # batch size vs the starting batch size
      lowercase__    :						Tuple       =			observed_batch_size // batch_size

      # And then set it in the Accelerator directly:
      lowercase__    :						Any       =			num_gradient_accumulation_steps

      # Next we need to free all of the stored model references in the Accelerator each time
      accelerator.free_memory()

      # And set the seed so our results are reproducable each reset
      set_seed(A       )

      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      lowercase__    :						str       =			AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""		,					return_dict=A       )

      # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
      # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
      # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
      lowercase__    :						List[Any]       =			model.to(accelerator.device       )

      # Instantiate optimizer
      lowercase__    :						List[Any]       =			AdamW(params=model.parameters()		,					lr=A       )
      lowercase__  ,       lowercase__    :						List[str]       =			get_dataloaders(A		,					A       )

      # Instantiate scheduler
      lowercase__    :						Any       =			get_linear_schedule_with_warmup(
          optimizer=A		,					num_warmup_steps=100		,					num_training_steps=(len(A       ) * num_epochs)		,					)

      # Prepare everything
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__  ,       lowercase__    :						List[Any]       =			accelerator.prepare(
          A		,					A		,					A		,					A		,					A       )

      # Now we train the model
      for epoch in range(A       ):
         model.train()
         for step, batch in enumerate(A       ):
            # And perform gradient accumulation
            with accelerator.accumulate(A       ):
               # We could avoid this line since we set the accelerator with `device_placement=True`.
               batch.to(accelerator.device       )
               lowercase__    :						int       =			model(**A       )
               lowercase__    :						Optional[int]       =			outputs.loss
               accelerator.backward(A       )
               optimizer.step()
               lr_scheduler.step()
               optimizer.zero_grad()

         model.eval()
         for step, batch in enumerate(A       ):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device       )
            with torch.no_grad():
               lowercase__    :						Optional[Any]       =			model(**A       )
            lowercase__    :						Tuple       =			outputs.logits.argmax(dim=-1       )
            lowercase__  ,       lowercase__    :						Dict       =			accelerator.gather_for_metrics((predictions, batch["""labels"""])       )
            metric.add_batch(
                predictions=A		,					references=A		,					)

         lowercase__    :						Dict       =			metric.compute()
         # Use accelerator.print to print only on the main process.
         accelerator.print(F'''epoch {epoch}:'''		,					A       )

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
   inner_training_loop()
def    a__    (	):
   lowercase__    :						Optional[int]       =			argparse.ArgumentParser(description="""Simple example of training script."""       )
   parser.add_argument(
       """--mixed_precision"""		,					type=A		,					default=A		,					choices=["""no""", """fp16""", """bf16""", """fp8"""]		,					help="""Whether to use mixed precision. Choose"""
       """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
       """and an Nvidia Ampere GPU."""		,					)
   parser.add_argument("""--cpu"""		,					action="""store_true"""		,					help="""If passed, will train on the CPU."""       )
   lowercase__    :						Optional[Any]       =			parser.parse_args()
   # New Code #
   # We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
   lowercase__    :						str       =			{"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 256}
   training_function(A		,					A       )


if __name__ == "__main__":
 main()










from datetime import date

import requests

a 			=							'https://api.carbonintensity.org.uk/intensity'
def    a__    (	):
   lowercase__    :						Union[str, Any]       =			requests.get(A       ).json()["""data"""][0]
   return last_half_hour["intensity"]["actual"]
def    a__    (	A		,					A       ):
   return requests.get(F'''{BASE_URL}/{start}/{end}'''       ).json()["data"]


if __name__ == "__main__":
 for entry in fetch_from_to(start=date(2020, 10, 1), end=date(2020, 10, 3)):
  print('from {from} to {to}: {intensity[actual]}'.format(**entry))
 print(F'{fetch_last_half_hour() = }')


