



"""simple docstring"""






import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

lowerCamelCase				:     Dict								=  3_2


def 							A__ (     UpperCamelCase__       ,							UpperCamelCase__ = 16    ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			AutoTokenizer.from_pretrained('''bert-base-cased'''    )
  _SCREAMING_SNAKE_CASE        =			load_dataset('''glue'''       ,							'''mrpc'''    )

  def tokenize_function(UpperCamelCase__    ):
    # max_length=None => use the model max length (it's actually the default)
    _SCREAMING_SNAKE_CASE        =			tokenizer(examples['''sentence1''']       ,							examples['''sentence2''']       ,							truncation=UpperCamelCase__       ,							max_length=UpperCamelCase__    )
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    _SCREAMING_SNAKE_CASE        =			datasets.map(
        UpperCamelCase__       ,							batched=UpperCamelCase__       ,							remove_columns=['''idx''', '''sentence1''', '''sentence2''']       ,							)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  _SCREAMING_SNAKE_CASE        =			tokenized_datasets.rename_column('''label'''       ,							'''labels'''    )

  def collate_fn(UpperCamelCase__    ):
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      _SCREAMING_SNAKE_CASE        =			16
    elif accelerator.mixed_precision != "no":
      _SCREAMING_SNAKE_CASE        =			8
    else:
      _SCREAMING_SNAKE_CASE        =			None

    return tokenizer.pad(
        UpperCamelCase__       ,							padding='''longest'''       ,							pad_to_multiple_of=UpperCamelCase__       ,							return_tensors='''pt'''       ,							)

  # Instantiate dataloaders.
  _SCREAMING_SNAKE_CASE        =			DataLoader(
      tokenized_datasets['''train''']       ,							shuffle=UpperCamelCase__       ,							collate_fn=UpperCamelCase__       ,							batch_size=UpperCamelCase__    )
  _SCREAMING_SNAKE_CASE        =			DataLoader(
      tokenized_datasets['''validation''']       ,							shuffle=UpperCamelCase__       ,							collate_fn=UpperCamelCase__       ,							batch_size=UpperCamelCase__    )

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
    from accelerate.test_utils.training import mocked_dataloaders

    lowerCamelCase				:     Tuple								=  mocked_dataloaders  # noqa: F811


def 							A__ (     UpperCamelCase__       ,							UpperCamelCase__    ):


  '''simple docstring'''


  if os.environ.get('''TESTING_MOCKED_DATALOADERS'''       ,							UpperCamelCase__    ) == "1":
    _SCREAMING_SNAKE_CASE        =			2
  # Initialize accelerator
  _SCREAMING_SNAKE_CASE        =			Accelerator(cpu=args.cpu       ,							mixed_precision=args.mixed_precision    )
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  _SCREAMING_SNAKE_CASE        =			config['''lr''']
  _SCREAMING_SNAKE_CASE        =			int(config['''num_epochs''']    )
  _SCREAMING_SNAKE_CASE        =			int(config['''seed''']    )
  _SCREAMING_SNAKE_CASE        =			int(config['''batch_size''']    )

  _SCREAMING_SNAKE_CASE        =			evaluate.load('''glue'''       ,							'''mrpc'''    )

  # New Code #
  # We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
  # to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
  # half each time. From this, we can calculate the number of gradient accumulation steps needed
  # and modify the Accelerator object as a result
  @find_executable_batch_size(starting_batch_size=int(UpperCamelCase__    )    )
  def inner_training_loop(UpperCamelCase__    ):
    # Since we need to modify the outside accelerator object, we need to bring it
    # to the local scope
    nonlocal accelerator

    # We can calculate the number of gradient accumulation steps based on the current
    # batch size vs the starting batch size
    _SCREAMING_SNAKE_CASE        =			observed_batch_size // batch_size

    # And then set it in the Accelerator directly:
    _SCREAMING_SNAKE_CASE        =			num_gradient_accumulation_steps

    # Next we need to free all of the stored model references in the Accelerator each time
    accelerator.free_memory()

    # And set the seed so our results are reproducable each reset
    set_seed(UpperCamelCase__    )

    # Instantiate the model (we build the model here so that the seed also control new weights initialization)
    _SCREAMING_SNAKE_CASE        =			AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''       ,							return_dict=UpperCamelCase__    )

    # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
    # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
    # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
    _SCREAMING_SNAKE_CASE        =			model.to(accelerator.device    )

    # Instantiate optimizer
    _SCREAMING_SNAKE_CASE        =			AdamW(params=model.parameters()       ,							lr=UpperCamelCase__    )
    _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			get_dataloaders(UpperCamelCase__       ,							UpperCamelCase__    )

    # Instantiate scheduler
    _SCREAMING_SNAKE_CASE        =			get_linear_schedule_with_warmup(
        optimizer=UpperCamelCase__       ,							num_warmup_steps=100       ,							num_training_steps=(len(UpperCamelCase__    ) * num_epochs)       ,							)

    # Prepare everything
    # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
    # prepare method.
    _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			accelerator.prepare(
        UpperCamelCase__       ,							UpperCamelCase__       ,							UpperCamelCase__       ,							UpperCamelCase__       ,							UpperCamelCase__    )

    # Now we train the model
    for epoch in range(UpperCamelCase__    ):
      model.train()
      for step, batch in enumerate(UpperCamelCase__    ):
        # And perform gradient accumulation
        with accelerator.accumulate(UpperCamelCase__    ):
          # We could avoid this line since we set the accelerator with `device_placement=True`.
          batch.to(accelerator.device    )
          _SCREAMING_SNAKE_CASE        =			model(**UpperCamelCase__    )
          _SCREAMING_SNAKE_CASE        =			outputs.loss
          accelerator.backward(UpperCamelCase__    )
          optimizer.step()
          lr_scheduler.step()
          optimizer.zero_grad()

      model.eval()
      for step, batch in enumerate(UpperCamelCase__    ):
        # We could avoid this line since we set the accelerator with `device_placement=True`.
        batch.to(accelerator.device    )
        with torch.no_grad():
          _SCREAMING_SNAKE_CASE        =			model(**UpperCamelCase__    )
        _SCREAMING_SNAKE_CASE        =			outputs.logits.argmax(dim=-1    )
        _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			accelerator.gather_for_metrics((predictions, batch['''labels'''])    )
        metric.add_batch(
            predictions=UpperCamelCase__       ,							references=UpperCamelCase__       ,							)

      _SCREAMING_SNAKE_CASE        =			metric.compute()
      # Use accelerator.print to print only on the main process.
      accelerator.print(F'''epoch {epoch}:'''       ,							UpperCamelCase__    )

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
  inner_training_loop()


def 							A__ (     ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			argparse.ArgumentParser(description='''Simple example of training script.'''    )
  parser.add_argument(
      '''--mixed_precision'''       ,							type=UpperCamelCase__       ,							default=UpperCamelCase__       ,							choices=['''no''', '''fp16''', '''bf16''', '''fp8''']       ,							help='''Whether to use mixed precision. Choose'''
      '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
      '''and an Nvidia Ampere GPU.'''       ,							)
  parser.add_argument('''--cpu'''       ,							action='''store_true'''       ,							help='''If passed, will train on the CPU.'''    )
  _SCREAMING_SNAKE_CASE        =			parser.parse_args()
  # New Code #
  # We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
  _SCREAMING_SNAKE_CASE        =			{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 256}
  training_function(UpperCamelCase__       ,							UpperCamelCase__    )


if __name__ == "__main__":
    main()





"""simple docstring"""






from transformers import ProcessorMixin




class    __snake_case(					__A       ):
    _A       = '''AutoFeatureExtractor'''
    _A       = '''AutoTokenizer'''
