


import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_A		       =						16
_A		       =						32

def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Accelerator			,     SCREAMING_SNAKE_CASE__			:					DatasetDict			,     SCREAMING_SNAKE_CASE__			:					List[int]			,     SCREAMING_SNAKE_CASE__			:					List[int]			,     SCREAMING_SNAKE_CASE__			:					int = 16				):
     __UpperCamelCase    							=AutoTokenizer.from_pretrained('bert-base-cased'				)
     __UpperCamelCase    							=DatasetDict(
         {
             'train': dataset['train'].select(SCREAMING_SNAKE_CASE__				),
             'validation': dataset['train'].select(SCREAMING_SNAKE_CASE__				),
             'test': dataset['validation'],
         }				)

     def tokenize_function(SCREAMING_SNAKE_CASE__			:					Union[str, Any]				):
          # max_length=None => use the model max length (it's actually the default)
          __UpperCamelCase    							=tokenizer(examples['sentence1']			,     examples['sentence2']			,     truncation=SCREAMING_SNAKE_CASE__			,     max_length=SCREAMING_SNAKE_CASE__				)
          return outputs

     # Apply the method we just defined to all the examples in all the splits of the dataset
     # starting with the main process first:
     with accelerator.main_process_first():
          __UpperCamelCase    							=datasets.map(
              SCREAMING_SNAKE_CASE__			,     batched=SCREAMING_SNAKE_CASE__			,     remove_columns=['idx', 'sentence1', 'sentence2']			,     )

     # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
     # transformers library
     __UpperCamelCase    							=tokenized_datasets.rename_column('label'			,     'labels'				)

     def collate_fn(SCREAMING_SNAKE_CASE__			:					List[str]				):
          # On TPU it's best to pad everything to the same length or training will be very slow.
          __UpperCamelCase    							=1_28 if accelerator.distributed_type == DistributedType.TPU else None
          # When using mixed precision we want round multiples of 8/16
          if accelerator.mixed_precision == "fp8":
               __UpperCamelCase    							=16
          elif accelerator.mixed_precision != "no":
               __UpperCamelCase    							=8
          else:
               __UpperCamelCase    							=None

          return tokenizer.pad(
              SCREAMING_SNAKE_CASE__			,     padding='longest'			,     max_length=SCREAMING_SNAKE_CASE__			,     pad_to_multiple_of=SCREAMING_SNAKE_CASE__			,     return_tensors='pt'			,     )

     # Instantiate dataloaders.
     __UpperCamelCase    							=DataLoader(
         tokenized_datasets['train']			,     shuffle=SCREAMING_SNAKE_CASE__			,     collate_fn=SCREAMING_SNAKE_CASE__			,     batch_size=SCREAMING_SNAKE_CASE__				)
     __UpperCamelCase    							=DataLoader(
         tokenized_datasets['validation']			,     shuffle=SCREAMING_SNAKE_CASE__			,     collate_fn=SCREAMING_SNAKE_CASE__			,     batch_size=SCREAMING_SNAKE_CASE__				)

     __UpperCamelCase    							=DataLoader(
         tokenized_datasets['test']			,     shuffle=SCREAMING_SNAKE_CASE__			,     collate_fn=SCREAMING_SNAKE_CASE__			,     batch_size=SCREAMING_SNAKE_CASE__				)

     return train_dataloader, eval_dataloader, test_dataloader

def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Optional[Any]			,     SCREAMING_SNAKE_CASE__			:					Dict				):
     # New Code #
     __UpperCamelCase    							=[]
     # Download the dataset
     __UpperCamelCase    							=load_dataset('glue'			,     'mrpc'				)
     # Create our splits
     __UpperCamelCase    							=StratifiedKFold(n_splits=int(args.num_folds				)				)
     # Initialize accelerator
     __UpperCamelCase    							=Accelerator(cpu=args.cpu			,     mixed_precision=args.mixed_precision				)
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     __UpperCamelCase    							=config['lr']
     __UpperCamelCase    							=int(config['num_epochs']				)
     __UpperCamelCase    							=int(config['seed']				)
     __UpperCamelCase    							=int(config['batch_size']				)

     __UpperCamelCase    							=evaluate.load('glue'			,     'mrpc'				)

     # If the batch size is too big we use gradient accumulation
     __UpperCamelCase    							=1
     if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
          __UpperCamelCase    							=batch_size // MAX_GPU_BATCH_SIZE
          __UpperCamelCase    							=MAX_GPU_BATCH_SIZE

     set_seed(SCREAMING_SNAKE_CASE__				)

     # New Code #
     # Create our folds:
     __UpperCamelCase    							=kfold.split(np.zeros(datasets['train'].num_rows				)			,     datasets['train']['label']				)
     __UpperCamelCase    							=[]
     # Iterate over them
     for i, (train_idxs, valid_idxs) in enumerate(SCREAMING_SNAKE_CASE__				):
          __UpperCamelCase							,      __UpperCamelCase							,      __UpperCamelCase    							=get_fold_dataloaders(
              SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__			,     )
          # Instantiate the model (we build the model here so that the seed also control new weights initialization)
          __UpperCamelCase    							=AutoModelForSequenceClassification.from_pretrained('bert-base-cased'			,     return_dict=SCREAMING_SNAKE_CASE__				)

          # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
          # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
          # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
          __UpperCamelCase    							=model.to(accelerator.device				)

          # Instantiate optimizer
          __UpperCamelCase    							=AdamW(params=model.parameters()			,     lr=SCREAMING_SNAKE_CASE__				)

          # Instantiate scheduler
          __UpperCamelCase    							=get_linear_schedule_with_warmup(
              optimizer=SCREAMING_SNAKE_CASE__			,     num_warmup_steps=1_00			,     num_training_steps=(len(SCREAMING_SNAKE_CASE__				) * num_epochs) // gradient_accumulation_steps			,     )

          # Prepare everything
          # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
          # prepare method.
          __UpperCamelCase							,      __UpperCamelCase							,      __UpperCamelCase							,      __UpperCamelCase							,      __UpperCamelCase    							=accelerator.prepare(
              SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)

          # Now we train the model
          for epoch in range(SCREAMING_SNAKE_CASE__				):
               model.train()
               for step, batch in enumerate(SCREAMING_SNAKE_CASE__				):
                    # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device				)
                    __UpperCamelCase    							=model(**SCREAMING_SNAKE_CASE__				)
                    __UpperCamelCase    							=outputs.loss
                    __UpperCamelCase    							=loss / gradient_accumulation_steps
                    accelerator.backward(SCREAMING_SNAKE_CASE__				)
                    if step % gradient_accumulation_steps == 0:
                         optimizer.step()
                         lr_scheduler.step()
                         optimizer.zero_grad()

               model.eval()
               for step, batch in enumerate(SCREAMING_SNAKE_CASE__				):
                    # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device				)
                    with torch.no_grad():
                         __UpperCamelCase    							=model(**SCREAMING_SNAKE_CASE__				)
                    __UpperCamelCase    							=outputs.logits.argmax(dim=-1				)
                    __UpperCamelCase							,      __UpperCamelCase    							=accelerator.gather_for_metrics((predictions, batch['labels'])				)
                    metric.add_batch(
                        predictions=SCREAMING_SNAKE_CASE__			,     references=SCREAMING_SNAKE_CASE__			,     )

               __UpperCamelCase    							=metric.compute()
               # Use accelerator.print to print only on the main process.
               accelerator.print(F'epoch {epoch}:'			,     SCREAMING_SNAKE_CASE__				)

          # New Code #
          # We also run predictions on the test set at the very end
          __UpperCamelCase    							=[]
          for step, batch in enumerate(SCREAMING_SNAKE_CASE__				):
               # We could avoid this line since we set the accelerator with `device_placement=True`.
               batch.to(accelerator.device				)
               with torch.no_grad():
                    __UpperCamelCase    							=model(**SCREAMING_SNAKE_CASE__				)
               __UpperCamelCase    							=outputs.logits
               __UpperCamelCase							,      __UpperCamelCase    							=accelerator.gather_for_metrics((predictions, batch['labels'])				)
               fold_predictions.append(predictions.cpu()				)
               if i == 0:
                    # We need all of the test predictions
                    test_references.append(references.cpu()				)
        # Use accelerator.print to print only on the main process.
          test_predictions.append(torch.cat(SCREAMING_SNAKE_CASE__			,     dim=0				)				)
          # We now need to release all our memory and get rid of the current model, optimizer, etc
          accelerator.free_memory()
     # New Code #
     # Finally we check the accuracy of our folded results:
     __UpperCamelCase    							=torch.cat(SCREAMING_SNAKE_CASE__			,     dim=0				)
     __UpperCamelCase    							=torch.stack(SCREAMING_SNAKE_CASE__			,     dim=0				).sum(dim=0				).div(int(args.num_folds				)				).argmax(dim=-1				)
     __UpperCamelCase    							=metric.compute(predictions=SCREAMING_SNAKE_CASE__			,     references=SCREAMING_SNAKE_CASE__				)
     accelerator.print('Average test metrics from all folds:'			,     SCREAMING_SNAKE_CASE__				)

def 						_UpperCAmelCase				(			):
     __UpperCamelCase    							=argparse.ArgumentParser(description='Simple example of training script.'				)
     parser.add_argument(
         '--mixed_precision'			,     type=SCREAMING_SNAKE_CASE__			,     default=SCREAMING_SNAKE_CASE__			,     choices=['no', 'fp16', 'bf16', 'fp8']			,     help='Whether to use mixed precision. Choose'
         'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
         'and an Nvidia Ampere GPU.'			,     )
     parser.add_argument('--cpu'			,     action='store_true'			,     help='If passed, will train on the CPU.'				)
     # New Code #
     parser.add_argument('--num_folds'			,     type=SCREAMING_SNAKE_CASE__			,     default=3			,     help='The number of splits to perform across the dataset'				)
     __UpperCamelCase    							=parser.parse_args()
     __UpperCamelCase    							={'lr': 2E-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
     training_function(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)


if __name__ == "__main__":
   main()






'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_:     Dict  =logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:     Tuple  ={}




class        __A      ( UpperCamelCase__	):
    a__       :						int          =  """llama"""
    a__       :						Any          =  ["""past_key_values"""]



    def __init__(self    :    List[str]   ,			__a    :    List[str]=32000   ,			__a    :    Tuple=4096   ,			__a    :    List[Any]=11008   ,			__a    :    Dict=32   ,			__a    :    Tuple=32   ,			__a    :    Any=None   ,			__a    :    Any="silu"   ,			__a    :    List[Any]=2048   ,			__a    :    List[Any]=0.02   ,			__a    :    str=1E-6   ,			__a    :    Optional[Any]=True   ,			__a    :    Union[str, Any]=0   ,			__a    :    Any=1   ,			__a    :    Dict=2   ,			__a    :    Dict=1   ,			__a    :    str=False   ,			__a    :    str=None   ,			**__a    :    Optional[Any]   ,			):
         UpperCAmelCase_           =   vocab_size
         UpperCAmelCase_           =   max_position_embeddings
         UpperCAmelCase_           =   hidden_size
         UpperCAmelCase_           =   intermediate_size
         UpperCAmelCase_           =   num_hidden_layers
         UpperCAmelCase_           =   num_attention_heads

         # for backward compatibility
         if num_key_value_heads is None:
              UpperCAmelCase_           =   num_attention_heads

         UpperCAmelCase_           =   num_key_value_heads
         UpperCAmelCase_           =   hidden_act
         UpperCAmelCase_           =   initializer_range
         UpperCAmelCase_           =   rms_norm_eps
         UpperCAmelCase_           =   pretraining_tp
         UpperCAmelCase_           =   use_cache
         UpperCAmelCase_           =   rope_scaling
         self._rope_scaling_validation()

         super().__init__(
             pad_token_id=__a   ,			bos_token_id=__a   ,			eos_token_id=__a   ,			tie_word_embeddings=__a   ,			**__a   ,			)






    def 					_lowercase						(self    :    List[str] ):
         if self.rope_scaling is None:
              return

         if not isinstance(self.rope_scaling   ,			__a ) or len(self.rope_scaling ) != 2:
              raise ValueError(
                  "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
                  f"""got {self.rope_scaling}""" )
         UpperCAmelCase_           =   self.rope_scaling.get("type"   ,			__a )
         UpperCAmelCase_           =   self.rope_scaling.get("factor"   ,			__a )
         if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
              raise ValueError(
                  f"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}""" )
         if rope_scaling_factor is None or not isinstance(__a   ,			__a ) or rope_scaling_factor <= 1.0:
              raise ValueError(f"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}""" )




