


'''simple docstring'''




import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__snake_case    						=16
__snake_case    						=32





def 	a_					( lowerCamelCase	:     Accelerator	, lowerCamelCase	:     DatasetDict	, lowerCamelCase	:     List[int]	, lowerCamelCase	:     List[int]	, lowerCamelCase	:     int = 16				):
   lowerCAmelCase     =			AutoTokenizer.from_pretrained('bert-base-cased'				)
   lowerCAmelCase     =			DatasetDict(
       {
           'train': dataset['train'].select(lowerCamelCase				),
           'validation': dataset['train'].select(lowerCamelCase				),
           'test': dataset['validation'],
       }				)

   def tokenize_function(lowerCamelCase	:     str				):
      # max_length=None => use the model max length (it's actually the default)
      lowerCAmelCase     =			tokenizer(examples['sentence1']	, examples['sentence2']	, truncation=lowerCamelCase	, max_length=lowerCamelCase				)
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      lowerCAmelCase     =			datasets.map(
          lowerCamelCase	, batched=lowerCamelCase	, remove_columns=['idx', 'sentence1', 'sentence2']	, )

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   lowerCAmelCase     =			tokenized_datasets.rename_column('label'	, 'labels'				)

   def collate_fn(lowerCamelCase	:     Dict				):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      lowerCAmelCase     =			128 if accelerator.distributed_type == DistributedType.TPU else None
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         lowerCAmelCase     =			16
      elif accelerator.mixed_precision != "no":
         lowerCAmelCase     =			8
      else:
         lowerCAmelCase     =			None

      return tokenizer.pad(
          lowerCamelCase	, padding='longest'	, max_length=lowerCamelCase	, pad_to_multiple_of=lowerCamelCase	, return_tensors='pt'	, )

   # Instantiate dataloaders.
   lowerCAmelCase     =			DataLoader(
       tokenized_datasets['train']	, shuffle=lowerCamelCase	, collate_fn=lowerCamelCase	, batch_size=lowerCamelCase				)
   lowerCAmelCase     =			DataLoader(
       tokenized_datasets['validation']	, shuffle=lowerCamelCase	, collate_fn=lowerCamelCase	, batch_size=lowerCamelCase				)

   lowerCAmelCase     =			DataLoader(
       tokenized_datasets['test']	, shuffle=lowerCamelCase	, collate_fn=lowerCamelCase	, batch_size=lowerCamelCase				)

   return train_dataloader, eval_dataloader, test_dataloader





def 	a_					( lowerCamelCase	:     Tuple	, lowerCamelCase	:     List[Any]				):
   # New Code #
   lowerCAmelCase     =			[]
   # Download the dataset
   lowerCAmelCase     =			load_dataset('glue'	, 'mrpc'				)
   # Create our splits
   lowerCAmelCase     =			StratifiedKFold(n_splits=int(args.num_folds				)				)
   # Initialize accelerator
   lowerCAmelCase     =			Accelerator(cpu=args.cpu	, mixed_precision=args.mixed_precision				)
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   lowerCAmelCase     =			config['lr']
   lowerCAmelCase     =			int(config['num_epochs']				)
   lowerCAmelCase     =			int(config['seed']				)
   lowerCAmelCase     =			int(config['batch_size']				)

   lowerCAmelCase     =			evaluate.load('glue'	, 'mrpc'				)

   # If the batch size is too big we use gradient accumulation
   lowerCAmelCase     =			1
   if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
      lowerCAmelCase     =			batch_size // MAX_GPU_BATCH_SIZE
      lowerCAmelCase     =			MAX_GPU_BATCH_SIZE

   set_seed(lowerCamelCase				)

   # New Code #
   # Create our folds:
   lowerCAmelCase     =			kfold.split(np.zeros(datasets['train'].num_rows				)	, datasets['train']['label']				)
   lowerCAmelCase     =			[]
   # Iterate over them
   for i, (train_idxs, valid_idxs) in enumerate(lowerCamelCase				):
      lowerCAmelCase     ,  lowerCAmelCase     ,  lowerCAmelCase     =			get_fold_dataloaders(
          lowerCamelCase	, lowerCamelCase	, lowerCamelCase	, lowerCamelCase	, )
      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      lowerCAmelCase     =			AutoModelForSequenceClassification.from_pretrained('bert-base-cased'	, return_dict=lowerCamelCase				)

      # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
      # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
      # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
      lowerCAmelCase     =			model.to(accelerator.device				)

      # Instantiate optimizer
      lowerCAmelCase     =			AdamW(params=model.parameters()	, lr=lowerCamelCase				)

      # Instantiate scheduler
      lowerCAmelCase     =			get_linear_schedule_with_warmup(
          optimizer=lowerCamelCase	, num_warmup_steps=100	, num_training_steps=(len(lowerCamelCase				) * num_epochs) // gradient_accumulation_steps	, )

      # Prepare everything
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      lowerCAmelCase     ,  lowerCAmelCase     ,  lowerCAmelCase     ,  lowerCAmelCase     ,  lowerCAmelCase     =			accelerator.prepare(
          lowerCamelCase	, lowerCamelCase	, lowerCamelCase	, lowerCamelCase	, lowerCamelCase				)

      # Now we train the model
      for epoch in range(lowerCamelCase				):
         model.train()
         for step, batch in enumerate(lowerCamelCase				):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device				)
            lowerCAmelCase     =			model(**lowerCamelCase				)
            lowerCAmelCase     =			outputs.loss
            lowerCAmelCase     =			loss / gradient_accumulation_steps
            accelerator.backward(lowerCamelCase				)
            if step % gradient_accumulation_steps == 0:
               optimizer.step()
               lr_scheduler.step()
               optimizer.zero_grad()

         model.eval()
         for step, batch in enumerate(lowerCamelCase				):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device				)
            with torch.no_grad():
               lowerCAmelCase     =			model(**lowerCamelCase				)
            lowerCAmelCase     =			outputs.logits.argmax(dim=-1				)
            lowerCAmelCase     ,  lowerCAmelCase     =			accelerator.gather_for_metrics((predictions, batch['labels'])				)
            metric.add_batch(
                predictions=lowerCamelCase	, references=lowerCamelCase	, )

         lowerCAmelCase     =			metric.compute()
         # Use accelerator.print to print only on the main process.
         accelerator.print(f'''epoch {epoch}:'''	, lowerCamelCase				)

      # New Code #
      # We also run predictions on the test set at the very end
      lowerCAmelCase     =			[]
      for step, batch in enumerate(lowerCamelCase				):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device				)
         with torch.no_grad():
            lowerCAmelCase     =			model(**lowerCamelCase				)
         lowerCAmelCase     =			outputs.logits
         lowerCAmelCase     ,  lowerCAmelCase     =			accelerator.gather_for_metrics((predictions, batch['labels'])				)
         fold_predictions.append(predictions.cpu()				)
         if i == 0:
            # We need all of the test predictions
            test_references.append(references.cpu()				)
        # Use accelerator.print to print only on the main process.
      test_predictions.append(torch.cat(lowerCamelCase	, dim=0				)				)
      # We now need to release all our memory and get rid of the current model, optimizer, etc
      accelerator.free_memory()
   # New Code #
   # Finally we check the accuracy of our folded results:
   lowerCAmelCase     =			torch.cat(lowerCamelCase	, dim=0				)
   lowerCAmelCase     =			torch.stack(lowerCamelCase	, dim=0				).sum(dim=0				).div(int(args.num_folds				)				).argmax(dim=-1				)
   lowerCAmelCase     =			metric.compute(predictions=lowerCamelCase	, references=lowerCamelCase				)
   accelerator.print('Average test metrics from all folds:'	, lowerCamelCase				)





def 	a_					( ):
   lowerCAmelCase     =			argparse.ArgumentParser(description='Simple example of training script.'				)
   parser.add_argument(
       '--mixed_precision'	, type=lowerCamelCase	, default=lowerCamelCase	, choices=['no', 'fp16', 'bf16', 'fp8']	, help='Whether to use mixed precision. Choose'
       'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
       'and an Nvidia Ampere GPU.'	, )
   parser.add_argument('--cpu'	, action='store_true'	, help='If passed, will train on the CPU.'				)
   # New Code #
   parser.add_argument('--num_folds'	, type=lowerCamelCase	, default=3	, help='The number of splits to perform across the dataset'				)
   lowerCAmelCase     =			parser.parse_args()
   lowerCAmelCase     =			{'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
   training_function(lowerCamelCase	, lowerCamelCase				)


if __name__ == "__main__":
       main()












import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A									=				logging.get_logger(__name__)

__A									=				{
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			"unispeech"

 def __init__(self				:   Any       , UpperCAmelCase_				:   Any=32       , UpperCAmelCase_				:   List[str]=768       , UpperCAmelCase_				:   Any=12       , UpperCAmelCase_				:   Union[str, Any]=12       , UpperCAmelCase_				:   Optional[Any]=3_072       , UpperCAmelCase_				:   List[Any]="gelu"       , UpperCAmelCase_				:   int=0.1       , UpperCAmelCase_				:   Optional[int]=0.1       , UpperCAmelCase_				:   int=0.1       , UpperCAmelCase_				:   Any=0.0       , UpperCAmelCase_				:   str=0.0       , UpperCAmelCase_				:   Dict=0.1       , UpperCAmelCase_				:   Optional[int]=0.1       , UpperCAmelCase_				:   Optional[Any]=0.02       , UpperCAmelCase_				:   Union[str, Any]=1E-5       , UpperCAmelCase_				:   str="group"       , UpperCAmelCase_				:   List[Any]="gelu"       , UpperCAmelCase_				:   Tuple=(512, 512, 512, 512, 512, 512, 512)       , UpperCAmelCase_				:   str=(5, 2, 2, 2, 2, 2, 2)       , UpperCAmelCase_				:   Any=(10, 3, 3, 3, 3, 2, 2)       , UpperCAmelCase_				:   Optional[Any]=False       , UpperCAmelCase_				:   str=128       , UpperCAmelCase_				:   int=16       , UpperCAmelCase_				:   Dict=False       , UpperCAmelCase_				:   Optional[int]=True       , UpperCAmelCase_				:   Dict=0.05       , UpperCAmelCase_				:   Optional[int]=10       , UpperCAmelCase_				:   Tuple=2       , UpperCAmelCase_				:   Union[str, Any]=0.0       , UpperCAmelCase_				:   int=10       , UpperCAmelCase_				:   List[Any]=0       , UpperCAmelCase_				:   Optional[Any]=320       , UpperCAmelCase_				:   int=2       , UpperCAmelCase_				:   Union[str, Any]=0.1       , UpperCAmelCase_				:   str=100       , UpperCAmelCase_				:   Any=256       , UpperCAmelCase_				:   int=256       , UpperCAmelCase_				:   Optional[Any]=0.1       , UpperCAmelCase_				:   str="mean"       , UpperCAmelCase_				:   Union[str, Any]=False       , UpperCAmelCase_				:   List[str]=False       , UpperCAmelCase_				:   List[Any]=256       , UpperCAmelCase_				:   Optional[int]=80       , UpperCAmelCase_				:   Optional[int]=0       , UpperCAmelCase_				:   Optional[Any]=1       , UpperCAmelCase_				:   Union[str, Any]=2       , UpperCAmelCase_				:   Dict=0.5       , **UpperCAmelCase_				:   Optional[int]       , )       ->str:



   '''simple docstring'''



   super().__init__(**UpperCAmelCase_       , pad_token_id=UpperCAmelCase_       , bos_token_id=UpperCAmelCase_       , eos_token_id=UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =hidden_size
   lowerCamelCase__:      List[str]			  =feat_extract_norm
   lowerCamelCase__:      Dict			  =feat_extract_activation
   lowerCamelCase__:      Optional[Any]			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Any			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Dict			  =conv_bias
   lowerCamelCase__:      Optional[Any]			  =num_conv_pos_embeddings
   lowerCamelCase__:      Dict			  =num_conv_pos_embedding_groups
   lowerCamelCase__:      int			  =len(self.conv_dim)
   lowerCamelCase__:      Union[str, Any]			  =num_hidden_layers
   lowerCamelCase__:      Union[str, Any]			  =intermediate_size
   lowerCamelCase__:      Dict			  =hidden_act
   lowerCamelCase__:      List[Any]			  =num_attention_heads
   lowerCamelCase__:      Dict			  =hidden_dropout
   lowerCamelCase__:      Optional[Any]			  =attention_dropout
   lowerCamelCase__:      Optional[Any]			  =activation_dropout
   lowerCamelCase__:      Tuple			  =feat_proj_dropout
   lowerCamelCase__:      int			  =final_dropout
   lowerCamelCase__:      Optional[Any]			  =layerdrop
   lowerCamelCase__:      Dict			  =layer_norm_eps
   lowerCamelCase__:      Optional[Any]			  =initializer_range
   lowerCamelCase__:      int			  =num_ctc_classes
   lowerCamelCase__:      Tuple			  =vocab_size
   lowerCamelCase__:      Dict			  =do_stable_layer_norm
   lowerCamelCase__:      List[Any]			  =use_weighted_layer_sum
   lowerCamelCase__:      Dict			  =classifier_proj_size

   if (
       (len(self.conv_stride) != self.num_feat_extract_layers)
       or (len(self.conv_kernel) != self.num_feat_extract_layers)
       or (len(self.conv_dim) != self.num_feat_extract_layers)
   ):
     raise ValueError(
         "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
         " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
         F""" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"""
         F""" `len(config.conv_kernel) = {len(self.conv_kernel)}`.""")

   # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
   lowerCamelCase__:      int			  =apply_spec_augment
   lowerCamelCase__:      List[str]			  =mask_time_prob
   lowerCamelCase__:      Union[str, Any]			  =mask_time_length
   lowerCamelCase__:      List[Any]			  =mask_time_min_masks
   lowerCamelCase__:      Any			  =mask_feature_prob
   lowerCamelCase__:      Optional[Any]			  =mask_feature_length
   lowerCamelCase__:      List[str]			  =mask_feature_min_masks

   # parameters for pretraining with codevector quantized representations
   lowerCamelCase__:      Optional[Any]			  =num_codevectors_per_group
   lowerCamelCase__:      str			  =num_codevector_groups
   lowerCamelCase__:      Tuple			  =contrastive_logits_temperature
   lowerCamelCase__:      int			  =feat_quantizer_dropout
   lowerCamelCase__:      Any			  =num_negatives
   lowerCamelCase__:      List[str]			  =codevector_dim
   lowerCamelCase__:      Union[str, Any]			  =proj_codevector_dim
   lowerCamelCase__:      Any			  =diversity_loss_weight

   # ctc loss
   lowerCamelCase__:      Any			  =ctc_loss_reduction
   lowerCamelCase__:      Dict			  =ctc_zero_infinity

   # pretraining loss
   lowerCamelCase__:      Dict			  =replace_prob







 @property
 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Optional[Any]:



   '''simple docstring'''



   return functools.reduce(operator.mul       , self.conv_stride       , 1)

