


'''simple docstring'''




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
       from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
       __snake_case    						=True
       from torch.cuda.amp import autocast


__snake_case    						=logging.getLogger(__name__)

@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	str		=     field(
         metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}		)
     lowerCamelCase				:	Optional[str]		=     field(
         default=__lowercase       ,	metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}       ,	)
     lowerCamelCase				:	Optional[bool]		=     field(
         default=__lowercase       ,	metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}		)
     lowerCamelCase				:	Optional[bool]		=     field(
         default=__lowercase       ,	metadata={'''help''': '''Whether to log verbose messages or not.'''}       ,	)
     lowerCamelCase				:	Optional[float]		=     field(
         default=2.0       ,	metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}		)
     lowerCamelCase				:	Optional[float]		=     field(
         default=0.5       ,	metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}		)
     lowerCamelCase				:	Optional[float]		=     field(
         default=0.9_9_9_9_9_5       ,	metadata={'''help''': '''Decay of gumbel temperature during training.'''}		)





def 	a_					( lowerCamelCase	:     ModelArguments	, lowerCamelCase	:     TrainingArguments				):
   logging.basicConfig(
       format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'	, datefmt='%m/%d/%Y %H:%M:%S'	, handlers=[logging.StreamHandler(sys.stdout				)]	, )
   lowerCAmelCase     =			logging.WARNING
   if model_args.verbose_logging:
      lowerCAmelCase     =			logging.DEBUG
   elif trainer_utils.is_main_process(training_args.local_rank				):
      lowerCAmelCase     =			logging.INFO
   logger.setLevel(lowerCamelCase				)

@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	str		=     field(
         default=__lowercase       ,	metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}		)
     lowerCamelCase				:	Optional[str]		=     field(
         default=__lowercase       ,	metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}		)
     lowerCamelCase				:	Optional[str]		=     field(
         default='''train'''       ,	metadata={
             '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
         }       ,	)
     lowerCamelCase				:	Optional[str]		=     field(
         default='''validation'''       ,	metadata={
             '''help''': (
                 '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
             )
         }       ,	)
     lowerCamelCase				:	Optional[str]		=     field(
         default='''file'''       ,	metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}       ,	)
     lowerCamelCase				:	bool		=     field(
         default=__lowercase       ,	metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}		)
     lowerCamelCase				:	Optional[int]		=     field(
         default=1       ,	metadata={
             '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
         }       ,	)
     lowerCamelCase				:	Optional[int]		=     field(
         default=__lowercase       ,	metadata={'''help''': '''The number of processes to use for the preprocessing.'''}       ,	)
     lowerCamelCase				:	Optional[float]		=     field(
         default=2_0.0       ,	metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}		)




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	WavaVecaForPreTraining
     lowerCamelCase				:	WavaVecaFeatureExtractor
     lowerCamelCase				:	Union[bool, str]		=     "longest"
     lowerCamelCase				:	Optional[int]		=     None
     lowerCamelCase				:	Optional[int]		=     None





     def __call__( self       :      Union[str, Any]    ,      UpperCAmelCase__       :      List[Dict[str, Union[List[int], torch.Tensor]]]    )				->  Dict[str, torch.Tensor]:
        # reformat list to dict and set to pytorch format
        lowerCAmelCase     =			self.feature_extractor.pad(
            UpperCAmelCase__    ,      max_length=self.max_length    ,      padding=self.padding    ,      pad_to_multiple_of=self.pad_to_multiple_of    ,      return_tensors='pt'    ,      )
        lowerCAmelCase     =			self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1]    )

        lowerCAmelCase     =			batch['input_values'].shape[0]

        # make sure that no loss is computed on padded inputs
        if batch["attention_mask"] is not None:
           # compute real output lengths according to convolution formula
           lowerCAmelCase     =			self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1    )    ).to(
               torch.long    )

           lowerCAmelCase     =			torch.zeros(
               (batch_size, mask_indices_seq_length)    ,      dtype=torch.long    ,      device=batch['input_values'].device    )

           # these two operations makes sure that all values
           # before the output lengths indices are attended to
           lowerCAmelCase     =			1
           lowerCAmelCase     =			attention_mask.flip([-1]    ).cumsum(-1    ).flip([-1]    ).bool()

        # sample randomly masked indices
        lowerCAmelCase     =			_compute_mask_indices(
            (batch_size, mask_indices_seq_length)    ,      self.model.config.mask_time_prob    ,      self.model.config.mask_time_length    ,      attention_mask=UpperCAmelCase__    ,      min_masks=2    ,      )

        return batch




class 		UpperCAmelCase_      (  __lowercase		):





     def __init__( self       :      Optional[int]    ,      *UpperCAmelCase__       :      Optional[int]    ,      UpperCAmelCase__       :      Optional[int]=1    ,      UpperCAmelCase__       :      Dict=0    ,      UpperCAmelCase__       :      List[str]=1.0    ,      **UpperCAmelCase__       :      int    )				->  int:
        super().__init__(*UpperCAmelCase__    ,      **UpperCAmelCase__    )
        lowerCAmelCase     =			0
        lowerCAmelCase     =			max_gumbel_temp
        lowerCAmelCase     =			min_gumbel_temp
        lowerCAmelCase     =			gumbel_temp_decay





     def 							__UpperCAmelCase      ( self       :      Any    ,      UpperCAmelCase__       :      nn.Module    ,      UpperCAmelCase__       :      Dict[str, Union[torch.Tensor, Any]]    )				->  torch.Tensor:
        model.train()
        lowerCAmelCase     =			self._prepare_inputs(UpperCAmelCase__    )

        if self.use_amp:
           with autocast():
              lowerCAmelCase     =			self.compute_loss(UpperCAmelCase__    ,      UpperCAmelCase__    )
        else:
           lowerCAmelCase     =			self.compute_loss(UpperCAmelCase__    ,      UpperCAmelCase__    )

        if self.args.n_gpu > 1 or self.deepspeed:
           if model.module.config.ctc_loss_reduction == "mean":
              lowerCAmelCase     =			loss.mean()
           elif model.module.config.ctc_loss_reduction == "sum":
              lowerCAmelCase     =			loss.sum() / (inputs['mask_time_indices']).sum()
           else:
              raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''    )

        if self.args.gradient_accumulation_steps > 1:
           lowerCAmelCase     =			loss / self.args.gradient_accumulation_steps

        if self.use_amp:
           self.scaler.scale(UpperCAmelCase__    ).backward()
        elif self.use_apex:
           with amp.scale_loss(UpperCAmelCase__    ,      self.optimizer    ) as scaled_loss:
              scaled_loss.backward()
        elif self.deepspeed:
           self.deepspeed.backward(UpperCAmelCase__    )
        else:
           loss.backward()

        self.num_update_step += 1
        # make sure gumbel softmax temperature is decayed
        if self.args.n_gpu > 1 or self.deepspeed:
           model.module.set_gumbel_temperature(
               max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,      self.min_gumbel_temp    )    )
        else:
           model.set_gumbel_temperature(
               max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,      self.min_gumbel_temp    )    )

        return loss.detach()





def 	a_					( ):
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   lowerCAmelCase     =			HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)

   lowerCAmelCase     ,  lowerCAmelCase     ,  lowerCAmelCase     =			parser.parse_args_into_dataclasses()
   configure_logger(lowerCamelCase	, lowerCamelCase				)

   # Downloading and loading a dataset from the hub.
   lowerCAmelCase     =			load_dataset(data_args.dataset_name	, data_args.dataset_config_name	, cache_dir=model_args.cache_dir				)

   if "validation" not in datasets.keys():
      # make sure only "validation" and "train" keys remain"
      lowerCAmelCase     =			DatasetDict()
      lowerCAmelCase     =			load_dataset(
          data_args.dataset_name	, data_args.dataset_config_name	, split=f'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''	, cache_dir=model_args.cache_dir	, )
      lowerCAmelCase     =			load_dataset(
          data_args.dataset_name	, data_args.dataset_config_name	, split=f'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''	, cache_dir=model_args.cache_dir	, )
   else:
      # make sure only "validation" and "train" keys remain"
      lowerCAmelCase     =			DatasetDict()
      lowerCAmelCase     =			load_dataset(
          data_args.dataset_name	, data_args.dataset_config_name	, split='validation'	, cache_dir=model_args.cache_dir	, )
      lowerCAmelCase     =			load_dataset(
          data_args.dataset_name	, data_args.dataset_config_name	, split=f'''{data_args.train_split_name}'''	, cache_dir=model_args.cache_dir	, )

   # only normalized-inputs-training is supported
   lowerCAmelCase     =			WavaVecaFeatureExtractor.from_pretrained(
       model_args.model_name_or_path	, cache_dir=model_args.cache_dir	, do_normalize=lowerCamelCase				)

   def prepare_dataset(lowerCamelCase	:     Optional[Any]				):
      # check that all files have the correct sampling rate
      lowerCAmelCase     ,  lowerCAmelCase     =			librosa.load(batch[data_args.speech_file_column]	, sr=feature_extractor.sampling_rate				)
      return batch

   # load audio files into numpy arrays
   lowerCAmelCase     =			datasets.map(
       lowerCamelCase	, num_proc=data_args.preprocessing_num_workers	, remove_columns=datasets['train'].column_names				)

   # filter audio files that are too long
   lowerCAmelCase     =			vectorized_datasets.filter(
       lambda lowerCamelCase				: len(data['speech']				) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate				)				)

   def normalize(lowerCamelCase	:     Dict				):
      return feature_extractor(batch['speech']	, sampling_rate=feature_extractor.sampling_rate				)

   # normalize and transform to `BatchFeatures`
   lowerCAmelCase     =			vectorized_datasets.map(
       lowerCamelCase	, batched=lowerCamelCase	, num_proc=data_args.preprocessing_num_workers	, load_from_cache_file=not data_args.overwrite_cache	, remove_columns=vectorized_datasets['train'].column_names	, )

   # pretraining is only supported for "newer" stable layer norm architecture
   # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
   lowerCAmelCase     =			WavaVecaConfig.from_pretrained(
       model_args.model_name_or_path	, cache_dir=model_args.cache_dir	, gradient_checkpointing=training_args.gradient_checkpointing	, )

   if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
      raise ValueError(
          'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
          ' ``config.feat_extract_norm=\'layer\''				)

   lowerCAmelCase     =			WavaVecaForPreTraining(lowerCamelCase				)

   lowerCAmelCase     =			DataCollatorForWavaVecaPretraining(model=lowerCamelCase	, feature_extractor=lowerCamelCase				)

   lowerCAmelCase     =			WavaVecaPreTrainer(
       model=lowerCamelCase	, data_collator=lowerCamelCase	, args=lowerCamelCase	, train_dataset=vectorized_datasets['train']	, eval_dataset=vectorized_datasets['validation']	, tokenizer=lowerCamelCase	, max_gumbel_temp=model_args.max_gumbel_temperature	, min_gumbel_temp=model_args.min_gumbel_temperature	, gumbel_temp_decay=model_args.gumbel_temperature_decay	, )
   trainer.train()


if __name__ == "__main__":
       main()









'''simple docstring'''




class 		UpperCAmelCase_      :





     def __init__( self       :      List[str]    ,      UpperCAmelCase__       :      list[int]    )				->  None:
        lowerCAmelCase     =			len(UpperCAmelCase__    )
        lowerCAmelCase     =			[0] * len_array

        if len_array > 0:
           lowerCAmelCase     =			array[0]

        for i in range(1    ,      UpperCAmelCase__    ):
           lowerCAmelCase     =			self.prefix_sum[i - 1] + array[i]





     def 							__UpperCAmelCase      ( self       :      Optional[Any]    ,      UpperCAmelCase__       :      int    ,      UpperCAmelCase__       :      int    )				->  int:
        if start == 0:
           return self.prefix_sum[end]

        return self.prefix_sum[end] - self.prefix_sum[start - 1]





     def 							__UpperCAmelCase      ( self       :      int    ,      UpperCAmelCase__       :      int    )				->  bool:
        lowerCAmelCase     =			{0}
        for sum_item in self.prefix_sum:
           if sum_item - target_sum in sums:
              return True

           sums.add(UpperCAmelCase__    )

        return False


if __name__ == "__main__":
       import doctest

       doctest.testmod()





