






"""simple docstring"""



import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
						from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
						__A		:       List[str]  				=		True
						from torch.cuda.amp import autocast


__A		:       str  				=		logging.getLogger(__name__)





@dataclass
class 					_UpperCAmelCase       :
					SCREAMING_SNAKE_CASE_		:				str							    =   field(
					    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}							)
					SCREAMING_SNAKE_CASE_		:				Optional[str]							    =   field(
					    default=_A   ,						metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}   ,						)
					SCREAMING_SNAKE_CASE_		:				Optional[bool]							    =   field(
					    default=_A   ,						metadata={"help": "Whether to freeze the feature extractor layers of the model."}							)
					SCREAMING_SNAKE_CASE_		:				Optional[bool]							    =   field(
					    default=_A   ,						metadata={"help": "Whether to log verbose messages or not."}   ,						)
					SCREAMING_SNAKE_CASE_		:				Optional[float]							    =   field(
					    default=2.0   ,						metadata={"help": "Maximum temperature for gumbel softmax."}							)
					SCREAMING_SNAKE_CASE_		:				Optional[float]							    =   field(
					    default=0.5   ,						metadata={"help": "Minimum temperature for gumbel softmax."}							)
					SCREAMING_SNAKE_CASE_		:				Optional[float]							    =   field(
					    default=0.999_995   ,						metadata={"help": "Decay of gumbel temperature during training."}							)



def 						lowercase    (				__snake_case				: ModelArguments		,					__snake_case				: TrainingArguments	):
		logging.basicConfig(
		    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''		,					datefmt='''%m/%d/%Y %H:%M:%S'''		,					handlers=[logging.StreamHandler(sys.stdout	)]		,					)
		lowercase_ : int							       =   logging.WARNING
		if model_args.verbose_logging:
				lowercase_ : Any							       =   logging.DEBUG
		elif trainer_utils.is_main_process(training_args.local_rank	):
				lowercase_ : Tuple							       =   logging.INFO
		logger.setLevel(__snake_case	)





@dataclass
class 					_UpperCAmelCase       :
					SCREAMING_SNAKE_CASE_		:				str							    =   field(
					    default=_A   ,						metadata={"help": "The name of the dataset to use (via the datasets library)."}							)
					SCREAMING_SNAKE_CASE_		:				Optional[str]							    =   field(
					    default=_A   ,						metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}							)
					SCREAMING_SNAKE_CASE_		:				Optional[str]							    =   field(
					    default="train"   ,						metadata={
					        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
					    }   ,						)
					SCREAMING_SNAKE_CASE_		:				Optional[str]							    =   field(
					    default="validation"   ,						metadata={
					        "help": (
					            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
					        )
					    }   ,						)
					SCREAMING_SNAKE_CASE_		:				Optional[str]							    =   field(
					    default="file"   ,						metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}   ,						)
					SCREAMING_SNAKE_CASE_		:				bool							    =   field(
					    default=_A   ,						metadata={"help": "Overwrite the cached preprocessed datasets or not."}							)
					SCREAMING_SNAKE_CASE_		:				Optional[int]							    =   field(
					    default=1   ,						metadata={
					        "help": "The percentage of the train set used as validation set in case there's no validation split"
					    }   ,						)
					SCREAMING_SNAKE_CASE_		:				Optional[int]							    =   field(
					    default=_A   ,						metadata={"help": "The number of processes to use for the preprocessing."}   ,						)
					SCREAMING_SNAKE_CASE_		:				Optional[float]							    =   field(
					    default=20.0   ,						metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}							)





@dataclass
class 					_UpperCAmelCase       :
					SCREAMING_SNAKE_CASE_		:				WavaVecaForPreTraining
					SCREAMING_SNAKE_CASE_		:				WavaVecaFeatureExtractor
					SCREAMING_SNAKE_CASE_		:				Union[bool, str]							    =   "longest"
					SCREAMING_SNAKE_CASE_		:				Optional[int]							    =   None
					SCREAMING_SNAKE_CASE_		:				Optional[int]							    =   None




					def __call__(				self     :						Union[str, Any]    ,						A     :						List[Dict[str, Union[List[int], torch.Tensor]]]   )  ->  Dict[str, torch.Tensor]:
							# reformat list to dict and set to pytorch format
							lowercase_ : List[str]							       =   self.feature_extractor.pad(
							    A    ,						max_length=self.max_length    ,						padding=self.padding    ,						pad_to_multiple_of=self.pad_to_multiple_of    ,						return_tensors='''pt'''    ,						)
							lowercase_ : Tuple							       =   self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]   )

							lowercase_ : Union[str, Any]							       =   batch['''input_values'''].shape[0]

							# make sure that no loss is computed on padded inputs
							if batch["attention_mask"] is not None:
									# compute real output lengths according to convolution formula
									lowercase_ : Any							       =   self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1   )   ).to(
									    torch.long   )

									lowercase_ : str							       =   torch.zeros(
									    (batch_size, mask_indices_seq_length)    ,						dtype=torch.long    ,						device=batch['''input_values'''].device   )

									# these two operations makes sure that all values
									# before the output lengths indices are attended to
									lowercase_ : Optional[Any]							       =   1
									lowercase_ : Tuple							       =   attention_mask.flip([-1]   ).cumsum(-1   ).flip([-1]   ).bool()

							# sample randomly masked indices
							lowercase_ : Union[str, Any]							       =   _compute_mask_indices(
							    (batch_size, mask_indices_seq_length)    ,						self.model.config.mask_time_prob    ,						self.model.config.mask_time_length    ,						attention_mask=A    ,						min_masks=2    ,						)

							return batch





class 					_UpperCAmelCase       ( _A							):




					def __init__(				self     :						Optional[Any]    ,						*A     :						Dict    ,						A     :						Any=1    ,						A     :						Dict=0    ,						A     :						Optional[int]=1.0    ,						**A     :						str   )  ->  Tuple:
							super().__init__(*A    ,						**A   )
							lowercase_ : Any							       =   0
							lowercase_ : Tuple							       =   max_gumbel_temp
							lowercase_ : int							       =   min_gumbel_temp
							lowercase_ : Optional[Any]							       =   gumbel_temp_decay






					def   A  (				self     :						str    ,						A     :						nn.Module    ,						A     :						Dict[str, Union[torch.Tensor, Any]]   )  ->  torch.Tensor:
							model.train()
							lowercase_ : Any							       =   self._prepare_inputs(A   )

							if self.use_amp:
									with autocast():
											lowercase_ : Union[str, Any]							       =   self.compute_loss(A    ,						A   )
							else:
									lowercase_ : str							       =   self.compute_loss(A    ,						A   )

							if self.args.n_gpu > 1 or self.deepspeed:
									if model.module.config.ctc_loss_reduction == "mean":
											lowercase_ : Optional[int]							       =   loss.mean()
									elif model.module.config.ctc_loss_reduction == "sum":
											lowercase_ : List[Any]							       =   loss.sum() / (inputs['''mask_time_indices''']).sum()
									else:
											raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''   )

							if self.args.gradient_accumulation_steps > 1:
									lowercase_ : int							       =   loss / self.args.gradient_accumulation_steps

							if self.use_amp:
									self.scaler.scale(A   ).backward()
							elif self.use_apex:
									with amp.scale_loss(A    ,						self.optimizer   ) as scaled_loss:
											scaled_loss.backward()
							elif self.deepspeed:
									self.deepspeed.backward(A   )
							else:
									loss.backward()

							self.num_update_step += 1
							# make sure gumbel softmax temperature is decayed
							if self.args.n_gpu > 1 or self.deepspeed:
									model.module.set_gumbel_temperature(
									    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,						self.min_gumbel_temp   )   )
							else:
									model.set_gumbel_temperature(
									    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,						self.min_gumbel_temp   )   )

							return loss.detach()



def 						lowercase    (				):
		# See all possible arguments in src/transformers/training_args.py
		# or by passing the --help flag to this script.
		# We now keep distinct sets of args, for a cleaner separation of concerns.

		lowercase_ : int							       =   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)	)

		lowercase_       ,					lowercase_       ,					lowercase_ : Optional[Any]							       =   parser.parse_args_into_dataclasses()
		configure_logger(__snake_case		,					__snake_case	)

		# Downloading and loading a dataset from the hub.
		lowercase_ : Union[str, Any]							       =   load_dataset(data_args.dataset_name		,					data_args.dataset_config_name		,					cache_dir=model_args.cache_dir	)

		if "validation" not in datasets.keys():
				# make sure only "validation" and "train" keys remain"
				lowercase_ : Optional[int]							       =   DatasetDict()
				lowercase_ : int							       =   load_dataset(
				    data_args.dataset_name		,					data_args.dataset_config_name		,					split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''		,					cache_dir=model_args.cache_dir		,					)
				lowercase_ : Any							       =   load_dataset(
				    data_args.dataset_name		,					data_args.dataset_config_name		,					split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''		,					cache_dir=model_args.cache_dir		,					)
		else:
				# make sure only "validation" and "train" keys remain"
				lowercase_ : Dict							       =   DatasetDict()
				lowercase_ : List[Any]							       =   load_dataset(
				    data_args.dataset_name		,					data_args.dataset_config_name		,					split='''validation'''		,					cache_dir=model_args.cache_dir		,					)
				lowercase_ : Union[str, Any]							       =   load_dataset(
				    data_args.dataset_name		,					data_args.dataset_config_name		,					split=F'''{data_args.train_split_name}'''		,					cache_dir=model_args.cache_dir		,					)

		# only normalized-inputs-training is supported
		lowercase_ : str							       =   WavaVecaFeatureExtractor.from_pretrained(
		    model_args.model_name_or_path		,					cache_dir=model_args.cache_dir		,					do_normalize=__snake_case	)

		def prepare_dataset(__snake_case				: int	):
				# check that all files have the correct sampling rate
				lowercase_       ,					lowercase_ : List[str]							       =   librosa.load(batch[data_args.speech_file_column]		,					sr=feature_extractor.sampling_rate	)
				return batch

		# load audio files into numpy arrays
		lowercase_ : Dict							       =   datasets.map(
		    __snake_case		,					num_proc=data_args.preprocessing_num_workers		,					remove_columns=datasets['''train'''].column_names	)

		# filter audio files that are too long
		lowercase_ : Union[str, Any]							       =   vectorized_datasets.filter(
		    lambda __snake_case	: len(data['''speech''']	) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate	)	)

		def normalize(__snake_case				: Tuple	):
				return feature_extractor(batch['''speech''']		,					sampling_rate=feature_extractor.sampling_rate	)

		# normalize and transform to `BatchFeatures`
		lowercase_ : Union[str, Any]							       =   vectorized_datasets.map(
		    __snake_case		,					batched=__snake_case		,					num_proc=data_args.preprocessing_num_workers		,					load_from_cache_file=not data_args.overwrite_cache		,					remove_columns=vectorized_datasets['''train'''].column_names		,					)

		# pretraining is only supported for "newer" stable layer norm architecture
		# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
		lowercase_ : str							       =   WavaVecaConfig.from_pretrained(
		    model_args.model_name_or_path		,					cache_dir=model_args.cache_dir		,					gradient_checkpointing=training_args.gradient_checkpointing		,					)

		if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
				raise ValueError(
				    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
				    ''' ``config.feat_extract_norm=\'layer\''''	)

		lowercase_ : Optional[int]							       =   WavaVecaForPreTraining(__snake_case	)

		lowercase_ : Union[str, Any]							       =   DataCollatorForWavaVecaPretraining(model=__snake_case		,					feature_extractor=__snake_case	)

		lowercase_ : Dict							       =   WavaVecaPreTrainer(
		    model=__snake_case		,					data_collator=__snake_case		,					args=__snake_case		,					train_dataset=vectorized_datasets['''train''']		,					eval_dataset=vectorized_datasets['''validation''']		,					tokenizer=__snake_case		,					max_gumbel_temp=model_args.max_gumbel_temperature		,					min_gumbel_temp=model_args.min_gumbel_temperature		,					gumbel_temp_decay=model_args.gumbel_temperature_decay		,					)
		trainer.train()


if __name__ == "__main__":
						main()











"""simple docstring"""



import argparse
import os
import re

import packaging.version


__A		:       List[str]  				=		'''examples/'''
__A		:       int  				=		{
    '''examples''': (re.compile(R'''^check_min_version\("[^"]+"\)\s*$''', re.MULTILINE), '''check_min_version("VERSION")\n'''),
    '''init''': (re.compile(R'''^__version__\s+=\s+"([^"]+)"\s*$''', re.MULTILINE), '''__version__ = "VERSION"\n'''),
    '''setup''': (re.compile(R'''^(\s*)version\s*=\s*"[^"]+",''', re.MULTILINE), R'''\1version="VERSION",'''),
    '''doc''': (re.compile(R'''^(\s*)release\s*=\s*"[^"]+"$''', re.MULTILINE), '''release = "VERSION"\n'''),
}
__A		:       Dict  				=		{
    '''init''': '''src/transformers/__init__.py''',
    '''setup''': '''setup.py''',
}
__A		:       Optional[int]  				=		'''README.md'''



def 						lowercase    (				__snake_case				: int		,					__snake_case				: Any		,					__snake_case				: int	):
		with open(__snake_case		,					'''r'''		,					encoding='''utf-8'''		,					newline='''\n'''	) as f:
				lowercase_ : int							       =   f.read()
		lowercase_       ,					lowercase_ : List[str]							       =   REPLACE_PATTERNS[pattern]
		lowercase_ : Union[str, Any]							       =   replace.replace('''VERSION'''		,					__snake_case	)
		lowercase_ : Optional[Any]							       =   re_pattern.sub(__snake_case		,					__snake_case	)
		with open(__snake_case		,					'''w'''		,					encoding='''utf-8'''		,					newline='''\n'''	) as f:
				f.write(__snake_case	)



def 						lowercase    (				__snake_case				: int	):
		for folder, directories, fnames in os.walk(__snake_case	):
				# Removing some of the folders with non-actively maintained examples from the walk
				if "research_projects" in directories:
						directories.remove('''research_projects'''	)
				if "legacy" in directories:
						directories.remove('''legacy'''	)
				for fname in fnames:
						if fname.endswith('''.py'''	):
								update_version_in_file(os.path.join(__snake_case		,					__snake_case	)		,					__snake_case		,					pattern='''examples'''	)



def 						lowercase    (				__snake_case				: Optional[Any]		,					__snake_case				: Optional[Any]=False	):
		for pattern, fname in REPLACE_FILES.items():
				update_version_in_file(__snake_case		,					__snake_case		,					__snake_case	)
		if not patch:
				update_version_in_examples(__snake_case	)



def 						lowercase    (				):
		lowercase_ : Union[str, Any]							       =   '''🤗 Transformers currently provides the following architectures'''
		lowercase_ : Union[str, Any]							       =   '''1. Want to contribute a new model?'''
		with open(__snake_case		,					'''r'''		,					encoding='''utf-8'''		,					newline='''\n'''	) as f:
				lowercase_ : List[str]							       =   f.readlines()

		# Find the start of the list.
		lowercase_ : Optional[Any]							       =   0
		while not lines[start_index].startswith(_start_prompt	):
				start_index += 1
		start_index += 1

		lowercase_ : str							       =   start_index
		# Update the lines in the model list.
		while not lines[index].startswith(_end_prompt	):
				if lines[index].startswith('''1.'''	):
						lowercase_ : str							       =   lines[index].replace(
						    '''https://huggingface.co/docs/transformers/main/model_doc'''		,					'''https://huggingface.co/docs/transformers/model_doc'''		,					)
				index += 1

		with open(__snake_case		,					'''w'''		,					encoding='''utf-8'''		,					newline='''\n'''	) as f:
				f.writelines(__snake_case	)



def 						lowercase    (				):
		with open(REPLACE_FILES['''init''']		,					'''r'''	) as f:
				lowercase_ : List[Any]							       =   f.read()
		lowercase_ : List[str]							       =   REPLACE_PATTERNS['''init'''][0].search(__snake_case	).groups()[0]
		return packaging.version.parse(__snake_case	)



def 						lowercase    (				__snake_case				: Optional[Any]=False	):
		lowercase_ : str							       =   get_version()
		if patch and default_version.is_devrelease:
				raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''	)
		if default_version.is_devrelease:
				lowercase_ : Optional[Any]							       =   default_version.base_version
		elif patch:
				lowercase_ : Optional[int]							       =   F'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
		else:
				lowercase_ : Optional[int]							       =   F'''{default_version.major}.{default_version.minor + 1}.0'''

		# Now let's ask nicely if that's the right one.
		lowercase_ : int							       =   input(F'''Which version are you releasing? [{default_version}]'''	)
		if len(__snake_case	) == 0:
				lowercase_ : Dict							       =   default_version

		print(F'''Updating version to {version}.'''	)
		global_version_update(__snake_case		,					patch=__snake_case	)
		if not patch:
				print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''	)
				clean_main_ref_in_model_list()



def 						lowercase    (				):
		lowercase_ : List[Any]							       =   get_version()
		lowercase_ : List[str]							       =   F'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
		lowercase_ : Any							       =   current_version.base_version

		# Check with the user we got that right.
		lowercase_ : Tuple							       =   input(F'''Which version are we developing now? [{dev_version}]'''	)
		if len(__snake_case	) == 0:
				lowercase_ : str							       =   dev_version

		print(F'''Updating version to {version}.'''	)
		global_version_update(__snake_case	)
		print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''	)
		clean_main_ref_in_model_list()


if __name__ == "__main__":
						__A		:       int  				=		argparse.ArgumentParser()
						parser.add_argument('''--post_release''', action='''store_true''', help='''Whether this is pre or post release.''')
						parser.add_argument('''--patch''', action='''store_true''', help='''Whether or not this is a patch release.''')
						__A		:       Any  				=		parser.parse_args()
						if not args.post_release:
												pre_release_work(patch=args.patch)
						elif args.patch:
												print('''Nothing to do after a patch :-)''')
						else:
												post_release_work()



