




import torch

from accelerate import Accelerator, DistributedType


class        UpperCAmelCase__				:



     '''simple docstring'''
     def __enter__(				self	):
            if self.enabled:
                   a			   =self.model.no_sync()
                   self.model_sync_obj.__enter__()

            return self
     def __exit__(				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	):
            if self.enabled:
                   # Average all models on exit
                   self._sync_and_avg_model_params()
                   self.model_sync_obj.__exit__(_lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase	)
     def __init__(				self ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase = True	):
            if accelerator.distributed_type not in [
                DistributedType.NO,
                DistributedType.MULTI_CPU,
                DistributedType.MULTI_GPU,
            ]:
                   raise NotImplementedError("""LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"""	)
            a			   =enabled and accelerator.distributed_type != DistributedType.NO
            a			   =0
            if self.enabled:
                   a			   =accelerator
                   a			   =model
                   a			   =local_sgd_steps
     def        lowerCAmelCase__  (				self	):
            self.num_steps += 1
            if not self.enabled:
                   return

            if self.num_steps % self.local_sgd_steps == 0:
                   self._sync_and_avg_model_params()






     def        lowerCAmelCase__  (				self	):
            self.accelerator.wait_for_everyone()
            with self.accelerator.autocast():
                   for param in self.model.parameters():
                          a			   =self.accelerator.reduce(param.data ,    reduction="""mean"""	)






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase							=      logging.get_logger(__name__)

_lowerCamelCase							=      {
    '''facebook/hubert-base-ls960''': '''https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json''',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}


class        UpperCAmelCase__				(							UpperCAmelCase__				):



     '''simple docstring'''
     _SCREAMING_SNAKE_CASE      :					Dict     					=    "hubert"
     def __init__(				self ,    _lowerCAmelCase=32 ,    _lowerCAmelCase=768 ,    _lowerCAmelCase=12 ,    _lowerCAmelCase=12 ,    _lowerCAmelCase=3_072 ,    _lowerCAmelCase="gelu" ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=True ,    _lowerCAmelCase=0.0 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.02 ,    _lowerCAmelCase=1E-5 ,    _lowerCAmelCase="group" ,    _lowerCAmelCase="gelu" ,    _lowerCAmelCase=(512, 512, 512, 512, 512, 512, 512) ,    _lowerCAmelCase=(5, 2, 2, 2, 2, 2, 2) ,    _lowerCAmelCase=(10, 3, 3, 3, 3, 2, 2) ,    _lowerCAmelCase=False ,    _lowerCAmelCase=128 ,    _lowerCAmelCase=16 ,    _lowerCAmelCase=False ,    _lowerCAmelCase=True ,    _lowerCAmelCase=0.05 ,    _lowerCAmelCase=10 ,    _lowerCAmelCase=2 ,    _lowerCAmelCase=0.0 ,    _lowerCAmelCase=10 ,    _lowerCAmelCase=0 ,    _lowerCAmelCase="sum" ,    _lowerCAmelCase=False ,    _lowerCAmelCase=False ,    _lowerCAmelCase=256 ,    _lowerCAmelCase=0 ,    _lowerCAmelCase=1 ,    _lowerCAmelCase=2 ,    **_lowerCAmelCase ,    ):
            super().__init__(**_lowerCAmelCase ,    pad_token_id=_lowerCAmelCase ,    bos_token_id=_lowerCAmelCase ,    eos_token_id=_lowerCAmelCase	)
            a			   =hidden_size
            a			   =feat_extract_norm
            a			   =feat_extract_activation
            a			   =list(_lowerCAmelCase	)
            a			   =list(_lowerCAmelCase	)
            a			   =list(_lowerCAmelCase	)
            a			   =conv_bias
            a			   =num_conv_pos_embeddings
            a			   =num_conv_pos_embedding_groups
            a			   =len(self.conv_dim	)
            a			   =num_hidden_layers
            a			   =intermediate_size
            a			   =hidden_act
            a			   =num_attention_heads
            a			   =hidden_dropout
            a			   =attention_dropout
            a			   =activation_dropout
            a			   =feat_proj_layer_norm
            a			   =feat_proj_dropout
            a			   =final_dropout
            a			   =layerdrop
            a			   =layer_norm_eps
            a			   =initializer_range
            a			   =vocab_size
            a			   =do_stable_layer_norm
            a			   =use_weighted_layer_sum
            a			   =classifier_proj_size

            if (
                (len(self.conv_stride	) != self.num_feat_extract_layers)
                or (len(self.conv_kernel	) != self.num_feat_extract_layers)
                or (len(self.conv_dim	) != self.num_feat_extract_layers)
            ):
                   raise ValueError(
                       """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                       """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                       F''' {len(self.conv_dim	)}`, `len(config.conv_stride) = {len(self.conv_stride	)}`,'''
                       F''' `len(config.conv_kernel) = {len(self.conv_kernel	)}`.'''	)

            # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
            a			   =apply_spec_augment
            a			   =mask_time_prob
            a			   =mask_time_length
            a			   =mask_time_min_masks
            a			   =mask_feature_prob
            a			   =mask_feature_length
            a			   =mask_feature_min_masks

            # ctc loss
            a			   =ctc_loss_reduction
            a			   =ctc_zero_infinity






     @property
     def        lowerCAmelCase__  (				self	):
            return functools.reduce(operator.mul ,    self.conv_stride ,    1	)
