



import torch

from accelerate import Accelerator, DistributedType


class 	snake_case_			:




       """simple docstring"""






       def __enter__(  self):

             """simple docstring"""





             if self.enabled:
                   UpperCAmelCase_	:						str					   =		self.model.no_sync()
                   self.model_sync_obj.__enter__()

             return self



       def __exit__(  self  ,lowercase  ,lowercase  ,lowercase):

             """simple docstring"""





             if self.enabled:
                   # Average all models on exit
                   self._sync_and_avg_model_params()
                   self.model_sync_obj.__exit__(lowercase  ,lowercase  ,lowercase)



       def __init__(  self  ,lowercase  ,lowercase  ,lowercase  ,lowercase = True):

             """simple docstring"""





             if accelerator.distributed_type not in [
                 DistributedType.NO,
                 DistributedType.MULTI_CPU,
                 DistributedType.MULTI_GPU,
             ]:
                   raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)")
             UpperCAmelCase_	:						int					   =		enabled and accelerator.distributed_type != DistributedType.NO
             UpperCAmelCase_	:						Any					   =		0
             if self.enabled:
                   UpperCAmelCase_	:						str					   =		accelerator
                   UpperCAmelCase_	:						int					   =		model
                   UpperCAmelCase_	:						List[str]					   =		local_sgd_steps



       def 			A_ (  self):

             """simple docstring"""





             self.num_steps += 1
             if not self.enabled:
                   return

             if self.num_steps % self.local_sgd_steps == 0:
                   self._sync_and_avg_model_params()



       def 			A_ (  self):

             """simple docstring"""





             self.accelerator.wait_for_everyone()
             with self.accelerator.autocast():
                   for param in self.model.parameters():
                         UpperCAmelCase_	:						List[Any]					   =		self.accelerator.reduce(param.data  ,reduction="mean")





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase								=      logging.get_logger(__name__)

__lowerCamelCase								=      {
    '''MCG-NJU/videomae-base''': '''https://huggingface.co/MCG-NJU/videomae-base/resolve/main/config.json''',
}


class 	snake_case_			(lowercase__	):




       """simple docstring"""






       _lowerCamelCase           =					"""videomae"""



       def __init__(  self  ,lowercase=224  ,lowercase=16  ,lowercase=3  ,lowercase=16  ,lowercase=2  ,lowercase=768  ,lowercase=12  ,lowercase=12  ,lowercase=3072  ,lowercase="gelu"  ,lowercase=0.0  ,lowercase=0.0  ,lowercase=0.02  ,lowercase=1E-12  ,lowercase=True  ,lowercase=True  ,lowercase=6  ,lowercase=384  ,lowercase=4  ,lowercase=1536  ,lowercase=True  ,**lowercase  ,):

             """simple docstring"""





             super().__init__(**lowercase)

             UpperCAmelCase_	:						Optional[int]					   =		image_size
             UpperCAmelCase_	:						Optional[Any]					   =		patch_size
             UpperCAmelCase_	:						Tuple					   =		num_channels
             UpperCAmelCase_	:						Optional[int]					   =		num_frames
             UpperCAmelCase_	:						str					   =		tubelet_size

             UpperCAmelCase_	:						Any					   =		hidden_size
             UpperCAmelCase_	:						List[str]					   =		num_hidden_layers
             UpperCAmelCase_	:						List[str]					   =		num_attention_heads
             UpperCAmelCase_	:						Union[str, Any]					   =		intermediate_size
             UpperCAmelCase_	:						List[str]					   =		hidden_act
             UpperCAmelCase_	:						Tuple					   =		hidden_dropout_prob
             UpperCAmelCase_	:						Optional[Any]					   =		attention_probs_dropout_prob
             UpperCAmelCase_	:						Optional[Any]					   =		initializer_range
             UpperCAmelCase_	:						Dict					   =		layer_norm_eps
             UpperCAmelCase_	:						Tuple					   =		qkv_bias
             UpperCAmelCase_	:						int					   =		use_mean_pooling

             UpperCAmelCase_	:						List[Any]					   =		decoder_num_attention_heads
             UpperCAmelCase_	:						Optional[int]					   =		decoder_hidden_size
             UpperCAmelCase_	:						Any					   =		decoder_num_hidden_layers
             UpperCAmelCase_	:						Dict					   =		decoder_intermediate_size
             UpperCAmelCase_	:						str					   =		norm_pix_loss
