






'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


lowerCamelCase_												=					logging.get_logger(__name__)

lowerCamelCase_												=					{
    '''microsoft/focalnet-tiny''': '''https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json''',
}

class    _UpperCAmelCase			(     snake_case_							,    snake_case_						):




   """simple docstring"""

   snake_case    				=						'''focalnet'''

   def __init__(		self :	Dict      ,      __UpperCAmelCase :	Optional[Any]=224      ,      __UpperCAmelCase :	Optional[Any]=4      ,      __UpperCAmelCase :	int=3      ,      __UpperCAmelCase :	Optional[int]=96      ,      __UpperCAmelCase :	Any=False      ,      __UpperCAmelCase :	int=[192, 384, 768, 768]      ,      __UpperCAmelCase :	str=[2, 2, 6, 2]      ,      __UpperCAmelCase :	int=[2, 2, 2, 2]      ,      __UpperCAmelCase :	Optional[int]=[3, 3, 3, 3]      ,      __UpperCAmelCase :	Any="gelu"      ,      __UpperCAmelCase :	Dict=4.0      ,      __UpperCAmelCase :	Any=0.0      ,      __UpperCAmelCase :	List[str]=0.1      ,      __UpperCAmelCase :	int=False      ,      __UpperCAmelCase :	Dict=1E-4      ,      __UpperCAmelCase :	List[Any]=False      ,      __UpperCAmelCase :	str=False      ,      __UpperCAmelCase :	Any=False      ,      __UpperCAmelCase :	int=0.02      ,      __UpperCAmelCase :	Optional[int]=1E-5      ,      __UpperCAmelCase :	Dict=32      ,      __UpperCAmelCase :	Any=None      ,      __UpperCAmelCase :	Optional[int]=None      ,      **__UpperCAmelCase :	Any      ,      ):





         '''simple docstring'''





         super().__init__(**__UpperCAmelCase		)

         _A			     =	image_size
         _A			     =	patch_size
         _A			     =	num_channels
         _A			     =	embed_dim
         _A			     =	use_conv_embed
         _A			     =	hidden_sizes
         _A			     =	depths
         _A			     =	focal_levels
         _A			     =	focal_windows
         _A			     =	hidden_act
         _A			     =	mlp_ratio
         _A			     =	hidden_dropout_prob
         _A			     =	drop_path_rate
         _A			     =	use_layerscale
         _A			     =	layerscale_value
         _A			     =	use_post_layernorm
         _A			     =	use_post_layernorm_in_modulation
         _A			     =	normalize_modulator
         _A			     =	initializer_range
         _A			     =	layer_norm_eps
         _A			     =	encoder_stride
         _A			     =	["stem"] + [f'''stage{idx}''' for idx in range(1      ,      len(self.depths		) + 1		)]
         _A     ,	_A			     =	get_aligned_output_features_output_indices(
             out_features=__UpperCAmelCase      ,      out_indices=__UpperCAmelCase      ,      stage_names=self.stage_names		)









'''simple docstring'''

import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
 lowerCamelCase							:   List[str]  						=	argparse.ArgumentParser(
     description=(
         'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
         ' Distillation'
     )
 )
 parser.add_argument('--model_type', default='bert', choices=['bert'])
 parser.add_argument('--model_name', default='bert-base-uncased', type=str)
 parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
 parser.add_argument('--vocab_transform', action='store_true')
 lowerCamelCase							:   str  						=	parser.parse_args()

 if args.model_type == "bert":
  lowerCamelCase							:   List[Any]  						=	BertForMaskedLM.from_pretrained(args.model_name)
  lowerCamelCase							:   Any  						=	'bert'
 else:
  raise ValueError('args.model_type should be "bert".')

 lowerCamelCase							:   int  						=	model.state_dict()
 lowerCamelCase							:   int  						=	{}

 for w in ["word_embeddings", "position_embeddings"]:
  lowerCamelCase							:   Any  						=	state_dict[f"""{prefix}.embeddings.{w}.weight"""]
 for w in ["weight", "bias"]:
  lowerCamelCase							:   Any  						=	state_dict[f"""{prefix}.embeddings.LayerNorm.{w}"""]

 lowerCamelCase							:   Tuple  						=	0
 for teacher_idx in [0, 2, 4, 7, 9, 11]:
  for w in ["weight", "bias"]:
   lowerCamelCase							:   Dict  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"""
   ]
   lowerCamelCase							:   Dict  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"""
   ]
   lowerCamelCase							:   List[Any]  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"""
   ]

   lowerCamelCase							:   Tuple  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"""
   ]
   lowerCamelCase							:   Optional[int]  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"""
   ]

   lowerCamelCase							:   Optional[Any]  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"""
   ]
   lowerCamelCase							:   Dict  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"""
   ]
   lowerCamelCase							:   Any  						=	state_dict[
       f"""{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"""
   ]
  std_idx += 1

 lowerCamelCase							:   Optional[int]  						=	state_dict['cls.predictions.decoder.weight']
 lowerCamelCase							:   str  						=	state_dict['cls.predictions.bias']
 if args.vocab_transform:
  for w in ["weight", "bias"]:
   lowerCamelCase							:   str  						=	state_dict[f"""cls.predictions.transform.dense.{w}"""]
   lowerCamelCase							:   Any  						=	state_dict[f"""cls.predictions.transform.LayerNorm.{w}"""]

 print(f"""N layers selected for distillation: {std_idx}""")
 print(f"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

 print(f"""Save transferred checkpoint to {args.dump_checkpoint}.""")
 torch.save(compressed_sd, args.dump_checkpoint)




