




'''simple docstring'''



import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME




def    lowercase__(       __UpperCamelCase: Optional[int]  ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE					:	str   =		[
          "encoder.version",
          "decoder.version",
          "model.encoder.version",
          "model.decoder.version",
          "decoder.output_projection.weight",
          "_float_tensor",
          "encoder.embed_positions._float_tensor",
          "decoder.embed_positions._float_tensor",
      ]
      for k in ignore_keys:
            state_dict.pop(_lowerCAmelCase	,_lowerCAmelCase  )




def    lowercase__(       __UpperCamelCase: Optional[int]  ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE					:	str   =		emb.weight.shape
      SCREAMING_SNAKE_CASE					:	int   =		nn.Linear(_lowerCAmelCase	,_lowerCAmelCase	,bias=_lowerCAmelCase  )
      SCREAMING_SNAKE_CASE					:	str   =		emb.weight.data
      return lin_layer




def    lowercase__(       __UpperCamelCase: List[Any]	,__UpperCamelCase: int=None  ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE					:	Optional[int]   =		{}
      for old_key in state_dict.keys():
            SCREAMING_SNAKE_CASE					:	Optional[Any]   =		old_key
            if "moe_layer.experts." in key:
                  if expert_idx is not None:
                        SCREAMING_SNAKE_CASE					:	Any   =		key.replace('moe_layer.experts.0'	,f"ffn.experts.expert_{expert_idx}"  )
                  else:
                        SCREAMING_SNAKE_CASE					:	List[Any]   =		key.replace('moe_layer.experts.'	,'ffn.experts.expert_'  )
            if "gate" in key:
                  SCREAMING_SNAKE_CASE					:	List[Any]   =		key.replace('.moe_layer.gate.wg'	,'.ffn.router.classifier'  )
            if "fc2" and "experts" not in key:
                  SCREAMING_SNAKE_CASE					:	Optional[Any]   =		key.replace('.fc2.'	,'.ffn.fc2.'  )
            if "fc1" and "experts" not in key:
                  SCREAMING_SNAKE_CASE					:	Dict   =		key.replace('.fc1.'	,'.ffn.fc1.'  )
            if ".encoder_attn." in key:
                  SCREAMING_SNAKE_CASE					:	List[str]   =		key.replace('.encoder_attn.'	,'.cross_attention.'  )
            if "encoder_attn_layer_norm" in key:
                  SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		key.replace('encoder_attn_layer_norm'	,'cross_attention_layer_norm'  )
            if "final_layer_norm" in key:
                  SCREAMING_SNAKE_CASE					:	Dict   =		key.replace('final_layer_norm'	,'ff_layer_norm'  )
            SCREAMING_SNAKE_CASE					:	Optional[int]   =		state_dict[old_key]
      return new_dict




def    lowercase__(       __UpperCamelCase: int	,__UpperCamelCase: Tuple	,__UpperCamelCase: Union[str, Any]	,__UpperCamelCase: Tuple	,__UpperCamelCase: Dict = WEIGHTS_NAME  ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE					:	Optional[Any]   =		[]
      SCREAMING_SNAKE_CASE					:	Optional[int]   =		0
      os.makedirs(_lowerCAmelCase	,exist_ok=_lowerCAmelCase  )

      for expert in range(_lowerCAmelCase  ):
            SCREAMING_SNAKE_CASE					:	Tuple   =		switch_checkpoint_path + f"-rank-{expert}.pt"
            if os.path.isfile(_lowerCAmelCase  ):
                  SCREAMING_SNAKE_CASE					:	str   =		torch.load(_lowerCAmelCase  )["model"]
                  remove_ignore_keys_(_lowerCAmelCase  )
                  SCREAMING_SNAKE_CASE					:	List[Any]   =		rename_fairseq_keys(_lowerCAmelCase	,_lowerCAmelCase  )
                  SCREAMING_SNAKE_CASE					:	Optional[int]   =		os.path.join(
                      _lowerCAmelCase	,weights_name.replace('.bin'	,f"-{len(_lowerCAmelCase  )+1:05d}-of-???.bin"  )  )
                  torch.save(_lowerCAmelCase	,_lowerCAmelCase  )
                  sharded_state_dicts.append(expert_state.keys()  )
                  total_size += sum([value.numel() for key, value in expert_state.items()]  ) * dtype_byte_size(
                      expert_state[list(_lowerCAmelCase  )[0]].dtype  )

                   # Add the last block
      SCREAMING_SNAKE_CASE					:	int   =		os.path.join(_lowerCAmelCase	,weights_name.replace('.bin'	,f"-{len(_lowerCAmelCase  )+1:05d}-of-???.bin"  )  )
      SCREAMING_SNAKE_CASE					:	Optional[int]   =		torch.load(switch_checkpoint_path + '-shared.pt'  )["model"]
      remove_ignore_keys_(_lowerCAmelCase  )
      SCREAMING_SNAKE_CASE					:	List[Any]   =		rename_fairseq_keys(_lowerCAmelCase	,_lowerCAmelCase  )
      SCREAMING_SNAKE_CASE					:	List[str]   =		shared_weights["decoder.embed_tokens.weight"]
      sharded_state_dicts.append(shared_weights.keys()  )

      # If we only have the shared weights (dummy model/experts saved on the same file)
      if len(_lowerCAmelCase  ) == 1:
            SCREAMING_SNAKE_CASE					:	Tuple   =		os.path.join(_lowerCAmelCase	,_lowerCAmelCase  )
            torch.save(_lowerCAmelCase	,_lowerCAmelCase  )
            return {weights_name: sharded_state_dicts[0]}, None
      else:
            torch.save(_lowerCAmelCase	,_lowerCAmelCase  )
      # Otherwise, let's build the index
      SCREAMING_SNAKE_CASE					:	List[str]   =		{}
      for idx, shard in enumerate(_lowerCAmelCase  ):
            SCREAMING_SNAKE_CASE					:	Optional[int]   =		weights_name.replace('.bin'	,f"-{idx+1:05d}-of-{len(_lowerCAmelCase  ):05d}.bin"  )
            SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		os.path.join(_lowerCAmelCase	,weights_name.replace('.bin'	,f"-{idx+1:05d}-of-???.bin"  )  )
            os.rename(_lowerCAmelCase	,os.path.join(_lowerCAmelCase	,_lowerCAmelCase  )  )
            for key in shard:
                  SCREAMING_SNAKE_CASE					:	Optional[Any]   =		shard_file

                   # Add the metadata
      SCREAMING_SNAKE_CASE					:	Tuple   =		{"total_size": total_size}
      SCREAMING_SNAKE_CASE					:	str   =		{"metadata": metadata, "weight_map": weight_map}

      with open(os.path.join(_lowerCAmelCase	,_lowerCAmelCase  )	,'w'	,encoding='utf-8'  ) as f:
            SCREAMING_SNAKE_CASE					:	str   =		json.dumps(_lowerCAmelCase	,indent=2	,sort_keys=_lowerCAmelCase  ) + "\n"
            f.write(_lowerCAmelCase  )

      return metadata, index


if __name__ == "__main__":
      UpperCamelCase_       =							argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          "--nllb_moe_checkpoint_path",
          default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000",
          type=str,
          required=False,
          help="Path to a directory containing a folder per layer. Follows the original Google format.",
      )
      parser.add_argument("--dtype", default="float32", type=str, required=False, help="dtype of the saved model")
      parser.add_argument(
          "--pytorch_dump_folder_path",
          default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b",
          type=str,
          required=False,
          help="Path to the output pytorch model.",
      )
      UpperCamelCase_       =							parser.parse_args()
      UpperCamelCase_       =							shard_on_the_fly(
          args.nllb_moe_checkpoint_path,
          args.pytorch_dump_folder_path,
          1_2_8,
          args.dtype,
      )

      UpperCamelCase_       =							NllbMoeConfig.from_pretrained(
          "facebook/nllb-200-3.3B", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=1_2_8
      )
      config.save_pretrained(args.pytorch_dump_folder_path)
      UpperCamelCase_       =							NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
      print("Done")
      model.save_pretrained(args.pytorch_dump_folder_path)









def 		A_				(     _lowerCAmelCase = 50     )						-> int:
 UpperCamelCase	:					List[Any]       	=   [[0] * 3 for _ in range(length + 1     )]

 for row_length in range(length + 1     ):
  for tile_length in range(2							,   5     ):
   for tile_start in range(row_length - tile_length + 1     ):
    different_colour_ways_number[row_length][tile_length - 2] += (
        different_colour_ways_number[row_length - tile_start - tile_length][
            tile_length - 2
        ]
        + 1
    )

 return sum(different_colour_ways_number[length]     )


if __name__ == "__main__":
    print(f"""{solution() = }""")
