
'''simple docstring'''





import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextConfig, ConvNextForImageClassification, ConvNextImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
A							     = logging.get_logger(__name__)
def 	snake_case_							(				_a  :				Tuple      ):
      snake_case_    							=				ConvNextConfig()

      if "tiny" in checkpoint_url:
            snake_case_    							=				[3, 3, 9, 3]
            snake_case_    							=				[96, 192, 384, 768]
      if "small" in checkpoint_url:
            snake_case_    							=				[3, 3, 27, 3]
            snake_case_    							=				[96, 192, 384, 768]
      if "base" in checkpoint_url:
            snake_case_    							=				[3, 3, 27, 3]
            snake_case_    							=				[128, 256, 512, 1_024]
      if "large" in checkpoint_url:
            snake_case_    							=				[3, 3, 27, 3]
            snake_case_    							=				[192, 384, 768, 1_536]
      if "xlarge" in checkpoint_url:
            snake_case_    							=				[3, 3, 27, 3]
            snake_case_    							=				[256, 512, 1_024, 2_048]

      if "1k" in checkpoint_url:
            snake_case_    							=				1_000
            snake_case_    							=				"imagenet-1k-id2label.json"
            snake_case_    							=				(1, 1_000)
      else:
            snake_case_    							=				21_841
            snake_case_    							=				"imagenet-22k-id2label.json"
            snake_case_    							=				(1, 21_841)

      snake_case_    							=				"huggingface/label-files"
      snake_case_    							=				num_labels
      snake_case_    							=				json.load(open(hf_hub_download(_a			,       _a			,       repo_type="dataset"      )			,       "r"      )      )
      snake_case_    							=				{int(_a      ): v for k, v in idalabel.items()}
      if "1k" not in checkpoint_url:
            # this dataset contains 21843 labels but the model only has 21841
            # we delete the classes as mentioned in https://github.com/google-research/big_transfer/issues/18
            del idalabel[9_205]
            del idalabel[15_027]
      snake_case_    							=				idalabel
      snake_case_    							=				{v: k for k, v in idalabel.items()}
      snake_case_    							=				hidden_sizes
      snake_case_    							=				depths

      return config, expected_shape
def 	snake_case_							(				_a  :				List[str]      ):
      if "downsample_layers.0.0" in name:
            snake_case_    							=				name.replace("downsample_layers.0.0"			,       "embeddings.patch_embeddings"      )
      if "downsample_layers.0.1" in name:
            snake_case_    							=				name.replace("downsample_layers.0.1"			,       "embeddings.norm"      )  # we rename to layernorm later on
      if "downsample_layers.1.0" in name:
            snake_case_    							=				name.replace("downsample_layers.1.0"			,       "stages.1.downsampling_layer.0"      )
      if "downsample_layers.1.1" in name:
            snake_case_    							=				name.replace("downsample_layers.1.1"			,       "stages.1.downsampling_layer.1"      )
      if "downsample_layers.2.0" in name:
            snake_case_    							=				name.replace("downsample_layers.2.0"			,       "stages.2.downsampling_layer.0"      )
      if "downsample_layers.2.1" in name:
            snake_case_    							=				name.replace("downsample_layers.2.1"			,       "stages.2.downsampling_layer.1"      )
      if "downsample_layers.3.0" in name:
            snake_case_    							=				name.replace("downsample_layers.3.0"			,       "stages.3.downsampling_layer.0"      )
      if "downsample_layers.3.1" in name:
            snake_case_    							=				name.replace("downsample_layers.3.1"			,       "stages.3.downsampling_layer.1"      )
      if "stages" in name and "downsampling_layer" not in name:
            # stages.0.0. for instance should be renamed to stages.0.layers.0.
            snake_case_    							=				name[: len("stages.0"      )] + ".layers" + name[len("stages.0"      ) :]
      if "stages" in name:
            snake_case_    							=				name.replace("stages"			,       "encoder.stages"      )
      if "norm" in name:
            snake_case_    							=				name.replace("norm"			,       "layernorm"      )
      if "gamma" in name:
            snake_case_    							=				name.replace("gamma"			,       "layer_scale_parameter"      )
      if "head" in name:
            snake_case_    							=				name.replace("head"			,       "classifier"      )

      return name
def 	snake_case_							(				):
      snake_case_    							=				"http://images.cocodataset.org/val2017/000000039769.jpg"
      snake_case_    							=				Image.open(requests.get(_a			,       stream=_a      ).raw      )
      return im
@torch.no_grad()
def 	snake_case_							(				_a  :				Any			,       _a  :				List[Any]      ):
      snake_case_							,						snake_case_    							=				get_convnext_config(_a      )
      # load original state_dict from URL
      snake_case_    							=				torch.hub.load_state_dict_from_url(_a      )["model"]
      # rename keys
      for key in state_dict.copy().keys():
            snake_case_    							=				state_dict.pop(_a      )
            snake_case_    							=				val
      # add prefix to all keys expect classifier head
      for key in state_dict.copy().keys():
            snake_case_    							=				state_dict.pop(_a      )
            if not key.startswith("classifier"      ):
                  snake_case_    							=				"convnext." + key
            snake_case_    							=				val

      # load HuggingFace model
      snake_case_    							=				ConvNextForImageClassification(_a      )
      model.load_state_dict(_a      )
      model.eval()

      # Check outputs on an image, prepared by ConvNextImageProcessor
      snake_case_    							=				224 if "224" in checkpoint_url else 384
      snake_case_    							=				ConvNextImageProcessor(size=_a      )
      snake_case_    							=				image_processor(images=prepare_img()			,       return_tensors="pt"      ).pixel_values

      snake_case_    							=				model(_a      ).logits

      # note: the logits below were obtained without center cropping
      if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth":
            snake_case_    							=				torch.tensor([-0.1_2_1_0, -0.6_6_0_5, 0.1_9_1_8]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_small_1k_224_ema.pth":
            snake_case_    							=				torch.tensor([-0.4_4_7_3, -0.1_8_4_7, -0.6_3_6_5]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_224_ema.pth":
            snake_case_    							=				torch.tensor([0.4_5_2_5, 0.7_5_3_9, 0.0_3_0_8]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_384.pth":
            snake_case_    							=				torch.tensor([0.3_5_6_1, 0.6_3_5_0, -0.0_3_8_4]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_224_ema.pth":
            snake_case_    							=				torch.tensor([0.4_1_7_4, -0.0_9_8_9, 0.1_4_8_9]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_384.pth":
            snake_case_    							=				torch.tensor([0.2_5_1_3, -0.1_3_4_9, -0.1_6_1_3]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_224.pth":
            snake_case_    							=				torch.tensor([1.2_9_8_0, 0.3_6_3_1, -0.1_1_9_8]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_224.pth":
            snake_case_    							=				torch.tensor([1.2_9_6_3, 0.1_2_2_7, 0.1_7_2_3]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_224.pth":
            snake_case_    							=				torch.tensor([1.7_9_5_6, 0.8_3_9_0, 0.2_8_2_0]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_224.pth":
            snake_case_    							=				torch.tensor([-0.2_8_2_2, -0.0_5_0_2, -0.0_8_7_8]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_384.pth":
            snake_case_    							=				torch.tensor([-0.5_6_7_2, -0.0_7_3_0, -0.4_3_4_8]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_224.pth":
            snake_case_    							=				torch.tensor([0.2_6_8_1, 0.2_3_6_5, 0.6_2_4_6]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_384.pth":
            snake_case_    							=				torch.tensor([-0.2_6_4_2, 0.3_9_3_1, 0.5_1_1_6]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_224_ema.pth":
            snake_case_    							=				torch.tensor([-0.6_6_7_7, -0.1_8_7_3, -0.8_3_7_9]      )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_384_ema.pth":
            snake_case_    							=				torch.tensor([-0.7_7_4_9, -0.2_9_6_7, -0.6_4_4_4]      )
      else:
            raise ValueError(F'''Unknown URL: {checkpoint_url}'''      )

      assert torch.allclose(logits[0, :3]			,       _a			,       atol=1E-3      )
      assert logits.shape == expected_shape

      Path(_a      ).mkdir(exist_ok=_a      )
      print(F'''Saving model to {pytorch_dump_folder_path}'''      )
      model.save_pretrained(_a      )
      print(F'''Saving image processor to {pytorch_dump_folder_path}'''      )
      image_processor.save_pretrained(_a      )

      print("Pushing model to the hub..."      )
      snake_case_    							=				"convnext"
      if "tiny" in checkpoint_url:
            model_name += "-tiny"
      elif "small" in checkpoint_url:
            model_name += "-small"
      elif "base" in checkpoint_url:
            model_name += "-base"
      elif "xlarge" in checkpoint_url:
            model_name += "-xlarge"
      elif "large" in checkpoint_url:
            model_name += "-large"
      if "224" in checkpoint_url:
            model_name += "-224"
      elif "384" in checkpoint_url:
            model_name += "-384"
      if "22k" in checkpoint_url and "1k" not in checkpoint_url:
            model_name += "-22k"
      if "22k" in checkpoint_url and "1k" in checkpoint_url:
            model_name += "-22k-1k"

      model.push_to_hub(
          repo_path_or_name=Path(_a			,       _a      )			,       organization="nielsr"			,       commit_message="Add model"			,       )


if __name__ == "__main__":
     A							     = argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         "--checkpoint_url",
         default="https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth",
         type=str,
         help="URL of the original ConvNeXT checkpoint you'd like to convert.",
     )
     parser.add_argument(
         "--pytorch_dump_folder_path",
         default=None,
         type=str,
         required=True,
         help="Path to the output PyTorch model directory.",
     )

     A							     = parser.parse_args()
     convert_convnext_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)



'''simple docstring'''





import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch
def 	snake_case_							(				):
      snake_case_    							=				Dataset.from_dict({"i": range(17      )}      )
      snake_case_    							=				len(_a      )
      snake_case_    							=				3
      snake_case_    							=				[
          split_dataset_by_node(_a			,       rank=_a			,       world_size=_a      ) for rank in range(_a      )
      ]
      assert sum(len(_a      ) for ds in datasets_per_rank      ) == full_size
      assert len({tuple(x.values()      ) for ds in datasets_per_rank for x in ds}      ) == full_size
def 	snake_case_							(				):
      def gen():
            return ({"i": i} for i in range(17      ))

      snake_case_    							=				3
      snake_case_    							=				IterableDataset.from_generator(_a      )
      snake_case_    							=				len(list(_a      )      )
      snake_case_    							=				[
          split_dataset_by_node(_a			,       rank=_a			,       world_size=_a      ) for rank in range(_a      )
      ]
      assert sum(len(list(_a      )      ) for ds in datasets_per_rank      ) == full_size
      assert len({tuple(x.values()      ) for ds in datasets_per_rank for x in ds}      ) == full_size
@pytest.mark.parametrize("shards_per_node"			,       [1, 2, 3]      )
def 	snake_case_							(				_a  :				List[Any]      ):
      def gen(_a  :				Dict      ):
            for shard in shards:
                  yield from ({"i": i, "shard": shard} for i in range(17      ))

      snake_case_    							=				3
      snake_case_    							=				shards_per_node * world_size
      snake_case_    							=				{"shards": [F'''shard_{shard_idx}.txt''' for shard_idx in range(_a      )]}
      snake_case_    							=				IterableDataset.from_generator(_a			,       gen_kwargs=_a      )
      snake_case_    							=				len(list(_a      )      )
      assert full_ds.n_shards == world_size * shards_per_node
      snake_case_    							=				[
          split_dataset_by_node(_a			,       rank=_a			,       world_size=_a      ) for rank in range(_a      )
      ]
      assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
      assert sum(len(list(_a      )      ) for ds in datasets_per_rank      ) == full_size
      assert len({tuple(x.values()      ) for ds in datasets_per_rank for x in ds}      ) == full_size
def 	snake_case_							(				):
      def gen():
            return ({"i": i} for i in range(17      ))

      snake_case_    							=				2
      snake_case_    							=				IterableDataset.from_generator(_a      )
      snake_case_    							=				len(list(_a      )      )

      snake_case_    							=				split_dataset_by_node(_a			,       rank=0			,       world_size=_a      ).shuffle(seed=42      )
      assert len(list(_a      )      ) == 1 + full_size // world_size
      with pytest.raises(_a      ):
            split_dataset_by_node(_a			,       rank=0			,       world_size=_a      ).shuffle()

      snake_case_    							=				split_dataset_by_node(full_ds.shuffle(seed=42      )			,       rank=0			,       world_size=_a      )
      assert len(list(_a      )      ) == 1 + full_size // world_size
      with pytest.raises(_a      ):
            split_dataset_by_node(full_ds.shuffle()			,       rank=0			,       world_size=_a      )
@pytest.mark.parametrize("streaming"			,       [False, True]      )
@require_torch
@pytest.mark.skipif(os.name == "nt"			,       reason="execute_subprocess_async doesn't support windows"      )
@pytest.mark.integration
def 	snake_case_							(				_a  :				Any      ):
      snake_case_    							=				2
      snake_case_    							=				get_torch_dist_unique_port()
      snake_case_    							=				Path(_a      ).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
      snake_case_    							=				F'''
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    '''.split()
      snake_case_    							=				F'''
        --streaming={streaming}
    '''.split()
      snake_case_    							=				[sys.executable] + distributed_args + args
      execute_subprocess_async(_a			,       env=os.environ.copy()      )




@pytest.mark.parametrize(
    "nproc_per_node, num_workers"			,       [
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ]			,       )
@require_torch
@pytest.mark.skipif(os.name == "nt"			,       reason="execute_subprocess_async doesn't support windows"      )
@pytest.mark.integration
def 	snake_case_							(				_a  :				int			,       _a  :				List[Any]      ):
      snake_case_    							=				True
      snake_case_    							=				get_torch_dist_unique_port()
      snake_case_    							=				Path(_a      ).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
      snake_case_    							=				F'''
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    '''.split()
      snake_case_    							=				F'''
        --streaming={streaming}
        --num_workers={num_workers}
    '''.split()
      snake_case_    							=				[sys.executable] + distributed_args + args
      execute_subprocess_async(_a			,       env=os.environ.copy()      )

