






'''simple docstring'''




import argparse
import json
from collections import OrderedDict
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    SegformerConfig,
    SegformerForImageClassification,
    SegformerForSemanticSegmentation,
    SegformerImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
lowerCamelCase_												=					logging.get_logger(__name__)

def 					__lowercase						(			__lowercase   ,  __lowercase=False       )       ->						List[str]:

      '''simple docstring'''


      _A			     =	OrderedDict()
      for key, value in state_dict.items():
            if encoder_only and not key.startswith("head"       ):
                  _A			     =	"segformer.encoder." + key
            if key.startswith("backbone"       ):
                  _A			     =	key.replace("backbone"   ,  "segformer.encoder"       )
            if "patch_embed" in key:
                  # replace for example patch_embed1 by patch_embeddings.0
                  _A			     =	key[key.find("patch_embed"       ) + len("patch_embed"       )]
                  _A			     =	key.replace(F'''patch_embed{idx}'''   ,  F'''patch_embeddings.{int(__lowercase       )-1}'''       )
            if "norm" in key:
                  _A			     =	key.replace("norm"   ,  "layer_norm"       )
            if "segformer.encoder.layer_norm" in key:
                  # replace for example layer_norm1 by layer_norm.0
                  _A			     =	key[key.find("segformer.encoder.layer_norm"       ) + len("segformer.encoder.layer_norm"       )]
                  _A			     =	key.replace(F'''layer_norm{idx}'''   ,  F'''layer_norm.{int(__lowercase       )-1}'''       )
            if "layer_norm1" in key:
                  _A			     =	key.replace("layer_norm1"   ,  "layer_norm_1"       )
            if "layer_norm2" in key:
                  _A			     =	key.replace("layer_norm2"   ,  "layer_norm_2"       )
            if "block" in key:
                  # replace for example block1 by block.0
                  _A			     =	key[key.find("block"       ) + len("block"       )]
                  _A			     =	key.replace(F'''block{idx}'''   ,  F'''block.{int(__lowercase       )-1}'''       )
            if "attn.q" in key:
                  _A			     =	key.replace("attn.q"   ,  "attention.self.query"       )
            if "attn.proj" in key:
                  _A			     =	key.replace("attn.proj"   ,  "attention.output.dense"       )
            if "attn" in key:
                  _A			     =	key.replace("attn"   ,  "attention.self"       )
            if "fc1" in key:
                  _A			     =	key.replace("fc1"   ,  "dense1"       )
            if "fc2" in key:
                  _A			     =	key.replace("fc2"   ,  "dense2"       )
            if "linear_pred" in key:
                  _A			     =	key.replace("linear_pred"   ,  "classifier"       )
            if "linear_fuse" in key:
                  _A			     =	key.replace("linear_fuse.conv"   ,  "linear_fuse"       )
                  _A			     =	key.replace("linear_fuse.bn"   ,  "batch_norm"       )
            if "linear_c" in key:
                  # replace for example linear_c4 by linear_c.3
                  _A			     =	key[key.find("linear_c"       ) + len("linear_c"       )]
                  _A			     =	key.replace(F'''linear_c{idx}'''   ,  F'''linear_c.{int(__lowercase       )-1}'''       )
            if key.startswith("head"       ):
                  _A			     =	key.replace("head"   ,  "classifier"       )
            _A			     =	value

      return new_state_dict

def 					__lowercase						(			__lowercase   ,  __lowercase       )       ->						Optional[Any]:

      '''simple docstring'''


      for i in range(config.num_encoder_blocks       ):
            for j in range(config.depths[i]       ):
                  # read in weights + bias of keys and values (which is a single matrix in the original implementation)
                  _A			     =	state_dict.pop(F'''segformer.encoder.block.{i}.{j}.attention.self.kv.weight'''       )
                  _A			     =	state_dict.pop(F'''segformer.encoder.block.{i}.{j}.attention.self.kv.bias'''       )
                  # next, add keys and values (in that order) to the state dict
                  _A			     =	kv_weight[
                      : config.hidden_sizes[i], :
                  ]
                  _A			     =	kv_bias[: config.hidden_sizes[i]]
                  _A			     =	kv_weight[
                      config.hidden_sizes[i] :, :
                  ]
                  _A			     =	kv_bias[
                      config.hidden_sizes[i] :
                  ]

def 					__lowercase						(			)       ->						List[str]:

      '''simple docstring'''


      _A			     =	"http://images.cocodataset.org/val2017/000000039769.jpg"
      _A			     =	Image.open(requests.get(__lowercase   ,  stream=__lowercase       ).raw       )

      return image

@torch.no_grad()
def 					__lowercase						(			__lowercase   ,  __lowercase   ,  __lowercase       )       ->						str:

      '''simple docstring'''


      _A			     =	SegformerConfig()
      _A			     =	False

      # set attributes based on model_name
      _A			     =	"huggingface/label-files"
      if "segformer" in model_name:
            _A			     =	model_name[len("segformer."       ) : len("segformer."       ) + 2]
            if "ade" in model_name:
                  _A			     =	150
                  _A			     =	"ade20k-id2label.json"
                  _A			     =	(1, 150, 128, 128)
            elif "city" in model_name:
                  _A			     =	19
                  _A			     =	"cityscapes-id2label.json"
                  _A			     =	(1, 19, 128, 128)
            else:
                  raise ValueError(F'''Model {model_name} not supported'''       )
      elif "mit" in model_name:
            _A			     =	True
            _A			     =	model_name[4:6]
            _A			     =	1000
            _A			     =	"imagenet-1k-id2label.json"
            _A			     =	(1, 1000)
      else:
            raise ValueError(F'''Model {model_name} not supported'''       )

      # set config attributes
      _A			     =	json.load(open(hf_hub_download(__lowercase   ,  __lowercase   ,  repo_type="dataset"       )   ,  "r"       )       )
      _A			     =	{int(__lowercase       ): v for k, v in idalabel.items()}
      _A			     =	idalabel
      _A			     =	{v: k for k, v in idalabel.items()}
      if size == "b0":
            pass
      elif size == "b1":
            _A			     =	[64, 128, 320, 512]
            _A			     =	256
      elif size == "b2":
            _A			     =	[64, 128, 320, 512]
            _A			     =	768
            _A			     =	[3, 4, 6, 3]
      elif size == "b3":
            _A			     =	[64, 128, 320, 512]
            _A			     =	768
            _A			     =	[3, 4, 18, 3]
      elif size == "b4":
            _A			     =	[64, 128, 320, 512]
            _A			     =	768
            _A			     =	[3, 8, 27, 3]
      elif size == "b5":
            _A			     =	[64, 128, 320, 512]
            _A			     =	768
            _A			     =	[3, 6, 40, 3]
      else:
            raise ValueError(F'''Size {size} not supported'''       )

      # load image processor (only resize + normalize)
      _A			     =	SegformerImageProcessor(
          image_scale=(512, 512)   ,  keep_ratio=__lowercase   ,  align=__lowercase   ,  do_random_crop=__lowercase       )

      # prepare image
      _A			     =	prepare_img()
      _A			     =	image_processor(images=__lowercase   ,  return_tensors="pt"       ).pixel_values

      logger.info(F'''Converting model {model_name}...'''       )

      # load original state dict
      if encoder_only:
            _A			     =	torch.load(__lowercase   ,  map_location=torch.device("cpu"       )       )
      else:
            _A			     =	torch.load(__lowercase   ,  map_location=torch.device("cpu"       )       )["state_dict"]

      # rename keys
      _A			     =	rename_keys(__lowercase   ,  encoder_only=__lowercase       )
      if not encoder_only:
            del state_dict["decode_head.conv_seg.weight"]
            del state_dict["decode_head.conv_seg.bias"]

      # key and value matrices need special treatment
      read_in_k_v(__lowercase   ,  __lowercase       )

      # create HuggingFace model and load state dict
      if encoder_only:
            _A			     =	False
            _A			     =	SegformerForImageClassification(__lowercase       )
      else:
            _A			     =	SegformerForSemanticSegmentation(__lowercase       )
      model.load_state_dict(__lowercase       )
      model.eval()

      # forward pass
      _A			     =	model(__lowercase       )
      _A			     =	outputs.logits

      # set expected_slice based on model name
      # ADE20k checkpoints
      if model_name == "segformer.b0.512x512.ade.160k":
            _A			     =	torch.tensor(
                [
                    [[-4.6310, -5.5232, -6.2356], [-5.1921, -6.1444, -6.5996], [-5.4424, -6.2790, -6.7574]],
                    [[-12.1391, -13.3122, -13.9554], [-12.8732, -13.9352, -14.3563], [-12.9438, -13.8226, -14.2513]],
                    [[-12.5134, -13.4686, -14.4915], [-12.8669, -14.4343, -14.7758], [-13.2523, -14.5819, -15.0694]],
                ]       )
      elif model_name == "segformer.b1.512x512.ade.160k":
            _A			     =	torch.tensor(
                [
                    [[-7.5820, -8.7231, -8.3215], [-8.0600, -10.3529, -10.0304], [-7.5208, -9.4103, -9.6239]],
                    [[-12.6918, -13.8994, -13.7137], [-13.3196, -15.7523, -15.4789], [-12.9343, -14.8757, -14.9689]],
                    [[-11.1911, -11.9421, -11.3243], [-11.3342, -13.6839, -13.3581], [-10.3909, -12.1832, -12.4858]],
                ]       )
      elif model_name == "segformer.b2.512x512.ade.160k":
            _A			     =	torch.tensor(
                [
                    [[-11.8173, -14.3850, -16.3128], [-14.5648, -16.5804, -18.6568], [-14.7223, -15.7387, -18.4218]],
                    [[-15.7290, -17.9171, -19.4423], [-18.3105, -19.9448, -21.4661], [-17.9296, -18.6497, -20.7910]],
                    [[-15.0783, -17.0336, -18.2789], [-16.8771, -18.6870, -20.1612], [-16.2454, -17.1426, -19.5055]],
                ]       )
      elif model_name == "segformer.b3.512x512.ade.160k":
            _A			     =	torch.tensor(
                [
                    [[-9.0878, -10.2081, -10.1891], [-9.3144, -10.7941, -10.9843], [-9.2294, -10.3855, -10.5704]],
                    [[-12.2316, -13.9068, -13.6102], [-12.9161, -14.3702, -14.3235], [-12.5233, -13.7174, -13.7932]],
                    [[-14.6275, -15.2490, -14.9727], [-14.3400, -15.9687, -16.2827], [-14.1484, -15.4033, -15.8937]],
                ]       )
      elif model_name == "segformer.b4.512x512.ade.160k":
            _A			     =	torch.tensor(
                [
                    [[-12.3144, -13.2447, -14.0802], [-13.3614, -14.5816, -15.6117], [-13.3340, -14.4433, -16.2219]],
                    [[-19.2781, -20.4128, -20.7506], [-20.6153, -21.6566, -22.0998], [-19.9800, -21.0430, -22.1494]],
                    [[-18.8739, -19.7804, -21.1834], [-20.1233, -21.6765, -23.2944], [-20.0315, -21.2641, -23.6944]],
                ]       )
      elif model_name == "segformer.b5.640x640.ade.160k":
            _A			     =	torch.tensor(
                [
                    [[-9.5524, -12.0835, -11.7348], [-10.5229, -13.6446, -14.5662], [-9.5842, -12.8851, -13.9414]],
                    [[-15.3432, -17.5323, -17.0818], [-16.3330, -18.9255, -19.2101], [-15.1340, -17.7848, -18.3971]],
                    [[-12.6072, -14.9486, -14.6631], [-13.7629, -17.0907, -17.7745], [-12.7899, -16.1695, -17.1671]],
                ]       )
      # Cityscapes checkpoints
      elif model_name == "segformer.b0.1024x1024.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-11.9295, -13.4057, -14.8106], [-13.3431, -14.8179, -15.3781], [-14.2836, -15.5942, -16.1588]],
                    [[-11.4906, -12.8067, -13.6564], [-13.1189, -14.0500, -14.1543], [-13.8748, -14.5136, -14.8789]],
                    [[0.5374, 0.1067, -0.4742], [0.1141, -0.2255, -0.7099], [-0.3000, -0.5924, -1.3105]],
                ]       )
      elif model_name == "segformer.b0.512x1024.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-7.8217, -9.8767, -10.1717], [-9.4438, -10.9058, -11.4047], [-9.7939, -12.3495, -12.1079]],
                    [[-7.1514, -9.5336, -10.0860], [-9.7776, -11.6822, -11.8439], [-10.1411, -12.7655, -12.8972]],
                    [[0.3021, 0.0805, -0.2310], [-0.0328, -0.1605, -0.2714], [-0.1408, -0.5477, -0.6976]],
                ]       )
      elif model_name == "segformer.b0.640x1280.city.160k":
            _A			     =	torch.tensor(
                [
                    [
                        [-1.1372e01, -1.2787e01, -1.3477e01],
                        [-1.2536e01, -1.4194e01, -1.4409e01],
                        [-1.3217e01, -1.4888e01, -1.5327e01],
                    ],
                    [
                        [-1.4791e01, -1.7122e01, -1.8277e01],
                        [-1.7163e01, -1.9192e01, -1.9533e01],
                        [-1.7897e01, -1.9991e01, -2.0315e01],
                    ],
                    [
                        [7.6723e-01, 4.1921e-01, -7.7878e-02],
                        [4.7772e-01, 9.5557e-03, -2.8082e-01],
                        [3.6032e-01, -2.4826e-01, -5.1168e-01],
                    ],
                ]       )
      elif model_name == "segformer.b0.768x768.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-9.4959, -11.3087, -11.7479], [-11.0025, -12.6540, -12.3319], [-11.4064, -13.0487, -12.9905]],
                    [[-9.8905, -11.3084, -12.0854], [-11.1726, -12.7698, -12.9583], [-11.5985, -13.3278, -14.1774]],
                    [[0.2213, 0.0192, -0.2466], [-0.1731, -0.4213, -0.4874], [-0.3126, -0.6541, -1.1389]],
                ]       )
      elif model_name == "segformer.b1.1024x1024.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-13.5748, -13.9111, -12.6500], [-14.3500, -15.3683, -14.2328], [-14.7532, -16.0424, -15.6087]],
                    [[-17.1651, -15.8725, -12.9653], [-17.2580, -17.3718, -14.8223], [-16.6058, -16.8783, -16.7452]],
                    [[-3.6456, -3.0209, -1.4203], [-3.0797, -3.1959, -2.0000], [-1.8757, -1.9217, -1.6997]],
                ]       )
      elif model_name == "segformer.b2.1024x1024.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-16.0976, -16.4856, -17.3962], [-16.6234, -19.0342, -19.7685], [-16.0900, -18.0661, -19.1180]],
                    [[-18.4750, -18.8488, -19.5074], [-19.4030, -22.1570, -22.5977], [-19.1191, -20.8486, -22.3783]],
                    [[-4.5178, -5.5037, -6.5109], [-5.0884, -7.2174, -8.0334], [-4.4156, -5.8117, -7.2970]],
                ]       )
      elif model_name == "segformer.b3.1024x1024.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-14.2081, -14.4732, -14.1977], [-14.5867, -16.4423, -16.6356], [-13.4441, -14.9685, -16.8696]],
                    [[-14.4576, -14.7073, -15.0451], [-15.0816, -17.6237, -17.9873], [-14.4213, -16.0199, -18.5992]],
                    [[-4.7349, -4.9588, -5.0966], [-4.3210, -6.9325, -7.2591], [-3.4312, -4.7484, -7.1917]],
                ]       )
      elif model_name == "segformer.b4.1024x1024.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-11.7737, -11.9526, -11.3273], [-13.6692, -14.4574, -13.8878], [-13.8937, -14.6924, -15.9345]],
                    [[-14.6706, -14.5330, -14.1306], [-16.1502, -16.8180, -16.4269], [-16.8338, -17.8939, -20.1746]],
                    [[1.0491, 0.8289, 1.0310], [1.1044, 0.5219, 0.8055], [1.0899, 0.6926, 0.5590]],
                ]       )
      elif model_name == "segformer.b5.1024x1024.city.160k":
            _A			     =	torch.tensor(
                [
                    [[-12.5641, -13.4777, -13.0684], [-13.9587, -15.8983, -16.6557], [-13.3109, -15.7350, -16.3141]],
                    [[-14.7074, -15.4352, -14.5944], [-16.6353, -18.1663, -18.6120], [-15.1702, -18.0329, -18.1547]],
                    [[-1.7990, -2.0951, -1.7784], [-2.6397, -3.8245, -3.9686], [-1.5264, -2.8126, -2.9316]],
                ]       )
      else:
            _A			     =	logits.argmax(-1       ).item()
            print("Predicted class:"   ,  model.config.idalabel[predicted_class_idx]       )

      # verify logits
      if not encoder_only:
            assert logits.shape == expected_shape
            assert torch.allclose(logits[0, :3, :3, :3]   ,  __lowercase   ,  atol=1e-2       )

      # finally, save model and image processor
      logger.info(F'''Saving PyTorch model and image processor to {pytorch_dump_folder_path}...'''       )
      Path(__lowercase       ).mkdir(exist_ok=__lowercase       )
      model.save_pretrained(__lowercase       )
      image_processor.save_pretrained(__lowercase       )


if __name__ == "__main__":
    lowerCamelCase_												=					argparse.ArgumentParser()

    parser.add_argument(
        '''--model_name''',
        default='''segformer.b0.512x512.ade.160k''',
        type=str,
        help='''Name of the model you\'d like to convert.''',
    )
    parser.add_argument(
        '''--checkpoint_path''', default=None, type=str, help='''Path to the original PyTorch checkpoint (.pth file).'''
    )
    parser.add_argument(
        '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the folder to output PyTorch model.'''
    )
    lowerCamelCase_												=					parser.parse_args()
    convert_segformer_checkpoint(args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path)







'''simple docstring'''




import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


lowercase			:				str    		=			get_tests_dir('fixtures/dummy_feature_extractor_config.json')
lowercase			:				str    		=			get_tests_dir('fixtures/vocab.json')
lowercase			:				int    		=			get_tests_dir('fixtures')





class  A				(							unittest.TestCase						):
     __magic_name__						=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





     def 				__lowerCAmelCase							(							self	) ->				List[str]:






         """simple docstring"""






         A       :   Tuple					=						0





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   List[Any]					=						AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''	)
         self.assertIsInstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				List[str]:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmpdirname:
             A       :   Union[str, Any]					=						WavaVecaConfig()
             A       :   List[str]					=						AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''	)

             # save in new folder
             model_config.save_pretrained(SCREAMING_SNAKE_CASE	)
             processor.save_pretrained(SCREAMING_SNAKE_CASE	)

             A       :   List[str]					=						AutoProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)

         self.assertIsInstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				int:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmpdirname:
             # copy relevant files
             copyfile(SCREAMING_SNAKE_CASE						,			os.path.join(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)	)
             copyfile(SCREAMING_SNAKE_CASE						,			os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.json'''	)	)

             A       :   Optional[Any]					=						AutoProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)

         self.assertIsInstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Union[str, Any]:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmpdirname:
             A       :   Dict					=						WavaVecaFeatureExtractor()
             A       :   List[str]					=						AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''	)

             A       :   str					=						WavaVecaProcessor(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

             # save in new folder
             processor.save_pretrained(SCREAMING_SNAKE_CASE	)

             # drop `processor_class` in tokenizer
             with open(os.path.join(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)						,			'''r'''	) as f:
                 A       :   Dict					=						json.load(SCREAMING_SNAKE_CASE	)
                 config_dict.pop('''processor_class'''	)

             with open(os.path.join(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)						,			'''w'''	) as f:
                 f.write(json.dumps(SCREAMING_SNAKE_CASE	)	)

             A       :   Optional[Any]					=						AutoProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)

         self.assertIsInstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				int:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmpdirname:
             A       :   List[Any]					=						WavaVecaFeatureExtractor()
             A       :   List[Any]					=						AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''	)

             A       :   str					=						WavaVecaProcessor(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

             # save in new folder
             processor.save_pretrained(SCREAMING_SNAKE_CASE	)

             # drop `processor_class` in feature extractor
             with open(os.path.join(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)						,			'''r'''	) as f:
                 A       :   str					=						json.load(SCREAMING_SNAKE_CASE	)
                 config_dict.pop('''processor_class'''	)

             with open(os.path.join(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)						,			'''w'''	) as f:
                 f.write(json.dumps(SCREAMING_SNAKE_CASE	)	)

             A       :   str					=						AutoProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)

         self.assertIsInstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmpdirname:
             A       :   str					=						WavaVecaConfig(processor_class='''Wav2Vec2Processor'''	)
             model_config.save_pretrained(SCREAMING_SNAKE_CASE	)
             # copy relevant files
             copyfile(SCREAMING_SNAKE_CASE						,			os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.json'''	)	)
             # create emtpy sample processor
             with open(os.path.join(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)						,			'''w'''	) as f:
                 f.write('''{}'''	)

             A       :   List[str]					=						AutoProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)

         self.assertIsInstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         with self.assertRaises(SCREAMING_SNAKE_CASE	):
             A       :   Optional[Any]					=						AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''	)
         # If remote code is disabled, we can't load this config.
         with self.assertRaises(SCREAMING_SNAKE_CASE	):
             A       :   Union[str, Any]					=						AutoProcessor.from_pretrained(
                 '''hf-internal-testing/test_dynamic_processor'''						,			trust_remote_code=SCREAMING_SNAKE_CASE	)

         A       :   Union[str, Any]					=						AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''						,			trust_remote_code=SCREAMING_SNAKE_CASE	)
         self.assertTrue(processor.special_attribute_present	)
         self.assertEqual(processor.__class__.__name__						,			'''NewProcessor'''	)

         A       :   List[str]					=						processor.feature_extractor
         self.assertTrue(feature_extractor.special_attribute_present	)
         self.assertEqual(feature_extractor.__class__.__name__						,			'''NewFeatureExtractor'''	)

         A       :   Tuple					=						processor.tokenizer
         self.assertTrue(tokenizer.special_attribute_present	)
         if is_tokenizers_available():
             self.assertEqual(tokenizer.__class__.__name__						,			'''NewTokenizerFast'''	)

             # Test we can also load the slow version
             A       :   List[str]					=						AutoProcessor.from_pretrained(
                 '''hf-internal-testing/test_dynamic_processor'''						,			trust_remote_code=SCREAMING_SNAKE_CASE						,			use_fast=SCREAMING_SNAKE_CASE	)
             A       :   int					=						new_processor.tokenizer
             self.assertTrue(new_tokenizer.special_attribute_present	)
             self.assertEqual(new_tokenizer.__class__.__name__						,			'''NewTokenizer'''	)
         else:
             self.assertEqual(tokenizer.__class__.__name__						,			'''NewTokenizer'''	)





     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         try:
             AutoConfig.register('''custom'''						,			SCREAMING_SNAKE_CASE	)
             AutoFeatureExtractor.register(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)
             AutoTokenizer.register(SCREAMING_SNAKE_CASE						,			slow_tokenizer_class=SCREAMING_SNAKE_CASE	)
             AutoProcessor.register(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)
             # Trying to register something existing in the Transformers library will raise an error
             with self.assertRaises(SCREAMING_SNAKE_CASE	):
                 AutoProcessor.register(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

             # Now that the config is registered, it can be used as any other config with the auto-API
             A       :   List[Any]					=						CustomFeatureExtractor.from_pretrained(SCREAMING_SNAKE_CASE	)

             with tempfile.TemporaryDirectory() as tmp_dir:
                 A       :   Tuple					=						os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.txt'''	)
                 with open(SCREAMING_SNAKE_CASE						,			'''w'''						,			encoding='''utf-8'''	) as vocab_writer:
                     vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]	)	)
                 A       :   Optional[int]					=						CustomTokenizer(SCREAMING_SNAKE_CASE	)

             A       :   Any					=						CustomProcessor(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

             with tempfile.TemporaryDirectory() as tmp_dir:
                 processor.save_pretrained(SCREAMING_SNAKE_CASE	)
                 A       :   List[str]					=						AutoProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)
                 self.assertIsInstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

         finally:
             if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
             if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                 del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
             if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
             if CustomConfig in PROCESSOR_MAPPING._extra_content:
                 del PROCESSOR_MAPPING._extra_content[CustomConfig]





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         class  A				(							__snake_case						):
              __magic_name__						=		False





         class  A				(							__snake_case						):
              __magic_name__						=		False





         class  A				(							__snake_case						):
              __magic_name__						=		'''AutoFeatureExtractor'''
              __magic_name__						=		'''AutoTokenizer'''
              __magic_name__						=		False

         try:
             AutoConfig.register('''custom'''						,			SCREAMING_SNAKE_CASE	)
             AutoFeatureExtractor.register(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)
             AutoTokenizer.register(SCREAMING_SNAKE_CASE						,			slow_tokenizer_class=SCREAMING_SNAKE_CASE	)
             AutoProcessor.register(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)
             # If remote code is not set, the default is to use local classes.
             A       :   Union[str, Any]					=						AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''	)
             self.assertEqual(processor.__class__.__name__						,			'''NewProcessor'''	)
             self.assertFalse(processor.special_attribute_present	)
             self.assertFalse(processor.feature_extractor.special_attribute_present	)
             self.assertFalse(processor.tokenizer.special_attribute_present	)

             # If remote code is disabled, we load the local ones.
             A       :   Optional[int]					=						AutoProcessor.from_pretrained(
                 '''hf-internal-testing/test_dynamic_processor'''						,			trust_remote_code=SCREAMING_SNAKE_CASE	)
             self.assertEqual(processor.__class__.__name__						,			'''NewProcessor'''	)
             self.assertFalse(processor.special_attribute_present	)
             self.assertFalse(processor.feature_extractor.special_attribute_present	)
             self.assertFalse(processor.tokenizer.special_attribute_present	)

             # If remote is enabled, we load from the Hub.
             A       :   Tuple					=						AutoProcessor.from_pretrained(
                 '''hf-internal-testing/test_dynamic_processor'''						,			trust_remote_code=SCREAMING_SNAKE_CASE	)
             self.assertEqual(processor.__class__.__name__						,			'''NewProcessor'''	)
             self.assertTrue(processor.special_attribute_present	)
             self.assertTrue(processor.feature_extractor.special_attribute_present	)
             self.assertTrue(processor.tokenizer.special_attribute_present	)

         finally:
             if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
             if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                 del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
             if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
             if CustomConfig in PROCESSOR_MAPPING._extra_content:
                 del PROCESSOR_MAPPING._extra_content[CustomConfig]





     def 				__lowerCAmelCase							(							self	) ->				str:






         """simple docstring"""






         A       :   int					=						AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-bert'''	)
         self.assertEqual(processor.__class__.__name__						,			'''BertTokenizerFast'''	)





     def 				__lowerCAmelCase							(							self	) ->				str:






         """simple docstring"""






         A       :   Optional[int]					=						AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-convnext'''	)
         self.assertEqual(processor.__class__.__name__						,			'''ConvNextImageProcessor'''	)





@is_staging_test
class  A				(							unittest.TestCase						):
     __magic_name__						=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





     @classmethod
     def 				__lowerCAmelCase							(							cls	) ->				Dict:






         """simple docstring"""






         A       :   Optional[int]					=						TOKEN
         HfFolder.save_token(SCREAMING_SNAKE_CASE	)





     @classmethod
     def 				__lowerCAmelCase							(							cls	) ->				Any:






         """simple docstring"""






         try:
             delete_repo(token=cls._token						,			repo_id='''test-processor'''	)
         except HTTPError:
             pass

         try:
             delete_repo(token=cls._token						,			repo_id='''valid_org/test-processor-org'''	)
         except HTTPError:
             pass

         try:
             delete_repo(token=cls._token						,			repo_id='''test-dynamic-processor'''	)
         except HTTPError:
             pass





     def 				__lowerCAmelCase							(							self	) ->				Dict:






         """simple docstring"""






         A       :   Union[str, Any]					=						WavaVecaProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)
         with tempfile.TemporaryDirectory() as tmp_dir:
             processor.save_pretrained(
                 os.path.join(SCREAMING_SNAKE_CASE						,			'''test-processor'''	)						,			push_to_hub=SCREAMING_SNAKE_CASE						,			use_auth_token=self._token	)

             A       :   int					=						WavaVecaProcessor.from_pretrained(F'{USER}/test-processor'	)
             for k, v in processor.feature_extractor.__dict__.items():
                 self.assertEqual(SCREAMING_SNAKE_CASE						,			getattr(new_processor.feature_extractor						,			SCREAMING_SNAKE_CASE	)	)
             self.assertDictEqual(new_processor.tokenizer.get_vocab()						,			processor.tokenizer.get_vocab()	)





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   Tuple					=						WavaVecaProcessor.from_pretrained(SCREAMING_SNAKE_CASE	)

         with tempfile.TemporaryDirectory() as tmp_dir:
             processor.save_pretrained(
                 os.path.join(SCREAMING_SNAKE_CASE						,			'''test-processor-org'''	)						,			push_to_hub=SCREAMING_SNAKE_CASE						,			use_auth_token=self._token						,			organization='''valid_org'''						,			)

             A       :   int					=						WavaVecaProcessor.from_pretrained('''valid_org/test-processor-org'''	)
             for k, v in processor.feature_extractor.__dict__.items():
                 self.assertEqual(SCREAMING_SNAKE_CASE						,			getattr(new_processor.feature_extractor						,			SCREAMING_SNAKE_CASE	)	)
             self.assertDictEqual(new_processor.tokenizer.get_vocab()						,			processor.tokenizer.get_vocab()	)





     def 				__lowerCAmelCase							(							self	) ->				str:






         """simple docstring"""






         CustomFeatureExtractor.register_for_auto_class()
         CustomTokenizer.register_for_auto_class()
         CustomProcessor.register_for_auto_class()

         A       :   Any					=						CustomFeatureExtractor.from_pretrained(SCREAMING_SNAKE_CASE	)

         with tempfile.TemporaryDirectory() as tmp_dir:
             A       :   Union[str, Any]					=						os.path.join(SCREAMING_SNAKE_CASE						,			'''vocab.txt'''	)
             with open(SCREAMING_SNAKE_CASE						,			'''w'''						,			encoding='''utf-8'''	) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]	)	)
             A       :   str					=						CustomTokenizer(SCREAMING_SNAKE_CASE	)

         A       :   Union[str, Any]					=						CustomProcessor(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

         with tempfile.TemporaryDirectory() as tmp_dir:
             create_repo(F'{USER}/test-dynamic-processor'						,			token=self._token	)
             A       :   List[str]					=						Repository(SCREAMING_SNAKE_CASE						,			clone_from=F'{USER}/test-dynamic-processor'						,			token=self._token	)
             processor.save_pretrained(SCREAMING_SNAKE_CASE	)

             # This has added the proper auto_map field to the feature extractor config
             self.assertDictEqual(
                 processor.feature_extractor.auto_map						,			{
                     '''AutoFeatureExtractor''': '''custom_feature_extraction.CustomFeatureExtractor''',
                     '''AutoProcessor''': '''custom_processing.CustomProcessor''',
                 }						,			)

             # This has added the proper auto_map field to the tokenizer config
             with open(os.path.join(SCREAMING_SNAKE_CASE						,			'''tokenizer_config.json'''	)	) as f:
                 A       :   Dict					=						json.load(SCREAMING_SNAKE_CASE	)
             self.assertDictEqual(
                 tokenizer_config['''auto_map''']						,			{
                     '''AutoTokenizer''': ['''custom_tokenization.CustomTokenizer''', None],
                     '''AutoProcessor''': '''custom_processing.CustomProcessor''',
                 }						,			)

             # The code has been copied from fixtures
             self.assertTrue(os.path.isfile(os.path.join(SCREAMING_SNAKE_CASE						,			'''custom_feature_extraction.py'''	)	)	)
             self.assertTrue(os.path.isfile(os.path.join(SCREAMING_SNAKE_CASE						,			'''custom_tokenization.py'''	)	)	)
             self.assertTrue(os.path.isfile(os.path.join(SCREAMING_SNAKE_CASE						,			'''custom_processing.py'''	)	)	)

             repo.push_to_hub()

         A       :   Optional[int]					=						AutoProcessor.from_pretrained(F'{USER}/test-dynamic-processor'						,			trust_remote_code=SCREAMING_SNAKE_CASE	)
         # Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
         self.assertEqual(new_processor.__class__.__name__						,			'''CustomProcessor'''	)




