




import argparse
import json
from collections import OrderedDict
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    SegformerConfig,
    SegformerForImageClassification,
    SegformerForSemanticSegmentation,
    SegformerImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
lowerCamelCase       :       Any        =    logging.get_logger(__name__)




def 						SCREAMING_SNAKE_CASE__					(    lowercase						,lowercase=False     )  ->   List[str]:
   snake_case      :				Any				=       OrderedDict()
   for key, value in state_dict.items():
      if encoder_only and not key.startswith("""head"""     ):
         snake_case      :				Optional[int]				=       'segformer.encoder.' + key
      if key.startswith("""backbone"""     ):
         snake_case      :				Tuple				=       key.replace("""backbone"""						,"""segformer.encoder"""     )
      if "patch_embed" in key:
         # replace for example patch_embed1 by patch_embeddings.0
         snake_case      :				str				=       key[key.find("""patch_embed"""     ) + len("""patch_embed"""     )]
         snake_case      :				Union[str, Any]				=       key.replace(f"""patch_embed{idx}"""						,f"""patch_embeddings.{int(_UpperCAmelCase     )-1}"""     )
      if "norm" in key:
         snake_case      :				Any				=       key.replace("""norm"""						,"""layer_norm"""     )
      if "segformer.encoder.layer_norm" in key:
         # replace for example layer_norm1 by layer_norm.0
         snake_case      :				Dict				=       key[key.find("""segformer.encoder.layer_norm"""     ) + len("""segformer.encoder.layer_norm"""     )]
         snake_case      :				Tuple				=       key.replace(f"""layer_norm{idx}"""						,f"""layer_norm.{int(_UpperCAmelCase     )-1}"""     )
      if "layer_norm1" in key:
         snake_case      :				Optional[Any]				=       key.replace("""layer_norm1"""						,"""layer_norm_1"""     )
      if "layer_norm2" in key:
         snake_case      :				List[str]				=       key.replace("""layer_norm2"""						,"""layer_norm_2"""     )
      if "block" in key:
         # replace for example block1 by block.0
         snake_case      :				List[Any]				=       key[key.find("""block"""     ) + len("""block"""     )]
         snake_case      :				Optional[int]				=       key.replace(f"""block{idx}"""						,f"""block.{int(_UpperCAmelCase     )-1}"""     )
      if "attn.q" in key:
         snake_case      :				Dict				=       key.replace("""attn.q"""						,"""attention.self.query"""     )
      if "attn.proj" in key:
         snake_case      :				str				=       key.replace("""attn.proj"""						,"""attention.output.dense"""     )
      if "attn" in key:
         snake_case      :				Any				=       key.replace("""attn"""						,"""attention.self"""     )
      if "fc1" in key:
         snake_case      :				Tuple				=       key.replace("""fc1"""						,"""dense1"""     )
      if "fc2" in key:
         snake_case      :				Any				=       key.replace("""fc2"""						,"""dense2"""     )
      if "linear_pred" in key:
         snake_case      :				Optional[Any]				=       key.replace("""linear_pred"""						,"""classifier"""     )
      if "linear_fuse" in key:
         snake_case      :				Optional[Any]				=       key.replace("""linear_fuse.conv"""						,"""linear_fuse"""     )
         snake_case      :				Union[str, Any]				=       key.replace("""linear_fuse.bn"""						,"""batch_norm"""     )
      if "linear_c" in key:
         # replace for example linear_c4 by linear_c.3
         snake_case      :				Tuple				=       key[key.find("""linear_c"""     ) + len("""linear_c"""     )]
         snake_case      :				Tuple				=       key.replace(f"""linear_c{idx}"""						,f"""linear_c.{int(_UpperCAmelCase     )-1}"""     )
      if key.startswith("""head"""     ):
         snake_case      :				str				=       key.replace("""head"""						,"""classifier"""     )
      snake_case      :				Union[str, Any]				=       value

   return new_state_dict




def 						SCREAMING_SNAKE_CASE__					(    lowercase						,lowercase     )  ->   Union[str, Any]:
   # for each of the encoder blocks:
   for i in range(config.num_encoder_blocks     ):
      for j in range(config.depths[i]     ):
         # read in weights + bias of keys and values (which is a single matrix in the original implementation)
         snake_case      :				List[Any]				=       state_dict.pop(f"""segformer.encoder.block.{i}.{j}.attention.self.kv.weight"""     )
         snake_case      :				List[Any]				=       state_dict.pop(f"""segformer.encoder.block.{i}.{j}.attention.self.kv.bias"""     )
         # next, add keys and values (in that order) to the state dict
         snake_case      :				int				=       kv_weight[
             : config.hidden_sizes[i], :
         ]
         snake_case      :				List[str]				=       kv_bias[: config.hidden_sizes[i]]
         snake_case      :				Union[str, Any]				=       kv_weight[
             config.hidden_sizes[i] :, :
         ]
         snake_case      :				str				=       kv_bias[
             config.hidden_sizes[i] :
         ]




def 						SCREAMING_SNAKE_CASE__					(    )  ->   Union[str, Any]:
   snake_case      :				Dict				=       'http://images.cocodataset.org/val2017/000000039769.jpg'
   snake_case      :				Any				=       Image.open(requests.get(_UpperCAmelCase						,stream=_UpperCAmelCase     ).raw     )

   return image




@torch.no_grad()
def 						SCREAMING_SNAKE_CASE__					(    lowercase						,lowercase						,lowercase     )  ->   List[str]:
   snake_case      :				List[str]				=       SegformerConfig()
   snake_case      :				Optional[int]				=       False

   # set attributes based on model_name
   snake_case      :				Optional[int]				=       'huggingface/label-files'
   if "segformer" in model_name:
      snake_case      :				Optional[Any]				=       model_name[len("""segformer."""     ) : len("""segformer."""     ) + 2]
      if "ade" in model_name:
         snake_case      :				Optional[Any]				=       150
         snake_case      :				List[str]				=       'ade20k-id2label.json'
         snake_case      :				int				=       (1, 150, 128, 128)
      elif "city" in model_name:
         snake_case      :				str				=       19
         snake_case      :				Tuple				=       'cityscapes-id2label.json'
         snake_case      :				Optional[int]				=       (1, 19, 128, 128)
      else:
         raise ValueError(f"""Model {model_name} not supported"""     )
   elif "mit" in model_name:
      snake_case      :				int				=       True
      snake_case      :				Tuple				=       model_name[4:6]
      snake_case      :				Tuple				=       1000
      snake_case      :				Union[str, Any]				=       'imagenet-1k-id2label.json'
      snake_case      :				int				=       (1, 1000)
   else:
      raise ValueError(f"""Model {model_name} not supported"""     )

   # set config attributes
   snake_case      :				Dict				=       json.load(open(hf_hub_download(_UpperCAmelCase						,_UpperCAmelCase						,repo_type="""dataset"""     )						,"""r"""     )     )
   snake_case      :				Optional[Any]				=       {int(_UpperCAmelCase     ): v for k, v in idalabel.items()}
   snake_case      :				Union[str, Any]				=       idalabel
   snake_case      :				List[Any]				=       {v: k for k, v in idalabel.items()}
   if size == "b0":
      pass
   elif size == "b1":
      snake_case      :				int				=       [64, 128, 320, 512]
      snake_case      :				Any				=       256
   elif size == "b2":
      snake_case      :				str				=       [64, 128, 320, 512]
      snake_case      :				Any				=       768
      snake_case      :				int				=       [3, 4, 6, 3]
   elif size == "b3":
      snake_case      :				List[str]				=       [64, 128, 320, 512]
      snake_case      :				Optional[Any]				=       768
      snake_case      :				List[Any]				=       [3, 4, 18, 3]
   elif size == "b4":
      snake_case      :				int				=       [64, 128, 320, 512]
      snake_case      :				Dict				=       768
      snake_case      :				List[str]				=       [3, 8, 27, 3]
   elif size == "b5":
      snake_case      :				Optional[Any]				=       [64, 128, 320, 512]
      snake_case      :				List[Any]				=       768
      snake_case      :				List[str]				=       [3, 6, 40, 3]
   else:
      raise ValueError(f"""Size {size} not supported"""     )

   # load image processor (only resize + normalize)
   snake_case      :				Tuple				=       SegformerImageProcessor(
       image_scale=(512, 512)						,keep_ratio=_UpperCAmelCase						,align=_UpperCAmelCase						,do_random_crop=_UpperCAmelCase     )

   # prepare image
   snake_case      :				List[str]				=       prepare_img()
   snake_case      :				int				=       image_processor(images=_UpperCAmelCase						,return_tensors="""pt"""     ).pixel_values

   logger.info(f"""Converting model {model_name}..."""     )

   # load original state dict
   if encoder_only:
      snake_case      :				int				=       torch.load(_UpperCAmelCase						,map_location=torch.device("""cpu"""     )     )
   else:
      snake_case      :				Tuple				=       torch.load(_UpperCAmelCase						,map_location=torch.device("""cpu"""     )     )['state_dict']

   # rename keys
   snake_case      :				Optional[Any]				=       rename_keys(_UpperCAmelCase						,encoder_only=_UpperCAmelCase     )
   if not encoder_only:
      del state_dict["decode_head.conv_seg.weight"]
      del state_dict["decode_head.conv_seg.bias"]

   # key and value matrices need special treatment
   read_in_k_v(_UpperCAmelCase						,_UpperCAmelCase     )

   # create HuggingFace model and load state dict
   if encoder_only:
      snake_case      :				List[Any]				=       False
      snake_case      :				Optional[Any]				=       SegformerForImageClassification(_UpperCAmelCase     )
   else:
      snake_case      :				str				=       SegformerForSemanticSegmentation(_UpperCAmelCase     )
   model.load_state_dict(_UpperCAmelCase     )
   model.eval()

   # forward pass
   snake_case      :				Union[str, Any]				=       model(_UpperCAmelCase     )
   snake_case      :				Optional[Any]				=       outputs.logits

   # set expected_slice based on model name
   # ADE20k checkpoints
   if model_name == "segformer.b0.512x512.ade.160k":
      snake_case      :				List[Any]				=       torch.tensor(
          [
              [[-4.6310, -5.5232, -6.2356], [-5.1921, -6.1444, -6.5996], [-5.4424, -6.2790, -6.7574]],
              [[-12.1391, -13.3122, -13.9554], [-12.8732, -13.9352, -14.3563], [-12.9438, -13.8226, -14.2513]],
              [[-12.5134, -13.4686, -14.4915], [-12.8669, -14.4343, -14.7758], [-13.2523, -14.5819, -15.0694]],
          ]     )
   elif model_name == "segformer.b1.512x512.ade.160k":
      snake_case      :				int				=       torch.tensor(
          [
              [[-7.5820, -8.7231, -8.3215], [-8.0600, -10.3529, -10.0304], [-7.5208, -9.4103, -9.6239]],
              [[-12.6918, -13.8994, -13.7137], [-13.3196, -15.7523, -15.4789], [-12.9343, -14.8757, -14.9689]],
              [[-11.1911, -11.9421, -11.3243], [-11.3342, -13.6839, -13.3581], [-10.3909, -12.1832, -12.4858]],
          ]     )
   elif model_name == "segformer.b2.512x512.ade.160k":
      snake_case      :				Tuple				=       torch.tensor(
          [
              [[-11.8173, -14.3850, -16.3128], [-14.5648, -16.5804, -18.6568], [-14.7223, -15.7387, -18.4218]],
              [[-15.7290, -17.9171, -19.4423], [-18.3105, -19.9448, -21.4661], [-17.9296, -18.6497, -20.7910]],
              [[-15.0783, -17.0336, -18.2789], [-16.8771, -18.6870, -20.1612], [-16.2454, -17.1426, -19.5055]],
          ]     )
   elif model_name == "segformer.b3.512x512.ade.160k":
      snake_case      :				str				=       torch.tensor(
          [
              [[-9.0878, -10.2081, -10.1891], [-9.3144, -10.7941, -10.9843], [-9.2294, -10.3855, -10.5704]],
              [[-12.2316, -13.9068, -13.6102], [-12.9161, -14.3702, -14.3235], [-12.5233, -13.7174, -13.7932]],
              [[-14.6275, -15.2490, -14.9727], [-14.3400, -15.9687, -16.2827], [-14.1484, -15.4033, -15.8937]],
          ]     )
   elif model_name == "segformer.b4.512x512.ade.160k":
      snake_case      :				int				=       torch.tensor(
          [
              [[-12.3144, -13.2447, -14.0802], [-13.3614, -14.5816, -15.6117], [-13.3340, -14.4433, -16.2219]],
              [[-19.2781, -20.4128, -20.7506], [-20.6153, -21.6566, -22.0998], [-19.9800, -21.0430, -22.1494]],
              [[-18.8739, -19.7804, -21.1834], [-20.1233, -21.6765, -23.2944], [-20.0315, -21.2641, -23.6944]],
          ]     )
   elif model_name == "segformer.b5.640x640.ade.160k":
      snake_case      :				Dict				=       torch.tensor(
          [
              [[-9.5524, -12.0835, -11.7348], [-10.5229, -13.6446, -14.5662], [-9.5842, -12.8851, -13.9414]],
              [[-15.3432, -17.5323, -17.0818], [-16.3330, -18.9255, -19.2101], [-15.1340, -17.7848, -18.3971]],
              [[-12.6072, -14.9486, -14.6631], [-13.7629, -17.0907, -17.7745], [-12.7899, -16.1695, -17.1671]],
          ]     )
   # Cityscapes checkpoints
   elif model_name == "segformer.b0.1024x1024.city.160k":
      snake_case      :				int				=       torch.tensor(
          [
              [[-11.9295, -13.4057, -14.8106], [-13.3431, -14.8179, -15.3781], [-14.2836, -15.5942, -16.1588]],
              [[-11.4906, -12.8067, -13.6564], [-13.1189, -14.0500, -14.1543], [-13.8748, -14.5136, -14.8789]],
              [[0.5374, 0.1067, -0.4742], [0.1141, -0.2255, -0.7099], [-0.3000, -0.5924, -1.3105]],
          ]     )
   elif model_name == "segformer.b0.512x1024.city.160k":
      snake_case      :				int				=       torch.tensor(
          [
              [[-7.8217, -9.8767, -10.1717], [-9.4438, -10.9058, -11.4047], [-9.7939, -12.3495, -12.1079]],
              [[-7.1514, -9.5336, -10.0860], [-9.7776, -11.6822, -11.8439], [-10.1411, -12.7655, -12.8972]],
              [[0.3021, 0.0805, -0.2310], [-0.0328, -0.1605, -0.2714], [-0.1408, -0.5477, -0.6976]],
          ]     )
   elif model_name == "segformer.b0.640x1280.city.160k":
      snake_case      :				Optional[int]				=       torch.tensor(
          [
              [
                  [-1.1_372E01, -1.2_787E01, -1.3_477E01],
                  [-1.2_536E01, -1.4_194E01, -1.4_409E01],
                  [-1.3_217E01, -1.4_888E01, -1.5_327E01],
              ],
              [
                  [-1.4_791E01, -1.7_122E01, -1.8_277E01],
                  [-1.7_163E01, -1.9_192E01, -1.9_533E01],
                  [-1.7_897E01, -1.9_991E01, -2.0_315E01],
              ],
              [
                  [7.6_723E-01, 4.1_921E-01, -7.7_878E-02],
                  [4.7_772E-01, 9.5_557E-03, -2.8_082E-01],
                  [3.6_032E-01, -2.4_826E-01, -5.1_168E-01],
              ],
          ]     )
   elif model_name == "segformer.b0.768x768.city.160k":
      snake_case      :				Any				=       torch.tensor(
          [
              [[-9.4959, -11.3087, -11.7479], [-11.0025, -12.6540, -12.3319], [-11.4064, -13.0487, -12.9905]],
              [[-9.8905, -11.3084, -12.0854], [-11.1726, -12.7698, -12.9583], [-11.5985, -13.3278, -14.1774]],
              [[0.2213, 0.0192, -0.2466], [-0.1731, -0.4213, -0.4874], [-0.3126, -0.6541, -1.1389]],
          ]     )
   elif model_name == "segformer.b1.1024x1024.city.160k":
      snake_case      :				Optional[int]				=       torch.tensor(
          [
              [[-13.5748, -13.9111, -12.6500], [-14.3500, -15.3683, -14.2328], [-14.7532, -16.0424, -15.6087]],
              [[-17.1651, -15.8725, -12.9653], [-17.2580, -17.3718, -14.8223], [-16.6058, -16.8783, -16.7452]],
              [[-3.6456, -3.0209, -1.4203], [-3.0797, -3.1959, -2.0000], [-1.8757, -1.9217, -1.6997]],
          ]     )
   elif model_name == "segformer.b2.1024x1024.city.160k":
      snake_case      :				Union[str, Any]				=       torch.tensor(
          [
              [[-16.0976, -16.4856, -17.3962], [-16.6234, -19.0342, -19.7685], [-16.0900, -18.0661, -19.1180]],
              [[-18.4750, -18.8488, -19.5074], [-19.4030, -22.1570, -22.5977], [-19.1191, -20.8486, -22.3783]],
              [[-4.5178, -5.5037, -6.5109], [-5.0884, -7.2174, -8.0334], [-4.4156, -5.8117, -7.2970]],
          ]     )
   elif model_name == "segformer.b3.1024x1024.city.160k":
      snake_case      :				Union[str, Any]				=       torch.tensor(
          [
              [[-14.2081, -14.4732, -14.1977], [-14.5867, -16.4423, -16.6356], [-13.4441, -14.9685, -16.8696]],
              [[-14.4576, -14.7073, -15.0451], [-15.0816, -17.6237, -17.9873], [-14.4213, -16.0199, -18.5992]],
              [[-4.7349, -4.9588, -5.0966], [-4.3210, -6.9325, -7.2591], [-3.4312, -4.7484, -7.1917]],
          ]     )
   elif model_name == "segformer.b4.1024x1024.city.160k":
      snake_case      :				Optional[Any]				=       torch.tensor(
          [
              [[-11.7737, -11.9526, -11.3273], [-13.6692, -14.4574, -13.8878], [-13.8937, -14.6924, -15.9345]],
              [[-14.6706, -14.5330, -14.1306], [-16.1502, -16.8180, -16.4269], [-16.8338, -17.8939, -20.1746]],
              [[1.0491, 0.8289, 1.0310], [1.1044, 0.5219, 0.8055], [1.0899, 0.6926, 0.5590]],
          ]     )
   elif model_name == "segformer.b5.1024x1024.city.160k":
      snake_case      :				Any				=       torch.tensor(
          [
              [[-12.5641, -13.4777, -13.0684], [-13.9587, -15.8983, -16.6557], [-13.3109, -15.7350, -16.3141]],
              [[-14.7074, -15.4352, -14.5944], [-16.6353, -18.1663, -18.6120], [-15.1702, -18.0329, -18.1547]],
              [[-1.7990, -2.0951, -1.7784], [-2.6397, -3.8245, -3.9686], [-1.5264, -2.8126, -2.9316]],
          ]     )
   else:
      snake_case      :				Union[str, Any]				=       logits.argmax(-1     ).item()
      print("""Predicted class:"""						,model.config.idalabel[predicted_class_idx]     )

   # verify logits
   if not encoder_only:
      assert logits.shape == expected_shape
      assert torch.allclose(logits[0, :3, :3, :3]						,_UpperCAmelCase						,atol=1E-2     )

   # finally, save model and image processor
   logger.info(f"""Saving PyTorch model and image processor to {pytorch_dump_folder_path}..."""     )
   Path(_UpperCAmelCase     ).mkdir(exist_ok=_UpperCAmelCase     )
   model.save_pretrained(_UpperCAmelCase     )
   image_processor.save_pretrained(_UpperCAmelCase     )


if __name__ == "__main__":
       lowerCamelCase       :       Optional[int]        =    argparse.ArgumentParser()

       parser.add_argument(
           '--model_name',
           default='segformer.b0.512x512.ade.160k',
           type=str,
           help='Name of the model you\'d like to convert.',
       )
       parser.add_argument(
           '--checkpoint_path', default=None, type=str, help='Path to the original PyTorch checkpoint (.pth file).'
       )
       parser.add_argument(
           '--pytorch_dump_folder_path', default=None, type=str, help='Path to the folder to output PyTorch model.'
       )
       lowerCamelCase       :       str        =    parser.parse_args()
       convert_segformer_checkpoint(args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path)












import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_nllb import NllbTokenizer
else:
    _UpperCAmelCase				:       int	   =				None


_UpperCAmelCase				:       Dict	   =				logging.get_logger(__name__)


_UpperCAmelCase				:       Optional[int]	   =				{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase				:       List[Any]	   =				{
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase				:       List[str]	   =				{
    """facebook/nllb-large-en-ro""": 10_24,
    """facebook/nllb-200-distilled-600M""": 10_24,
}

# fmt: off
_UpperCAmelCase				:       Optional[int]	   =				["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]

class    lowerCAmelCase							(					__UpperCamelCase			):
 UpperCAmelCase__													=		VOCAB_FILES_NAMES
 UpperCAmelCase__													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__													=		PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__													=		["""input_ids""", """attention_mask"""]
 UpperCAmelCase__													=		NllbTokenizer

 UpperCAmelCase__													=		[]
 UpperCAmelCase__													=		[]
 def __init__(      self  :      Tuple			,     UpperCAmelCase  :      int=None			,     UpperCAmelCase  :      Any=None			,     UpperCAmelCase  :      str="<s>"			,     UpperCAmelCase  :      Optional[Any]="</s>"			,     UpperCAmelCase  :      str="</s>"			,     UpperCAmelCase  :      Tuple="<s>"			,     UpperCAmelCase  :      Optional[Any]="<unk>"			,     UpperCAmelCase  :      List[str]="<pad>"			,     UpperCAmelCase  :      Union[str, Any]="<mask>"			,     UpperCAmelCase  :      Tuple=None			,     UpperCAmelCase  :      int=None			,     UpperCAmelCase  :      Dict=None			,     UpperCAmelCase  :      Any=False			,     **UpperCAmelCase  :      Optional[int]			,     )			->	Tuple:
     # Mask token behave like a normal word, i.e. include the space before it
     lowerCamelCase__    :      List[Any]					 =		AddedToken(UpperCAmelCase			,     lstrip=UpperCAmelCase			,     rstrip=UpperCAmelCase	) if isinstance(UpperCAmelCase			,     UpperCAmelCase	) else mask_token
     lowerCamelCase__    :      Union[str, Any]					 =		legacy_behaviour
     super().__init__(
         vocab_file=UpperCAmelCase			,     tokenizer_file=UpperCAmelCase			,     bos_token=UpperCAmelCase			,     eos_token=UpperCAmelCase			,     sep_token=UpperCAmelCase			,     cls_token=UpperCAmelCase			,     unk_token=UpperCAmelCase			,     pad_token=UpperCAmelCase			,     mask_token=UpperCAmelCase			,     src_lang=UpperCAmelCase			,     tgt_lang=UpperCAmelCase			,     additional_special_tokens=UpperCAmelCase			,     legacy_behaviour=UpperCAmelCase			,     **UpperCAmelCase			,     )

     lowerCamelCase__    :      List[Any]					 =		vocab_file
     lowerCamelCase__    :      Dict					 =		False if not self.vocab_file else True

     lowerCamelCase__    :      Optional[Any]					 =		FAIRSEQ_LANGUAGE_CODES.copy()

     if additional_special_tokens is not None:
         # Only add those special tokens if they are not already there.
         _additional_special_tokens.extend(
             [t for t in additional_special_tokens if t not in _additional_special_tokens]	)

     self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}	)
     lowerCamelCase__    :      str					 =		{
         lang_code: self.convert_tokens_to_ids(UpperCAmelCase	) for lang_code in FAIRSEQ_LANGUAGE_CODES
     }

     lowerCamelCase__    :      int					 =		src_lang if src_lang is not None else 'eng_Latn'
     lowerCamelCase__    :      List[Any]					 =		self.convert_tokens_to_ids(self._src_lang	)
     lowerCamelCase__    :      str					 =		tgt_lang
     self.set_src_lang_special_tokens(self._src_lang	)
 @property
 def      A_	(      self  :      int	)			->	str:
     return self._src_lang
 @src_lang.setter
 def      A_	(      self  :      List[Any]			,     UpperCAmelCase  :      str	)			->	None:
     lowerCamelCase__    :      Any					 =		new_src_lang
     self.set_src_lang_special_tokens(self._src_lang	)
 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      List[int]			,     UpperCAmelCase  :      Optional[List[int]] = None	)			->	List[int]:
     if token_ids_a is None:
         return self.prefix_tokens + token_ids_a + self.suffix_tokens
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      List[int]			,     UpperCAmelCase  :      Optional[List[int]] = None	)			->	List[int]:
     lowerCamelCase__    :      Dict					 =		[self.sep_token_id]
     lowerCamelCase__    :      List[str]					 =		[self.cls_token_id]

     if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]
 def      A_	(      self  :      int			,     UpperCAmelCase  :      int			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[str]			,     UpperCAmelCase  :      Optional[str]			,     **UpperCAmelCase  :      List[str]	)			->	Dict:
     if src_lang is None or tgt_lang is None:
         raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
     lowerCamelCase__    :      Optional[int]					 =		src_lang
     lowerCamelCase__    :      Optional[int]					 =		self(UpperCAmelCase			,     add_special_tokens=UpperCAmelCase			,     return_tensors=UpperCAmelCase			,     **UpperCAmelCase	)
     lowerCamelCase__    :      Optional[Any]					 =		self.convert_tokens_to_ids(UpperCAmelCase	)
     lowerCamelCase__    :      Union[str, Any]					 =		tgt_lang_id
     return inputs
 def      A_	(      self  :      Dict			,     UpperCAmelCase  :      List[str]			,     UpperCAmelCase  :      str = "eng_Latn"			,     UpperCAmelCase  :      Optional[List[str]] = None			,     UpperCAmelCase  :      str = "fra_Latn"			,     **UpperCAmelCase  :      Dict			,     )			->	BatchEncoding:
     lowerCamelCase__    :      Any					 =		src_lang
     lowerCamelCase__    :      int					 =		tgt_lang
     return super().prepare_seqaseq_batch(UpperCAmelCase			,     UpperCAmelCase			,     **UpperCAmelCase	)
 def      A_	(      self  :      Union[str, Any]	)			->	Optional[int]:
     return self.set_src_lang_special_tokens(self.src_lang	)
 def      A_	(      self  :      Any	)			->	Union[str, Any]:
     return self.set_tgt_lang_special_tokens(self.tgt_lang	)
 def      A_	(      self  :      str			,     UpperCAmelCase  :      Optional[Any]	)			->	None:
     lowerCamelCase__    :      int					 =		self.convert_tokens_to_ids(UpperCAmelCase	)

     if self.legacy_behaviour:
         lowerCamelCase__    :      int					 =		[]
         lowerCamelCase__    :      str					 =		[self.eos_token_id, self.cur_lang_code]
     else:
         lowerCamelCase__    :      int					 =		[self.cur_lang_code]
         lowerCamelCase__    :      Tuple					 =		[self.eos_token_id]

     lowerCamelCase__    :      Any					 =		self.convert_ids_to_tokens(self.prefix_tokens	)
     lowerCamelCase__    :      Optional[Any]					 =		self.convert_ids_to_tokens(self.suffix_tokens	)

     lowerCamelCase__    :      str					 =		processors.TemplateProcessing(
         single=prefix_tokens_str + ['$A'] + suffix_tokens_str			,     pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str			,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,     self.prefix_tokens + self.suffix_tokens	)	)			,     )
 def      A_	(      self  :      int			,     UpperCAmelCase  :      str	)			->	None:
     lowerCamelCase__    :      Union[str, Any]					 =		self.convert_tokens_to_ids(UpperCAmelCase	)
     if self.legacy_behaviour:
         lowerCamelCase__    :      Dict					 =		[]
         lowerCamelCase__    :      Union[str, Any]					 =		[self.eos_token_id, self.cur_lang_code]
     else:
         lowerCamelCase__    :      Any					 =		[self.cur_lang_code]
         lowerCamelCase__    :      Optional[Any]					 =		[self.eos_token_id]

     lowerCamelCase__    :      Union[str, Any]					 =		self.convert_ids_to_tokens(self.prefix_tokens	)
     lowerCamelCase__    :      List[Any]					 =		self.convert_ids_to_tokens(self.suffix_tokens	)

     lowerCamelCase__    :      Optional[int]					 =		processors.TemplateProcessing(
         single=prefix_tokens_str + ['$A'] + suffix_tokens_str			,     pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str			,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,     self.prefix_tokens + self.suffix_tokens	)	)			,     )







 def      A_	(      self  :      Union[str, Any]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[str] = None	)			->	Tuple[str]:
     if not self.can_save_slow_tokenizer:
         raise ValueError(
             'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
             'tokenizer.'	)

     if not os.path.isdir(UpperCAmelCase	):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""	)
         return
     lowerCamelCase__    :      int					 =		os.path.join(
         UpperCAmelCase			,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

     if os.path.abspath(self.vocab_file	) != os.path.abspath(UpperCAmelCase	):
         copyfile(self.vocab_file			,     UpperCAmelCase	)

     return (out_vocab_file,)





