




import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_snake_case										=       logging.get_logger(__name__)


_snake_case										=       {name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}







def 				lowerCAmelCase_	(     snake_case_,snake_case_,snake_case_,snake_case_     ):
      if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
            raise ValueError(f'''Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()     )}.'''     )

      if tokenizer_name is None:
            _A							:     List[str]			=						TOKENIZER_CLASSES
      else:
            _A							:     Union[str, Any]			=						{tokenizer_name: getattr(snake_case_,tokenizer_name + """Fast"""     )}

      logger.info(f'''Loading tokenizer classes: {tokenizer_names}'''     )

      for tokenizer_name in tokenizer_names:
            _A							:     Tuple			=						TOKENIZER_CLASSES[tokenizer_name]

            _A							:     Optional[int]			=						True
            if checkpoint_name is None:
                  _A							:     int			=						list(tokenizer_class.max_model_input_sizes.keys()     )
            else:
                  _A							:     Dict			=						[checkpoint_name]

            logger.info(f'''For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'''     )

            for checkpoint in checkpoint_names:
                  logger.info(f'''Loading {tokenizer_class.__class__.__name__} {checkpoint}'''     )

                  # Load tokenizer
                  _A							:     List[Any]			=						tokenizer_class.from_pretrained(snake_case_,force_download=snake_case_     )

                  # Save fast tokenizer
                  logger.info(f'''Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'''     )

                  # For organization names we create sub-directories
                  if "/" in checkpoint:
                        _A			,   _A							:     Any			=						checkpoint.split("""/"""     )
                        _A							:     Optional[Any]			=						os.path.join(snake_case_,snake_case_     )
                  elif add_prefix:
                        _A							:     Optional[int]			=						checkpoint
                        _A							:     List[str]			=						dump_path
                  else:
                        _A							:     List[Any]			=						None
                        _A							:     int			=						dump_path

                  logger.info(f'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''     )

                  if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()     )[0]:
                        _A							:     Tuple			=						list(tokenizer.pretrained_vocab_files_map.values()     )[0][checkpoint]
                        _A							:     List[Any]			=						file_path.split(snake_case_     )[-1][0]
                        if next_char == "/":
                              _A							:     Optional[Any]			=						os.path.join(snake_case_,snake_case_     )
                              _A							:     List[str]			=						None

                        logger.info(f'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''     )

                  _A							:     Optional[Any]			=						tokenizer.save_pretrained(
                      snake_case_,legacy_format=snake_case_,filename_prefix=snake_case_     )
                  logger.info(f'''=> File names {file_names}'''     )

                  for file_name in file_names:
                        if not file_name.endswith("""tokenizer.json"""     ):
                              os.remove(snake_case_     )
                              logger.info(f'''=> removing {file_name}'''     )


if __name__ == "__main__":
  _snake_case										=       argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
  )
  parser.add_argument(
      "--tokenizer_name",
      default=None,
      type=str,
      help=(
          f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
          "download and convert all the checkpoints from AWS."
      ),
  )
  parser.add_argument(
      "--checkpoint_name",
      default=None,
      type=str,
      help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
  )
  parser.add_argument(
      "--force_download",
      action="store_true",
      help="Re-download checkpoints.",
  )
  _snake_case										=       parser.parse_args()

  convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)







import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


_snake_case										=       logging.get_logger(__name__)

_snake_case										=       OrderedDict(
    [
        ("audio-spectrogram-transformer", "ASTFeatureExtractor"),
        ("beit", "BeitFeatureExtractor"),
        ("chinese_clip", "ChineseCLIPFeatureExtractor"),
        ("clap", "ClapFeatureExtractor"),
        ("clip", "CLIPFeatureExtractor"),
        ("clipseg", "ViTFeatureExtractor"),
        ("conditional_detr", "ConditionalDetrFeatureExtractor"),
        ("convnext", "ConvNextFeatureExtractor"),
        ("cvt", "ConvNextFeatureExtractor"),
        ("data2vec-audio", "Wav2Vec2FeatureExtractor"),
        ("data2vec-vision", "BeitFeatureExtractor"),
        ("deformable_detr", "DeformableDetrFeatureExtractor"),
        ("deit", "DeiTFeatureExtractor"),
        ("detr", "DetrFeatureExtractor"),
        ("dinat", "ViTFeatureExtractor"),
        ("donut-swin", "DonutFeatureExtractor"),
        ("dpt", "DPTFeatureExtractor"),
        ("encodec", "EncodecFeatureExtractor"),
        ("flava", "FlavaFeatureExtractor"),
        ("glpn", "GLPNFeatureExtractor"),
        ("groupvit", "CLIPFeatureExtractor"),
        ("hubert", "Wav2Vec2FeatureExtractor"),
        ("imagegpt", "ImageGPTFeatureExtractor"),
        ("layoutlmv2", "LayoutLMv2FeatureExtractor"),
        ("layoutlmv3", "LayoutLMv3FeatureExtractor"),
        ("levit", "LevitFeatureExtractor"),
        ("maskformer", "MaskFormerFeatureExtractor"),
        ("mctct", "MCTCTFeatureExtractor"),
        ("mobilenet_v1", "MobileNetV1FeatureExtractor"),
        ("mobilenet_v2", "MobileNetV2FeatureExtractor"),
        ("mobilevit", "MobileViTFeatureExtractor"),
        ("nat", "ViTFeatureExtractor"),
        ("owlvit", "OwlViTFeatureExtractor"),
        ("perceiver", "PerceiverFeatureExtractor"),
        ("poolformer", "PoolFormerFeatureExtractor"),
        ("regnet", "ConvNextFeatureExtractor"),
        ("resnet", "ConvNextFeatureExtractor"),
        ("segformer", "SegformerFeatureExtractor"),
        ("sew", "Wav2Vec2FeatureExtractor"),
        ("sew-d", "Wav2Vec2FeatureExtractor"),
        ("speech_to_text", "Speech2TextFeatureExtractor"),
        ("speecht5", "SpeechT5FeatureExtractor"),
        ("swiftformer", "ViTFeatureExtractor"),
        ("swin", "ViTFeatureExtractor"),
        ("swinv2", "ViTFeatureExtractor"),
        ("table-transformer", "DetrFeatureExtractor"),
        ("timesformer", "VideoMAEFeatureExtractor"),
        ("tvlt", "TvltFeatureExtractor"),
        ("unispeech", "Wav2Vec2FeatureExtractor"),
        ("unispeech-sat", "Wav2Vec2FeatureExtractor"),
        ("van", "ConvNextFeatureExtractor"),
        ("videomae", "VideoMAEFeatureExtractor"),
        ("vilt", "ViltFeatureExtractor"),
        ("vit", "ViTFeatureExtractor"),
        ("vit_mae", "ViTFeatureExtractor"),
        ("vit_msn", "ViTFeatureExtractor"),
        ("wav2vec2", "Wav2Vec2FeatureExtractor"),
        ("wav2vec2-conformer", "Wav2Vec2FeatureExtractor"),
        ("wavlm", "Wav2Vec2FeatureExtractor"),
        ("whisper", "WhisperFeatureExtractor"),
        ("xclip", "CLIPFeatureExtractor"),
        ("yolos", "YolosFeatureExtractor"),
    ]
)

_snake_case										=       _LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)







def 				lowerCAmelCase_	(     snake_case_     ):
      for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
            if class_name in extractors:
                  _A							:     List[str]			=						model_type_to_module_name(snake_case_     )

                  _A							:     List[Any]			=						importlib.import_module(f'''.{module_name}''',"""transformers.models"""     )
                  try:
                        return getattr(snake_case_,snake_case_     )
                  except AttributeError:
                        continue

      for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
            if getattr(snake_case_,"""__name__""",snake_case_     ) == class_name:
                  return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
      _A							:     List[Any]			=						importlib.import_module("""transformers"""     )
      if hasattr(snake_case_,snake_case_     ):
            return getattr(snake_case_,snake_case_     )

      return None







def 				lowerCAmelCase_	(     snake_case_,snake_case_ = None,snake_case_ = False,snake_case_ = False,snake_case_ = None,snake_case_ = None,snake_case_ = None,snake_case_ = False,**snake_case_,):
      _A							:     Optional[int]			=						get_file_from_repo(
          snake_case_,snake_case_,cache_dir=snake_case_,force_download=snake_case_,resume_download=snake_case_,proxies=snake_case_,use_auth_token=snake_case_,revision=snake_case_,local_files_only=snake_case_,)
      if resolved_config_file is None:
            logger.info(
                """Could not locate the feature extractor configuration file, will try to use the model config instead."""     )
            return {}

      with open(snake_case_,encoding="""utf-8"""     ) as reader:
            return json.load(snake_case_     )







class 						lowercase							:

     def __init__(  self				)						->							List[Any]:
           raise EnvironmentError(
               """AutoFeatureExtractor is designed to be instantiated """
               """using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method."""				)

     @classmethod
     @replace_list_option_in_docstrings(_a				)
     def 					a__	(  cls		,     _a		,     **_a				)						->							Any:
           _A							:     Tuple			=						kwargs.pop("""config"""		,     _a				)
           _A							:     Tuple			=						kwargs.pop("""trust_remote_code"""		,     _a				)
           _A							:     List[Any]			=						True

           _A			,   _A							:     Tuple			=						FeatureExtractionMixin.get_feature_extractor_dict(_a		,     **_a				)
           _A							:     Tuple			=						config_dict.get("""feature_extractor_type"""		,     _a				)
           _A							:     int			=						None
           if "AutoFeatureExtractor" in config_dict.get("""auto_map"""		,     {}				):
                 _A							:     Optional[int]			=						config_dict["""auto_map"""]["""AutoFeatureExtractor"""]

           # If we don't find the feature extractor class in the feature extractor config, let's try the model config.
           if feature_extractor_class is None and feature_extractor_auto_map is None:
                 if not isinstance(_a		,     _a				):
                       _A							:     int			=						AutoConfig.from_pretrained(_a		,     **_a				)
                 # It could be in `config.feature_extractor_type``
                 _A							:     Optional[int]			=						getattr(_a		,     """feature_extractor_type"""		,     _a				)
                 if hasattr(_a		,     """auto_map"""				) and "AutoFeatureExtractor" in config.auto_map:
                       _A							:     Tuple			=						config.auto_map["""AutoFeatureExtractor"""]

           if feature_extractor_class is not None:
                 _A							:     Optional[Any]			=						feature_extractor_class_from_name(_a				)

           _A							:     List[Any]			=						feature_extractor_auto_map is not None
           _A							:     Union[str, Any]			=						feature_extractor_class is not None or type(_a				) in FEATURE_EXTRACTOR_MAPPING
           _A							:     Optional[int]			=						resolve_trust_remote_code(
               _a		,     _a		,     _a		,     _a				)

           if has_remote_code and trust_remote_code:
                 _A							:     Dict			=						get_class_from_dynamic_module(
                     _a		,     _a		,     **_a				)
                 _A							:     str			=						kwargs.pop("""code_revision"""		,     _a				)
                 if os.path.isdir(_a				):
                       feature_extractor_class.register_for_auto_class()
                 return feature_extractor_class.from_dict(_a		,     **_a				)
           elif feature_extractor_class is not None:
                 return feature_extractor_class.from_dict(_a		,     **_a				)
           # Last try: we use the FEATURE_EXTRACTOR_MAPPING.
           elif type(_a				) in FEATURE_EXTRACTOR_MAPPING:
                 _A							:     Dict			=						FEATURE_EXTRACTOR_MAPPING[type(_a				)]
                 return feature_extractor_class.from_dict(_a		,     **_a				)

           raise ValueError(
               F'''Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a '''
               F'''`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following '''
               F'''`model_type` keys in its {CONFIG_NAME}: {", ".join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()				)}'''				)



     @staticmethod
     def 					a__	(  _a		,     _a				)						->							Optional[int]:
           FEATURE_EXTRACTOR_MAPPING.register(_a		,     _a				)

