


"""simple docstring"""





import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

snake_case__						:   Tuple			 = logging.get_logger(__name__)


snake_case__						:   Tuple			 = {name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}


def    _snake_case  (  _snake_case :	Optional[Any]       ,      _snake_case :	List[Any]       ,      _snake_case :	Any       ,      _snake_case :	Dict	):
      if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
            raise ValueError(f'''Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()	)}.'''	)

      if tokenizer_name is None:
            lowerCAmelCase    :		List[Any]				=      TOKENIZER_CLASSES
      else:
            lowerCAmelCase    :		str				=      {tokenizer_name: getattr(_snake_case       ,      tokenizer_name + '''Fast'''	)}

      logger.info(f'''Loading tokenizer classes: {tokenizer_names}'''	)

      for tokenizer_name in tokenizer_names:
            lowerCAmelCase    :		str				=      TOKENIZER_CLASSES[tokenizer_name]

            lowerCAmelCase    :		Any				=      True
            if checkpoint_name is None:
                  lowerCAmelCase    :		List[str]				=      list(tokenizer_class.max_model_input_sizes.keys()	)
            else:
                  lowerCAmelCase    :		List[Any]				=      [checkpoint_name]

            logger.info(f'''For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'''	)

            for checkpoint in checkpoint_names:
                  logger.info(f'''Loading {tokenizer_class.__class__.__name__} {checkpoint}'''	)

                  # Load tokenizer
                  lowerCAmelCase    :		Optional[int]				=      tokenizer_class.from_pretrained(_snake_case       ,      force_download=_snake_case	)

                  # Save fast tokenizer
                  logger.info(f'''Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'''	)

                  # For organization names we create sub-directories
                  if "/" in checkpoint:
                        lowerCAmelCase,					lowerCAmelCase    :		List[Any]				=      checkpoint.split('''/'''	)
                        lowerCAmelCase    :		Any				=      os.path.join(_snake_case       ,      _snake_case	)
                  elif add_prefix:
                        lowerCAmelCase    :		int				=      checkpoint
                        lowerCAmelCase    :		Optional[Any]				=      dump_path
                  else:
                        lowerCAmelCase    :		Optional[Any]				=      None
                        lowerCAmelCase    :		List[str]				=      dump_path

                  logger.info(f'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''	)

                  if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()	)[0]:
                        lowerCAmelCase    :		List[Any]				=      list(tokenizer.pretrained_vocab_files_map.values()	)[0][checkpoint]
                        lowerCAmelCase    :		Optional[int]				=      file_path.split(_snake_case	)[-1][0]
                        if next_char == "/":
                              lowerCAmelCase    :		Dict				=      os.path.join(_snake_case       ,      _snake_case	)
                              lowerCAmelCase    :		Optional[int]				=      None

                        logger.info(f'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''	)

                  lowerCAmelCase    :		Union[str, Any]				=      tokenizer.save_pretrained(
                      _snake_case       ,      legacy_format=_snake_case       ,      filename_prefix=_snake_case	)
                  logger.info(f'''=> File names {file_names}'''	)

                  for file_name in file_names:
                        if not file_name.endswith('''tokenizer.json'''	):
                              os.remove(_snake_case	)
                              logger.info(f'''=> removing {file_name}'''	)


if __name__ == "__main__":
 snake_case__						:   List[Any]			 = argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
 )
 parser.add_argument(
     '''--tokenizer_name''',
     default=None,
     type=str,
     help=(
         f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
         '''download and convert all the checkpoints from AWS.'''
     ),
 )
 parser.add_argument(
     '''--checkpoint_name''',
     default=None,
     type=str,
     help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
 )
 parser.add_argument(
     '''--force_download''',
     action='''store_true''',
     help='''Re-download checkpoints.''',
 )
 snake_case__						:   Optional[Any]			 = parser.parse_args()

 convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)






"""simple docstring"""





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
snake_case__						:   Optional[Any]			 = '''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
snake_case__						:   Dict			 = direct_transformers_import(PATH_TO_TRANSFORMERS)

snake_case__						:   Optional[int]			 = transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
snake_case__						:   Optional[int]			 = re.compile(R'''\[(.+?)\]\((https://huggingface\.co/.+?)\)''')


snake_case__						:   int			 = {
    '''DecisionTransformerConfig''',
    '''EncoderDecoderConfig''',
    '''MusicgenConfig''',
    '''RagConfig''',
    '''SpeechEncoderDecoderConfig''',
    '''TimmBackboneConfig''',
    '''VisionEncoderDecoderConfig''',
    '''VisionTextDualEncoderConfig''',
    '''LlamaConfig''',
}


def    _snake_case  (  _snake_case :	List[str]	):
      lowerCAmelCase    :		Dict				=      None

      # source code of `config_class`
      lowerCAmelCase    :		Union[str, Any]				=      inspect.getsource(_snake_case	)
      lowerCAmelCase    :		List[Any]				=      _re_checkpoint.findall(_snake_case	)

      # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
      # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
      for ckpt_name, ckpt_link in checkpoints:
            # allow the link to end with `/`
            if ckpt_link.endswith('''/'''	):
                  lowerCAmelCase    :		List[str]				=      ckpt_link[:-1]

            # verify the checkpoint name corresponds to the checkpoint link
            lowerCAmelCase    :		Optional[int]				=      f'''https://huggingface.co/{ckpt_name}'''
            if ckpt_link == ckpt_link_from_name:
                  lowerCAmelCase    :		List[str]				=      ckpt_name
                  break

      return checkpoint


def    _snake_case  (  ):
      lowerCAmelCase    :		List[Any]				=      []

      for config_class in list(CONFIG_MAPPING.values()	):
            # Skip deprecated models
            if "models.deprecated" in config_class.__module__:
                  continue
            lowerCAmelCase    :		int				=      get_checkpoint_from_config_class(_snake_case	)

            lowerCAmelCase    :		int				=      config_class.__name__
            if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
                  configs_without_checkpoint.append(_snake_case	)

      if len(_snake_case	) > 0:
            lowerCAmelCase    :		Dict				=      '''\n'''.join(sorted(_snake_case	)	)
            raise ValueError(f'''The following configurations don\'t contain any valid checkpoint:\n{message}'''	)


if __name__ == "__main__":
 check_config_docstrings_have_checkpoints()


