


'''simple docstring'''







from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Namespace  )  ->			List[str]:
       return ConvertCommand(
           args.model_type    ,args.tf_checkpoint    ,args.pytorch_dump_output    ,args.config    ,args.finetuning_task_name  )


A__:    Any					    =       '''\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n'''







class A__       (						_snake_case      ):






      @staticmethod
      def __UpperCAmelCase       ( SCREAMING_SNAKE_CASE			:Any	)       ->					Tuple:




             '''simple docstring'''
             _a    : Optional[Any]     					=parser.add_parser(
                 """convert"""     ,						help="""CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."""     ,						)
             train_parser.add_argument("""--model_type"""     ,						type=_UpperCamelCase     ,						required=_UpperCamelCase     ,						help="""Model\'s type."""	)
             train_parser.add_argument(
                 """--tf_checkpoint"""     ,						type=_UpperCamelCase     ,						required=_UpperCamelCase     ,						help="""TensorFlow checkpoint path or folder."""	)
             train_parser.add_argument(
                 """--pytorch_dump_output"""     ,						type=_UpperCamelCase     ,						required=_UpperCamelCase     ,						help="""Path to the PyTorch saved model output."""	)
             train_parser.add_argument("""--config"""     ,						type=_UpperCamelCase     ,						default=""""""     ,						help="""Configuration file path or folder."""	)
             train_parser.add_argument(
                 """--finetuning_task_name"""     ,						type=_UpperCamelCase     ,						default=_UpperCamelCase     ,						help="""Optional fine-tuning task name if the TF model was a finetuned model."""     ,						)
             train_parser.set_defaults(func=_UpperCamelCase	)






      def __init__( self			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:Any     ,						SCREAMING_SNAKE_CASE			:Dict     ,						SCREAMING_SNAKE_CASE			:str     ,						SCREAMING_SNAKE_CASE			:List[str]     ,						*SCREAMING_SNAKE_CASE			:Any     ,						)       ->					Union[str, Any]:




             '''simple docstring'''
             _a    : Dict     					=logging.get_logger("""transformers-cli/converting"""	)

             self._logger.info(f"Loading model {model_type}"	)
             _a    : Dict     					=model_type
             _a    : Optional[int]     					=tf_checkpoint
             _a    : Tuple     					=pytorch_dump_output
             _a    : int     					=config
             _a    : Optional[int]     					=finetuning_task_name






      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					Optional[Any]:




             '''simple docstring'''
             if self._model_type == "albert":
                    try:
                           from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                               convert_tf_checkpoint_to_pytorch,
                           )
                    except ImportError:
                           raise ImportError(_UpperCamelCase	)

                    convert_tf_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output	)
             elif self._model_type == "bert":
                    try:
                           from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                               convert_tf_checkpoint_to_pytorch,
                           )
                    except ImportError:
                           raise ImportError(_UpperCamelCase	)

                    convert_tf_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output	)
             elif self._model_type == "funnel":
                    try:
                           from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                               convert_tf_checkpoint_to_pytorch,
                           )
                    except ImportError:
                           raise ImportError(_UpperCamelCase	)

                    convert_tf_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output	)
             elif self._model_type == "t5":
                    try:
                           from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
                    except ImportError:
                           raise ImportError(_UpperCamelCase	)

                    convert_tf_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output	)
             elif self._model_type == "gpt":
                    from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
                        convert_openai_checkpoint_to_pytorch,
                    )

                    convert_openai_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output	)
             elif self._model_type == "transfo_xl":
                    try:
                           from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
                               convert_transfo_xl_checkpoint_to_pytorch,
                           )
                    except ImportError:
                           raise ImportError(_UpperCamelCase	)

                    if "ckpt" in self._tf_checkpoint.lower():
                           _a    : Union[str, Any]     					=self._tf_checkpoint
                           _a    : Tuple     					=''
                    else:
                           _a    : List[Any]     					=self._tf_checkpoint
                           _a    : str     					=''
                    convert_transfo_xl_checkpoint_to_pytorch(
                        _UpperCamelCase     ,						self._config     ,						self._pytorch_dump_output     ,						_UpperCamelCase	)
             elif self._model_type == "gpt2":
                    try:
                           from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
                               convert_gpta_checkpoint_to_pytorch,
                           )
                    except ImportError:
                           raise ImportError(_UpperCamelCase	)

                    convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output	)
             elif self._model_type == "xlnet":
                    try:
                           from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                               convert_xlnet_checkpoint_to_pytorch,
                           )
                    except ImportError:
                           raise ImportError(_UpperCamelCase	)

                    convert_xlnet_checkpoint_to_pytorch(
                        self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output     ,						self._finetuning_task_name	)
             elif self._model_type == "xlm":
                    from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
                        convert_xlm_checkpoint_to_pytorch,
                    )

                    convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._pytorch_dump_output	)
             elif self._model_type == "lxmert":
                    from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
                        convert_lxmert_checkpoint_to_pytorch,
                    )

                    convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._pytorch_dump_output	)
             elif self._model_type == "rembert":
                    from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
                        convert_rembert_tf_checkpoint_to_pytorch,
                    )

                    convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint     ,						self._config     ,						self._pytorch_dump_output	)
             else:
                    raise ValueError(
                        """--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]"""	)










import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__UpperCAmelCase								=						logging.get_logger(__name__)

__UpperCAmelCase								=						{'vocab_file': 'vocab.txt'}

__UpperCAmelCase								=						{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

__UpperCAmelCase								=						{
    'YituTech/conv-bert-base': 512,
    'YituTech/conv-bert-medium-small': 512,
    'YituTech/conv-bert-small': 512,
}


__UpperCAmelCase								=						{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}



class   lowerCamelCase      (_snake_case			):







		'''simple docstring'''



		_snake_case    :    Optional[int]       						=  VOCAB_FILES_NAMES
		_snake_case    :    int       						=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case    :    Dict       						=  PRETRAINED_INIT_CONFIGURATION
		_snake_case    :    List[str]       						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case    :    Any       						=  ConvBertTokenizer



		def __init__(							self				,		_UpperCamelCase=None				,		_UpperCamelCase=None				,		_UpperCamelCase=True				,		_UpperCamelCase="[UNK]"				,		_UpperCamelCase="[SEP]"				,		_UpperCamelCase="[PAD]"				,		_UpperCamelCase="[CLS]"				,		_UpperCamelCase="[MASK]"				,		_UpperCamelCase=True				,		_UpperCamelCase=None				,		**_UpperCamelCase				,		)	->							Dict:
							super().__init__(
							    _UpperCamelCase				,		tokenizer_file=_UpperCamelCase				,		do_lower_case=_UpperCamelCase				,		unk_token=_UpperCamelCase				,		sep_token=_UpperCamelCase				,		pad_token=_UpperCamelCase				,		cls_token=_UpperCamelCase				,		mask_token=_UpperCamelCase				,		tokenize_chinese_chars=_UpperCamelCase				,		strip_accents=_UpperCamelCase				,		**_UpperCamelCase				,		)

							UpperCAmelCase_    :      Optional[int]            =						json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
							if (
							    normalizer_state.get('lowercase'				,		_UpperCamelCase   ) != do_lower_case
							    or normalizer_state.get('strip_accents'				,		_UpperCamelCase   ) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'				,		_UpperCamelCase   ) != tokenize_chinese_chars
							):
												UpperCAmelCase_    :      Any            =						getattr(_UpperCamelCase				,		normalizer_state.pop('type'   )   )
												UpperCAmelCase_    :      str            =						do_lower_case
												UpperCAmelCase_    :      List[Any]            =						strip_accents
												UpperCAmelCase_    :      str            =						tokenize_chinese_chars
												UpperCAmelCase_    :      Tuple            =						normalizer_class(**_UpperCamelCase   )

							UpperCAmelCase_    :      Any            =						do_lower_case



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase=None   )	->							List[str]:
							UpperCAmelCase_    :      int            =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

							return output



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							List[int]:
							UpperCAmelCase_    :      Union[str, Any]            =						[self.sep_token_id]
							UpperCAmelCase_    :      int            =						[self.cls_token_id]
							if token_ids_a is None:
												return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							Tuple[str]:
							UpperCAmelCase_    :      Any            =						self._tokenizer.model.save(_UpperCamelCase				,		name=_UpperCamelCase   )
							return tuple(_UpperCamelCase   )






