





'''simple docstring'''
import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
    from cookiecutter.main import cookiecutter

    a__							:							str =True
except ImportError:
    a__							:							Tuple =False

a__							:							int =logging.get_logger(__name__)  # pylint: disable=invalid-name
def 				lowercase__   (   __lowercase						:  Namespace     ) ->      List[str]:



       """simple docstring"""






       return AddNewModelCommand(args.testing				,  args.testing_file				,  path=args.path     )


class      snake_case     (    __lowerCamelCase       ):




 """simple docstring"""



 @staticmethod
 def        _lowerCamelCase					(       __A       :							ArgumentParser	):
        __UpperCamelCase       =  parser.add_parser('add-new-model'	)
        add_new_model_parser.add_argument('--testing'							,	action='store_true'							,	help='If in testing mode.'	)
        add_new_model_parser.add_argument('--testing_file'							,	type=__A							,	help='Configuration file on which to run.'	)
        add_new_model_parser.add_argument(
            '--path'							,	type=__A							,	help='Path to cookiecutter. Should only be used for testing purposes.'	)
        add_new_model_parser.set_defaults(func=__A	)






 def __init__(       self       :							Dict							,	__A       :							bool							,	__A       :							str							,	__A       :							Any=None							,	*__A       :							Optional[int]	):
        __UpperCamelCase       =  testing
        __UpperCamelCase       =  testing_file
        __UpperCamelCase       =  path






 def        _lowerCamelCase					(       self       :							str	):
        warnings.warn(
            'The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '
            'It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '
            'checks, you should use `transformers-cli add-new-model-like` instead.'	)
        if not _has_cookiecutter:
               raise ImportError(
                   'Model creation dependencies are required to use the `add_new_model` command. Install them by running '
                   'the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'	)
        # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
        __UpperCamelCase       =  [directory for directory in os.listdir() if 'cookiecutter-template-' == directory[:2_2]]
        if len(__A	) > 0:
               raise ValueError(
                   'Several directories starting with `cookiecutter-template-` in current working directory. '
                   'Please clean your directory by removing all folders starting with `cookiecutter-template-` or '
                   'change your working directory.'	)

        __UpperCamelCase       =  (
            Path(__A	).parent.parent.parent.parent if self._path is None else Path(self._path	).parent.parent
        )
        __UpperCamelCase       =  path_to_transformer_root / 'templates' / 'adding_a_new_model'

        # Execute cookiecutter
        if not self._testing:
               cookiecutter(str(__A	)	)
        else:
               with open(self._testing_file							,	'r'	) as configuration_file:
                      __UpperCamelCase       =  json.load(__A	)

               cookiecutter(
                   str(path_to_cookiecutter if self._path is None else self._path	)							,	no_input=__A							,	extra_context=__A							,	)

        __UpperCamelCase       =  [directory for directory in os.listdir() if 'cookiecutter-template-' in directory[:2_2]][0]

        # Retrieve configuration
        with open(directory + '/configuration.json'							,	'r'	) as configuration_file:
               __UpperCamelCase       =  json.load(__A	)

        __UpperCamelCase       =  configuration['lowercase_modelname']
        __UpperCamelCase       =  configuration['generate_tensorflow_pytorch_and_flax']
        os.remove(f'''{directory}/configuration.json'''	)

        __UpperCamelCase       =  'PyTorch' in generate_tensorflow_pytorch_and_flax
        __UpperCamelCase       =  'TensorFlow' in generate_tensorflow_pytorch_and_flax
        __UpperCamelCase       =  'Flax' in generate_tensorflow_pytorch_and_flax

        __UpperCamelCase       =  f'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
        os.makedirs(__A							,	exist_ok=__A	)
        os.makedirs(f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''							,	exist_ok=__A	)

        # Tests require submodules as they have parent imports
        with open(f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''							,	'w'	):
               pass

        shutil.move(
            f'''{directory}/__init__.py'''							,	f'''{model_dir}/__init__.py'''							,	)
        shutil.move(
            f'''{directory}/configuration_{lowercase_model_name}.py'''							,	f'''{model_dir}/configuration_{lowercase_model_name}.py'''							,	)

        def remove_copy_lines(__A       :							Any	):
               with open(__A							,	'r'	) as f:
                      __UpperCamelCase       =  f.readlines()
               with open(__A							,	'w'	) as f:
                      for line in lines:
                             if "# Copied from transformers." not in line:
                                    f.write(__A	)

        if output_pytorch:
               if not self._testing:
                      remove_copy_lines(f'''{directory}/modeling_{lowercase_model_name}.py'''	)

               shutil.move(
                   f'''{directory}/modeling_{lowercase_model_name}.py'''							,	f'''{model_dir}/modeling_{lowercase_model_name}.py'''							,	)

               shutil.move(
                   f'''{directory}/test_modeling_{lowercase_model_name}.py'''							,	f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''							,	)
        else:
               os.remove(f'''{directory}/modeling_{lowercase_model_name}.py'''	)
               os.remove(f'''{directory}/test_modeling_{lowercase_model_name}.py'''	)

        if output_tensorflow:
               if not self._testing:
                      remove_copy_lines(f'''{directory}/modeling_tf_{lowercase_model_name}.py'''	)

               shutil.move(
                   f'''{directory}/modeling_tf_{lowercase_model_name}.py'''							,	f'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''							,	)

               shutil.move(
                   f'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''							,	f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''							,	)
        else:
               os.remove(f'''{directory}/modeling_tf_{lowercase_model_name}.py'''	)
               os.remove(f'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''	)

        if output_flax:
               if not self._testing:
                      remove_copy_lines(f'''{directory}/modeling_flax_{lowercase_model_name}.py'''	)

               shutil.move(
                   f'''{directory}/modeling_flax_{lowercase_model_name}.py'''							,	f'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''							,	)

               shutil.move(
                   f'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''							,	f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''							,	)
        else:
               os.remove(f'''{directory}/modeling_flax_{lowercase_model_name}.py'''	)
               os.remove(f'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''	)

        shutil.move(
            f'''{directory}/{lowercase_model_name}.md'''							,	f'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''							,	)

        shutil.move(
            f'''{directory}/tokenization_{lowercase_model_name}.py'''							,	f'''{model_dir}/tokenization_{lowercase_model_name}.py'''							,	)

        shutil.move(
            f'''{directory}/tokenization_fast_{lowercase_model_name}.py'''							,	f'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''							,	)

        from os import fdopen, remove
        from shutil import copymode, move
        from tempfile import mkstemp

        def replace(__A       :							str							,	__A       :							str							,	__A       :							List[str]	):
               # Create temp file
               __UpperCamelCase       ,						__UpperCamelCase       =  mkstemp()
               __UpperCamelCase       =  False
               with fdopen(__A							,	'w'	) as new_file:
                      with open(__A	) as old_file:
                             for line in old_file:
                                    new_file.write(__A	)
                                    if line_to_copy_below in line:
                                           __UpperCamelCase       =  True
                                           for line_to_copy in lines_to_copy:
                                                  new_file.write(__A	)

               if not line_found:
                      raise ValueError(f'''Line {line_to_copy_below} was not found in file.'''	)

               # Copy the file permissions from the old file to the new file
               copymode(__A							,	__A	)
               # Remove original file
               remove(__A	)
               # Move new file
               move(__A							,	__A	)

        def skip_units(__A       :							Dict	):
               return (
                   ("generating PyTorch" in line and not output_pytorch)
                   or ("generating TensorFlow" in line and not output_tensorflow)
                   or ("generating Flax" in line and not output_flax)
               )

        def replace_in_files(__A       :							Optional[Any]	):
               with open(__A	) as datafile:
                      __UpperCamelCase       =  []
                      __UpperCamelCase       =  False
                      __UpperCamelCase       =  False
                      for line in datafile:
                             if "# To replace in: " in line and "##" not in line:
                                    __UpperCamelCase       =  line.split('"'	)[1]
                                    __UpperCamelCase       =  skip_units(__A	)
                             elif "# Below: " in line and "##" not in line:
                                    __UpperCamelCase       =  line.split('"'	)[1]
                                    __UpperCamelCase       =  skip_units(__A	)
                             elif "# End." in line and "##" not in line:
                                    if not skip_file and not skip_snippet:
                                           replace(__A							,	__A							,	__A	)

                                    __UpperCamelCase       =  []
                             elif "# Replace with" in line and "##" not in line:
                                    __UpperCamelCase       =  []
                             elif "##" not in line:
                                    lines_to_copy.append(__A	)

               remove(__A	)

        replace_in_files(f'''{directory}/to_replace_{lowercase_model_name}.py'''	)
        os.rmdir(__A	)


import json
import os
import shutil
import tempfile
import unittest

import numpy as np

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
   from PIL import Image

   from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor






@require_tokenizers
@require_vision
class   UpperCAmelCase		(						unittest.TestCase ):



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	List[str]:


          '''simple docstring'''


          snake_case       :   int							=      tempfile.mkdtemp()

          # fmt: off
          snake_case       :   Optional[int]							=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "want", "##want", "##ed", "wa", "un", "runn", "##ing", ",", "low", "lowest"]
          # fmt: on
          snake_case       :   List[str]							=      os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"] )
          with open(self.vocab_file    ,						"w"    ,						encoding="utf-8" ) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )

          snake_case       :   int							=      {
              "do_resize": True,
              "size": {"height": 18, "width": 18},
              "do_normalize": True,
              "image_mean": [0.5, 0.5, 0.5],
              "image_std": [0.5, 0.5, 0.5],
          }
          snake_case       :   Optional[Any]							=      os.path.join(self.tmpdirname    ,						snake_case__ )
          with open(self.image_processor_file    ,						"w"    ,						encoding="utf-8" ) as fp:
                 json.dump(snake_case__    ,						snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any]    ,						**snake_case__		:					str )		->	Optional[int]:


          '''simple docstring'''


          return BertTokenizer.from_pretrained(self.tmpdirname    ,						**snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any]    ,						**snake_case__		:					List[str] )		->	int:


          '''simple docstring'''


          return ViTImageProcessor.from_pretrained(self.tmpdirname    ,						**snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	Dict:


          '''simple docstring'''


          shutil.rmtree(self.tmpdirname )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	str:


          '''simple docstring'''


          snake_case       :   List[Any]							=      [np.random.randint(2_55    ,						size=(3, 30, 4_00)    ,						dtype=np.uinta )]

          snake_case       :   Optional[int]							=      [Image.fromarray(np.moveaxis(snake_case__    ,						0    ,						-1 ) ) for x in image_inputs]

          return image_inputs



   def 				_SCREAMING_SNAKE_CASE   (self		:					Union[str, Any] )		->	Union[str, Any]:


          '''simple docstring'''


          snake_case       :   Dict							=      self.get_tokenizer()
          snake_case       :   Optional[Any]							=      self.get_image_processor()

          snake_case       :   Optional[Any]							=      VisionTextDualEncoderProcessor(tokenizer=snake_case__    ,						image_processor=snake_case__ )

          processor.save_pretrained(self.tmpdirname )
          snake_case       :   Any							=      VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname )

          self.assertEqual(processor.tokenizer.get_vocab()    ,						tokenizer.get_vocab() )
          self.assertIsInstance(processor.tokenizer    ,						(BertTokenizer, BertTokenizerFast) )

          self.assertEqual(processor.image_processor.to_json_string()    ,						image_processor.to_json_string() )
          self.assertIsInstance(processor.image_processor    ,						snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	Optional[Any]:


          '''simple docstring'''


          snake_case       :   str							=      VisionTextDualEncoderProcessor(
              tokenizer=self.get_tokenizer()    ,						image_processor=self.get_image_processor() )
          processor.save_pretrained(self.tmpdirname )

          snake_case       :   Optional[int]							=      self.get_tokenizer(bos_token="(BOS)"    ,						eos_token="(EOS)" )
          snake_case       :   Tuple							=      self.get_image_processor(do_normalize=snake_case__    ,						padding_value=1.0 )

          snake_case       :   List[str]							=      VisionTextDualEncoderProcessor.from_pretrained(
              self.tmpdirname    ,						bos_token="(BOS)"    ,						eos_token="(EOS)"    ,						do_normalize=snake_case__    ,						padding_value=1.0 )

          self.assertEqual(processor.tokenizer.get_vocab()    ,						tokenizer_add_kwargs.get_vocab() )
          self.assertIsInstance(processor.tokenizer    ,						(BertTokenizer, BertTokenizerFast) )

          self.assertEqual(processor.image_processor.to_json_string()    ,						image_processor_add_kwargs.to_json_string() )
          self.assertIsInstance(processor.image_processor    ,						snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Union[str, Any] )		->	int:


          '''simple docstring'''


          snake_case       :   str							=      self.get_image_processor()
          snake_case       :   Optional[int]							=      self.get_tokenizer()

          snake_case       :   List[Any]							=      VisionTextDualEncoderProcessor(tokenizer=snake_case__    ,						image_processor=snake_case__ )

          snake_case       :   Optional[Any]							=      self.prepare_image_inputs()

          snake_case       :   str							=      image_processor(snake_case__    ,						return_tensors="np" )
          snake_case       :   Any							=      processor(images=snake_case__    ,						return_tensors="np" )

          for key in input_feat_extract.keys():
                 self.assertAlmostEqual(input_feat_extract[key].sum()    ,						input_processor[key].sum()    ,						delta=1e-2 )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Tuple )		->	Optional[Any]:


          '''simple docstring'''


          snake_case       :   Dict							=      self.get_image_processor()
          snake_case       :   int							=      self.get_tokenizer()

          snake_case       :   Any							=      VisionTextDualEncoderProcessor(tokenizer=snake_case__    ,						image_processor=snake_case__ )

          snake_case       :   Tuple							=      "lower newer"

          snake_case       :   Tuple							=      processor(text=snake_case__ )

          snake_case       :   Union[str, Any]							=      tokenizer(snake_case__ )

          for key in encoded_tok.keys():
                 self.assertListEqual(encoded_tok[key]    ,						encoded_processor[key] )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	Optional[int]:


          '''simple docstring'''


          snake_case       :   List[Any]							=      self.get_image_processor()
          snake_case       :   Dict							=      self.get_tokenizer()

          snake_case       :   Dict							=      VisionTextDualEncoderProcessor(tokenizer=snake_case__    ,						image_processor=snake_case__ )

          snake_case       :   int							=      "lower newer"
          snake_case       :   Dict							=      self.prepare_image_inputs()

          snake_case       :   Union[str, Any]							=      processor(text=snake_case__    ,						images=snake_case__ )

          self.assertListEqual(list(inputs.keys() )    ,						["input_ids", "token_type_ids", "attention_mask", "pixel_values"] )

          # test if it raises when no input is passed
          with self.assertRaises(snake_case__ ):
                 processor()



   def 				_SCREAMING_SNAKE_CASE   (self		:					str )		->	Tuple:


          '''simple docstring'''


          snake_case       :   Tuple							=      self.get_image_processor()
          snake_case       :   Optional[Any]							=      self.get_tokenizer()

          snake_case       :   Tuple							=      VisionTextDualEncoderProcessor(tokenizer=snake_case__    ,						image_processor=snake_case__ )

          snake_case       :   List[str]							=      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          snake_case       :   List[Any]							=      processor.batch_decode(snake_case__ )
          snake_case       :   Union[str, Any]							=      tokenizer.batch_decode(snake_case__ )

          self.assertListEqual(snake_case__    ,						snake_case__ )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	List[str]:


          '''simple docstring'''


          snake_case       :   str							=      self.get_image_processor()
          snake_case       :   Union[str, Any]							=      self.get_tokenizer()

          snake_case       :   Any							=      VisionTextDualEncoderProcessor(tokenizer=snake_case__    ,						image_processor=snake_case__ )

          snake_case       :   Optional[Any]							=      "lower newer"
          snake_case       :   List[Any]							=      self.prepare_image_inputs()

          snake_case       :   Tuple							=      processor(text=snake_case__    ,						images=snake_case__ )

          self.assertListEqual(list(inputs.keys() )    ,						processor.model_input_names )





