





import os
import re
import shutil
import sys
import tempfile
import unittest

import black


_UpperCAmelCase				:       Optional[int]	   =				os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
_UpperCAmelCase				:       Union[str, Any]	   =				"""    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
"""

class    lowerCAmelCase							(					unittest.TestCase			):
 def      A_	(      self  :      str	)			->	Optional[Any]:
     lowerCamelCase__    :      Any					 =		tempfile.mkdtemp()
     os.makedirs(os.path.join(self.transformer_dir			,     'models/bert/'	)	)
     lowerCamelCase__    :      str					 =		self.transformer_dir
     shutil.copy(
         os.path.join(UpperCAmelCase			,     'src/transformers/models/bert/modeling_bert.py'	)			,     os.path.join(self.transformer_dir			,     'models/bert/modeling_bert.py'	)			,     )
 def      A_	(      self  :      Optional[int]	)			->	Dict:
     lowerCamelCase__    :      Optional[Any]					 =		'src/transformers'
     shutil.rmtree(self.transformer_dir	)
 def      A_	(      self  :      Tuple			,     UpperCAmelCase  :      Union[str, Any]			,     UpperCAmelCase  :      int			,     UpperCAmelCase  :      Dict			,     UpperCAmelCase  :      Tuple=None	)			->	List[Any]:
     lowerCamelCase__    :      List[str]					 =		comment + F"""\nclass {class_name}(nn.Module):\n""" + class_code
     if overwrite_result is not None:
         lowerCamelCase__    :      Tuple					 =		comment + F"""\nclass {class_name}(nn.Module):\n""" + overwrite_result
     lowerCamelCase__    :      int					 =		black.Mode(target_versions={black.TargetVersion.PYaa}			,     line_length=119	)
     lowerCamelCase__    :      int					 =		black.format_str(UpperCAmelCase			,     mode=UpperCAmelCase	)
     lowerCamelCase__    :      List[str]					 =		os.path.join(self.transformer_dir			,     'new_code.py'	)
     with open(UpperCAmelCase			,     'w'			,     newline='\n'	) as f:
         f.write(UpperCAmelCase	)
     if overwrite_result is None:
         self.assertTrue(len(check_copies.is_copy_consistent(UpperCAmelCase	)	) == 0	)
     else:
         check_copies.is_copy_consistent(f.name			,     overwrite=UpperCAmelCase	)
         with open(UpperCAmelCase			,     'r'	) as f:
             self.assertTrue(f.read()			,     UpperCAmelCase	)
 def      A_	(      self  :      Optional[int]	)			->	str:
     lowerCamelCase__    :      Optional[Any]					 =		check_copies.find_code_in_transformers('models.bert.modeling_bert.BertLMPredictionHead'	)
     self.assertEqual(UpperCAmelCase			,     UpperCAmelCase	)
 def      A_	(      self  :      Optional[Any]	)			->	Dict:
     # Base copy consistency
     self.check_copy_consistency(
         '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'			,     'BertLMPredictionHead'			,     REFERENCE_CODE + '\n'			,     )

     # With no empty line at the end
     self.check_copy_consistency(
         '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'			,     'BertLMPredictionHead'			,     UpperCAmelCase			,     )

     # Copy consistency with rename
     self.check_copy_consistency(
         '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'			,     'TestModelLMPredictionHead'			,     re.sub('Bert'			,     'TestModel'			,     UpperCAmelCase	)			,     )

     # Copy consistency with a really long name
     lowerCamelCase__    :      Tuple					 =		'TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason'
     self.check_copy_consistency(
         F"""# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}"""			,     F"""{long_class_name}LMPredictionHead"""			,     re.sub('Bert'			,     UpperCAmelCase			,     UpperCAmelCase	)			,     )

     # Copy consistency with overwrite
     self.check_copy_consistency(
         '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'			,     'TestModelLMPredictionHead'			,     UpperCAmelCase			,     overwrite_result=re.sub('Bert'			,     'TestModel'			,     UpperCAmelCase	)			,     )







 def      A_	(      self  :      Any	)			->	Optional[Any]:
     lowerCamelCase__    :      Tuple					 =		check_copies.LOCALIZED_READMES['README_zh-hans.md']

     lowerCamelCase__    :      Optional[Any]					 =		(
         '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
         ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
         ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
         ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1.'
         ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),'
         ' released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
         ' lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same'
         ' method has been applied to compress GPT2 into'
         ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
         ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
         ' Multilingual BERT into'
         ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
         ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**'
         ' (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders'
         ' as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang'
         ' Luong, Quoc V. Le, Christopher D. Manning.'
     )
     lowerCamelCase__    :      Any					 =		(
         '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
         ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
         ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
         ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
     )
     lowerCamelCase__    :      Optional[int]					 =		(
         '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
         ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
         ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
         ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1.'
         ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文'
         ' [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
         ' lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same'
         ' method has been applied to compress GPT2 into'
         ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
         ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
         ' Multilingual BERT into'
         ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
         ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自'
         ' Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather'
         ' than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,'
         ' Christopher D. Manning 发布。\n'
     )

     lowerCamelCase__ ,							lowerCamelCase__    :      Optional[int]					 =		check_copies.convert_to_localized_md(
         UpperCAmelCase			,     UpperCAmelCase			,     localized_readme['format_model_list']	)

     self.assertFalse(UpperCAmelCase	)
     self.assertEqual(UpperCAmelCase			,     UpperCAmelCase	)

     lowerCamelCase__ ,							lowerCamelCase__    :      Tuple					 =		check_copies.convert_to_localized_md(
         UpperCAmelCase			,     UpperCAmelCase			,     localized_readme['format_model_list']	)

     # Check whether the number of models is equal to README.md after conversion.
     self.assertTrue(UpperCAmelCase	)

     lowerCamelCase__    :      Dict					 =		(
         '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
         ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
         ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
         ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.'
     )
     lowerCamelCase__    :      List[Any]					 =		(
         '1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and'
         ' the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
         ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
         ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
     )
     lowerCamelCase__    :      Tuple					 =		(
         '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
         ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
         ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
         ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
     )

     lowerCamelCase__ ,							lowerCamelCase__    :      Any					 =		check_copies.convert_to_localized_md(
         UpperCAmelCase			,     UpperCAmelCase			,     localized_readme['format_model_list']	)

     # Check if the model link is synchronized.
     self.assertEqual(UpperCAmelCase			,     UpperCAmelCase	)












import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment


def        SCREAMING_SNAKE_CASE      (       _UpperCAmelCase						,	_UpperCAmelCase=()						,	_UpperCAmelCase=None						,	_UpperCAmelCase="no"						,	_UpperCAmelCase="29500" )			->     Tuple:
    lowerCamelCase__    :      Dict					 =		False
    lowerCamelCase__    :      Dict					 =		False
    if any(key.startswith('KAGGLE' ) for key in os.environ.keys() ):
        lowerCamelCase__    :      Optional[Any]					 =		True
    elif "IPython" in sys.modules:
        lowerCamelCase__    :      Optional[Any]					 =		'google.colab' in str(sys.modules['IPython'].get_ipython() )

    try:
        lowerCamelCase__    :      List[str]					 =		PrecisionType(mixed_precision.lower() )
    except ValueError:
        raise ValueError(
            F"""Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.""" )

    if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME'						,	_UpperCAmelCase ) is not None):
        # TPU launch
        import torch_xla.distributed.xla_multiprocessing as xmp

        if len(AcceleratorState._shared_state ) > 0:
            raise ValueError(
                'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
                'your training function. Restart your notebook and make sure no cells initializes an '
                '`Accelerator`.' )
        if num_processes is None:
            lowerCamelCase__    :      Optional[Any]					 =		8

        lowerCamelCase__    :      List[str]					 =		PrepareForLaunch(_UpperCAmelCase						,	distributed_type='TPU' )
        print(F"""Launching a training on {num_processes} TPU cores.""" )
        xmp.spawn(_UpperCAmelCase						,	args=_UpperCAmelCase						,	nprocs=_UpperCAmelCase						,	start_method='fork' )
    elif in_colab:
        # No need for a distributed launch otherwise as it's either CPU or one GPU.
        if torch.cuda.is_available():
            print('Launching training on one GPU.' )
        else:
            print('Launching training on one CPU.' )
        function(*_UpperCAmelCase )
    else:
        if num_processes is None:
            raise ValueError(
                'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.' )

        if num_processes > 1:
            # Multi-GPU launch
            from torch.multiprocessing import start_processes
            from torch.multiprocessing.spawn import ProcessRaisedException

            if len(AcceleratorState._shared_state ) > 0:
                raise ValueError(
                    'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
                    'inside your training function. Restart your notebook and make sure no cells initializes an '
                    '`Accelerator`.' )

            if torch.cuda.is_initialized():
                raise ValueError(
                    'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
                    'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
                    'function.' )

            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
            # process here (the other ones will be set be the launcher).
            with patch_environment(
                world_size=_UpperCAmelCase						,	master_addr='127.0.01'						,	master_port=_UpperCAmelCase						,	mixed_precision=_UpperCAmelCase ):
                lowerCamelCase__    :      Optional[int]					 =		PrepareForLaunch(_UpperCAmelCase						,	distributed_type='MULTI_GPU' )
                print(F"""Launching training on {num_processes} GPUs.""" )
                try:
                    start_processes(_UpperCAmelCase						,	args=_UpperCAmelCase						,	nprocs=_UpperCAmelCase						,	start_method='fork' )
                except ProcessRaisedException as e:
                    if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
                        raise RuntimeError(
                            'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
                            'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
                            'Please review your imports and test them when running the `notebook_launcher()` to identify '
                            'which one is problematic.' ) from e

        else:
            # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
            if is_mps_available():
                lowerCamelCase__    :      int					 =		'1'
                print('Launching training on MPS.' )
            elif torch.cuda.is_available():
                print('Launching training on one GPU.' )
            else:
                print('Launching training on CPU.' )
            function(*_UpperCAmelCase )


def        SCREAMING_SNAKE_CASE      (       _UpperCAmelCase						,	_UpperCAmelCase=()						,	_UpperCAmelCase=2 )			->     Optional[Any]:
    from torch.multiprocessing import start_processes

    with tempfile.NamedTemporaryFile() as tmp_file:
        # torch.distributed will expect a few environment variable to be here. We set the ones common to each
        # process here (the other ones will be set be the launcher).
        with patch_environment(
            world_size=_UpperCAmelCase						,	master_addr='127.0.01'						,	master_port='29500'						,	accelerate_mixed_precision='no'						,	accelerate_debug_rdv_file=tmp_file.name						,	accelerate_use_cpu='yes'						,	):
            lowerCamelCase__    :      Optional[Any]					 =		PrepareForLaunch(_UpperCAmelCase						,	debug=_UpperCAmelCase )
            start_processes(_UpperCAmelCase						,	args=_UpperCAmelCase						,	nprocs=_UpperCAmelCase						,	start_method='fork' )





