

"""simple docstring"""




import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to ensure out-of-memory errors never
# interrupt training, and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A_							:	Any      =       16
A_							:	Any      =       32

def 		A				(   snake_case__						, snake_case__ = 16			):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									=					AutoTokenizer.from_pretrained("""bert-base-cased"""			)
			SCREAMING_SNAKE_CASE__									=					load_dataset("""glue"""						, """mrpc"""			)

			def tokenize_function(snake_case__			):
						# max_length=None => use the model max length (it's actually the default)
						SCREAMING_SNAKE_CASE__									=					tokenizer(examples["""sentence1"""]						, examples["""sentence2"""]						, truncation=lowercase_						, max_length=lowercase_			)
						return outputs

			# Apply the method we just defined to all the examples in all the splits of the dataset
			# starting with the main process first:
			with accelerator.main_process_first():
						SCREAMING_SNAKE_CASE__									=					datasets.map(
						    lowercase_						, batched=lowercase_						, remove_columns=["""idx""", """sentence1""", """sentence2"""]						, )

			# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
			# transformers library
			SCREAMING_SNAKE_CASE__									=					tokenized_datasets.rename_column("""label"""						, """labels"""			)

			def collate_fn(snake_case__			):
						# On TPU it's best to pad everything to the same length or training will be very slow.
						SCREAMING_SNAKE_CASE__									=					1_28 if accelerator.distributed_type == DistributedType.TPU else None
						# When using mixed precision we want round multiples of 8/16
						if accelerator.mixed_precision == "fp8":
									SCREAMING_SNAKE_CASE__									=					16
						elif accelerator.mixed_precision != "no":
									SCREAMING_SNAKE_CASE__									=					8
						else:
									SCREAMING_SNAKE_CASE__									=					None

						return tokenizer.pad(
						    lowercase_						, padding="""longest"""						, max_length=lowercase_						, pad_to_multiple_of=lowercase_						, return_tensors="""pt"""						, )

			# Instantiate dataloaders.
			SCREAMING_SNAKE_CASE__									=					DataLoader(
			    tokenized_datasets["""train"""]						, shuffle=lowercase_						, collate_fn=lowercase_						, batch_size=lowercase_			)
			SCREAMING_SNAKE_CASE__									=					DataLoader(
			    tokenized_datasets["""validation"""]						, shuffle=lowercase_						, collate_fn=lowercase_						, batch_size=lowercase_			)

			return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
						from accelerate.test_utils.training import mocked_dataloaders

						A_							:	Union[str, Any]      =       mocked_dataloaders  # noqa: F811

def 		A				(   snake_case__						, snake_case__			):




			'''simple docstring'''


			if os.environ.get("""TESTING_MOCKED_DATALOADERS"""						, lowercase_			) == "1":
						SCREAMING_SNAKE_CASE__									=					2
			# Initialize accelerator
			SCREAMING_SNAKE_CASE__									=					Accelerator(cpu=args.cpu						, mixed_precision=args.mixed_precision			)
			# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
			SCREAMING_SNAKE_CASE__									=					config["""lr"""]
			SCREAMING_SNAKE_CASE__									=					int(config["""num_epochs"""]			)
			SCREAMING_SNAKE_CASE__									=					int(config["""seed"""]			)
			SCREAMING_SNAKE_CASE__									=					int(config["""batch_size"""]			)

			SCREAMING_SNAKE_CASE__									=					evaluate.load("""glue"""						, """mrpc"""			)

			# New Code #
			# We now can define an inner training loop function. It should take a batch size as the only parameter,
			# and build the dataloaders in there.
			# It also gets our decorator
			@find_executable_batch_size(starting_batch_size=lowercase_			)
			def inner_training_loop(snake_case__			):
						# And now just move everything below under this function
						# We need to bring in the Accelerator object from earlier
						nonlocal accelerator
						# And reset all of its attributes that could hold onto any memory:
						accelerator.free_memory()

						# Then we can declare the model, optimizer, and everything else:
						set_seed(lowercase_			)

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						SCREAMING_SNAKE_CASE__									=					AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""						, return_dict=lowercase_			)

						# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
						# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
						# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
						SCREAMING_SNAKE_CASE__									=					model.to(accelerator.device			)

						# Instantiate optimizer
						SCREAMING_SNAKE_CASE__									=					AdamW(params=model.parameters()						, lr=lowercase_			)
						SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__									=					get_dataloaders(lowercase_						, lowercase_			)

						# Instantiate scheduler
						SCREAMING_SNAKE_CASE__									=					get_linear_schedule_with_warmup(
						    optimizer=lowercase_						, num_warmup_steps=1_00						, num_training_steps=(len(lowercase_			) * num_epochs)						, )

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__									=					accelerator.prepare(
						    lowercase_						, lowercase_						, lowercase_						, lowercase_						, lowercase_			)

						# Now we train the model
						for epoch in range(lowercase_			):
									model.train()
									for step, batch in enumerate(lowercase_			):
												# We could avoid this line since we set the accelerator with `device_placement=True`.
												batch.to(accelerator.device			)
												SCREAMING_SNAKE_CASE__									=					model(**lowercase_			)
												SCREAMING_SNAKE_CASE__									=					outputs.loss
												accelerator.backward(lowercase_			)
												optimizer.step()
												lr_scheduler.step()
												optimizer.zero_grad()

									model.eval()
									for step, batch in enumerate(lowercase_			):
												# We could avoid this line since we set the accelerator with `device_placement=True`.
												batch.to(accelerator.device			)
												with torch.no_grad():
															SCREAMING_SNAKE_CASE__									=					model(**lowercase_			)
												SCREAMING_SNAKE_CASE__									=					outputs.logits.argmax(dim=-1			)
												SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__									=					accelerator.gather_for_metrics((predictions, batch["""labels"""])			)
												metric.add_batch(
												    predictions=lowercase_						, references=lowercase_						, )

									SCREAMING_SNAKE_CASE__									=					metric.compute()
									# Use accelerator.print to print only on the main process.
									accelerator.print(f"""epoch {epoch}:"""						, lowercase_			)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
			inner_training_loop()

def 		A				(   ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									=					argparse.ArgumentParser(description="""Simple example of training script."""			)
			parser.add_argument(
			    """--mixed_precision"""						, type=lowercase_						, default=lowercase_						, choices=["""no""", """fp16""", """bf16""", """fp8"""]						, help="""Whether to use mixed precision. Choose"""
			    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
			    """and an Nvidia Ampere GPU."""						, )
			parser.add_argument("""--cpu"""						, action="""store_true"""						, help="""If passed, will train on the CPU."""			)
			SCREAMING_SNAKE_CASE__									=					parser.parse_args()
			SCREAMING_SNAKE_CASE__									=					{"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
			training_function(lowercase_						, lowercase_			)


if __name__ == "__main__":
						main()








import os
import sys
import unittest


_lowerCamelCase	:		Optional[Any]							       =   os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import get_test_info  # noqa: E402
from get_test_info import (  # noqa: E402
    get_model_to_test_mapping,
    get_model_to_tester_mapping,
    get_test_to_tester_mapping,
)


_lowerCamelCase	:		Any							       =   os.path.join("""tests""", """models""", """bert""", """test_modeling_bert.py""")
_lowerCamelCase	:		str							       =   os.path.join("""tests""", """models""", """blip""", """test_modeling_blip.py""")


class 				UpperCamelCase_    ( unittest.TestCase     ):

    '''simple docstring'''






    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple)						->Tuple:





           '''simple docstring'''

           A__       =    get_test_to_tester_mapping(UpperCAmelCase__)
           A__       =    get_test_to_tester_mapping(UpperCAmelCase__)

           A__       =    {'''BertModelTest''': '''BertModelTester'''}

           A__       =    {
               '''BlipModelTest''': '''BlipModelTester''',
               '''BlipTextImageModelTest''': '''BlipTextImageModelsModelTester''',
               '''BlipTextModelTest''': '''BlipTextModelTester''',
               '''BlipTextRetrievalModelTest''': '''BlipTextRetrievalModelTester''',
               '''BlipVQAModelTest''': '''BlipVQAModelTester''',
               '''BlipVisionModelTest''': '''BlipVisionModelTester''',
           }

           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)
           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple)						->List[Any]:





           '''simple docstring'''

           A__       =    get_model_to_test_mapping(UpperCAmelCase__)
           A__       =    get_model_to_test_mapping(UpperCAmelCase__)

           A__       =    {
               '''BertForMaskedLM''': ['''BertModelTest'''],
               '''BertForMultipleChoice''': ['''BertModelTest'''],
               '''BertForNextSentencePrediction''': ['''BertModelTest'''],
               '''BertForPreTraining''': ['''BertModelTest'''],
               '''BertForQuestionAnswering''': ['''BertModelTest'''],
               '''BertForSequenceClassification''': ['''BertModelTest'''],
               '''BertForTokenClassification''': ['''BertModelTest'''],
               '''BertLMHeadModel''': ['''BertModelTest'''],
               '''BertModel''': ['''BertModelTest'''],
           }

           A__       =    {
               '''BlipForConditionalGeneration''': ['''BlipTextImageModelTest'''],
               '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTest'''],
               '''BlipForQuestionAnswering''': ['''BlipVQAModelTest'''],
               '''BlipModel''': ['''BlipModelTest'''],
               '''BlipTextModel''': ['''BlipTextModelTest'''],
               '''BlipVisionModel''': ['''BlipVisionModelTest'''],
           }

           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)
           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)





    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any])						->str:





           '''simple docstring'''

           A__       =    get_model_to_tester_mapping(UpperCAmelCase__)
           A__       =    get_model_to_tester_mapping(UpperCAmelCase__)

           A__       =    {
               '''BertForMaskedLM''': ['''BertModelTester'''],
               '''BertForMultipleChoice''': ['''BertModelTester'''],
               '''BertForNextSentencePrediction''': ['''BertModelTester'''],
               '''BertForPreTraining''': ['''BertModelTester'''],
               '''BertForQuestionAnswering''': ['''BertModelTester'''],
               '''BertForSequenceClassification''': ['''BertModelTester'''],
               '''BertForTokenClassification''': ['''BertModelTester'''],
               '''BertLMHeadModel''': ['''BertModelTester'''],
               '''BertModel''': ['''BertModelTester'''],
           }

           A__       =    {
               '''BlipForConditionalGeneration''': ['''BlipTextImageModelsModelTester'''],
               '''BlipForImageTextRetrieval''': ['''BlipTextRetrievalModelTester'''],
               '''BlipForQuestionAnswering''': ['''BlipVQAModelTester'''],
               '''BlipModel''': ['''BlipModelTester'''],
               '''BlipTextModel''': ['''BlipTextModelTester'''],
               '''BlipVisionModel''': ['''BlipVisionModelTester'''],
           }

           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)
           self.assertEqual(get_test_info.to_json(UpperCAmelCase__)					, UpperCAmelCase__)


