

import argparse
import gc
import json
import os

import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowerCAmelCase_       	=	16
lowerCAmelCase_       	=	32


def   __SCREAMING_SNAKE_CASE					(SCREAMING_SNAKE_CASE__       ):
						return int(x / 2**20       )





class 	snake_case_   :







			'''simple docstring'''

			def __enter__(     self :				Any							)      ->Optional[Any]:
									gc.collect()
									torch.cuda.empty_cache()
									torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
									snake_case_   =	torch.cuda.memory_allocated()
									return self







			def __exit__(     self :				List[str]							,   *_UpperCamelCase :				str							)      ->List[Any]:
									gc.collect()
									torch.cuda.empty_cache()
									snake_case_   =	torch.cuda.memory_allocated()
									snake_case_   =	torch.cuda.max_memory_allocated()
									snake_case_   =	bamb(self.end - self.begin							)
									snake_case_   =	bamb(self.peak - self.begin							)
									# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")






def   __SCREAMING_SNAKE_CASE					(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__ = 16  ,		SCREAMING_SNAKE_CASE__ = "bert-base-cased"  ,		SCREAMING_SNAKE_CASE__ = 320  ,		SCREAMING_SNAKE_CASE__ = 160  ,		):
						snake_case_   =	AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__       )
						snake_case_   =	load_dataset(
						    '''glue'''  ,		'''mrpc'''  ,		split={'''train''': F'''train[:{n_train}]''', '''validation''': F'''validation[:{n_val}]'''}       )

						def tokenize_function(SCREAMING_SNAKE_CASE__       ):
												# max_length=None => use the model max length (it's actually the default)
												snake_case_   =	tokenizer(examples['''sentence1''']  ,		examples['''sentence2''']  ,		truncation=SCREAMING_SNAKE_CASE__  ,		max_length=SCREAMING_SNAKE_CASE__       )
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						snake_case_   =	datasets.map(
						    SCREAMING_SNAKE_CASE__  ,		batched=SCREAMING_SNAKE_CASE__  ,		remove_columns=['''idx''', '''sentence1''', '''sentence2''']  ,		load_from_cache_file=SCREAMING_SNAKE_CASE__       )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						snake_case_   =	tokenized_datasets.rename_column('''label'''  ,		'''labels'''       )

						def collate_fn(SCREAMING_SNAKE_CASE__       ):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												if accelerator.distributed_type == DistributedType.TPU:
																		return tokenizer.pad(SCREAMING_SNAKE_CASE__  ,		padding='''max_length'''  ,		max_length=128  ,		return_tensors='''pt'''       )
												return tokenizer.pad(SCREAMING_SNAKE_CASE__  ,		padding='''longest'''  ,		return_tensors='''pt'''       )

						# Instantiate dataloaders.
						snake_case_   =	DataLoader(
						    tokenized_datasets['''train''']  ,		shuffle=SCREAMING_SNAKE_CASE__  ,		collate_fn=SCREAMING_SNAKE_CASE__  ,		batch_size=SCREAMING_SNAKE_CASE__       )
						snake_case_   =	DataLoader(
						    tokenized_datasets['''validation''']  ,		shuffle=SCREAMING_SNAKE_CASE__  ,		collate_fn=SCREAMING_SNAKE_CASE__  ,		batch_size=SCREAMING_SNAKE_CASE__       )

						return train_dataloader, eval_dataloader


def   __SCREAMING_SNAKE_CASE					(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       ):
						# Initialize accelerator
						snake_case_   =	Accelerator()

						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						snake_case_   =	config['''lr''']
						snake_case_   =	int(config['''num_epochs''']       )
						snake_case_   =	int(config['''seed''']       )
						snake_case_   =	int(config['''batch_size''']       )
						snake_case_   =	args.model_name_or_path

						set_seed(SCREAMING_SNAKE_CASE__       )
						snake_case_,    snake_case_   =	get_dataloaders(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		args.n_train  ,		args.n_val       )

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						snake_case_   =	AutoModelForSequenceClassification.from_pretrained(SCREAMING_SNAKE_CASE__  ,		return_dict=SCREAMING_SNAKE_CASE__       )

						# Instantiate optimizer
						snake_case_   =	(
						    AdamW
						    if accelerator.state.deepspeed_plugin is None
						    or '''optimizer''' not in accelerator.state.deepspeed_plugin.deepspeed_config
						    else DummyOptim
						)
						snake_case_   =	optimizer_cls(params=model.parameters()  ,		lr=SCREAMING_SNAKE_CASE__       )

						if accelerator.state.deepspeed_plugin is not None:
												snake_case_   =	accelerator.state.deepspeed_plugin.deepspeed_config[
												    '''gradient_accumulation_steps'''
												]
						else:
												snake_case_   =	1
						snake_case_   =	(len(SCREAMING_SNAKE_CASE__       ) * num_epochs) // gradient_accumulation_steps

						# Instantiate scheduler
						if (
						    accelerator.state.deepspeed_plugin is None
						    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
						):
												snake_case_   =	get_linear_schedule_with_warmup(
												    optimizer=SCREAMING_SNAKE_CASE__  ,		num_warmup_steps=0  ,		num_training_steps=SCREAMING_SNAKE_CASE__  ,		)
						else:
												snake_case_   =	DummyScheduler(SCREAMING_SNAKE_CASE__  ,		total_num_steps=SCREAMING_SNAKE_CASE__  ,		warmup_num_steps=0       )

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						snake_case_,    snake_case_,    snake_case_,    snake_case_,    snake_case_   =	accelerator.prepare(
						    SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       )

						# We need to keep track of how many total steps we have iterated over
						snake_case_   =	0
						# We also need to keep track of the stating epoch so files are named properly
						snake_case_   =	0

						# Now we train the model
						snake_case_   =	{}
						for epoch in range(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       ):
												with TorchTracemalloc() as tracemalloc:
																		model.train()
																		for step, batch in enumerate(SCREAMING_SNAKE_CASE__       ):
																								snake_case_   =	model(**SCREAMING_SNAKE_CASE__       )
																								snake_case_   =	outputs.loss
																								snake_case_   =	loss / gradient_accumulation_steps
																								accelerator.backward(SCREAMING_SNAKE_CASE__       )
																								if step % gradient_accumulation_steps == 0:
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()

																								overall_step += 1

        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
												accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin       )       )       )
												accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used       )       )
												accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked       )       )
												accelerator.print(
												    '''Total Peak Memory consumed during the train (max): {}'''.format(
												        tracemalloc.peaked + bamb(tracemalloc.begin       )       )       )
												snake_case_   =	tracemalloc.peaked + bamb(tracemalloc.begin       )
												if args.peak_memory_upper_bound is not None:
																		assert (
																		    train_total_peak_memory[F'''epoch-{epoch}'''] <= args.peak_memory_upper_bound
																		), "Peak memory usage exceeded the upper bound"

						accelerator.wait_for_everyone()
						if accelerator.is_main_process:
												with open(os.path.join(args.output_dir  ,		'''peak_memory_utilization.json'''       )  ,		'''w'''       ) as f:
																		json.dump(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       )


def   __SCREAMING_SNAKE_CASE					():
						snake_case_   =	argparse.ArgumentParser(description='''Simple example of training script tracking peak GPU memory usage.'''       )
						parser.add_argument(
						    '''--model_name_or_path'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default='''bert-base-cased'''  ,		help='''Path to pretrained model or model identifier from huggingface.co/models.'''  ,		required=SCREAMING_SNAKE_CASE__  ,		)
						parser.add_argument(
						    '''--output_dir'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default='''.'''  ,		help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''  ,		)
						parser.add_argument(
						    '''--peak_memory_upper_bound'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default=SCREAMING_SNAKE_CASE__  ,		help='''The upper bound of peak memory usage in MB. If set, the training will throw an error if the peak memory usage exceeds this value.'''  ,		)
						parser.add_argument(
						    '''--n_train'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default=320  ,		help='''Number of training examples to use.'''  ,		)
						parser.add_argument(
						    '''--n_val'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default=160  ,		help='''Number of validation examples to use.'''  ,		)
						parser.add_argument(
						    '''--num_epochs'''  ,		type=SCREAMING_SNAKE_CASE__  ,		default=1  ,		help='''Number of train epochs.'''  ,		)
						snake_case_   =	parser.parse_args()
						snake_case_   =	{'''lr''': 2E-5, '''num_epochs''': args.num_epochs, '''seed''': 42, '''batch_size''': 16}
						training_function(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       )


if __name__ == "__main__":
					main()


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowerCAmelCase_       	=	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase_       	=	{
    '''vocab_file''': {
        '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''unc-nlp/lxmert-base-uncased''': (
            '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase_       	=	{
    '''unc-nlp/lxmert-base-uncased''': 5_12,
}

lowerCAmelCase_       	=	{
    '''unc-nlp/lxmert-base-uncased''': {'''do_lower_case''': True},
}





class 	snake_case_   (		__A					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Optional[Any]									=				VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE     :   Any									=				PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE     :   List[Any]									=				PRETRAINED_INIT_CONFIGURATION
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE     :   Any									=				LxmertTokenizer







			def __init__(     self :				Union[str, Any]							,   _UpperCamelCase :				int=None							,   _UpperCamelCase :				Optional[Any]=None							,   _UpperCamelCase :				Dict=True							,   _UpperCamelCase :				Any="[UNK]"							,   _UpperCamelCase :				Tuple="[SEP]"							,   _UpperCamelCase :				List[Any]="[PAD]"							,   _UpperCamelCase :				Union[str, Any]="[CLS]"							,   _UpperCamelCase :				str="[MASK]"							,   _UpperCamelCase :				List[str]=True							,   _UpperCamelCase :				List[str]=None							,   **_UpperCamelCase :				List[str]							,   )      ->Any:
									super().__init__(
									    _UpperCamelCase							,   tokenizer_file=_UpperCamelCase							,   do_lower_case=_UpperCamelCase							,   unk_token=_UpperCamelCase							,   sep_token=_UpperCamelCase							,   pad_token=_UpperCamelCase							,   cls_token=_UpperCamelCase							,   mask_token=_UpperCamelCase							,   tokenize_chinese_chars=_UpperCamelCase							,   strip_accents=_UpperCamelCase							,   **_UpperCamelCase							,   )

									snake_case_   =	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
									if (
									    normalizer_state.get('''lowercase'''							,   _UpperCamelCase							) != do_lower_case
									    or normalizer_state.get('''strip_accents'''							,   _UpperCamelCase							) != strip_accents
									    or normalizer_state.get('''handle_chinese_chars'''							,   _UpperCamelCase							) != tokenize_chinese_chars
									):
															snake_case_   =	getattr(_UpperCamelCase							,   normalizer_state.pop('''type'''							)							)
															snake_case_   =	do_lower_case
															snake_case_   =	strip_accents
															snake_case_   =	tokenize_chinese_chars
															snake_case_   =	normalizer_class(**_UpperCamelCase							)

									snake_case_   =	do_lower_case







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				List[Any]							,   _UpperCamelCase :				List[str]=None							)      ->List[Any]:
									snake_case_   =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output







			def 				snake_case__(     self :				int							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									snake_case_   =	[self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]
									return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







			def 				snake_case__(     self :				Any							,   _UpperCamelCase :				str							,   _UpperCamelCase :				Optional[str] = None							)      ->Tuple[str]:
									snake_case_   =	self._tokenizer.model.save(_UpperCamelCase							,   name=_UpperCamelCase							)
									return tuple(_UpperCamelCase							)