

"""simple docstring"""






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
__lowerCAmelCase				      =   logging.get_logger(__name__)
def 		A_	(   ):
		lowercase										=							DetaConfig(
		    num_queries=9_00       ,	encoder_ffn_dim=20_48       ,	decoder_ffn_dim=20_48       ,	num_feature_levels=5       ,	assign_first_stage=__UpperCamelCase       ,	with_box_refine=__UpperCamelCase       ,	two_stage=__UpperCamelCase       ,	)

		# set labels
		lowercase										=							91
		lowercase										=							'''huggingface/label-files'''
		lowercase										=							'''coco-detection-id2label.json'''
		lowercase										=							json.load(open(cached_download(hf_hub_url(__UpperCamelCase       ,	__UpperCamelCase       ,	repo_type='''dataset'''				)				)       ,	'''r'''				)				)
		lowercase										=							{int(__UpperCamelCase				): v for k, v in idalabel.items()}
		lowercase										=							idalabel
		lowercase										=							{v: k for k, v in idalabel.items()}

		return config
def 		A_	(   __UpperCamelCase     :				Union[str, Any]				):
		lowercase										=							[]

		# stem
		# fmt: off
		rename_keys.append(('''backbone.0.body.conv1.weight''', '''model.backbone.model.embedder.embedder.convolution.weight''')				)
		rename_keys.append(('''backbone.0.body.bn1.weight''', '''model.backbone.model.embedder.embedder.normalization.weight''')				)
		rename_keys.append(('''backbone.0.body.bn1.bias''', '''model.backbone.model.embedder.embedder.normalization.bias''')				)
		rename_keys.append(('''backbone.0.body.bn1.running_mean''', '''model.backbone.model.embedder.embedder.normalization.running_mean''')				)
		rename_keys.append(('''backbone.0.body.bn1.running_var''', '''model.backbone.model.embedder.embedder.normalization.running_var''')				)
		# stages
		for stage_idx in range(len(config.backbone_config.depths				)				):
				for layer_idx in range(config.backbone_config.depths[stage_idx]				):
						# shortcut
						if layer_idx == 0:
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.0.weight""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.convolution.weight""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.weight""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.weight""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.bias""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.bias""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_mean""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_mean""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_var""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_var""",
								    )				)
						# 3 convs
						for i in range(3				):
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.conv{i+1}.weight""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.convolution.weight""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.weight""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.weight""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.bias""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.bias""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_mean""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_mean""",
								    )				)
								rename_keys.append(
								    (
								        f"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_var""",
								        f"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_var""",
								    )				)
    # transformer encoder
		for i in range(config.encoder_layers				):
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight""", f"""model.encoder.layers.{i}.self_attn.sampling_offsets.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias""", f"""model.encoder.layers.{i}.self_attn.sampling_offsets.bias""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.attention_weights.weight""", f"""model.encoder.layers.{i}.self_attn.attention_weights.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.attention_weights.bias""", f"""model.encoder.layers.{i}.self_attn.attention_weights.bias""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.value_proj.weight""", f"""model.encoder.layers.{i}.self_attn.value_proj.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.value_proj.bias""", f"""model.encoder.layers.{i}.self_attn.value_proj.bias""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.output_proj.weight""", f"""model.encoder.layers.{i}.self_attn.output_proj.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.self_attn.output_proj.bias""", f"""model.encoder.layers.{i}.self_attn.output_proj.bias""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.norm1.weight""", f"""model.encoder.layers.{i}.self_attn_layer_norm.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.norm1.bias""", f"""model.encoder.layers.{i}.self_attn_layer_norm.bias""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.weight""", f"""model.encoder.layers.{i}.fc1.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.bias""", f"""model.encoder.layers.{i}.fc1.bias""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.weight""", f"""model.encoder.layers.{i}.fc2.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.bias""", f"""model.encoder.layers.{i}.fc2.bias""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.norm2.weight""", f"""model.encoder.layers.{i}.final_layer_norm.weight""")				)
				rename_keys.append((f"""transformer.encoder.layers.{i}.norm2.bias""", f"""model.encoder.layers.{i}.final_layer_norm.bias""")				)

		# transformer decoder
		for i in range(config.decoder_layers				):
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight""", f"""model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias""", f"""model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.attention_weights.weight""", f"""model.decoder.layers.{i}.encoder_attn.attention_weights.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.attention_weights.bias""", f"""model.decoder.layers.{i}.encoder_attn.attention_weights.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.value_proj.weight""", f"""model.decoder.layers.{i}.encoder_attn.value_proj.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.value_proj.bias""", f"""model.decoder.layers.{i}.encoder_attn.value_proj.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.output_proj.weight""", f"""model.decoder.layers.{i}.encoder_attn.output_proj.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.cross_attn.output_proj.bias""", f"""model.decoder.layers.{i}.encoder_attn.output_proj.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.norm1.weight""", f"""model.decoder.layers.{i}.encoder_attn_layer_norm.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.norm1.bias""", f"""model.decoder.layers.{i}.encoder_attn_layer_norm.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.self_attn.out_proj.weight""", f"""model.decoder.layers.{i}.self_attn.out_proj.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.self_attn.out_proj.bias""", f"""model.decoder.layers.{i}.self_attn.out_proj.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.norm2.weight""", f"""model.decoder.layers.{i}.self_attn_layer_norm.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.norm2.bias""", f"""model.decoder.layers.{i}.self_attn_layer_norm.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.weight""", f"""model.decoder.layers.{i}.fc1.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.bias""", f"""model.decoder.layers.{i}.fc1.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.weight""", f"""model.decoder.layers.{i}.fc2.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.bias""", f"""model.decoder.layers.{i}.fc2.bias""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.norm3.weight""", f"""model.decoder.layers.{i}.final_layer_norm.weight""")				)
				rename_keys.append((f"""transformer.decoder.layers.{i}.norm3.bias""", f"""model.decoder.layers.{i}.final_layer_norm.bias""")				)

		# fmt: on

		return rename_keys
def 		A_	(   __UpperCamelCase     :				Dict       ,	__UpperCamelCase     :				Tuple       ,	__UpperCamelCase     :				Optional[Any]				):
		lowercase										=							dct.pop(__UpperCamelCase				)
		lowercase										=							val
def 		A_	(   __UpperCamelCase     :				str       ,	__UpperCamelCase     :				Any				):
		# transformer decoder self-attention layers
		lowercase										=							config.d_model
		for i in range(config.decoder_layers				):
				# read in weights + bias of input projection layer of self-attention
				lowercase										=							state_dict.pop(f"""transformer.decoder.layers.{i}.self_attn.in_proj_weight"""				)
				lowercase										=							state_dict.pop(f"""transformer.decoder.layers.{i}.self_attn.in_proj_bias"""				)
				# next, add query, keys and values (in that order) to the state dict
				lowercase										=							in_proj_weight[:hidden_size, :]
				lowercase										=							in_proj_bias[:hidden_size]
				lowercase										=							in_proj_weight[
				    hidden_size : hidden_size * 2, :
				]
				lowercase										=							in_proj_bias[hidden_size : hidden_size * 2]
				lowercase										=							in_proj_weight[-hidden_size:, :]
				lowercase										=							in_proj_bias[-hidden_size:]
def 		A_	(   ):
		lowercase										=							'''http://images.cocodataset.org/val2017/000000039769.jpg'''
		lowercase										=							Image.open(requests.get(__UpperCamelCase       ,	stream=__UpperCamelCase				).raw				)

		return im
@torch.no_grad()
def 		A_	(   __UpperCamelCase     :				Tuple       ,	__UpperCamelCase     :				Dict       ,	__UpperCamelCase     :				List[Any]				):
		lowercase										=							get_deta_config()

		# load original state dict
		if model_name == "deta-resnet-50":
				lowercase										=							'''adet_checkpoint0011.pth'''
		elif model_name == "deta-resnet-50-24-epochs":
				lowercase										=							'''adet_2x_checkpoint0023.pth'''
		else:
				raise ValueError(f"""Model name {model_name} not supported"""				)
		lowercase										=							hf_hub_download(repo_id='''nielsr/deta-checkpoints'''       ,	filename=__UpperCamelCase				)
		lowercase										=							torch.load(__UpperCamelCase       ,	map_location='''cpu'''				)['''model''']

		# rename keys
		lowercase										=							create_rename_keys(__UpperCamelCase				)
		for src, dest in rename_keys:
				rename_key(__UpperCamelCase       ,	__UpperCamelCase       ,	__UpperCamelCase				)
		read_in_decoder_q_k_v(__UpperCamelCase       ,	__UpperCamelCase				)

		# fix some prefixes
		for key in state_dict.copy().keys():
				if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
						lowercase										=							state_dict.pop(__UpperCamelCase				)
						lowercase										=							val
				if "input_proj" in key:
						lowercase										=							state_dict.pop(__UpperCamelCase				)
						lowercase										=							val
				if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
						lowercase										=							state_dict.pop(__UpperCamelCase				)
						lowercase										=							val

    # finally, create HuggingFace model and load state dict
		lowercase										=							DetaForObjectDetection(__UpperCamelCase				)
		model.load_state_dict(__UpperCamelCase				)
		model.eval()

		lowercase										=							'''cuda''' if torch.cuda.is_available() else '''cpu'''
		model.to(__UpperCamelCase				)

		# load image processor
		lowercase										=							DetaImageProcessor(format='''coco_detection'''				)

		# verify our conversion on image
		lowercase										=							prepare_img()
		lowercase										=							processor(images=__UpperCamelCase       ,	return_tensors='''pt'''				)
		lowercase										=							encoding['''pixel_values''']
		lowercase										=							model(pixel_values.to(__UpperCamelCase				)				)

		# verify logits
		if model_name == "deta-resnet-50":
				lowercase										=							torch.tensor(
				    [[-7.39_78, -2.54_06, -4.16_68], [-8.26_84, -3.99_33, -3.80_96], [-7.05_15, -3.79_73, -5.85_16]]				)
				lowercase										=							torch.tensor([[0.50_43, 0.49_73, 0.99_98], [0.25_42, 0.54_89, 0.47_48], [0.54_90, 0.27_65, 0.05_70]]				)
		elif model_name == "deta-resnet-50-24-epochs":
				lowercase										=							torch.tensor(
				    [[-7.16_88, -2.48_57, -4.86_69], [-7.86_30, -3.81_54, -4.26_74], [-7.27_30, -4.18_65, -5.53_23]]				)
				lowercase										=							torch.tensor([[0.50_21, 0.49_71, 0.99_94], [0.25_46, 0.54_86, 0.47_31], [0.16_86, 0.19_86, 0.21_42]]				)

		assert torch.allclose(outputs.logits[0, :3, :3]       ,	expected_logits.to(__UpperCamelCase				)       ,	atol=1e-4				)
		assert torch.allclose(outputs.pred_boxes[0, :3, :3]       ,	expected_boxes.to(__UpperCamelCase				)       ,	atol=1e-4				)
		print('''Everything ok!'''				)

		if pytorch_dump_folder_path:
				# Save model and processor
				logger.info(f"""Saving PyTorch model and processor to {pytorch_dump_folder_path}..."""				)
				Path(__UpperCamelCase				).mkdir(exist_ok=__UpperCamelCase				)
				model.save_pretrained(__UpperCamelCase				)
				processor.save_pretrained(__UpperCamelCase				)

		# Push to hub
		if push_to_hub:
				print('''Pushing model and processor to hub...'''				)
				model.push_to_hub(f"""jozhang97/{model_name}"""				)
				processor.push_to_hub(f"""jozhang97/{model_name}"""				)


if __name__ == "__main__":
							__lowerCAmelCase				      =   argparse.ArgumentParser()

							parser.add_argument(
							    '''--model_name''',
							    type=str,
							    default='''deta-resnet-50''',
							    choices=['''deta-resnet-50''', '''deta-resnet-50-24-epochs'''],
							    help='''Name of the model you\'d like to convert.''',
							)
							parser.add_argument(
							    '''--pytorch_dump_folder_path''',
							    default=None,
							    type=str,
							    help='''Path to the folder to output PyTorch model.''',
							)
							parser.add_argument(
							    '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
							)
							__lowerCAmelCase				      =   parser.parse_args()
							convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)


"""simple docstring"""






import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
							import torch

							from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
							import pyanvml.pyanvml as nvml


__lowerCAmelCase				      =   logging.get_logger(__name__)
class 	_lowerCAmelCase    (      __snake_case       ):
							__lowerCAmelCase					:    PyTorchBenchmarkArguments
							__lowerCAmelCase					:    PretrainedConfig
							__lowerCAmelCase					:    str     =  "PyTorch"

							@property
							def 					_lowerCAmelCase  ( self       :    Dict    )   ->    Dict:

									"""simple docstring"""
									return torch.__version__

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_speed(_inference    )

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_memory(_inference    )

							def 					_lowerCAmelCase  ( self       :    Union[str, Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_speed(_train    )

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_memory(_train    )

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									if self.args.torchscript:
											lowercase										=							True

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_MAPPING[config.__class__](a    )

									model.eval()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )
											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									if self.args.torchscript:
											with torch.no_grad():
													lowercase										=							torch.jit.trace(a ,		a    )
									else:
											lowercase										=							model

									def encoder_decoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a ,		decoder_input_ids=a    )
											return outputs

									def encoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a    )
											return outputs

									lowercase										=							encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
									return _forward

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_WITH_LM_HEAD_MAPPING[config.__class__](a    )

									if self.args.torchscript:
											raise NotImplementedError('''Training for torchscript is currently not implemented'''    )
									else:
											lowercase										=							model

									model.train()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )

											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									def compute_loss_and_backprob_encoder():
											lowercase										=							train_model(a ,		labels=a    )[0]
											loss.backward()
											return loss

									def compute_loss_and_backprob_encoder_decoder():
											lowercase										=							train_model(a ,		decoder_input_ids=a ,		labels=a    )[0]
											loss.backward()
											return loss

									lowercase										=							(
									    compute_loss_and_backprob_encoder_decoder
									    if config.is_encoder_decoder
									    else compute_loss_and_backprob_encoder
									)
									return _train

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    Optional[Any]    )   ->    float:

									"""simple docstring"""
									try:
											if self.args.is_tpu or self.args.torchscript:
													# run additional 10 times to stabilize compilation for tpu and torchscript
													logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''    )
													timeit.repeat(
													    a ,		repeat=1 ,		number=5 ,		)

											# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
											lowercase										=							timeit.repeat(
											    a ,		repeat=self.args.repeat ,		number=10 ,		)

											if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
													import torch_xla.debug.metrics as met

													self.print_fn(met.metrics_report()    )

											return min(a    ) / 10.0
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A"

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    Callable[[], None]    )   ->    [Memory, MemorySummary]:

									"""simple docstring"""
									try:
											if self.args.trace_memory_line_by_line:
													lowercase										=							start_memory_tracing('''transformers'''    )

											if self.args.is_tpu:
													# tpu
													raise NotImplementedError(
													    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
													    ''' `--no-memory` or `args.memory=False`'''    )
											elif self.args.is_gpu:
													if not is_pyanvml_available():
															logger.warning(
															    '''py3nvml not installed, we won\'t log GPU memory usage. '''
															    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''    )
															lowercase										=							'''N/A'''
													else:
															logger.info(
															    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
															    ''' on the same GPU.'''    )
															# init nvml
															nvml.nvmlInit()
															func()
															lowercase										=							nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
															lowercase										=							nvml.nvmlDeviceGetMemoryInfo(a    )
															lowercase										=							meminfo.used
															lowercase										=							Memory(a    )
															# shutdown nvml
															nvml.nvmlShutdown()
											else:
													# cpu
													lowercase										=							measure_peak_memory_cpu(a    )
													lowercase										=							Memory(a    ) if isinstance(a ,		a    ) else memory_bytes

											if self.args.trace_memory_line_by_line:
													lowercase										=							stop_memory_tracing(a    )
											else:
													lowercase										=							None

											return memory, summary
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A", None