
'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import AlbertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import TF_MODEL_FOR_PRETRAINING_MAPPING
     from transformers.models.albert.modeling_tf_albert import (
         TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFAlbertForMaskedLM,
         TFAlbertForMultipleChoice,
         TFAlbertForPreTraining,
         TFAlbertForQuestionAnswering,
         TFAlbertForSequenceClassification,
         TFAlbertForTokenClassification,
         TFAlbertModel,
     )







class 			__A					:
 def __init__( self      :     Optional[int]					, lowerCamelCase      :     int					, lowerCamelCase      :     str=13					, lowerCamelCase      :     str=7					, lowerCamelCase      :     List[str]=True					, lowerCamelCase      :     List[str]=True					, lowerCamelCase      :     List[Any]=True					, lowerCamelCase      :     str=True					, lowerCamelCase      :     Dict=99					, lowerCamelCase      :     Union[str, Any]=16					, lowerCamelCase      :     Dict=32					, lowerCamelCase      :     Union[str, Any]=2					, lowerCamelCase      :     str=4					, lowerCamelCase      :     str=37					, lowerCamelCase      :     Union[str, Any]="gelu"					, lowerCamelCase      :     int=0.1					, lowerCamelCase      :     Optional[int]=0.1					, lowerCamelCase      :     List[Any]=5_12					, lowerCamelCase      :     Union[str, Any]=16					, lowerCamelCase      :     Any=2					, lowerCamelCase      :     Optional[int]=0.02					, lowerCamelCase      :     Optional[Any]=3					, lowerCamelCase      :     Union[str, Any]=4					, lowerCamelCase      :     Dict=None					, ):



   """simple docstring"""





   __A		:  List[str]       				=		parent
   __A		:  Any       				=		13
   __A		:  List[Any]       				=		7
   __A		:  Union[str, Any]       				=		True
   __A		:  Union[str, Any]       				=		True
   __A		:  List[Any]       				=		True
   __A		:  str       				=		True
   __A		:  List[str]       				=		99
   __A		:  List[str]       				=		16
   __A		:  Optional[int]       				=		32
   __A		:  Union[str, Any]       				=		2
   __A		:  int       				=		4
   __A		:  List[str]       				=		37
   __A		:  Union[str, Any]       				=		"""gelu"""
   __A		:  List[Any]       				=		0.1
   __A		:  str       				=		0.1
   __A		:  List[str]       				=		5_12
   __A		:  str       				=		16
   __A		:  int       				=		2
   __A		:  List[Any]       				=		0.02
   __A		:  Tuple       				=		3
   __A		:  Union[str, Any]       				=		4
   __A		:  List[str]       				=		None
 def 		lowercase_( self      :     Any							):



   """simple docstring"""





   __A		:  Optional[Any]       				=		ids_tensor([self.batch_size, self.seq_length]					, self.vocab_size							)

   __A		:  List[Any]       				=		None
   if self.use_input_mask:
     __A		:  Union[str, Any]       				=		random_attention_mask([self.batch_size, self.seq_length]							)

   __A		:  List[Any]       				=		None
   if self.use_token_type_ids:
     __A		:  int       				=		ids_tensor([self.batch_size, self.seq_length]					, self.type_vocab_size							)

   __A		:  Tuple       				=		None
   __A		:  str       				=		None
   __A		:  Union[str, Any]       				=		None
   if self.use_labels:
     __A		:  str       				=		ids_tensor([self.batch_size]					, self.type_sequence_label_size							)
     __A		:  Optional[Any]       				=		ids_tensor([self.batch_size, self.seq_length]					, self.num_labels							)
     __A		:  Dict       				=		ids_tensor([self.batch_size]					, self.num_choices							)

   __A		:  Tuple       				=		AlbertConfig(
       vocab_size=self.vocab_size					, hidden_size=self.hidden_size					, embedding_size=self.embedding_size					, num_hidden_layers=self.num_hidden_layers					, num_attention_heads=self.num_attention_heads					, intermediate_size=self.intermediate_size					, hidden_act=self.hidden_act					, hidden_dropout_prob=self.hidden_dropout_prob					, attention_probs_dropout_prob=self.attention_probs_dropout_prob					, max_position_embeddings=self.max_position_embeddings					, type_vocab_size=self.type_vocab_size					, initializer_range=self.initializer_range					, )

   return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 def 		lowercase_( self      :     Dict					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     Any					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     List[str]					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     Optional[int]							):



   """simple docstring"""





   __A		:  List[Any]       				=		TFAlbertModel(config=lowerCamelCase							)
   # inputs = {'input_ids': input_ids,
   #           'attention_mask': input_mask,
   #           'token_type_ids': token_type_ids}
   # sequence_output, pooled_output = model(**inputs)
   __A		:  List[Any]       				=		{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
   __A		:  List[str]       				=		model(lowerCamelCase							)

   __A		:  Tuple       				=		[input_ids, input_mask]
   __A		:  Any       				=		model(lowerCamelCase							)

   __A		:  List[Any]       				=		model(lowerCamelCase							)

   self.parent.assertEqual(result.last_hidden_state.shape					, (self.batch_size, self.seq_length, self.hidden_size)							)
   self.parent.assertEqual(result.pooler_output.shape					, (self.batch_size, self.hidden_size)							)
 def 		lowercase_( self      :     Optional[int]					, lowerCamelCase      :     int					, lowerCamelCase      :     List[str]					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     List[str]					, lowerCamelCase      :     str					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     List[str]							):



   """simple docstring"""





   __A		:  str       				=		self.num_labels
   __A		:  Optional[Any]       				=		TFAlbertForPreTraining(config=lowerCamelCase							)
   __A		:  List[Any]       				=		{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
   __A		:  Dict       				=		model(lowerCamelCase							)
   self.parent.assertEqual(result.prediction_logits.shape					, (self.batch_size, self.seq_length, self.vocab_size)							)
   self.parent.assertEqual(result.sop_logits.shape					, (self.batch_size, self.num_labels)							)
 def 		lowercase_( self      :     Union[str, Any]					, lowerCamelCase      :     Any					, lowerCamelCase      :     Dict					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     str					, lowerCamelCase      :     str					, lowerCamelCase      :     Tuple					, lowerCamelCase      :     str							):



   """simple docstring"""





   __A		:  List[str]       				=		TFAlbertForMaskedLM(config=lowerCamelCase							)
   __A		:  Optional[Any]       				=		{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
   __A		:  Optional[int]       				=		model(lowerCamelCase							)
   self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.seq_length, self.vocab_size)							)
 def 		lowercase_( self      :     Optional[Any]					, lowerCamelCase      :     int					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     List[Any]					, lowerCamelCase      :     Optional[Any]					, lowerCamelCase      :     Optional[Any]					, lowerCamelCase      :     Optional[int]							):



   """simple docstring"""





   __A		:  str       				=		self.num_labels
   __A		:  Union[str, Any]       				=		TFAlbertForSequenceClassification(config=lowerCamelCase							)
   __A		:  Union[str, Any]       				=		{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
   __A		:  List[Any]       				=		model(lowerCamelCase							)
   self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.num_labels)							)
 def 		lowercase_( self      :     List[Any]					, lowerCamelCase      :     Dict					, lowerCamelCase      :     Any					, lowerCamelCase      :     Optional[Any]					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     Dict					, lowerCamelCase      :     List[str]					, lowerCamelCase      :     List[str]							):



   """simple docstring"""





   __A		:  List[Any]       				=		TFAlbertForQuestionAnswering(config=lowerCamelCase							)
   __A		:  Tuple       				=		{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
   __A		:  Tuple       				=		model(lowerCamelCase							)
   self.parent.assertEqual(result.start_logits.shape					, (self.batch_size, self.seq_length)							)
   self.parent.assertEqual(result.end_logits.shape					, (self.batch_size, self.seq_length)							)
 def 		lowercase_( self      :     Union[str, Any]					, lowerCamelCase      :     str					, lowerCamelCase      :     Optional[Any]					, lowerCamelCase      :     str					, lowerCamelCase      :     str					, lowerCamelCase      :     Optional[Any]					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     List[Any]							):



   """simple docstring"""





   __A		:  Any       				=		self.num_choices
   __A		:  Any       				=		TFAlbertForMultipleChoice(config=lowerCamelCase							)
   __A		:  Union[str, Any]       				=		tf.tile(tf.expand_dims(lowerCamelCase					, 1							)					, (1, self.num_choices, 1)							)
   __A		:  Dict       				=		tf.tile(tf.expand_dims(lowerCamelCase					, 1							)					, (1, self.num_choices, 1)							)
   __A		:  Optional[Any]       				=		tf.tile(tf.expand_dims(lowerCamelCase					, 1							)					, (1, self.num_choices, 1)							)
   __A		:  Optional[Any]       				=		{
       """input_ids""": multiple_choice_inputs_ids,
       """attention_mask""": multiple_choice_input_mask,
       """token_type_ids""": multiple_choice_token_type_ids,
   }
   __A		:  Tuple       				=		model(lowerCamelCase							)
   self.parent.assertListEqual(list(result["""logits"""].shape							)					, [self.batch_size, self.num_choices]							)
 def 		lowercase_( self      :     List[str]					, lowerCamelCase      :     Dict					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     Any					, lowerCamelCase      :     Tuple					, lowerCamelCase      :     List[Any]					, lowerCamelCase      :     List[Any]					, lowerCamelCase      :     str							):



   """simple docstring"""





   __A		:  Any       				=		self.num_labels
   __A		:  List[str]       				=		TFAlbertForTokenClassification(config=lowerCamelCase							)
   __A		:  List[str]       				=		{
       """input_ids""": input_ids,
       """attention_mask""": input_mask,
       """token_type_ids""": token_type_ids,
   }
   __A		:  Dict       				=		model(lowerCamelCase							)
   self.parent.assertListEqual(list(result["""logits"""].shape							)					, [self.batch_size, self.seq_length, self.num_labels]							)






 def 		lowercase_( self      :     Tuple							):



   """simple docstring"""





   __A		:  List[str]       				=		self.prepare_config_and_inputs()
   (
       (
       __A  
   )				,       (
       __A  
   )				,       (
       __A  
   )				,       (
       __A  
   )				,       (
       __A  
   )				,       (
       __A  
   )				,       (
       __A  
   )				,         
   )		:  List[Any]       				=		config_and_inputs
   __A		:  str       				=		{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
   return config, inputs_dict







@require_tf
class 			__A					(			__snake_case     ,    __snake_case     ,    unittest.TestCase      ):
 lowerCamelCase												=(
     (
         TFAlbertModel,
         TFAlbertForPreTraining,
         TFAlbertForMaskedLM,
         TFAlbertForSequenceClassification,
         TFAlbertForQuestionAnswering,
         TFAlbertForTokenClassification,
         TFAlbertForMultipleChoice,
     )
     if is_tf_available()
     else ()
 )
 lowerCamelCase												=(
     {
         '''feature-extraction''': TFAlbertModel,
         '''fill-mask''': TFAlbertForMaskedLM,
         '''question-answering''': TFAlbertForQuestionAnswering,
         '''text-classification''': TFAlbertForSequenceClassification,
         '''token-classification''': TFAlbertForTokenClassification,
         '''zero-shot''': TFAlbertForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 lowerCamelCase												=False
 lowerCamelCase												=False
 def 		lowercase_( self      :     Dict					, lowerCamelCase      :     List[str]					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     str=False							):



   """simple docstring"""





   __A		:  str       				=		super()._prepare_for_class(lowerCamelCase					, lowerCamelCase					, return_labels=lowerCamelCase							)

   if return_labels:
     if model_class in get_values(lowerCamelCase							):
       __A		:  Optional[Any]       				=		tf.zeros(self.model_tester.batch_size					, dtype=tf.intaa							)

   return inputs_dict
 def 		lowercase_( self      :     Optional[Any]							):



   """simple docstring"""





   __A		:  List[Any]       				=		TFAlbertModelTester(self							)
   __A		:  Any       				=		ConfigTester(self					, config_class=lowerCamelCase					, hidden_size=37							)
 def 		lowercase_( self      :     Dict							):



   """simple docstring"""





   self.config_tester.run_common_tests()
 def 		lowercase_( self      :     Union[str, Any]							):



   """simple docstring"""





   __A		:  Optional[int]       				=		self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_albert_model(*lowerCamelCase							)
 def 		lowercase_( self      :     Tuple							):



   """simple docstring"""





   __A		:  Any       				=		self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_albert_for_pretraining(*lowerCamelCase							)
 def 		lowercase_( self      :     Dict							):



   """simple docstring"""





   __A		:  str       				=		self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_albert_for_masked_lm(*lowerCamelCase							)
 def 		lowercase_( self      :     List[str]							):



   """simple docstring"""





   __A		:  str       				=		self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_albert_for_multiple_choice(*lowerCamelCase							)
 def 		lowercase_( self      :     Optional[int]							):



   """simple docstring"""





   __A		:  int       				=		self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_albert_for_sequence_classification(*lowerCamelCase							)
 def 		lowercase_( self      :     str							):



   """simple docstring"""





   __A		:  int       				=		self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_albert_for_question_answering(*lowerCamelCase							)






 @slow
 def 		lowercase_( self      :     Any							):



   """simple docstring"""





   for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     __A		:  Optional[Any]       				=		TFAlbertModel.from_pretrained(lowerCamelCase							)
     self.assertIsNotNone(lowerCamelCase							)







@require_tf
class 			__A					(			unittest.TestCase      ):
 @slow
 def 		lowercase_( self      :     Optional[Any]							):



   """simple docstring"""





   __A		:  Optional[int]       				=		TFAlbertForPreTraining.from_pretrained("""albert-base-v2"""							)
   __A		:  Dict       				=		tf.constant([[0, 1, 2, 3, 4, 5]]							)
   __A		:  Tuple       				=		model(lowerCamelCase							)[0]

   __A		:  Optional[int]       				=		[1, 6, 3_00_00]
   self.assertEqual(output.shape					, lowerCamelCase							)

   __A		:  Optional[int]       				=		tf.constant(
       [
           [
               [4.595_668, 0.74_462_754, -1.818_147],
               [4.5_954_347, 0.7_454_184, -1.8_188_258],
               [4.5_954_905, 0.7_448_235, -1.8_182_316],
           ]
       ]							)
   tf.debugging.assert_near(output[:, :3, :3]					, lowerCamelCase					, atol=1e-4							)






"""simple docstring"""






import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
							import torch

							from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
							import pyanvml.pyanvml as nvml


__lowerCAmelCase				      =   logging.get_logger(__name__)
class 	_lowerCAmelCase    (      __snake_case       ):
							__lowerCAmelCase					:    PyTorchBenchmarkArguments
							__lowerCAmelCase					:    PretrainedConfig
							__lowerCAmelCase					:    str     =  "PyTorch"

							@property
							def 					_lowerCAmelCase  ( self       :    Dict    )   ->    Dict:

									"""simple docstring"""
									return torch.__version__

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_speed(_inference    )

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_memory(_inference    )

							def 					_lowerCAmelCase  ( self       :    Union[str, Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_speed(_train    )

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_memory(_train    )

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									if self.args.torchscript:
											lowercase										=							True

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_MAPPING[config.__class__](a    )

									model.eval()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )
											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									if self.args.torchscript:
											with torch.no_grad():
													lowercase										=							torch.jit.trace(a ,		a    )
									else:
											lowercase										=							model

									def encoder_decoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a ,		decoder_input_ids=a    )
											return outputs

									def encoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a    )
											return outputs

									lowercase										=							encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
									return _forward

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_WITH_LM_HEAD_MAPPING[config.__class__](a    )

									if self.args.torchscript:
											raise NotImplementedError('''Training for torchscript is currently not implemented'''    )
									else:
											lowercase										=							model

									model.train()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )

											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									def compute_loss_and_backprob_encoder():
											lowercase										=							train_model(a ,		labels=a    )[0]
											loss.backward()
											return loss

									def compute_loss_and_backprob_encoder_decoder():
											lowercase										=							train_model(a ,		decoder_input_ids=a ,		labels=a    )[0]
											loss.backward()
											return loss

									lowercase										=							(
									    compute_loss_and_backprob_encoder_decoder
									    if config.is_encoder_decoder
									    else compute_loss_and_backprob_encoder
									)
									return _train

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    Optional[Any]    )   ->    float:

									"""simple docstring"""
									try:
											if self.args.is_tpu or self.args.torchscript:
													# run additional 10 times to stabilize compilation for tpu and torchscript
													logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''    )
													timeit.repeat(
													    a ,		repeat=1 ,		number=5 ,		)

											# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
											lowercase										=							timeit.repeat(
											    a ,		repeat=self.args.repeat ,		number=10 ,		)

											if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
													import torch_xla.debug.metrics as met

													self.print_fn(met.metrics_report()    )

											return min(a    ) / 10.0
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A"

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    Callable[[], None]    )   ->    [Memory, MemorySummary]:

									"""simple docstring"""
									try:
											if self.args.trace_memory_line_by_line:
													lowercase										=							start_memory_tracing('''transformers'''    )

											if self.args.is_tpu:
													# tpu
													raise NotImplementedError(
													    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
													    ''' `--no-memory` or `args.memory=False`'''    )
											elif self.args.is_gpu:
													if not is_pyanvml_available():
															logger.warning(
															    '''py3nvml not installed, we won\'t log GPU memory usage. '''
															    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''    )
															lowercase										=							'''N/A'''
													else:
															logger.info(
															    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
															    ''' on the same GPU.'''    )
															# init nvml
															nvml.nvmlInit()
															func()
															lowercase										=							nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
															lowercase										=							nvml.nvmlDeviceGetMemoryInfo(a    )
															lowercase										=							meminfo.used
															lowercase										=							Memory(a    )
															# shutdown nvml
															nvml.nvmlShutdown()
											else:
													# cpu
													lowercase										=							measure_peak_memory_cpu(a    )
													lowercase										=							Memory(a    ) if isinstance(a ,		a    ) else memory_bytes

											if self.args.trace_memory_line_by_line:
													lowercase										=							stop_memory_tracing(a    )
											else:
													lowercase										=							None

											return memory, summary
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A", None