


import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
							import torch

							from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
							import pyanvml.pyanvml as nvml


SCREAMING_SNAKE_CASE__       			=				logging.get_logger(__name__)







class     __lowerCamelCase     (  snake_case_     ):


							"""simple docstring"""

							lowerCAmelCase__          =				42
							lowerCAmelCase__          =				42
							lowerCAmelCase__          =				"PyTorch"




							@property
							def 				A__      (  self  )     ->       Union[str, Any]:






								'''simple docstring'''






								return torch.__version__




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )     ->       float:






								'''simple docstring'''






								lowercase_			 = self._prepare_inference_func(UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )
								return self._measure_speed(_inference  )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )     ->       [Memory, Optional[MemorySummary]]:






								'''simple docstring'''






								lowercase_			 = self._prepare_inference_func(UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )
								return self._measure_memory(_inference  )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )     ->       float:






								'''simple docstring'''






								lowercase_			 = self._prepare_train_func(UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )
								return self._measure_speed(_train  )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )     ->       [Memory, Optional[MemorySummary]]:






								'''simple docstring'''






								lowercase_			 = self._prepare_train_func(UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )
								return self._measure_memory(_train  )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )     ->       Callable[[], None]:






								'''simple docstring'''






								lowercase_			 = self.config_dict[model_name]

								if self.args.torchscript:
									lowercase_			 = True

								lowercase_			 = (
								    hasattr(UpperCAmelCase						,       "architectures"  )
								    and isinstance(config.architectures						,       UpperCAmelCase  )
								    and len(config.architectures  ) > 0
								)
								if not self.args.only_pretrain_model and has_model_class_in_config:
									try:
										lowercase_			 = config.architectures[0]
										lowercase_			 = __import__("transformers"						,       fromlist=[model_class]  )
										lowercase_			 = getattr(UpperCAmelCase						,       UpperCAmelCase  )
										lowercase_			 = model_cls(UpperCAmelCase  )
									except ImportError:
										raise ImportError(
										    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
										    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."  )
								else:
									lowercase_			 = MODEL_MAPPING[config.__class__](UpperCAmelCase  )

								model.eval()
								model.to(self.args.device  )

								# encoder-decoder has vocab size saved differently
								lowercase_			 = config.vocab_size if hasattr(UpperCAmelCase						,       "vocab_size"  ) else config.encoder.vocab_size
								lowercase_			 = torch.randint(UpperCAmelCase						,       (batch_size, sequence_length)						,       dtype=torch.long						,       device=self.args.device  )

								if self.args.fpaa:
									logger.info("Running training in Mixed Precision..."  )
									if not self.args.is_gpu:
										raise ValueError("Mixed precision is possible only for GPU."  )
									# amp seems to have memory leaks so that memory usage
									# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
									model.half()

								if self.args.torchscript:
									with torch.no_grad():
										lowercase_			 = torch.jit.trace(UpperCAmelCase						,       UpperCAmelCase  )
								else:
									lowercase_			 = model

								def encoder_decoder_forward():
									with torch.no_grad():
										lowercase_			 = inference_model(UpperCAmelCase						,       decoder_input_ids=UpperCAmelCase  )
									return outputs

								def encoder_forward():
									with torch.no_grad():
										lowercase_			 = inference_model(UpperCAmelCase  )
									return outputs

								lowercase_			 = encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
								return _forward




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase						,       UpperCAmelCase  )     ->       Callable[[], None]:






								'''simple docstring'''






								lowercase_			 = self.config_dict[model_name]

								lowercase_			 = (
								    hasattr(UpperCAmelCase						,       "architectures"  )
								    and isinstance(config.architectures						,       UpperCAmelCase  )
								    and len(config.architectures  ) > 0
								)
								if not self.args.only_pretrain_model and has_model_class_in_config:
									try:
										lowercase_			 = config.architectures[0]
										lowercase_			 = __import__("transformers"						,       fromlist=[model_class]  )
										lowercase_			 = getattr(UpperCAmelCase						,       UpperCAmelCase  )
										lowercase_			 = model_cls(UpperCAmelCase  )
									except ImportError:
										raise ImportError(
										    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
										    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."  )
								else:
									lowercase_			 = MODEL_WITH_LM_HEAD_MAPPING[config.__class__](UpperCAmelCase  )

								if self.args.torchscript:
									raise NotImplementedError("Training for torchscript is currently not implemented"  )
								else:
									lowercase_			 = model

								model.train()
								model.to(self.args.device  )

								# encoder-decoder has vocab size saved differently
								lowercase_			 = config.vocab_size if hasattr(UpperCAmelCase						,       "vocab_size"  ) else config.encoder.vocab_size
								lowercase_			 = torch.randint(UpperCAmelCase						,       (batch_size, sequence_length)						,       dtype=torch.long						,       device=self.args.device  )

								if self.args.fpaa:
									logger.info("Running training in Mixed Precision..."  )
									if not self.args.is_gpu:
										raise ValueError("Mixed precision is possible only for GPU."  )

									# amp seems to have memory leaks so that memory usage
									# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
									model.half()

								def compute_loss_and_backprob_encoder():
									lowercase_			 = train_model(UpperCAmelCase						,       labels=UpperCAmelCase  )[0]
									loss.backward()
									return loss

								def compute_loss_and_backprob_encoder_decoder():
									lowercase_			 = train_model(UpperCAmelCase						,       decoder_input_ids=UpperCAmelCase						,       labels=UpperCAmelCase  )[0]
									loss.backward()
									return loss

								lowercase_			 = (
								    compute_loss_and_backprob_encoder_decoder
								    if config.is_encoder_decoder
								    else compute_loss_and_backprob_encoder
								)
								return _train




							def 				A__      (  self						,       UpperCAmelCase  )     ->       float:






								'''simple docstring'''






								try:
									if self.args.is_tpu or self.args.torchscript:
										# run additional 10 times to stabilize compilation for tpu and torchscript
										logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation"  )
										timeit.repeat(
										    UpperCAmelCase						,       repeat=1						,       number=5						,       )

									# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
									lowercase_			 = timeit.repeat(
									    UpperCAmelCase						,       repeat=self.args.repeat						,       number=10						,       )

									if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
										import torch_xla.debug.metrics as met

										self.print_fn(met.metrics_report()  )

									return min(UpperCAmelCase  ) / 10.0
								except RuntimeError as e:
									self.print_fn(F'Doesn\'t fit on GPU. {e}'  )
									return "N/A"




							def 				A__      (  self						,       UpperCAmelCase  )     ->       [Memory, MemorySummary]:






								'''simple docstring'''






								try:
									if self.args.trace_memory_line_by_line:
										lowercase_			 = start_memory_tracing("transformers"  )

									if self.args.is_tpu:
										# tpu
										raise NotImplementedError(
										    "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with"
										    " `--no-memory` or `args.memory=False`"  )
									elif self.args.is_gpu:
										if not is_pyanvml_available():
											logger.warning(
											    "py3nvml not installed, we won't log GPU memory usage. "
											    "Install py3nvml (pip install py3nvml) to log information about GPU."  )
											lowercase_			 = "N/A"
										else:
											logger.info(
											    "Measuring total GPU usage on GPU device. Make sure to not have additional processes running"
											    " on the same GPU."  )
											# init nvml
											nvml.nvmlInit()
											func()
											lowercase_			 = nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx  )
											lowercase_			 = nvml.nvmlDeviceGetMemoryInfo(UpperCAmelCase  )
											lowercase_			 = meminfo.used
											lowercase_			 = Memory(UpperCAmelCase  )
											# shutdown nvml
											nvml.nvmlShutdown()
									else:
										# cpu
										lowercase_			 = measure_peak_memory_cpu(UpperCAmelCase  )
										lowercase_			 = Memory(UpperCAmelCase  ) if isinstance(UpperCAmelCase						,       UpperCAmelCase  ) else memory_bytes

									if self.args.trace_memory_line_by_line:
										lowercase_			 = stop_memory_tracing(UpperCAmelCase  )
									else:
										lowercase_			 = None

									return memory, summary
								except RuntimeError as e:
									self.print_fn(F'Doesn\'t fit on GPU. {e}'  )
									return "N/A", None





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xglm import XGLMTokenizer
else:
							SCREAMING_SNAKE_CASE__       			=				None


SCREAMING_SNAKE_CASE__       			=				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       			=				{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__       			=				{
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json""",
    },
}

SCREAMING_SNAKE_CASE__       			=				{
    """facebook/xglm-564M""": 2_0_4_8,
}







class     __lowerCamelCase     (  snake_case_     ):


							"""simple docstring"""

							lowerCAmelCase__          =				VOCAB_FILES_NAMES
							lowerCAmelCase__          =				PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase__          =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase__          =				["input_ids", "attention_mask"]
							lowerCAmelCase__          =				XGLMTokenizer




							def __init__(  self						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase="<s>"						,       UpperCAmelCase="</s>"						,       UpperCAmelCase="</s>"						,       UpperCAmelCase="<s>"						,       UpperCAmelCase="<unk>"						,       UpperCAmelCase="<pad>"						,       **UpperCAmelCase						,       )     ->       Tuple:






								'''simple docstring'''






								lowercase_			 = 7
								lowercase_			 = [F'<madeupword{i}>' for i in range(self.num_madeup_words  )]

								lowercase_			 = kwargs.get("additional_special_tokens"						,       []  )
								kwargs["additional_special_tokens"] += [
								    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    UpperCAmelCase						,       tokenizer_file=UpperCAmelCase						,       bos_token=UpperCAmelCase						,       eos_token=UpperCAmelCase						,       sep_token=UpperCAmelCase						,       cls_token=UpperCAmelCase						,       unk_token=UpperCAmelCase						,       pad_token=UpperCAmelCase						,       **UpperCAmelCase						,       )

								lowercase_			 = vocab_file
								lowercase_			 = False if not self.vocab_file else True




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase = None  )     ->       List[int]:






								'''simple docstring'''






								if token_ids_a is None:
									return [self.sep_token_id] + token_ids_a
								lowercase_			 = [self.sep_token_id]
								return sep + token_ids_a + sep + sep + token_ids_a




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase = None  )     ->       List[int]:






								'''simple docstring'''






								lowercase_			 = [self.sep_token_id]

								if token_ids_a is None:
									return len(sep + token_ids_a  ) * [0]
								return len(sep + token_ids_a + sep + sep + token_ids_a  ) * [0]




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase = None  )     ->       Tuple[str]:






								'''simple docstring'''






								if not self.can_save_slow_tokenizer:
									raise ValueError(
									    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
									    "tokenizer."  )

								if not os.path.isdir(UpperCAmelCase  ):
									logger.error(F'Vocabulary path ({save_directory}) should be a directory.'  )
									return
								lowercase_			 = os.path.join(
								    UpperCAmelCase						,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

								if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCAmelCase  ):
									copyfile(self.vocab_file						,       UpperCAmelCase  )

								return (out_vocab_file,)

