
"""simple docstring"""







import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCamelCase__     =						16
lowerCamelCase__     =						32
def        __lowerCAmelCase							(_UpperCamelCase			,    _UpperCamelCase						):
	# Initialize accelerator
	if args.with_tracking:
		__lowerCAmelCase     :		Optional[int]		     = Accelerator(
		    cpu=args.cpu			,    mixed_precision=args.mixed_precision			,    log_with='all'			,    project_dir=args.project_dir						)
	else:
		__lowerCAmelCase     :		List[str]		     = Accelerator(cpu=args.cpu			,    mixed_precision=args.mixed_precision						)

	if hasattr(args.checkpointing_steps			,    'isdigit'						):
		if args.checkpointing_steps == "epoch":
			__lowerCAmelCase     :		List[Any]		     = args.checkpointing_steps
		elif args.checkpointing_steps.isdigit():
			__lowerCAmelCase     :		List[Any]		     = int(args.checkpointing_steps						)
		else:
			raise ValueError(
			    F"Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."						)
	else:
		__lowerCAmelCase     :		Dict		     = None
	# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
	__lowerCAmelCase     :		Union[str, Any]		     = config['lr']
	__lowerCAmelCase     :		Tuple		     = int(config['num_epochs']						)
	__lowerCAmelCase     :		Dict		     = int(config['seed']						)
	__lowerCAmelCase     :		Any		     = int(config['batch_size']						)

	# We need to initialize the trackers we use, and also store our configuration
	if args.with_tracking:
		__lowerCAmelCase     :		List[str]		     = os.path.split(_UpperCamelCase						)[-1].split('.'						)[0]
		accelerator.init_trackers(_UpperCamelCase			,    _UpperCamelCase						)

	__lowerCAmelCase     :		List[Any]		     = AutoTokenizer.from_pretrained('bert-base-cased'						)
	__lowerCAmelCase     :		Optional[Any]		     = load_dataset('glue'			,    'mrpc'						)
	__lowerCAmelCase     :		List[str]		     = evaluate.load('glue'			,    'mrpc'						)

	def tokenize_function(_UpperCamelCase						):
		# max_length=None => use the model max length (it's actually the default)
		__lowerCAmelCase     :		str		     = tokenizer(examples['sentence1']			,    examples['sentence2']			,    truncation=_UpperCamelCase			,    max_length=_UpperCamelCase						)
		return outputs

	# Apply the method we just defined to all the examples in all the splits of the dataset
	# starting with the main process first:
	with accelerator.main_process_first():
		__lowerCAmelCase     :		Tuple		     = datasets.map(
		    _UpperCamelCase			,    batched=_UpperCamelCase			,    remove_columns=['idx', 'sentence1', 'sentence2']			,    )

	# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
	# transformers library
	__lowerCAmelCase     :		Optional[Any]		     = tokenized_datasets.rename_column('label'			,    'labels'						)

	# If the batch size is too big we use gradient accumulation
	__lowerCAmelCase     :		Dict		     = 1
	if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
		__lowerCAmelCase     :		Optional[int]		     = batch_size // MAX_GPU_BATCH_SIZE
		__lowerCAmelCase     :		Optional[Any]		     = MAX_GPU_BATCH_SIZE

	def collate_fn(_UpperCamelCase						):
		# On TPU it's best to pad everything to the same length or training will be very slow.
		__lowerCAmelCase     :		Optional[int]		     = 128 if accelerator.distributed_type == DistributedType.TPU else None
		# When using mixed precision we want round multiples of 8/16
		if accelerator.mixed_precision == "fp8":
			__lowerCAmelCase     :		Optional[Any]		     = 16
		elif accelerator.mixed_precision != "no":
			__lowerCAmelCase     :		Any		     = 8
		else:
			__lowerCAmelCase     :		Dict		     = None

		return tokenizer.pad(
		    _UpperCamelCase			,    padding='longest'			,    max_length=_UpperCamelCase			,    pad_to_multiple_of=_UpperCamelCase			,    return_tensors='pt'			,    )

	# Instantiate dataloaders.
	__lowerCAmelCase     :		Optional[int]		     = DataLoader(
	    tokenized_datasets['train']			,    shuffle=_UpperCamelCase			,    collate_fn=_UpperCamelCase			,    batch_size=_UpperCamelCase						)
	__lowerCAmelCase     :		List[str]		     = DataLoader(
	    tokenized_datasets['validation']			,    shuffle=_UpperCamelCase			,    collate_fn=_UpperCamelCase			,    batch_size=_UpperCamelCase						)

	set_seed(_UpperCamelCase						)

	# Instantiate the model (we build the model here so that the seed also control new weights initialization)
	__lowerCAmelCase     :		int		     = AutoModelForSequenceClassification.from_pretrained('bert-base-cased'			,    return_dict=_UpperCamelCase						)

	# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
	# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
	# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
	__lowerCAmelCase     :		Any		     = model.to(accelerator.device						)

	# Instantiate optimizer
	__lowerCAmelCase     :		int		     = AdamW(params=model.parameters()			,    lr=_UpperCamelCase						)

	# Instantiate scheduler
	__lowerCAmelCase     :		Dict		     = get_linear_schedule_with_warmup(
	    optimizer=_UpperCamelCase			,    num_warmup_steps=100			,    num_training_steps=(len(_UpperCamelCase						) * num_epochs) // gradient_accumulation_steps			,    )

	# Prepare everything
	# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
	# prepare method.
	__lowerCAmelCase     ,  __lowerCAmelCase     ,  __lowerCAmelCase     ,  __lowerCAmelCase     ,  __lowerCAmelCase     :		str		     = accelerator.prepare(
	    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase						)

	# We need to keep track of how many total steps we have iterated over
	__lowerCAmelCase     :		int		     = 0
	# We also need to keep track of the stating epoch so files are named properly
	__lowerCAmelCase     :		Optional[int]		     = 0

	# Potentially load in the weights and states from a previous save
	if args.resume_from_checkpoint:
		if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
			accelerator.print(F"Resumed from checkpoint: {args.resume_from_checkpoint}"						)
			accelerator.load_state(args.resume_from_checkpoint						)
			__lowerCAmelCase     :		List[Any]		     = os.path.basename(args.resume_from_checkpoint						)
		else:
			# Get the most recent checkpoint
			__lowerCAmelCase     :		int		     = [f.name for f in os.scandir(os.getcwd()						) if f.is_dir()]
			dirs.sort(key=os.path.getctime						)
			__lowerCAmelCase     :		int		     = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
		# Extract `epoch_{i}` or `step_{i}`
		__lowerCAmelCase     :		Union[str, Any]		     = os.path.splitext(_UpperCamelCase						)[0]

		if "epoch" in training_difference:
			__lowerCAmelCase     :		int		     = int(training_difference.replace('epoch_'			,    ''						)						) + 1
			__lowerCAmelCase     :		int		     = None
		else:
			__lowerCAmelCase     :		Dict		     = int(training_difference.replace('step_'			,    ''						)						)
			__lowerCAmelCase     :		List[Any]		     = resume_step // len(_UpperCamelCase						)
			resume_step -= starting_epoch * len(_UpperCamelCase						)

    # Now we train the model
	for epoch in range(_UpperCamelCase			,    _UpperCamelCase						):
		model.train()
		if args.with_tracking:
			__lowerCAmelCase     :		Any		     = 0
		if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
			# We need to skip steps until we reach the resumed step
			__lowerCAmelCase     :		Optional[int]		     = accelerator.skip_first_batches(_UpperCamelCase			,    _UpperCamelCase						)
			overall_step += resume_step
		else:
			# After the first iteration though, we need to go back to the original dataloader
			__lowerCAmelCase     :		Dict		     = train_dataloader
		for step, batch in enumerate(_UpperCamelCase						):
			# We could avoid this line since we set the accelerator with `device_placement=True`.
			batch.to(accelerator.device						)
			__lowerCAmelCase     :		str		     = model(**_UpperCamelCase						)
			__lowerCAmelCase     :		List[Any]		     = outputs.loss
			__lowerCAmelCase     :		Dict		     = loss / gradient_accumulation_steps
			# We keep track of the loss at each epoch
			if args.with_tracking:
				total_loss += loss.detach().float()
			accelerator.backward(_UpperCamelCase						)
			if step % gradient_accumulation_steps == 0:
				optimizer.step()
				lr_scheduler.step()
				optimizer.zero_grad()

			overall_step += 1

			if isinstance(_UpperCamelCase			,    _UpperCamelCase						):
				__lowerCAmelCase     :		Tuple		     = F"step_{overall_step}"
				if overall_step % checkpointing_steps == 0:
					if args.output_dir is not None:
						__lowerCAmelCase     :		str		     = os.path.join(args.output_dir			,    _UpperCamelCase						)
					accelerator.save_state(_UpperCamelCase						)

		model.eval()
		for step, batch in enumerate(_UpperCamelCase						):
			# We could avoid this line since we set the accelerator with `device_placement=True`.
			batch.to(accelerator.device						)
			with torch.no_grad():
				__lowerCAmelCase     :		int		     = model(**_UpperCamelCase						)
			__lowerCAmelCase     :		Optional[int]		     = outputs.logits.argmax(dim=-1						)
			__lowerCAmelCase     ,  __lowerCAmelCase     :		Union[str, Any]		     = accelerator.gather_for_metrics((predictions, batch['labels'])						)
			metric.add_batch(
			    predictions=_UpperCamelCase			,    references=_UpperCamelCase			,    )

		__lowerCAmelCase     :		str		     = metric.compute()
		# Use accelerator.print to print only on the main process.
		accelerator.print(F"epoch {epoch}:"			,    _UpperCamelCase						)
		if args.with_tracking:
			accelerator.log(
			    {
			        'accuracy': eval_metric['accuracy'],
			        'f1': eval_metric['f1'],
			        'train_loss': total_loss.item() / len(_UpperCamelCase						),
			        'epoch': epoch,
			    }			,    step=_UpperCamelCase			,    )

		if checkpointing_steps == "epoch":
			__lowerCAmelCase     :		List[str]		     = F"epoch_{epoch}"
			if args.output_dir is not None:
				__lowerCAmelCase     :		List[Any]		     = os.path.join(args.output_dir			,    _UpperCamelCase						)
			accelerator.save_state(_UpperCamelCase						)

	if args.with_tracking:
		accelerator.end_training()
def        __lowerCAmelCase							():
	__lowerCAmelCase     :		str		     = argparse.ArgumentParser(description='Simple example of training script.'						)
	parser.add_argument(
	    '--mixed_precision'			,    type=_UpperCamelCase			,    default=_UpperCamelCase			,    choices=['no', 'fp16', 'bf16', 'fp8']			,    help='Whether to use mixed precision. Choose'
	    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
	    'and an Nvidia Ampere GPU.'			,    )
	parser.add_argument('--cpu'			,    action='store_true'			,    help='If passed, will train on the CPU.'						)
	parser.add_argument(
	    '--checkpointing_steps'			,    type=_UpperCamelCase			,    default=_UpperCamelCase			,    help='Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'			,    )
	parser.add_argument(
	    '--resume_from_checkpoint'			,    type=_UpperCamelCase			,    default=_UpperCamelCase			,    help='If the training should continue from a checkpoint folder.'			,    )
	parser.add_argument(
	    '--with_tracking'			,    action='store_true'			,    help='Whether to load in all available experiment trackers from the environment and use them for logging.'			,    )
	parser.add_argument(
	    '--output_dir'			,    type=_UpperCamelCase			,    default='.'			,    help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'			,    )
	parser.add_argument(
	    '--project_dir'			,    type=_UpperCamelCase			,    default='logs'			,    help='Location on where to store experiment tracking logs` and relevent project information'			,    )
	__lowerCAmelCase     :		Union[str, Any]		     = parser.parse_args()
	__lowerCAmelCase     :		Optional[int]		     = {'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
	training_function(_UpperCamelCase			,    _UpperCamelCase						)


if __name__ == "__main__":
							main()

"""simple docstring"""







from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
							from .data.data_collator import DataCollator
							from .modeling_utils import PreTrainedModel
							from .tokenization_utils_base import PreTrainedTokenizerBase
							from .trainer_callback import TrainerCallback
							from .trainer_utils import EvalPrediction, PredictionOutput
							from .training_args import TrainingArguments


lowerCamelCase__     =						logging.get_logger(__name__)




class 					A__			(      _lowerCamelCase):

		def __init__( self			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = (None, None)			,						_SCREAMING_SNAKE_CASE = None			,						):
			super().__init__(
			    model=_SCREAMING_SNAKE_CASE			,						args=_SCREAMING_SNAKE_CASE			,						data_collator=_SCREAMING_SNAKE_CASE			,						train_dataset=_SCREAMING_SNAKE_CASE			,						eval_dataset=_SCREAMING_SNAKE_CASE			,						tokenizer=_SCREAMING_SNAKE_CASE			,						model_init=_SCREAMING_SNAKE_CASE			,						compute_metrics=_SCREAMING_SNAKE_CASE			,						callbacks=_SCREAMING_SNAKE_CASE			,						optimizers=_SCREAMING_SNAKE_CASE			,						preprocess_logits_for_metrics=_SCREAMING_SNAKE_CASE			,						)

			# Override self.model.generation_config if a GenerationConfig is specified in args.
			# Priority: args.generation_config > model.generation_config > default GenerationConfig.
			if self.args.generation_config is not None:
				__lowerCAmelCase     :		Tuple		     = self.load_generation_config(self.args.generation_config					)
				__lowerCAmelCase     :		int		     = gen_config

		@staticmethod
		def 				__lowerCamelCase				( _SCREAMING_SNAKE_CASE					):

			# GenerationConfig provided, nothing to do
			if isinstance(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					):
				return deepcopy(_SCREAMING_SNAKE_CASE					)

			# str or Path
			__lowerCAmelCase     :		Dict		     = Path(_SCREAMING_SNAKE_CASE					) if isinstance(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					) else gen_config_arg
			__lowerCAmelCase     :		Any		     = None

			# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
			# This step is required in order to determine config_file_name
			if pretrained_model_name.is_file():
				__lowerCAmelCase     :		List[Any]		     = pretrained_model_name.name
				__lowerCAmelCase     :		Tuple		     = pretrained_model_name.parent
			# dir path
			elif pretrained_model_name.is_dir():
				pass
			# model id or URL
			else:
				__lowerCAmelCase     :		List[Any]		     = gen_config_arg

			__lowerCAmelCase     :		Optional[int]		     = GenerationConfig.from_pretrained(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					)
			return gen_config

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = "eval"			,						**_SCREAMING_SNAKE_CASE			,						):
			__lowerCAmelCase     :		Any		     = gen_kwargs.copy()
			if gen_kwargs.get('max_length'					) is None and gen_kwargs.get('max_new_tokens'					) is None:
				__lowerCAmelCase     :		Optional[Any]		     = self.args.generation_max_length
			__lowerCAmelCase     :		Tuple		     = (
			    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'					) is not None else self.args.generation_num_beams
			)
			__lowerCAmelCase     :		Optional[int]		     = gen_kwargs

			return super().evaluate(_SCREAMING_SNAKE_CASE			,						ignore_keys=_SCREAMING_SNAKE_CASE			,						metric_key_prefix=_SCREAMING_SNAKE_CASE					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = "test"			,						**_SCREAMING_SNAKE_CASE			,						):
			__lowerCAmelCase     :		Dict		     = gen_kwargs.copy()
			if gen_kwargs.get('max_length'					) is None and gen_kwargs.get('max_new_tokens'					) is None:
				__lowerCAmelCase     :		Optional[Any]		     = self.args.generation_max_length
			__lowerCAmelCase     :		int		     = (
			    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'					) is not None else self.args.generation_num_beams
			)
			__lowerCAmelCase     :		Tuple		     = gen_kwargs

			return super().predict(_SCREAMING_SNAKE_CASE			,						ignore_keys=_SCREAMING_SNAKE_CASE			,						metric_key_prefix=_SCREAMING_SNAKE_CASE					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None			,						**_SCREAMING_SNAKE_CASE			,						):

			if not self.args.predict_with_generate or prediction_loss_only:
				return super().prediction_step(
				    _SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						prediction_loss_only=_SCREAMING_SNAKE_CASE			,						ignore_keys=_SCREAMING_SNAKE_CASE					)

			__lowerCAmelCase     :		str		     = 'labels' in inputs
			__lowerCAmelCase     :		List[str]		     = self._prepare_inputs(_SCREAMING_SNAKE_CASE					)

			# XXX: adapt synced_gpus for fairscale as well
			# Priority (handled in generate):
			# gen_kwargs > model.generation_config > default GenerationConfig()

			if len(_SCREAMING_SNAKE_CASE					) == 0 and hasattr(self			,						'_gen_kwargs'					):
				__lowerCAmelCase     :		Dict		     = self._gen_kwargs.copy()

			if gen_kwargs.get('max_length'					) is None and gen_kwargs.get('max_new_tokens'					) is None:
				__lowerCAmelCase     :		Tuple		     = self.model.config.max_length
			__lowerCAmelCase     :		Optional[Any]		     = (
			    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'					) is not None else self.model.config.num_beams
			)
			__lowerCAmelCase     :		Union[str, Any]		     = True if is_deepspeed_zeroa_enabled() else False
			__lowerCAmelCase     :		int		     = (
			    gen_kwargs['synced_gpus'] if gen_kwargs.get('synced_gpus'					) is not None else default_synced_gpus
			)

			# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
			# (otherwise, it would continue generating from the padded `decoder_input_ids`)
			if (
			    "labels" in inputs
			    and "decoder_input_ids" in inputs
			    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
			):
				__lowerCAmelCase     :		Optional[int]		     = {k: v for k, v in inputs.items() if k != 'decoder_input_ids'}
			__lowerCAmelCase     :		Union[str, Any]		     = self.model.generate(**_SCREAMING_SNAKE_CASE			,						**_SCREAMING_SNAKE_CASE					)

			# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
			# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
			# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
			if self.model.generation_config._from_model_config:
				__lowerCAmelCase     :		str		     = False

			# Retrieves GenerationConfig from model.generation_config
			__lowerCAmelCase     :		List[Any]		     = self.model.generation_config
			# in case the batch is shorter than max length, the output should be padded
			if generated_tokens.shape[-1] < gen_config.max_length:
				__lowerCAmelCase     :		Any		     = self._pad_tensors_to_max_len(_SCREAMING_SNAKE_CASE			,						gen_config.max_length					)
			elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
				__lowerCAmelCase     :		Optional[Any]		     = self._pad_tensors_to_max_len(_SCREAMING_SNAKE_CASE			,						gen_config.max_new_tokens + 1					)

			with torch.no_grad():
				if has_labels:
					with self.compute_loss_context_manager():
						__lowerCAmelCase     :		List[Any]		     = model(**_SCREAMING_SNAKE_CASE					)
					if self.label_smoother is not None:
						__lowerCAmelCase     :		Optional[int]		     = self.label_smoother(_SCREAMING_SNAKE_CASE			,						inputs['labels']					).mean().detach()
					else:
						__lowerCAmelCase     :		Optional[int]		     = (outputs['loss'] if isinstance(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					) else outputs[0]).mean().detach()
				else:
					__lowerCAmelCase     :		Optional[int]		     = None

			if self.args.prediction_loss_only:
				return loss, None, None

			if has_labels:
				__lowerCAmelCase     :		Tuple		     = inputs['labels']
				if labels.shape[-1] < gen_config.max_length:
					__lowerCAmelCase     :		List[str]		     = self._pad_tensors_to_max_len(_SCREAMING_SNAKE_CASE			,						gen_config.max_length					)
				elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
					__lowerCAmelCase     :		int		     = self._pad_tensors_to_max_len(_SCREAMING_SNAKE_CASE			,						gen_config.max_new_tokens + 1					)
			else:
				__lowerCAmelCase     :		Optional[int]		     = None

			return loss, generated_tokens, labels







		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					):
			if self.tokenizer is not None and hasattr(self.tokenizer			,						'pad_token_id'					):
				# If PAD token is not defined at least EOS token has to be defined
				__lowerCAmelCase     :		Any		     = (
				    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
				)
			else:
				if self.model.config.pad_token_id is not None:
					__lowerCAmelCase     :		List[str]		     = self.model.config.pad_token_id
				else:
					raise ValueError('Pad_token_id must be set in the configuration of the model, in order to pad tensors'					)

			__lowerCAmelCase     :		Dict		     = pad_token_id * torch.ones(
			    (tensor.shape[0], max_length)			,						dtype=tensor.dtype			,						device=tensor.device					)
			__lowerCAmelCase     :		Tuple		     = tensor
			return padded_tensor