
from typing import Optional, Union

import torch
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_mobilenet_va import MobileNetVaConfig


__a  			=					logging.get_logger(__name__)


# General docstring
__a  			=					'MobileNetV1Config'

# Base docstring
__a  			=					'google/mobilenet_v1_1.0_224'
__a  			=					[1, 1_0_2_4, 7, 7]

# Image classification docstring
__a  			=					'google/mobilenet_v1_1.0_224'
__a  			=					'tabby, tabby cat'


__a  			=					[
    'google/mobilenet_v1_1.0_224',
    'google/mobilenet_v1_0.75_192',
    # See all MobileNetV1 models at https://huggingface.co/models?filter=mobilenet_v1
]




def 				a						(							snake_case__:					Union[str, Any]						,   snake_case__:					Union[str, Any]						,   snake_case__:					List[Any]=None					):


				'''simple docstring'''
				lowercase_       =					{}

				if isinstance(snake_case__						,   snake_case__					):
								lowercase_       =					model.mobilenet_va
				else:
								lowercase_       =					model

				lowercase_       =					'''MobilenetV1/Conv2d_0/'''
				lowercase_       =					backbone.conv_stem.convolution.weight
				lowercase_       =					backbone.conv_stem.normalization.bias
				lowercase_       =					backbone.conv_stem.normalization.weight
				lowercase_       =					backbone.conv_stem.normalization.running_mean
				lowercase_       =					backbone.conv_stem.normalization.running_var

				for i in range(13					):
								lowercase_       =					i + 1
								lowercase_       =					i * 2

								lowercase_       =					backbone.layer[pt_index]
								lowercase_       =					F'''MobilenetV1/Conv2d_{tf_index}_depthwise/'''
								lowercase_       =					pointer.convolution.weight
								lowercase_       =					pointer.normalization.bias
								lowercase_       =					pointer.normalization.weight
								lowercase_       =					pointer.normalization.running_mean
								lowercase_       =					pointer.normalization.running_var

								lowercase_       =					backbone.layer[pt_index + 1]
								lowercase_       =					F'''MobilenetV1/Conv2d_{tf_index}_pointwise/'''
								lowercase_       =					pointer.convolution.weight
								lowercase_       =					pointer.normalization.bias
								lowercase_       =					pointer.normalization.weight
								lowercase_       =					pointer.normalization.running_mean
								lowercase_       =					pointer.normalization.running_var

				if isinstance(snake_case__						,   snake_case__					):
								lowercase_       =					'''MobilenetV1/Logits/Conv2d_1c_1x1/'''
								lowercase_       =					model.classifier.weight
								lowercase_       =					model.classifier.bias

				return tf_to_pt_map




def 				a						(							snake_case__:					Dict						,   snake_case__:					List[str]						,   snake_case__:					List[Any]					):


				'''simple docstring'''
				try:
								import numpy as np
								import tensorflow as tf
				except ImportError:
								logger.error(
								    '''Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see '''
								    '''https://www.tensorflow.org/install/ for installation instructions.'''					)
								raise

				# Load weights from TF model
				lowercase_       =					tf.train.list_variables(snake_case__					)
				lowercase_       =					{}
				for name, shape in init_vars:
								logger.info(F'''Loading TF weight {name} with shape {shape}'''					)
								lowercase_       =					tf.train.load_variable(snake_case__						,   snake_case__					)
								lowercase_       =					array

				# Build TF to PyTorch weights loading map
				lowercase_       =					_build_tf_to_pytorch_map(snake_case__						,   snake_case__						,   snake_case__					)

				for name, pointer in tf_to_pt_map.items():
								logger.info(F'''Importing {name}'''					)
								if name not in tf_weights:
												logger.info(F'''{name} not in tf pre-trained weights, skipping'''					)
												continue

								lowercase_       =					tf_weights[name]

								if "depthwise_weights" in name:
												logger.info('''Transposing depthwise'''					)
												lowercase_       =					np.transpose(snake_case__						,   (2, 3, 0, 1)					)
								elif "weights" in name:
												logger.info('''Transposing'''					)
												if len(pointer.shape					) == 2:  # copying into linear layer
																lowercase_       =					array.squeeze().transpose()
												else:
																lowercase_       =					np.transpose(snake_case__						,   (3, 2, 0, 1)					)

								if pointer.shape != array.shape:
												raise ValueError(F'''Pointer shape {pointer.shape} and array shape {array.shape} mismatched'''					)

								logger.info(F'''Initialize PyTorch weight {name} {array.shape}'''					)
								lowercase_       =					torch.from_numpy(snake_case__					)

								tf_weights.pop(snake_case__						,   snake_case__					)
								tf_weights.pop(name + '''/RMSProp'''						,   snake_case__					)
								tf_weights.pop(name + '''/RMSProp_1'''						,   snake_case__					)
								tf_weights.pop(name + '''/ExponentialMovingAverage'''						,   snake_case__					)

				logger.info(F'''Weights not copied to PyTorch model: {', '.join(tf_weights.keys()					)}'''					)
				return model




def 				a						(							snake_case__:					torch.Tensor						,   snake_case__:					nn.Convad					):


				'''simple docstring'''
				lowercase_  ,	lowercase_       =					features.shape[-2:]
				lowercase_  ,	lowercase_       =					conv_layer.stride
				lowercase_  ,	lowercase_       =					conv_layer.kernel_size

				if in_height % stride_height == 0:
								lowercase_       =					max(kernel_height - stride_height						,   0					)
				else:
								lowercase_       =					max(kernel_height - (in_height % stride_height)						,   0					)

				if in_width % stride_width == 0:
								lowercase_       =					max(kernel_width - stride_width						,   0					)
				else:
								lowercase_       =					max(kernel_width - (in_width % stride_width)						,   0					)

				lowercase_       =					pad_along_width // 2
				lowercase_       =					pad_along_width - pad_left
				lowercase_       =					pad_along_height // 2
				lowercase_       =					pad_along_height - pad_top

				lowercase_       =					(pad_left, pad_right, pad_top, pad_bottom)
				return nn.functional.pad(snake_case__						,   snake_case__						,   '''constant'''						,   0.0					)
class  lowercase__(				nn.Module							):



	"""simple docstring"""




	def __init__(       self  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			MobileNetVaConfig       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			Optional[int] = 1       ,							SCREAMING_SNAKE_CASE_  :			Optional[int] = 1       ,							SCREAMING_SNAKE_CASE_  :			bool = False       ,							SCREAMING_SNAKE_CASE_  :			Optional[bool] = True       ,							SCREAMING_SNAKE_CASE_  :			Optional[bool or str] = True       ,							)       ->     None:
					super().__init__()
					lowercase_       =					config

					if in_channels % groups != 0:
									raise ValueError(f'''Input channels ({in_channels}) are not divisible by {groups} groups.'''       )
					if out_channels % groups != 0:
									raise ValueError(f'''Output channels ({out_channels}) are not divisible by {groups} groups.'''       )

					lowercase_       =					0 if config.tf_padding else int((kernel_size - 1) / 2       )

					lowercase_       =					nn.Convad(
					    in_channels=SCREAMING_SNAKE_CASE_       ,							out_channels=SCREAMING_SNAKE_CASE_       ,							kernel_size=SCREAMING_SNAKE_CASE_       ,							stride=SCREAMING_SNAKE_CASE_       ,							padding=SCREAMING_SNAKE_CASE_       ,							groups=SCREAMING_SNAKE_CASE_       ,							bias=SCREAMING_SNAKE_CASE_       ,							padding_mode='''zeros'''       ,							)

					if use_normalization:
									lowercase_       =					nn.BatchNormad(
									    num_features=SCREAMING_SNAKE_CASE_       ,							eps=config.layer_norm_eps       ,							momentum=0.99_97       ,							affine=SCREAMING_SNAKE_CASE_       ,							track_running_stats=SCREAMING_SNAKE_CASE_       ,							)
					else:
									lowercase_       =					None

					if use_activation:
									if isinstance(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ):
													lowercase_       =					ACTaFN[use_activation]
									elif isinstance(config.hidden_act       ,							SCREAMING_SNAKE_CASE_       ):
													lowercase_       =					ACTaFN[config.hidden_act]
									else:
													lowercase_       =					config.hidden_act
					else:
									lowercase_       =					None




	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			torch.Tensor       )       ->     torch.Tensor:
					if self.config.tf_padding:
									lowercase_       =					apply_tf_padding(SCREAMING_SNAKE_CASE_       ,							self.convolution       )
					lowercase_       =					self.convolution(SCREAMING_SNAKE_CASE_       )
					if self.normalization is not None:
									lowercase_       =					self.normalization(SCREAMING_SNAKE_CASE_       )
					if self.activation is not None:
									lowercase_       =					self.activation(SCREAMING_SNAKE_CASE_       )
					return features

class  lowercase__(				UpperCAmelCase							):



	"""simple docstring"""




	a			:Any         =	MobileNetVaConfig
	a			:Tuple         =	load_tf_weights_in_mobilenet_va
	a			:Optional[Any]         =	'mobilenet_v1'
	a			:Any         =	'pixel_values'
	a			:Optional[Any]         =	False
	def 						_lowercase  (       self  :			Union[str, Any]       ,							SCREAMING_SNAKE_CASE_  :			Union[nn.Linear, nn.Convad]       )       ->     None:
					if isinstance(SCREAMING_SNAKE_CASE_       ,							(nn.Linear, nn.Convad)       ):
									module.weight.data.normal_(mean=0.0       ,							std=self.config.initializer_range       )
									if module.bias is not None:
													module.bias.data.zero_()
					elif isinstance(SCREAMING_SNAKE_CASE_       ,							nn.BatchNormad       ):
									module.bias.data.zero_()
									module.weight.data.fill_(1.0       )

__a  			=					r'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`MobileNetV1Config`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

__a  			=					r'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`MobileNetV1ImageProcessor.__call__`] for details.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n'
@add_start_docstrings(
    'The bare MobileNetV1 model outputting raw hidden-states without any specific head on top.' ,						UpperCAmelCase ,						)
class  lowercase__(				UpperCAmelCase							):



	"""simple docstring"""




	def __init__(       self  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			MobileNetVaConfig       ,							SCREAMING_SNAKE_CASE_  :			bool = True       )       ->     Optional[int]:
					super().__init__(SCREAMING_SNAKE_CASE_       )
					lowercase_       =					config

					lowercase_       =					3_2
					lowercase_       =					max(int(depth * config.depth_multiplier       )       ,							config.min_depth       )

					lowercase_       =					MobileNetVaConvLayer(
					    SCREAMING_SNAKE_CASE_       ,							in_channels=config.num_channels       ,							out_channels=SCREAMING_SNAKE_CASE_       ,							kernel_size=3       ,							stride=2       ,							)

					lowercase_       =					[1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1]

					lowercase_       =					nn.ModuleList()
					for i in range(1_3       ):
									lowercase_       =					out_channels

									if strides[i] == 2 or i == 0:
													depth *= 2
													lowercase_       =					max(int(depth * config.depth_multiplier       )       ,							config.min_depth       )

									self.layer.append(
									    MobileNetVaConvLayer(
									        SCREAMING_SNAKE_CASE_       ,							in_channels=SCREAMING_SNAKE_CASE_       ,							out_channels=SCREAMING_SNAKE_CASE_       ,							kernel_size=3       ,							stride=strides[i]       ,							groups=SCREAMING_SNAKE_CASE_       ,							)       )

									self.layer.append(
									    MobileNetVaConvLayer(
									        SCREAMING_SNAKE_CASE_       ,							in_channels=SCREAMING_SNAKE_CASE_       ,							out_channels=SCREAMING_SNAKE_CASE_       ,							kernel_size=1       ,							)       )

					lowercase_       =					nn.AdaptiveAvgPoolad((1, 1)       ) if add_pooling_layer else None

					# Initialize weights and apply final processing
					self.post_init()
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			Dict       )       ->     Dict:
					raise NotImplementedError




	@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_       )
	@add_code_sample_docstrings(
	    checkpoint=_CHECKPOINT_FOR_DOC       ,							output_type=SCREAMING_SNAKE_CASE_       ,							config_class=_CONFIG_FOR_DOC       ,							modality='''vision'''       ,							expected_output=_EXPECTED_OUTPUT_SHAPE       ,							)
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			Optional[torch.Tensor] = None       ,							SCREAMING_SNAKE_CASE_  :			Optional[bool] = None       ,							SCREAMING_SNAKE_CASE_  :			Optional[bool] = None       ,							)       ->     Union[tuple, BaseModelOutputWithPoolingAndNoAttention]:
					lowercase_       =					(
					    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
					)
					lowercase_       =					return_dict if return_dict is not None else self.config.use_return_dict

					if pixel_values is None:
									raise ValueError('''You have to specify pixel_values'''       )

					lowercase_       =					self.conv_stem(SCREAMING_SNAKE_CASE_       )

					lowercase_       =					() if output_hidden_states else None

					for i, layer_module in enumerate(self.layer       ):
									lowercase_       =					layer_module(SCREAMING_SNAKE_CASE_       )

									if output_hidden_states:
													lowercase_       =					all_hidden_states + (hidden_states,)

					lowercase_       =					hidden_states

					if self.pooler is not None:
									lowercase_       =					torch.flatten(self.pooler(SCREAMING_SNAKE_CASE_       )       ,							start_dim=1       )
					else:
									lowercase_       =					None

					if not return_dict:
									return tuple(v for v in [last_hidden_state, pooled_output, all_hidden_states] if v is not None       )

					return BaseModelOutputWithPoolingAndNoAttention(
					    last_hidden_state=SCREAMING_SNAKE_CASE_       ,							pooler_output=SCREAMING_SNAKE_CASE_       ,							hidden_states=SCREAMING_SNAKE_CASE_       ,							)

@add_start_docstrings(
    '\n    MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    ' ,						UpperCAmelCase ,						)
class  lowercase__(				UpperCAmelCase							):



	"""simple docstring"""




	def __init__(       self  :			int       ,							SCREAMING_SNAKE_CASE_  :			MobileNetVaConfig       )       ->     None:
					super().__init__(SCREAMING_SNAKE_CASE_       )

					lowercase_       =					config.num_labels
					lowercase_       =					MobileNetVaModel(SCREAMING_SNAKE_CASE_       )

					lowercase_       =					self.mobilenet_va.layer[-1].convolution.out_channels

					# Classifier head
					lowercase_       =					nn.Dropout(config.classifier_dropout_prob       ,							inplace=SCREAMING_SNAKE_CASE_       )
					lowercase_       =					nn.Linear(SCREAMING_SNAKE_CASE_       ,							config.num_labels       ) if config.num_labels > 0 else nn.Identity()

					# Initialize weights and apply final processing
					self.post_init()




	@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_       )
	@add_code_sample_docstrings(
	    checkpoint=_IMAGE_CLASS_CHECKPOINT       ,							output_type=SCREAMING_SNAKE_CASE_       ,							config_class=_CONFIG_FOR_DOC       ,							expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT       ,							)
	def 						_lowercase  (       self  :			Tuple       ,							SCREAMING_SNAKE_CASE_  :			Optional[torch.Tensor] = None       ,							SCREAMING_SNAKE_CASE_  :			Optional[bool] = None       ,							SCREAMING_SNAKE_CASE_  :			Optional[torch.Tensor] = None       ,							SCREAMING_SNAKE_CASE_  :			Optional[bool] = None       ,							)       ->     Union[tuple, ImageClassifierOutputWithNoAttention]:
					lowercase_       =					return_dict if return_dict is not None else self.config.use_return_dict

					lowercase_       =					self.mobilenet_va(SCREAMING_SNAKE_CASE_       ,							output_hidden_states=SCREAMING_SNAKE_CASE_       ,							return_dict=SCREAMING_SNAKE_CASE_       )

					lowercase_       =					outputs.pooler_output if return_dict else outputs[1]

					lowercase_       =					self.classifier(self.dropout(SCREAMING_SNAKE_CASE_       )       )

					lowercase_       =					None
					if labels is not None:
									if self.config.problem_type is None:
													if self.num_labels == 1:
																	lowercase_       =					'''regression'''
													elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
																	lowercase_       =					'''single_label_classification'''
													else:
																	lowercase_       =					'''multi_label_classification'''

									if self.config.problem_type == "regression":
													lowercase_       =					MSELoss()
													if self.num_labels == 1:
																	lowercase_       =					loss_fct(logits.squeeze()       ,							labels.squeeze()       )
													else:
																	lowercase_       =					loss_fct(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )
									elif self.config.problem_type == "single_label_classification":
													lowercase_       =					CrossEntropyLoss()
													lowercase_       =					loss_fct(logits.view(-1       ,							self.num_labels       )       ,							labels.view(-1       )       )
									elif self.config.problem_type == "multi_label_classification":
													lowercase_       =					BCEWithLogitsLoss()
													lowercase_       =					loss_fct(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )

					if not return_dict:
									lowercase_       =					(logits,) + outputs[2:]
									return ((loss,) + output) if loss is not None else output

					return ImageClassifierOutputWithNoAttention(
					    loss=SCREAMING_SNAKE_CASE_       ,							logits=SCREAMING_SNAKE_CASE_       ,							hidden_states=outputs.hidden_states       ,							)






from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
						from fairscale.optim import OSS


__a  			=					logging.get_logger(__name__)

__a  			=					{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    'constant': get_constant_schedule,
    'constant_w_warmup': get_constant_schedule_with_warmup,
}
class  lowercase__(				UpperCAmelCase							):



	"""simple docstring"""




	def __init__(       self  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			List[str]=None       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]=None       ,							*SCREAMING_SNAKE_CASE_  :			str       ,							**SCREAMING_SNAKE_CASE_  :			Optional[Any]       )       ->     List[str]:
					super().__init__(*SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       )

					if config is None:
									assert isinstance(self.model       ,							SCREAMING_SNAKE_CASE_       ), (
									    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
									    f''' {self.model.__class__}'''
									)
									lowercase_       =					self.model.config
					else:
									lowercase_       =					config

					lowercase_       =					data_args
					lowercase_       =					self.config.tgt_vocab_size if isinstance(self.config       ,							SCREAMING_SNAKE_CASE_       ) else self.config.vocab_size

					if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
									assert self.config.pad_token_id is not None, (
									    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
									    " calculation or doing label smoothing."
									)

					if self.config.pad_token_id is None and self.config.eos_token_id is not None:
									logger.warning(
									    f'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
									    ''' padding..'''       )

					if self.args.label_smoothing == 0:
									lowercase_       =					torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id       )
					else:
									# dynamically import label_smoothed_nll_loss
									from utils import label_smoothed_nll_loss

									lowercase_       =					label_smoothed_nll_loss
	def 						_lowercase  (       self  :			int       ,							SCREAMING_SNAKE_CASE_  :			int       )       ->     Optional[Any]:
					if self.optimizer is None:
									lowercase_       =					['''bias''', '''LayerNorm.weight''']
									lowercase_       =					[
									    {
									        '''params''': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay       )],
									        '''weight_decay''': self.args.weight_decay,
									    },
									    {
									        '''params''': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay       )],
									        '''weight_decay''': 0.0,
									    },
									]
									lowercase_       =					Adafactor if self.args.adafactor else AdamW
									if self.args.adafactor:
													lowercase_       =					Adafactor
													lowercase_       =					{'''scale_parameter''': False, '''relative_step''': False}
									else:
													lowercase_       =					AdamW
													lowercase_       =					{
													    '''betas''': (self.args.adam_betaa, self.args.adam_betaa),
													    '''eps''': self.args.adam_epsilon,
													}
									lowercase_       =					self.args.learning_rate
									if self.sharded_ddp:
													lowercase_       =					OSS(
													    params=SCREAMING_SNAKE_CASE_       ,							optim=SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       ,							)
									else:
													lowercase_       =					optimizer_cls(SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       )

					if self.lr_scheduler is None:
									lowercase_       =					self._get_lr_scheduler(SCREAMING_SNAKE_CASE_       )
					else:  # ignoring --lr_scheduler
									logger.warning('''scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'''       )
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			Dict       )       ->     Dict:
					lowercase_       =					arg_to_scheduler[self.args.lr_scheduler]
					if self.args.lr_scheduler == "constant":
									lowercase_       =					schedule_func(self.optimizer       )
					elif self.args.lr_scheduler == "constant_w_warmup":
									lowercase_       =					schedule_func(self.optimizer       ,							num_warmup_steps=self.args.warmup_steps       )
					else:
									lowercase_       =					schedule_func(
									    self.optimizer       ,							num_warmup_steps=self.args.warmup_steps       ,							num_training_steps=SCREAMING_SNAKE_CASE_       )
					return scheduler
	def 						_lowercase  (       self  :			Tuple       )       ->     Optional[torch.utils.data.Sampler]:
					if isinstance(self.train_dataset       ,							torch.utils.data.IterableDataset       ):
									return None
					elif is_torch_tpu_available():
									return get_tpu_sampler(self.train_dataset       )
					else:
									if self.args.sortish_sampler:
													self.train_dataset.make_sortish_sampler(
													    self.args.per_device_train_batch_size       ,							distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)       ,							)

									return (
									    RandomSampler(self.train_dataset       )
									    if self.args.local_rank == -1
									    else DistributedSampler(self.train_dataset       )
									)
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			Dict       ,							SCREAMING_SNAKE_CASE_  :			List[Any]       )       ->     Any:
					if self.args.label_smoothing == 0:
									if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
													# force training to ignore pad token
													lowercase_       =					model(**SCREAMING_SNAKE_CASE_       ,							use_cache=SCREAMING_SNAKE_CASE_       )[0]
													lowercase_       =					self.loss_fn(logits.view(-1       ,							logits.shape[-1]       )       ,							labels.view(-1       )       )
									else:
													# compute usual loss via models
													lowercase_  ,	lowercase_       =					model(**SCREAMING_SNAKE_CASE_       ,							labels=SCREAMING_SNAKE_CASE_       ,							use_cache=SCREAMING_SNAKE_CASE_       )[:2]
					else:
									# compute label smoothed loss
									lowercase_       =					model(**SCREAMING_SNAKE_CASE_       ,							use_cache=SCREAMING_SNAKE_CASE_       )[0]
									lowercase_       =					torch.nn.functional.log_softmax(SCREAMING_SNAKE_CASE_       ,							dim=-1       )
									lowercase_  ,	lowercase_       =					self.loss_fn(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							self.args.label_smoothing       ,							ignore_index=self.config.pad_token_id       )
					return loss, logits
	def 						_lowercase  (       self  :			str       ,							SCREAMING_SNAKE_CASE_  :			Dict       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       )       ->     List[Any]:
					lowercase_       =					inputs.pop('''labels'''       )
					lowercase_  ,	lowercase_       =					self._compute_loss(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )
					return loss
	def 						_lowercase  (       self  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			nn.Module       ,							SCREAMING_SNAKE_CASE_  :			Dict[str, Union[torch.Tensor, Any]]       ,							SCREAMING_SNAKE_CASE_  :			bool       ,							SCREAMING_SNAKE_CASE_  :			Optional[List[str]] = None       ,							)       ->     Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
					lowercase_       =					self._prepare_inputs(SCREAMING_SNAKE_CASE_       )

					lowercase_       =					{
					    '''max_length''': self.data_args.val_max_target_length
					    if self.data_args is not None
					    else self.config.max_length,
					    '''num_beams''': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
					}

					if self.args.predict_with_generate and not self.args.prediction_loss_only:
									lowercase_       =					self.model.generate(
									    inputs['''input_ids''']       ,							attention_mask=inputs['''attention_mask''']       ,							**SCREAMING_SNAKE_CASE_       ,							)
									# in case the batch is shorter than max length, the output should be padded
									if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
													lowercase_       =					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_       ,							gen_kwargs['''max_length''']       )

					lowercase_       =					inputs.pop('''labels'''       )
					with torch.no_grad():
									# compute loss on predict data
									lowercase_  ,	lowercase_       =					self._compute_loss(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )

					lowercase_       =					loss.mean().detach()
					if self.args.prediction_loss_only:
									return (loss, None, None)

					lowercase_       =					generated_tokens if self.args.predict_with_generate else logits

					if labels.shape[-1] < gen_kwargs["max_length"]:
									lowercase_       =					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_       ,							gen_kwargs['''max_length''']       )

					return (loss, logits, labels)




	def 						_lowercase  (       self  :			str       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			int       )       ->     Tuple:
					# If PAD token is not defined at least EOS token has to be defined
					lowercase_       =					self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

					if pad_token_id is None:
									raise ValueError(
									    '''Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'''
									    f''' padded to `max_length`={max_length}'''       )

					lowercase_       =					pad_token_id * torch.ones(
					    (tensor.shape[0], max_length)       ,							dtype=tensor.dtype       ,							device=tensor.device       )
					lowercase_       =					tensor
					return padded_tensor




