


from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
	from fairscale.optim import OSS


A       :		Dict    =   logging.get_logger(__name__)

A       :		Tuple    =   {
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}



class 							A    (       __snake_case							):
					'''simple docstring'''






					def __init__(self    :						int      ,					_UpperCAmelCase    :						Any=None      ,					_UpperCAmelCase    :						Tuple=None      ,					*_UpperCAmelCase    :						int      ,					**_UpperCAmelCase    :						int		)						->   Tuple:




							"""simple docstring"""


							super().__init__(*A_      ,					**A_		)

							if config is None:
									assert isinstance(self.model      ,					A_		), (
									    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
									    f''' {self.model.__class__}'''
									)
									lowercase__     			=		self.model.config
							else:
									lowercase__     			=		config

							lowercase__     			=		data_args
							lowercase__     			=		self.config.tgt_vocab_size if isinstance(self.config      ,					A_		) else self.config.vocab_size

							if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
									assert self.config.pad_token_id is not None, (
									    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
									    " calculation or doing label smoothing."
									)

							if self.config.pad_token_id is None and self.config.eos_token_id is not None:
									logger.warning(
									    f'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
									    """ padding.."""		)

							if self.args.label_smoothing == 0:
									lowercase__     			=		torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id		)
							else:
									# dynamically import label_smoothed_nll_loss
									from utils import label_smoothed_nll_loss

									lowercase__     			=		label_smoothed_nll_loss

					def   lowerCamelCase__     (self    :						Tuple      ,					_UpperCAmelCase    :						List[str]		)						->   Tuple:




							"""simple docstring"""


							if self.optimizer is None:
									lowercase__     			=		["bias", "LayerNorm.weight"]
									lowercase__     			=		[
									    {
									        "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay		)],
									        "weight_decay": self.args.weight_decay,
									    },
									    {
									        "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay		)],
									        "weight_decay": 0.0,
									    },
									]
									lowercase__     			=		Adafactor if self.args.adafactor else AdamW
									if self.args.adafactor:
											lowercase__     			=		Adafactor
											lowercase__     			=		{"scale_parameter": False, "relative_step": False}
									else:
											lowercase__     			=		AdamW
											lowercase__     			=		{
											    "betas": (self.args.adam_betaa, self.args.adam_betaa),
											    "eps": self.args.adam_epsilon,
											}
									lowercase__     			=		self.args.learning_rate
									if self.sharded_ddp:
											lowercase__     			=		OSS(
											    params=A_      ,					optim=A_      ,					**A_      ,					)
									else:
											lowercase__     			=		optimizer_cls(A_      ,					**A_		)

							if self.lr_scheduler is None:
									lowercase__     			=		self._get_lr_scheduler(A_		)
							else:  # ignoring --lr_scheduler
									logger.warning("""scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored."""		)

					def   lowerCamelCase__     (self    :						Optional[Any]      ,					_UpperCAmelCase    :						Tuple		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		arg_to_scheduler[self.args.lr_scheduler]
							if self.args.lr_scheduler == "constant":
									lowercase__     			=		schedule_func(self.optimizer		)
							elif self.args.lr_scheduler == "constant_w_warmup":
									lowercase__     			=		schedule_func(self.optimizer      ,					num_warmup_steps=self.args.warmup_steps		)
							else:
									lowercase__     			=		schedule_func(
									    self.optimizer      ,					num_warmup_steps=self.args.warmup_steps      ,					num_training_steps=A_		)
							return scheduler

					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   Optional[int]:




							"""simple docstring"""


							if isinstance(self.train_dataset      ,					torch.utils.data.IterableDataset		):
									return None
							elif is_torch_tpu_available():
									return get_tpu_sampler(self.train_dataset		)
							else:
									if self.args.sortish_sampler:
											self.train_dataset.make_sortish_sampler(
											    self.args.per_device_train_batch_size      ,					distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)      ,					)

									return (
									    RandomSampler(self.train_dataset		)
									    if self.args.local_rank == -1
									    else DistributedSampler(self.train_dataset		)
									)

					def   lowerCamelCase__     (self    :						Any      ,					_UpperCAmelCase    :						List[Any]      ,					_UpperCAmelCase    :						Optional[int]      ,					_UpperCAmelCase    :						Optional[int]		)						->   str:




							"""simple docstring"""


							if self.args.label_smoothing == 0:
									if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
											# force training to ignore pad token
											lowercase__     			=		model(**A_      ,					use_cache=A_		)[0]
											lowercase__     			=		self.loss_fn(logits.view(-1      ,					logits.shape[-1]		)      ,					labels.view(-1		)		)
									else:
											# compute usual loss via models
											lowercase__     			=		model(**A_      ,					labels=A_      ,					use_cache=A_		)[:2]
							else:
									# compute label smoothed loss
									lowercase__     			=		model(**A_      ,					use_cache=A_		)[0]
									lowercase__     			=		torch.nn.functional.log_softmax(A_      ,					dim=-1		)
									lowercase__     			=		self.loss_fn(A_      ,					A_      ,					self.args.label_smoothing      ,					ignore_index=self.config.pad_token_id		)
							return loss, logits

					def   lowerCamelCase__     (self    :						Union[str, Any]      ,					_UpperCAmelCase    :						str      ,					_UpperCAmelCase    :						Union[str, Any]		)						->   List[Any]:




							"""simple docstring"""


							lowercase__     			=		inputs.pop("""labels"""		)
							lowercase__     			=		self._compute_loss(A_      ,					A_      ,					A_		)
							return loss

					def   lowerCamelCase__     (self    :						Dict      ,					_UpperCAmelCase    :						List[Any]      ,					_UpperCAmelCase    :						Dict      ,					_UpperCAmelCase    :						Dict      ,					_UpperCAmelCase    :						int = None      ,					)						->   str:




							"""simple docstring"""


							lowercase__     			=		self._prepare_inputs(A_		)

							lowercase__     			=		{
							    "max_length": self.data_args.val_max_target_length
							    if self.data_args is not None
							    else self.config.max_length,
							    "num_beams": self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
							}

							if self.args.predict_with_generate and not self.args.prediction_loss_only:
									lowercase__     			=		self.model.generate(
									    inputs["""input_ids"""]      ,					attention_mask=inputs["""attention_mask"""]      ,					**A_      ,					)
									# in case the batch is shorter than max length, the output should be padded
									if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
											lowercase__     			=		self._pad_tensors_to_max_len(A_      ,					gen_kwargs["""max_length"""]		)

							lowercase__     			=		inputs.pop("""labels"""		)
							with torch.no_grad():
									# compute loss on predict data
									lowercase__     			=		self._compute_loss(A_      ,					A_      ,					A_		)

							lowercase__     			=		loss.mean().detach()
							if self.args.prediction_loss_only:
									return (loss, None, None)

							lowercase__     			=		generated_tokens if self.args.predict_with_generate else logits

							if labels.shape[-1] < gen_kwargs["max_length"]:
									lowercase__     			=		self._pad_tensors_to_max_len(A_      ,					gen_kwargs["""max_length"""]		)

							return (loss, logits, labels)


					def   lowerCamelCase__     (self    :						List[str]      ,					_UpperCAmelCase    :						Tuple      ,					_UpperCAmelCase    :						Union[str, Any]		)						->   str:




							"""simple docstring"""


							lowercase__     			=		self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

							if pad_token_id is None:
									raise ValueError(
									    """Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be"""
									    f''' padded to `max_length`={max_length}'''		)

							lowercase__     			=		pad_token_id * torch.ones(
							    (tensor.shape[0], max_length)      ,					dtype=tensor.dtype      ,					device=tensor.device		)
							lowercase__     			=		tensor
							return padded_tensor








from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__lowerCamelCase       :     Optional[int]   =     {"""configuration_mmbt""": ["""MMBTConfig"""]}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase       :     List[Any]   =     ["""MMBTForClassification""", """MMBTModel""", """ModalEmbeddings"""]


if TYPE_CHECKING:
    from .configuration_mmbt import MMBTConfig

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings

else:
    import sys

    __lowerCamelCase       :     int   =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
