


# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
     import botoa  # noqa: F401
def 				__lowerCamelCase						( UpperCAmelCase_						:		Dict						):




      """simple docstring"""





      a						:Tuple 							=       botoa.client('''iam'''						)

      a						:List[Any] 							=       {
          '''Version''': '''2012-10-17''',
          '''Statement''': [
              {'''Effect''': '''Allow''', '''Principal''': {'''Service''': '''sagemaker.amazonaws.com'''}, '''Action''': '''sts:AssumeRole'''}
          ],
      }
      try:
            # create the role, associated with the chosen trust policy
            iam_client.create_role(
                RoleName=UpperCAmelCase_ ,  AssumeRolePolicyDocument=json.dumps(UpperCAmelCase_ ,  indent=2						)						)
            a						:List[Any] 							=       {
                '''Version''': '''2012-10-17''',
                '''Statement''': [
                    {
                        '''Effect''': '''Allow''',
                        '''Action''': [
                            '''sagemaker:*''',
                            '''ecr:GetDownloadUrlForLayer''',
                            '''ecr:BatchGetImage''',
                            '''ecr:BatchCheckLayerAvailability''',
                            '''ecr:GetAuthorizationToken''',
                            '''cloudwatch:PutMetricData''',
                            '''cloudwatch:GetMetricData''',
                            '''cloudwatch:GetMetricStatistics''',
                            '''cloudwatch:ListMetrics''',
                            '''logs:CreateLogGroup''',
                            '''logs:CreateLogStream''',
                            '''logs:DescribeLogStreams''',
                            '''logs:PutLogEvents''',
                            '''logs:GetLogEvents''',
                            '''s3:CreateBucket''',
                            '''s3:ListBucket''',
                            '''s3:GetBucketLocation''',
                            '''s3:GetObject''',
                            '''s3:PutObject''',
                        ],
                        '''Resource''': '''*''',
                    }
                ],
            }
            # attach policy to role
            iam_client.put_role_policy(
                RoleName=UpperCAmelCase_ ,  PolicyName=F'''{role_name}_policy_permission''' ,  PolicyDocument=json.dumps(UpperCAmelCase_ ,  indent=2						) ,  )
      except iam_client.exceptions.EntityAlreadyExistsException:
            print(F'''role {role_name} already exists. Using existing one'''						)
def 				__lowerCamelCase						( UpperCAmelCase_						:		Optional[int]						):




      """simple docstring"""





      a						:List[Any] 							=       botoa.client('''iam'''						)
      return iam_client.get_role(RoleName=UpperCAmelCase_						)["Role"]["Arn"]






def 				__lowerCamelCase						( ):




      """simple docstring"""





      a						:int 							=       _ask_options(
          '''How do you want to authorize?''' ,  ['''AWS Profile''', '''Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) '''] ,  UpperCAmelCase_ ,  )
      a						:Union[str, Any] 							=       None
      if credentials_configuration == 0:
            a						:Optional[Any] 							=       _ask_field('''Enter your AWS Profile name: [default] ''' ,  default='''default'''						)
            a						:List[Any] 							=       aws_profile
      else:
            print(
                '''Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,'''
                '''`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`'''						)
            a						:Dict 							=       _ask_field('''AWS Access Key ID: '''						)
            a						:Dict 							=       aws_access_key_id

            a						:Union[str, Any] 							=       _ask_field('''AWS Secret Access Key: '''						)
            a						:str 							=       aws_secret_access_key

      a						:List[Any] 							=       _ask_field('''Enter your AWS Region: [us-east-1]''' ,  default='''us-east-1'''						)
      a						:List[str] 							=       aws_region

      a						:Dict 							=       _ask_options(
          '''Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?''' ,  ['''Provide IAM Role name''', '''Create new IAM role using credentials'''] ,  UpperCAmelCase_ ,  )
      if role_management == 0:
            a						:Optional[int] 							=       _ask_field('''Enter your IAM role name: '''						)
      else:
            a						:Tuple 							=       '''accelerate_sagemaker_execution_role'''
            print(F'''Accelerate will create an iam role "{iam_role_name}" using the provided credentials'''						)
            _create_iam_role_for_sagemaker(UpperCAmelCase_						)

      a						:List[Any] 							=       _ask_field(
          '''Do you want to use custom Docker image? [yes/NO]: ''' ,  _convert_yes_no_to_bool ,  default=UpperCAmelCase_ ,  error_message='''Please enter yes or no.''' ,  )
      a						:Dict 							=       None
      if is_custom_docker_image:
            a						:List[Any] 							=       _ask_field('''Enter your Docker image: ''' ,  lambda UpperCAmelCase_						: str(UpperCAmelCase_						).lower()						)

      a						:str 							=       _ask_field(
          '''Do you want to provide SageMaker input channels with data locations? [yes/NO]: ''' ,  _convert_yes_no_to_bool ,  default=UpperCAmelCase_ ,  error_message='''Please enter yes or no.''' ,  )
      a						:int 							=       None
      if is_sagemaker_inputs_enabled:
            a						:Optional[int] 							=       _ask_field(
                '''Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): ''' ,  lambda UpperCAmelCase_						: str(UpperCAmelCase_						).lower() ,  )

      a						:Dict 							=       _ask_field(
          '''Do you want to enable SageMaker metrics? [yes/NO]: ''' ,  _convert_yes_no_to_bool ,  default=UpperCAmelCase_ ,  error_message='''Please enter yes or no.''' ,  )
      a						:Optional[int] 							=       None
      if is_sagemaker_metrics_enabled:
            a						:Tuple 							=       _ask_field(
                '''Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): ''' ,  lambda UpperCAmelCase_						: str(UpperCAmelCase_						).lower() ,  )

      a						:str 							=       _ask_options(
          '''What is the distributed mode?''' ,  ['''No distributed training''', '''Data parallelism'''] ,  _convert_sagemaker_distributed_mode ,  )
      a						:List[str] 							=       {}
      a						:Optional[int] 							=       _ask_field(
          '''Do you wish to optimize your script with torch dynamo?[yes/NO]:''' ,  _convert_yes_no_to_bool ,  default=UpperCAmelCase_ ,  error_message='''Please enter yes or no.''' ,  )
      if use_dynamo:
            a						:List[str] 							=       '''dynamo_'''
            a						:Optional[Any] 							=       _ask_options(
                '''Which dynamo backend would you like to use?''' ,  [x.lower() for x in DYNAMO_BACKENDS] ,  _convert_dynamo_backend ,  default=2 ,  )
            a						:List[str] 							=       _ask_field(
                '''Do you want to customize the defaults sent to torch.compile? [yes/NO]: ''' ,  _convert_yes_no_to_bool ,  default=UpperCAmelCase_ ,  error_message='''Please enter yes or no.''' ,  )

            if use_custom_options:
                  a						:Optional[int] 							=       _ask_options(
                      '''Which mode do you want to use?''' ,  UpperCAmelCase_ ,  lambda UpperCAmelCase_						: TORCH_DYNAMO_MODES[int(UpperCAmelCase_						)] ,  default='''default''' ,  )
                  a						:Optional[Any] 							=       _ask_field(
                      '''Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: ''' ,  _convert_yes_no_to_bool ,  default=UpperCAmelCase_ ,  error_message='''Please enter yes or no.''' ,  )
                  a						:Union[str, Any] 							=       _ask_field(
                      '''Do you want to enable dynamic shape tracing? [yes/NO]: ''' ,  _convert_yes_no_to_bool ,  default=UpperCAmelCase_ ,  error_message='''Please enter yes or no.''' ,  )
      a						:Tuple 							=       '''Which EC2 instance type you want to use for your training?'''
      if distributed_type != SageMakerDistributedType.NO:
            a						:int 							=       _ask_options(
                UpperCAmelCase_ ,  UpperCAmelCase_ ,  lambda UpperCAmelCase_						: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(UpperCAmelCase_						)]						)
      else:
            eca_instance_query += "? [ml.p3.2xlarge]:"
            a						:List[str] 							=       _ask_field(UpperCAmelCase_ ,  lambda UpperCAmelCase_						: str(UpperCAmelCase_						).lower() ,  default='''ml.p3.2xlarge'''						)

      a						:Dict 							=       1
      if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
            a						:Union[str, Any] 							=       _ask_field(
                '''How many machines do you want use? [1]: ''' ,  UpperCAmelCase_ ,  default=1 ,  )

      a						:Optional[int] 							=       _ask_options(
          '''Do you wish to use FP16 or BF16 (mixed precision)?''' ,  ['''no''', '''fp16''', '''bf16''', '''fp8'''] ,  _convert_mixed_precision ,  )

      if use_dynamo and mixed_precision == "no":
            print(
                '''Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts.'''						)

      return SageMakerConfig(
          image_uri=UpperCAmelCase_ ,  compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER ,  distributed_type=UpperCAmelCase_ ,  use_cpu=UpperCAmelCase_ ,  dynamo_config=UpperCAmelCase_ ,  eca_instance_type=UpperCAmelCase_ ,  profile=UpperCAmelCase_ ,  region=UpperCAmelCase_ ,  iam_role_name=UpperCAmelCase_ ,  mixed_precision=UpperCAmelCase_ ,  num_machines=UpperCAmelCase_ ,  sagemaker_inputs_file=UpperCAmelCase_ ,  sagemaker_metrics_file=UpperCAmelCase_ ,  )







from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
						from fairscale.optim import OSS


__a  			=					logging.get_logger(__name__)

__a  			=					{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    'constant': get_constant_schedule,
    'constant_w_warmup': get_constant_schedule_with_warmup,
}
class  lowercase__(				UpperCAmelCase							):



	"""simple docstring"""




	def __init__(       self  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			List[str]=None       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]=None       ,							*SCREAMING_SNAKE_CASE_  :			str       ,							**SCREAMING_SNAKE_CASE_  :			Optional[Any]       )       ->     List[str]:
					super().__init__(*SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       )

					if config is None:
									assert isinstance(self.model       ,							SCREAMING_SNAKE_CASE_       ), (
									    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
									    f''' {self.model.__class__}'''
									)
									lowercase_       =					self.model.config
					else:
									lowercase_       =					config

					lowercase_       =					data_args
					lowercase_       =					self.config.tgt_vocab_size if isinstance(self.config       ,							SCREAMING_SNAKE_CASE_       ) else self.config.vocab_size

					if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
									assert self.config.pad_token_id is not None, (
									    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
									    " calculation or doing label smoothing."
									)

					if self.config.pad_token_id is None and self.config.eos_token_id is not None:
									logger.warning(
									    f'''The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'''
									    ''' padding..'''       )

					if self.args.label_smoothing == 0:
									lowercase_       =					torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id       )
					else:
									# dynamically import label_smoothed_nll_loss
									from utils import label_smoothed_nll_loss

									lowercase_       =					label_smoothed_nll_loss
	def 						_lowercase  (       self  :			int       ,							SCREAMING_SNAKE_CASE_  :			int       )       ->     Optional[Any]:
					if self.optimizer is None:
									lowercase_       =					['''bias''', '''LayerNorm.weight''']
									lowercase_       =					[
									    {
									        '''params''': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay       )],
									        '''weight_decay''': self.args.weight_decay,
									    },
									    {
									        '''params''': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay       )],
									        '''weight_decay''': 0.0,
									    },
									]
									lowercase_       =					Adafactor if self.args.adafactor else AdamW
									if self.args.adafactor:
													lowercase_       =					Adafactor
													lowercase_       =					{'''scale_parameter''': False, '''relative_step''': False}
									else:
													lowercase_       =					AdamW
													lowercase_       =					{
													    '''betas''': (self.args.adam_betaa, self.args.adam_betaa),
													    '''eps''': self.args.adam_epsilon,
													}
									lowercase_       =					self.args.learning_rate
									if self.sharded_ddp:
													lowercase_       =					OSS(
													    params=SCREAMING_SNAKE_CASE_       ,							optim=SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       ,							)
									else:
													lowercase_       =					optimizer_cls(SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       )

					if self.lr_scheduler is None:
									lowercase_       =					self._get_lr_scheduler(SCREAMING_SNAKE_CASE_       )
					else:  # ignoring --lr_scheduler
									logger.warning('''scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'''       )
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			Dict       )       ->     Dict:
					lowercase_       =					arg_to_scheduler[self.args.lr_scheduler]
					if self.args.lr_scheduler == "constant":
									lowercase_       =					schedule_func(self.optimizer       )
					elif self.args.lr_scheduler == "constant_w_warmup":
									lowercase_       =					schedule_func(self.optimizer       ,							num_warmup_steps=self.args.warmup_steps       )
					else:
									lowercase_       =					schedule_func(
									    self.optimizer       ,							num_warmup_steps=self.args.warmup_steps       ,							num_training_steps=SCREAMING_SNAKE_CASE_       )
					return scheduler
	def 						_lowercase  (       self  :			Tuple       )       ->     Optional[torch.utils.data.Sampler]:
					if isinstance(self.train_dataset       ,							torch.utils.data.IterableDataset       ):
									return None
					elif is_torch_tpu_available():
									return get_tpu_sampler(self.train_dataset       )
					else:
									if self.args.sortish_sampler:
													self.train_dataset.make_sortish_sampler(
													    self.args.per_device_train_batch_size       ,							distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)       ,							)

									return (
									    RandomSampler(self.train_dataset       )
									    if self.args.local_rank == -1
									    else DistributedSampler(self.train_dataset       )
									)
	def 						_lowercase  (       self  :			List[Any]       ,							SCREAMING_SNAKE_CASE_  :			int       ,							SCREAMING_SNAKE_CASE_  :			Dict       ,							SCREAMING_SNAKE_CASE_  :			List[Any]       )       ->     Any:
					if self.args.label_smoothing == 0:
									if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
													# force training to ignore pad token
													lowercase_       =					model(**SCREAMING_SNAKE_CASE_       ,							use_cache=SCREAMING_SNAKE_CASE_       )[0]
													lowercase_       =					self.loss_fn(logits.view(-1       ,							logits.shape[-1]       )       ,							labels.view(-1       )       )
									else:
													# compute usual loss via models
													lowercase_  ,	lowercase_       =					model(**SCREAMING_SNAKE_CASE_       ,							labels=SCREAMING_SNAKE_CASE_       ,							use_cache=SCREAMING_SNAKE_CASE_       )[:2]
					else:
									# compute label smoothed loss
									lowercase_       =					model(**SCREAMING_SNAKE_CASE_       ,							use_cache=SCREAMING_SNAKE_CASE_       )[0]
									lowercase_       =					torch.nn.functional.log_softmax(SCREAMING_SNAKE_CASE_       ,							dim=-1       )
									lowercase_  ,	lowercase_       =					self.loss_fn(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							self.args.label_smoothing       ,							ignore_index=self.config.pad_token_id       )
					return loss, logits
	def 						_lowercase  (       self  :			str       ,							SCREAMING_SNAKE_CASE_  :			Dict       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]       )       ->     List[Any]:
					lowercase_       =					inputs.pop('''labels'''       )
					lowercase_  ,	lowercase_       =					self._compute_loss(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )
					return loss
	def 						_lowercase  (       self  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			nn.Module       ,							SCREAMING_SNAKE_CASE_  :			Dict[str, Union[torch.Tensor, Any]]       ,							SCREAMING_SNAKE_CASE_  :			bool       ,							SCREAMING_SNAKE_CASE_  :			Optional[List[str]] = None       ,							)       ->     Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
					lowercase_       =					self._prepare_inputs(SCREAMING_SNAKE_CASE_       )

					lowercase_       =					{
					    '''max_length''': self.data_args.val_max_target_length
					    if self.data_args is not None
					    else self.config.max_length,
					    '''num_beams''': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
					}

					if self.args.predict_with_generate and not self.args.prediction_loss_only:
									lowercase_       =					self.model.generate(
									    inputs['''input_ids''']       ,							attention_mask=inputs['''attention_mask''']       ,							**SCREAMING_SNAKE_CASE_       ,							)
									# in case the batch is shorter than max length, the output should be padded
									if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
													lowercase_       =					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_       ,							gen_kwargs['''max_length''']       )

					lowercase_       =					inputs.pop('''labels'''       )
					with torch.no_grad():
									# compute loss on predict data
									lowercase_  ,	lowercase_       =					self._compute_loss(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )

					lowercase_       =					loss.mean().detach()
					if self.args.prediction_loss_only:
									return (loss, None, None)

					lowercase_       =					generated_tokens if self.args.predict_with_generate else logits

					if labels.shape[-1] < gen_kwargs["max_length"]:
									lowercase_       =					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_       ,							gen_kwargs['''max_length''']       )

					return (loss, logits, labels)




	def 						_lowercase  (       self  :			str       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]       ,							SCREAMING_SNAKE_CASE_  :			int       )       ->     Tuple:
					# If PAD token is not defined at least EOS token has to be defined
					lowercase_       =					self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

					if pad_token_id is None:
									raise ValueError(
									    '''Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'''
									    f''' padded to `max_length`={max_length}'''       )

					lowercase_       =					pad_token_id * torch.ones(
					    (tensor.shape[0], max_length)       ,							dtype=tensor.dtype       ,							device=tensor.device       )
					lowercase_       =					tensor
					return padded_tensor




