

"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a       =							logging.get_logger(__name__)

_a       =							{
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}




class 					_lowerCAmelCase  (	lowercase   ):







 """simple docstring"""







 __UpperCAmelCase	:					Any   		=	"nllb-moe"
 __UpperCAmelCase	:					str   		=	["past_key_values"]
 __UpperCAmelCase	:					Any   		=	{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}


 def __init__(			self							:			List[Any],					UpperCAmelCase__							:			str=1_2_8_1_1_2,					UpperCAmelCase__							:			int=1_0_2_4,					UpperCAmelCase__							:			Optional[Any]=1_2,					UpperCAmelCase__							:			List[str]=4_0_9_6,					UpperCAmelCase__							:			Dict=1_6,					UpperCAmelCase__							:			Tuple=1_2,					UpperCAmelCase__							:			Union[str, Any]=4_0_9_6,					UpperCAmelCase__							:			int=1_6,					UpperCAmelCase__							:			Optional[int]=0.05,					UpperCAmelCase__							:			Union[str, Any]=0.05,					UpperCAmelCase__							:			Optional[int]=True,					UpperCAmelCase__							:			str=True,					UpperCAmelCase__							:			Tuple="relu",					UpperCAmelCase__							:			Optional[int]=1_0_2_4,					UpperCAmelCase__							:			str=0.1,					UpperCAmelCase__							:			Any=0.1,					UpperCAmelCase__							:			Optional[int]=0.0,					UpperCAmelCase__							:			Dict=0.02,					UpperCAmelCase__							:			str=2,					UpperCAmelCase__							:			List[Any]=True,					UpperCAmelCase__							:			int=False,					UpperCAmelCase__							:			Dict="float32",					UpperCAmelCase__							:			List[str]=False,					UpperCAmelCase__							:			Any=1_2_8,					UpperCAmelCase__							:			Any=6_4,					UpperCAmelCase__							:			str=4,					UpperCAmelCase__							:			Any=4,					UpperCAmelCase__							:			int=0.001,					UpperCAmelCase__							:			Optional[int]=0.001,					UpperCAmelCase__							:			Optional[Any]="all",					UpperCAmelCase__							:			Optional[int]=False,					UpperCAmelCase__							:			str=False,					UpperCAmelCase__							:			Dict=1.0,					UpperCAmelCase__							:			List[str]=0.2,					UpperCAmelCase__							:			Any=1,					UpperCAmelCase__							:			Optional[Any]=0,					UpperCAmelCase__							:			Dict=2,					UpperCAmelCase__							:			List[str]=False,					**UpperCAmelCase__							:			int,					):
    __lowercase								=						vocab_size
    __lowercase								=						max_position_embeddings
    __lowercase								=						d_model
    __lowercase								=						encoder_ffn_dim
    __lowercase								=						encoder_layers
    __lowercase								=						encoder_attention_heads
    __lowercase								=						decoder_ffn_dim
    __lowercase								=						decoder_layers
    __lowercase								=						decoder_attention_heads
    __lowercase								=						dropout
    __lowercase								=						attention_dropout
    __lowercase								=						activation_dropout
    __lowercase								=						activation_function
    __lowercase								=						init_std
    __lowercase								=						encoder_layerdrop
    __lowercase								=						decoder_layerdrop
    __lowercase								=						use_cache
    __lowercase								=						encoder_layers
    __lowercase								=						scale_embedding  # scale factor will be sqrt(d_model) if True
    __lowercase								=						router_z_loss_coef
    __lowercase								=						router_aux_loss_coef
    __lowercase								=						decoder_sparse_step
    __lowercase								=						encoder_sparse_step
    __lowercase								=						num_experts
    __lowercase								=						expert_capacity
    __lowercase								=						router_bias
    if router_dtype not in ["float32", "float16", "bfloat16"]:
       raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""	)
    __lowercase								=						router_dtype

    __lowercase								=						router_ignore_padding_tokens
    __lowercase								=						batch_prioritized_routing
    __lowercase								=						second_expert_policy
    __lowercase								=						normalize_router_prob_before_dropping
    __lowercase								=						moe_eval_capacity_token_fraction
    __lowercase								=						moe_token_dropout
    __lowercase								=						output_router_logits
    super().__init__(
        pad_token_id=UpperCAmelCase__,					bos_token_id=UpperCAmelCase__,					eos_token_id=UpperCAmelCase__,					is_encoder_decoder=UpperCAmelCase__,					decoder_start_token_id=UpperCAmelCase__,					**UpperCAmelCase__,					)







"""simple docstring"""





import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
   from sagemaker import Session, TrainingJobAnalytics
   from sagemaker.huggingface import HuggingFace




@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"     ,"False"   )   ) is not True     ,reason="Skipping test because should only be run when releasing minor transformers version"     ,)
@pytest.mark.usefixtures("sm_env"   )
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue_model_parallelism.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1_6_0_0, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1_6_0_0, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
    ]   )
class 					_lowerCAmelCase  (	unittest.TestCase   ):







 """simple docstring"""







 def 					_lowercase (			self							:			Optional[int]	):
    if self.framework == "pytorch":
       subprocess.run(
           F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split(),					encoding="utf-8",					check=UpperCAmelCase__,					)
    assert hasattr(self,					"env"	)


 def 					_lowercase (			self							:			str,					UpperCAmelCase__							:			List[Any]	):
    # configuration for running training on smdistributed Model Parallel
    __lowercase								=						{
        "enabled": True,
        "processes_per_host": 8,
    }
    __lowercase								=						{
        "enabled": True,
        "parameters": {
            "microbatches": 4,
            "placement_strategy": "spread",
            "pipeline": "interleaved",
            "optimize": "speed",
            "partitions": 4,
            "ddp": True,
        },
    }

    __lowercase								=						{"smdistributed": {"modelparallel": smp_options}, "mpi": mpi_options}

    __lowercase								=						"trainer" if self.script == "run_glue.py" else "smtrainer"
    # creates estimator
    return HuggingFace(
        entry_point=self.script,					source_dir=self.env.test_path,					role=self.env.role,					image_uri=self.env.image_uri,					base_job_name=F"""{self.env.base_job_name}-{instance_count}-smp-{name_extension}""",					instance_count=UpperCAmelCase__,					instance_type=self.instance_type,					debugger_hook_config=UpperCAmelCase__,					hyperparameters={
            **self.env.hyperparameters,
            "model_name_or_path": self.model_name_or_path,
            "max_steps": 5_0_0,
        },					metric_definitions=self.env.metric_definitions,					distribution=UpperCAmelCase__,					py_version="py36",					)


 def 					_lowercase (			self							:			Tuple,					UpperCAmelCase__							:			int	):
    TrainingJobAnalytics(UpperCAmelCase__	).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""	)



 @parameterized.expand([(1,)]	)
 def 					_lowercase (			self							:			str,					UpperCAmelCase__							:			Union[str, Any]	):
    # create estimator
    __lowercase								=						self.create_estimator(UpperCAmelCase__	)

    # run training
    estimator.fit()

    # result dataframe
    __lowercase								=						TrainingJobAnalytics(estimator.latest_training_job.name	).dataframe()

    # extract kpis
    __lowercase								=						list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]	)
    __lowercase								=						list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]	)
    # get train time from SageMaker job, this includes starting, preprocessing, stopping
    __lowercase								=						(
        Session().describe_training_job(estimator.latest_training_job.name	).get("TrainingTimeInSeconds",					9_9_9_9_9_9	)
    )

    # assert kpis
    assert train_runtime <= self.results["train_runtime"]
    assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy	)
    assert all(t <= self.results["eval_loss"] for t in eval_loss	)

    # dump tests result into json file to share in PR
    with open(F"""{estimator.latest_training_job.name}.json""",					"w"	) as outfile:
       json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss},					UpperCAmelCase__	)




