




'''simple docstring'''




# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


snake_case__ 							=    logging.get_logger(__name__)


if is_torch_available():
							import torch
							from torch import nn
							from torch.utils.data import Dataset

							from transformers import Trainer






							class      UpperCamelCase_  (a__  ):







													"""simple docstring"""


													def __init__(  self   :							int				,			_lowerCamelCase   :							int = 101     ):





																		"""simple docstring"""







																		A_			:  Optional[int]			=				length







													def __len__(  self   :							Any     ):





																		"""simple docstring"""







																		return self.length







													def __getitem__(  self   :							str				,			_lowerCamelCase   :							List[Any]     ):





																		"""simple docstring"""







																		return i






							class      UpperCamelCase_  :







													"""simple docstring"""


													def __call__(  self   :							str				,			_lowerCamelCase   :							str     ):





																		"""simple docstring"""







																		return {"input_ids": torch.tensor(_lowerCamelCase     ), "labels": torch.tensor(_lowerCamelCase     )}






							class      UpperCamelCase_  (nn.Module  ):







													"""simple docstring"""


													def __init__(  self   :							Tuple     ):





																		"""simple docstring"""







																		super().__init__()
																		# Add some (unused) params otherwise DDP will complain.
																		A_			:  Optional[Any]			=				nn.Linear(120				,			80     )







													def    _a     (  self   :							str				,			_lowerCamelCase   :							int				,			_lowerCamelCase   :							Any=None     ):





																		"""simple docstring"""







																		if labels is not None:
																							return torch.tensor(0.0				,			device=input_ids.device     ), input_ids
																		else:
																							return input_ids




def     snake_case__     (			) -> Optional[Any]:
					A_			:  int			=				HfArgumentParser((TrainingArguments,)      )
					sys.argv += ["--output_dir", "./examples"]
					A_			:  List[str]			=				parser.parse_args_into_dataclasses()[0]

					logger.warning(
					    f'Process rank: {training_args.local_rank}, device: {training_args.device}, '
					    f'tpu_num_cores: {training_args.tpu_num_cores}'      ,		)

					# Essentially, what we want to verify in the distributed case is
					# that we get all samples back, in the right order.
					# (this is crucial for prediction for instance)
					for dataset_length in [1_0_0_1, 2_5_6, 1_5]:
										A_			:  Optional[Any]			=				DummyDataset(lowerCamelCase__      )

										def compute_metrics(lowerCamelCase__					: EvalPrediction      ) -> Dict:
															A_			:  str			=				list(range(len(lowerCamelCase__      )      )      )
															A_			:  Tuple			=				p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
															return {"success": success}

										A_			:  Any			=				Trainer(
										    model=DummyModel()      ,		args=lowerCamelCase__      ,		data_collator=DummyDataCollator()      ,		eval_dataset=lowerCamelCase__      ,		compute_metrics=lowerCamelCase__      ,		)
										A_			:  str			=				trainer.evaluate()
										logger.info(lowerCamelCase__      )
										if metrics["eval_success"] is not True:
															logger.error(lowerCamelCase__      )
															exit(1      )

										A_			:  Union[str, Any]			=				trainer.predict(lowerCamelCase__      )
										logger.info(p.metrics      )
										if p.metrics["test_success"] is not True:
															logger.error(p.metrics      )
															exit(1      )

										A_			:  List[str]			=				2

										A_			:  int			=				trainer.evaluate()
										logger.info(lowerCamelCase__      )
										if metrics["eval_success"] is not True:
															logger.error(lowerCamelCase__      )
															exit(1      )

										A_			:  Optional[Any]			=				trainer.predict(lowerCamelCase__      )
										logger.info(p.metrics      )
										if p.metrics["test_success"] is not True:
															logger.error(p.metrics      )
															exit(1      )

										A_			:  List[Any]			=				None

					logger.info('''🔥 All distributed tests successful'''      )




def     snake_case__     (			lowerCamelCase__					: Optional[int]      ) -> Any:
					# For xla_spawn (TPUs)
					main()


if __name__ == "__main__":
							main()









'''simple docstring'''




import io
import json
import subprocess


snake_case__ 							=    [
    ["""en""", """ru"""],
    ["""ru""", """en"""],
    ["""en""", """de"""],
    ["""de""", """en"""],
]

snake_case__ 							=    8




def     snake_case__     (			lowerCamelCase__					: Union[str, Any]      ,		lowerCamelCase__					: Optional[int]      ) -> List[Any]:
					A_			:  Tuple			=				{}
					for src, tgt in pairs:
										A_			:  List[str]			=				f'{src}-{tgt}'
										A_			:  Optional[int]			=				f'sacrebleu -t wmt19 -l {pair} --echo src'.split()
										A_			:  int			=				subprocess.run(lowerCamelCase__      ,		stdout=subprocess.PIPE      ).stdout.decode('''utf-8'''      ).splitlines()
										A_			:  List[str]			=				f'sacrebleu -t wmt19 -l {pair} --echo ref'.split()
										A_			:  Any			=				subprocess.run(lowerCamelCase__      ,		stdout=subprocess.PIPE      ).stdout.decode('''utf-8'''      ).splitlines()
										A_			:  List[Any]			=				{'''src''': src_lines[:n_objs], '''tgt''': tgt_lines[:n_objs]}
					return text


snake_case__ 							=    get_all_data(pairs, n_objs)
snake_case__ 							=    """./fsmt_val_data.json"""
with io.open(filename, """w""", encoding="""utf-8""") as f:
							snake_case__ 							=    json.dump(text, f, indent=2, ensure_ascii=False)



