






'''simple docstring'''







# Usage:
# ./gen-card-allenai-wmt19.py

import os
from pathlib import Path





def 					SCREAMING_SNAKE_CASE_			(UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase ) ->			List[Any]:
							lowerCamelCase__   :      Any			    =			{
							    """en""": """Machine learning is great, isn't it?""",
							    """ru""": """Машинное обучение - это здорово, не так ли?""",
							    """de""": """Maschinelles Lernen ist großartig, nicht wahr?""",
							}

							# BLUE scores as follows:
							# "pair": [fairseq, transformers]
							lowerCamelCase__   :      List[str]			    =			{
							    """wmt19-de-en-6-6-base""": [0, 38.37],
							    """wmt19-de-en-6-6-big""": [0, 39.90],
							}
							lowerCamelCase__   :      int			    =			f'''{src_lang}-{tgt_lang}'''

							lowerCamelCase__   :      List[str]			    =			f'''
---

language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- allenai
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

2 models are available:

* [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big)
* [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "allenai/{model_name}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   |  transformers
-------|---------
{model_name}  |  {scores[model_name][1]}

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

'''
							model_card_dir.mkdir(parents=UpperCamelCase							,     exist_ok=UpperCamelCase )
							lowerCamelCase__   :      Union[str, Any]			    =			os.path.join(UpperCamelCase							,     """README.md""" )
							print(f'''Generating {path}''' )
							with open(UpperCamelCase							,     """w"""							,     encoding="""utf-8""" ) as f:
														f.write(UpperCamelCase )


# make sure we are under the root of the project
_A      :     Union[str, Any]       	=Path(__file__).resolve().parent.parent.parent
_A      :     Tuple       	=repo_dir / '''model_cards'''

for model_name in ["wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big"]:
			_A      :     int       	=model_cards_dir / '''allenai''' / model_name
			write_model_card(model_card_dir, src_lang='''de''', tgt_lang='''en''', model_name=model_name)









'''simple docstring'''







import json
import os
import unittest

from transformers import BartTokenizer, BartTokenizerFast, BatchEncoding
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors

@require_tokenizers
class _lowercase      (							_lowercase							, unittest.TestCase						):
							a							 =			BartTokenizer
							a							 =			BartTokenizerFast
							a							 =			True
							a							 =			filter_roberta_detectors




							def 						lowerCamelCase_							(				self:							Union[str, Any]	):
														super().setUp()
														lowerCamelCase__   :      int			    =			[
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """\u0120""",
														    """\u0120l""",
														    """\u0120n""",
														    """\u0120lo""",
														    """\u0120low""",
														    """er""",
														    """\u0120lowest""",
														    """\u0120newer""",
														    """\u0120wider""",
														    """<unk>""",
														]
														lowerCamelCase__   :      List[str]			    =			dict(zip(UpperCamelCase__     ,				range(len(UpperCamelCase__	)	)	)	)
														lowerCamelCase__   :      List[str]			    =			["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
														lowerCamelCase__   :      List[Any]			    =			{"""unk_token""": """<unk>"""}

														lowerCamelCase__   :      List[str]			    =			os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES["""vocab_file"""]	)
														lowerCamelCase__   :      List[str]			    =			os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES["""merges_file"""]	)
														with open(self.vocab_file     ,				"""w"""     ,				encoding="""utf-8"""	) as fp:
																					fp.write(json.dumps(UpperCamelCase__	) + """\n"""	)
														with open(self.merges_file     ,				"""w"""     ,				encoding="""utf-8"""	) as fp:
																					fp.write("""\n""".join(UpperCamelCase__	)	)




							def 						lowerCamelCase_							(				self:							Tuple     ,				**UpperCamelCase__:							Dict	):
														kwargs.update(self.special_tokens_map	)
														return self.tokenizer_class.from_pretrained(self.tmpdirname     ,				**UpperCamelCase__	)




							def 						lowerCamelCase_							(				self:							Optional[int]     ,				**UpperCamelCase__:							Tuple	):
														kwargs.update(self.special_tokens_map	)
														return self.rust_tokenizer_class.from_pretrained(self.tmpdirname     ,				**UpperCamelCase__	)




							def 						lowerCamelCase_							(				self:							List[str]     ,				UpperCamelCase__:							List[Any]	):
														return "lower newer", "lower newer"




							@cached_property
							def 						lowerCamelCase_							(				self:							str	):
														return BartTokenizer.from_pretrained("""facebook/bart-large"""	)




							@cached_property
							def 						lowerCamelCase_							(				self:							Union[str, Any]	):
														return BartTokenizerFast.from_pretrained("""facebook/bart-large"""	)




							@require_torch
							def 						lowerCamelCase_							(				self:							Optional[int]	):
														lowerCamelCase__   :      Any			    =			["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														lowerCamelCase__   :      List[Any]			    =			[0, 250, 251, 17_818, 13, 39_186, 1_938, 4, 2]

														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					lowerCamelCase__   :      Tuple			    =			tokenizer(UpperCamelCase__     ,				max_length=len(UpperCamelCase__	)     ,				padding=UpperCamelCase__     ,				return_tensors="""pt"""	)
																					self.assertIsInstance(UpperCamelCase__     ,				UpperCamelCase__	)

																					self.assertEqual((2, 9)     ,				batch.input_ids.shape	)
																					self.assertEqual((2, 9)     ,				batch.attention_mask.shape	)
																					lowerCamelCase__   :      Optional[Any]			    =			batch.input_ids.tolist()[0]
																					self.assertListEqual(UpperCamelCase__     ,				UpperCamelCase__	)
																					# Test that special tokens are reset




							@require_torch
							def 						lowerCamelCase_							(				self:							Tuple	):
														lowerCamelCase__   :      str			    =			["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					lowerCamelCase__   :      Union[str, Any]			    =			tokenizer(UpperCamelCase__     ,				padding=UpperCamelCase__     ,				return_tensors="""pt"""	)
																					# check if input_ids are returned and no labels
																					self.assertIn("""input_ids"""     ,				UpperCamelCase__	)
																					self.assertIn("""attention_mask"""     ,				UpperCamelCase__	)
																					self.assertNotIn("""labels"""     ,				UpperCamelCase__	)
																					self.assertNotIn("""decoder_attention_mask"""     ,				UpperCamelCase__	)




							@require_torch
							def 						lowerCamelCase_							(				self:							Dict	):
														lowerCamelCase__   :      str			    =			[
														    """Summary of the text.""",
														    """Another summary.""",
														]
														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					lowerCamelCase__   :      Any			    =			tokenizer(text_target=UpperCamelCase__     ,				max_length=32     ,				padding="""max_length"""     ,				return_tensors="""pt"""	)
																					self.assertEqual(32     ,				targets["""input_ids"""].shape[1]	)




							@require_torch
							def 						lowerCamelCase_							(				self:							str	):
														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					lowerCamelCase__   :      Union[str, Any]			    =			tokenizer(
																					    ["""I am a small frog""" * 1_024, """I am a small frog"""]     ,				padding=UpperCamelCase__     ,				truncation=UpperCamelCase__     ,				return_tensors="""pt"""	)
																					self.assertIsInstance(UpperCamelCase__     ,				UpperCamelCase__	)
																					self.assertEqual(batch.input_ids.shape     ,				(2, 1_024)	)




							@require_torch
							def 						lowerCamelCase_							(				self:							Optional[Any]	):
														lowerCamelCase__   :      Optional[Any]			    =			["""A long paragraph for summarization."""]
														lowerCamelCase__   :      Any			    =			[
														    """Summary of the text.""",
														]
														for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																					lowerCamelCase__   :      Union[str, Any]			    =			tokenizer(UpperCamelCase__     ,				return_tensors="""pt"""	)
																					lowerCamelCase__   :      Union[str, Any]			    =			tokenizer(text_target=UpperCamelCase__     ,				return_tensors="""pt"""	)
																					lowerCamelCase__   :      List[str]			    =			inputs["""input_ids"""]
																					lowerCamelCase__   :      Optional[int]			    =			targets["""input_ids"""]
																					self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()	)
																					self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()	)
																					self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()	)
																					self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()	)




							def 						lowerCamelCase_							(				self:							Optional[int]	):
														pass




							def 						lowerCamelCase_							(				self:							List[str]	):
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
																												lowerCamelCase__   :      int			    =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,				**UpperCamelCase__	)
																												lowerCamelCase__   :      int			    =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,				**UpperCamelCase__	)
																												lowerCamelCase__   :      Dict			    =			"""A, <mask> AllenNLP sentence."""
																												lowerCamelCase__   :      Dict			    =			tokenizer_r.encode_plus(UpperCamelCase__     ,				add_special_tokens=UpperCamelCase__     ,				return_token_type_ids=UpperCamelCase__	)
																												lowerCamelCase__   :      Union[str, Any]			    =			tokenizer_p.encode_plus(UpperCamelCase__     ,				add_special_tokens=UpperCamelCase__     ,				return_token_type_ids=UpperCamelCase__	)

																												# token_type_ids should put 0 everywhere
																												self.assertEqual(sum(tokens_r["""token_type_ids"""]	)     ,				sum(tokens_p["""token_type_ids"""]	)	)

																												# attention_mask should put 1 everywhere, so sum over length should be 1
																												self.assertEqual(
																												    sum(tokens_r["""attention_mask"""]	) / len(tokens_r["""attention_mask"""]	)     ,				sum(tokens_p["""attention_mask"""]	) / len(tokens_p["""attention_mask"""]	)     ,				)

																												lowerCamelCase__   :      Any			    =			tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]	)
																												lowerCamelCase__   :      Any			    =			tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]	)

																												# Rust correctly handles the space before the mask while python doesnt
																												self.assertSequenceEqual(tokens_p["""input_ids"""]     ,				[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]	)
																												self.assertSequenceEqual(tokens_r["""input_ids"""]     ,				[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]	)

																												self.assertSequenceEqual(
																												    UpperCamelCase__     ,				["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]	)
																												self.assertSequenceEqual(
																												    UpperCamelCase__     ,				["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]	)

