

# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path


def   __SCREAMING_SNAKE_CASE					(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       ):
						snake_case_   =	{
						    '''en''': '''Machine learning is great, isn\'t it?''',
						    '''ru''': '''Машинное обучение - это здорово, не так ли?''',
						    '''de''': '''Maschinelles Lernen ist großartig, oder?''',
						}

						# BLUE scores as follows:
						# "pair": [fairseq, transformers]
						snake_case_   =	{
						    '''ru-en''': ['''[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)''', '''39.20'''],
						    '''en-ru''': ['''[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)''', '''33.47'''],
						    '''en-de''': ['''[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)''', '''42.83'''],
						    '''de-en''': ['''[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)''', '''41.35'''],
						}
						snake_case_   =	F'''{src_lang}-{tgt_lang}'''

						snake_case_   =	F'''
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR\'s WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "facebook/wmt19-{src_lang}-{tgt_lang}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn\'t seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn\'t support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR\'s WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

'''
						os.makedirs(SCREAMING_SNAKE_CASE__  ,		exist_ok=SCREAMING_SNAKE_CASE__       )
						snake_case_   =	os.path.join(SCREAMING_SNAKE_CASE__  ,		'''README.md'''       )
						print(F'''Generating {path}'''       )
						with open(SCREAMING_SNAKE_CASE__  ,		'''w'''  ,		encoding='''utf-8'''       ) as f:
												f.write(SCREAMING_SNAKE_CASE__       )


# make sure we are under the root of the project
lowerCAmelCase_       	=	Path(__file__).resolve().parent.parent.parent
lowerCAmelCase_       	=	repo_dir / '''model_cards'''

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
					lowerCAmelCase_	,      lowerCAmelCase_	,      lowerCAmelCase_       	=	model_name.split('''-''')
					lowerCAmelCase_       	=	model_cards_dir / '''facebook''' / model_name
					write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)


import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_       	=	logging.get_logger(__name__)

lowerCAmelCase_       	=	{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase_       	=	{
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    }
}

lowerCAmelCase_       	=	{
    '''google/bigbird-roberta-base''': 40_96,
    '''google/bigbird-roberta-large''': 40_96,
    '''google/bigbird-base-trivia-itc''': 40_96,
}





class 	snake_case_   (		__A					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   List[Any]									=				VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE     :   List[Any]									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE     :   List[Any]									=				["input_ids", "attention_mask"]
			SCREAMING_SNAKE_CASE     :   List[int]									=				[]







			def __init__(     self :				List[str]							,   _UpperCamelCase :				List[str]							,   _UpperCamelCase :				Dict="<unk>"							,   _UpperCamelCase :				List[str]="<s>"							,   _UpperCamelCase :				Tuple="</s>"							,   _UpperCamelCase :				Any="<pad>"							,   _UpperCamelCase :				Any="[SEP]"							,   _UpperCamelCase :				Optional[Any]="[MASK]"							,   _UpperCamelCase :				Any="[CLS]"							,   _UpperCamelCase :				Optional[Dict[str, Any]] = None							,   **_UpperCamelCase :				Dict							,   )      ->None:
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else bos_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else eos_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else unk_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else pad_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else cls_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else sep_token

									# Mask token behave like a normal word, i.e. include the space before it
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else mask_token

									snake_case_   =	{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=_UpperCamelCase							,   eos_token=_UpperCamelCase							,   unk_token=_UpperCamelCase							,   pad_token=_UpperCamelCase							,   sep_token=_UpperCamelCase							,   mask_token=_UpperCamelCase							,   cls_token=_UpperCamelCase							,   sp_model_kwargs=self.sp_model_kwargs							,   **_UpperCamelCase							,   )

									snake_case_   =	vocab_file

									snake_case_   =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(_UpperCamelCase							)







			@property
			def 				snake_case__(     self :				str							)      ->List[Any]:
									return self.sp_model.get_piece_size()







			def 				snake_case__(     self :				int							)      ->Union[str, Any]:
									snake_case_   =	{self.convert_ids_to_tokens(_UpperCamelCase							): i for i in range(self.vocab_size							)}
									vocab.update(self.added_tokens_encoder							)
									return vocab







			def __getstate__(     self :				Tuple							)      ->Any:
									snake_case_   =	self.__dict__.copy()
									snake_case_   =	None
									return state







			def __setstate__(     self :				str							,   _UpperCamelCase :				List[Any]							)      ->List[str]:
									snake_case_   =	d

									# for backward compatibility
									if not hasattr(self							,   '''sp_model_kwargs'''							):
															snake_case_   =	{}

									snake_case_   =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(self.vocab_file							)







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				str							)      ->List[str]:
									return self.sp_model.encode(_UpperCamelCase							,   out_type=_UpperCamelCase							)







			def 				snake_case__(     self :				str							,   _UpperCamelCase :				List[str]							)      ->Tuple:
									return self.sp_model.piece_to_id(_UpperCamelCase							)







			def 				snake_case__(     self :				Union[str, Any]							,   _UpperCamelCase :				str							)      ->List[Any]:
									snake_case_   =	self.sp_model.IdToPiece(_UpperCamelCase							)
									return token







			def 				snake_case__(     self :				Dict							,   _UpperCamelCase :				Optional[int]							)      ->List[str]:
									snake_case_   =	[]
									snake_case_   =	''''''
									snake_case_   =	False
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					if not prev_is_special:
																											out_string += " "
																					out_string += self.sp_model.decode(_UpperCamelCase							) + token
																					snake_case_   =	True
																					snake_case_   =	[]
															else:
																					current_sub_tokens.append(_UpperCamelCase							)
																					snake_case_   =	False
									out_string += self.sp_model.decode(_UpperCamelCase							)
									return out_string.strip()







			def 				snake_case__(     self :				int							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				bool = False							,   _UpperCamelCase :				bool = None							,   _UpperCamelCase :				bool = True							,   **_UpperCamelCase :				List[str]							,   )      ->str:
									snake_case_   =	kwargs.pop('''use_source_tokenizer'''							,   _UpperCamelCase							)

									snake_case_   =	self.convert_ids_to_tokens(_UpperCamelCase							,   skip_special_tokens=_UpperCamelCase							)

									# To avoid mixing byte-level and unicode for byte-level BPT
									# we need to build string separately for added tokens and byte-level tokens
									# cf. https://github.com/huggingface/transformers/issues/1133
									snake_case_   =	[]
									snake_case_   =	[]
									for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																					continue
															if token in self.added_tokens_encoder:
																					if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase							)							)
																											snake_case_   =	[]
																					sub_texts.append(_UpperCamelCase							)
															else:
																					current_sub_text.append(_UpperCamelCase							)
									if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase							)							)

									# Mimic the behavior of the Rust tokenizer:
									# No space before [MASK] and [SEP]
									if spaces_between_special_tokens:
															snake_case_   =	re.sub(R''' (\[(MASK|SEP)\])'''							,   R'''\1'''							,   ''' '''.join(_UpperCamelCase							)							)
									else:
															snake_case_   =	''''''.join(_UpperCamelCase							)

									snake_case_   =	(
									    clean_up_tokenization_spaces
									    if clean_up_tokenization_spaces is not None
									    else self.clean_up_tokenization_spaces
									)
									if clean_up_tokenization_spaces:
															snake_case_   =	self.clean_up_tokenization(_UpperCamelCase							)
															return clean_text
									else:
															return text







			def 				snake_case__(     self :				List[Any]							,   _UpperCamelCase :				str							,   _UpperCamelCase :				Optional[str] = None							)      ->Tuple[str]:
									if not os.path.isdir(_UpperCamelCase							):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
															return
									snake_case_   =	os.path.join(
									    _UpperCamelCase							,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

									if os.path.abspath(self.vocab_file							) != os.path.abspath(_UpperCamelCase							) and os.path.isfile(self.vocab_file							):
															copyfile(self.vocab_file							,   _UpperCamelCase							)
									elif not os.path.isfile(self.vocab_file							):
															with open(_UpperCamelCase							,   '''wb'''							) as fi:
																					snake_case_   =	self.sp_model.serialized_model_proto()
																					fi.write(_UpperCamelCase							)

									return (out_vocab_file,)







			def 				snake_case__(     self :				Tuple							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									snake_case_   =	[self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep







			def 				snake_case__(     self :				List[str]							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							,   _UpperCamelCase :				bool = False							)      ->List[int]:
									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_UpperCamelCase							,   token_ids_a=_UpperCamelCase							,   already_has_special_tokens=_UpperCamelCase							)

									if token_ids_a is None:
															return [1] + ([0] * len(_UpperCamelCase							)) + [1]
									return [1] + ([0] * len(_UpperCamelCase							)) + [1] + ([0] * len(_UpperCamelCase							)) + [1]







			def 				snake_case__(     self :				List[Any]							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									snake_case_   =	[self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]
									return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]