
'''simple docstring'''







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase    :							List[str]      =					logging.get_logger(__name__)

lowercase    :							List[Any]      =					"▁"

lowercase    :							Optional[Any]      =					{
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowercase    :							Dict      =					{
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

lowercase    :							List[Any]      =					{
    "facebook/m2m100_418M": 1024,
}

# fmt: off
lowercase    :							Tuple      =					{
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}


class 						__UpperCAmelCase      (					_lowerCamelCase						):
							__lowercase      		=       VOCAB_FILES_NAMES
							__lowercase      		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase      		=       PRETRAINED_VOCAB_FILES_MAP
							__lowercase      		=       ["""input_ids""", """attention_mask"""]

							__lowercase      		=       []
							__lowercase      		=       []


							def __init__(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_="<s>"    ,   lowerCAmelCase_="</s>"    ,   lowerCAmelCase_="</s>"    ,   lowerCAmelCase_="<pad>"    ,   lowerCAmelCase_="<unk>"    ,   lowerCAmelCase_="m2m100"    ,   lowerCAmelCase_ = None    ,   lowerCAmelCase_=8    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		{} if sp_model_kwargs is None else sp_model_kwargs

								_snake_case   						=		language_codes
								_snake_case   						=		FAIRSEQ_LANGUAGE_CODES[language_codes]
								_snake_case   						=		{lang_code: F'__{lang_code}__' for lang_code in fairseq_language_code}

								_snake_case   						=		kwargs.get('additional_special_tokens'    ,   []  )
								kwargs["additional_special_tokens"] += [
								    self.get_lang_token(lowerCAmelCase_  )
								    for lang_code in fairseq_language_code
								    if self.get_lang_token(lowerCAmelCase_  ) not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    src_lang=lowerCAmelCase_    ,   tgt_lang=lowerCAmelCase_    ,   bos_token=lowerCAmelCase_    ,   eos_token=lowerCAmelCase_    ,   sep_token=lowerCAmelCase_    ,   unk_token=lowerCAmelCase_    ,   pad_token=lowerCAmelCase_    ,   language_codes=lowerCAmelCase_    ,   sp_model_kwargs=self.sp_model_kwargs    ,   num_madeup_words=lowerCAmelCase_    ,   **lowerCAmelCase_    ,   )

								_snake_case   						=		vocab_file
								_snake_case   						=		load_json(lowerCAmelCase_  )
								_snake_case   						=		{v: k for k, v in self.encoder.items()}
								_snake_case   						=		spm_file
								_snake_case   						=		load_spm(lowerCAmelCase_    ,   self.sp_model_kwargs  )

								_snake_case   						=		len(self.encoder  )

								_snake_case   						=		{
								    self.get_lang_token(lowerCAmelCase_  ): self.encoder_size + i for i, lang_code in enumerate(lowerCAmelCase_  )
								}
								_snake_case   						=		{lang_code: self.encoder_size + i for i, lang_code in enumerate(lowerCAmelCase_  )}
								_snake_case   						=		{v: k for k, v in self.lang_token_to_id.items()}

								_snake_case   						=		src_lang if src_lang is not None else 'en'
								_snake_case   						=		tgt_lang
								_snake_case   						=		self.get_lang_id(self._src_lang  )
								self.set_src_lang_special_tokens(self._src_lang  )

								_snake_case   						=		num_madeup_words


							@property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								return len(self.encoder  ) + len(self.lang_token_to_id  )


							@property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								return self._src_lang


							@src_lang.setter
							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		new_src_lang
								self.set_src_lang_special_tokens(self._src_lang  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								return self.sp_model.encode(lowerCAmelCase_    ,   out_type=lowerCAmelCase_  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								if token in self.lang_token_to_id:
									return self.lang_token_to_id[token]
								return self.encoder.get(lowerCAmelCase_    ,   self.encoder[self.unk_token]  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								if index in self.id_to_lang_token:
									return self.id_to_lang_token[index]
								return self.decoder.get(lowerCAmelCase_    ,   self.unk_token  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		[]
								_snake_case   						=		''
								for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
										out_string += self.sp_model.decode(lowerCAmelCase_  ) + token
										_snake_case   						=		[]
									else:
										current_sub_tokens.append(lowerCAmelCase_  )
								out_string += self.sp_model.decode(lowerCAmelCase_  )
								return out_string.strip()


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None    ,   lowerCAmelCase_ = False  ):





								"""simple docstring"""






								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowerCAmelCase_    ,   token_ids_a=lowerCAmelCase_    ,   already_has_special_tokens=lowerCAmelCase_  )

								_snake_case   						=		[1] * len(self.prefix_tokens  )
								_snake_case   						=		[1] * len(self.suffix_tokens  )
								if token_ids_a is None:
									return prefix_ones + ([0] * len(lowerCAmelCase_  )) + suffix_ones
								return prefix_ones + ([0] * len(lowerCAmelCase_  )) + ([0] * len(lowerCAmelCase_  )) + suffix_ones


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								if token_ids_a is None:
									return self.prefix_tokens + token_ids_a + self.suffix_tokens
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		{self.convert_ids_to_tokens(lowerCAmelCase_  ): i for i in range(self.vocab_size  )}
								vocab.update(self.added_tokens_encoder  )
								return vocab


							def __getstate__(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.__dict__.copy()
								_snake_case   						=		None
								return state


							def __setstate__(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		d

								# for backward compatibility
								if not hasattr(self    ,   'sp_model_kwargs'  ):
									_snake_case   						=		{}

								_snake_case   						=		load_spm(self.spm_file    ,   self.sp_model_kwargs  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		Path(lowerCAmelCase_  )
								if not save_dir.is_dir():
									raise OSError(F'{save_directory} should be a directory'  )
								_snake_case   						=		save_dir / (
								    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
								)
								_snake_case   						=		save_dir / (
								    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
								)

								save_json(self.encoder    ,   lowerCAmelCase_  )

								if os.path.abspath(self.spm_file  ) != os.path.abspath(lowerCAmelCase_  ) and os.path.isfile(self.spm_file  ):
									copyfile(self.spm_file    ,   lowerCAmelCase_  )
								elif not os.path.isfile(self.spm_file  ):
									with open(lowerCAmelCase_    ,   'wb'  ) as fi:
										_snake_case   						=		self.sp_model.serialized_model_proto()
										fi.write(lowerCAmelCase_  )

								return (str(lowerCAmelCase_  ), str(lowerCAmelCase_  ))


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = "en"    ,   lowerCAmelCase_ = None    ,   lowerCAmelCase_ = "ro"    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		src_lang
								_snake_case   						=		tgt_lang
								self.set_src_lang_special_tokens(self.src_lang  )
								return super().prepare_seqaseq_batch(lowerCAmelCase_    ,   lowerCAmelCase_    ,   **lowerCAmelCase_  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   **lowerCAmelCase_  ):





								"""simple docstring"""






								if src_lang is None or tgt_lang is None:
									raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'  )
								_snake_case   						=		src_lang
								_snake_case   						=		self(lowerCAmelCase_    ,   add_special_tokens=lowerCAmelCase_    ,   **lowerCAmelCase_  )
								_snake_case   						=		self.get_lang_id(lowerCAmelCase_  )
								_snake_case   						=		tgt_lang_id
								return inputs


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								self.set_src_lang_special_tokens(self.src_lang  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								self.set_tgt_lang_special_tokens(self.tgt_lang  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.get_lang_token(lowerCAmelCase_  )
								_snake_case   						=		self.lang_token_to_id[lang_token]
								_snake_case   						=		[self.cur_lang_id]
								_snake_case   						=		[self.eos_token_id]


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.get_lang_token(lowerCAmelCase_  )
								_snake_case   						=		self.lang_token_to_id[lang_token]
								_snake_case   						=		[self.cur_lang_id]
								_snake_case   						=		[self.eos_token_id]


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								return self.lang_code_to_token[lang]




							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.get_lang_token(lowerCAmelCase_  )
								return self.lang_token_to_id[lang_token]







def 				SCREAMING_SNAKE_CASE__	(   __A			,				__A	)       ->	sentencepiece.SentencePieceProcessor:
	_snake_case   						=		sentencepiece.SentencePieceProcessor(**__A	)
	spm.Load(str(__A	)	)
	return spm







def 				SCREAMING_SNAKE_CASE__	(   __A	)       ->	Union[Dict, List]:
	with open(__A			,				'r'	) as f:
		return json.load(__A	)







def 				SCREAMING_SNAKE_CASE__	(   __A			,				__A	)       ->	None:
	with open(__A			,				'w'	) as f:
		json.dump(__A			,				__A			,				indent=2	)



'''simple docstring'''







def 				SCREAMING_SNAKE_CASE__	(   __A	)       ->	bool:
	return number & 1 == 0


if __name__ == "__main__":
					import doctest

					doctest.testmod()

