

import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__			:  List[Any]					     =     logging.get_logger(__name__)

UpperCAmelCase__			:  int					     =     {
    """source_spm""": """source.spm""",
    """target_spm""": """target.spm""",
    """vocab""": """vocab.json""",
    """target_vocab_file""": """target_vocab.json""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

UpperCAmelCase__			:  Tuple					     =     {
    """source_spm""": {
        """Helsinki-NLP/opus-mt-en-de""": """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm"""
    },
    """target_spm""": {
        """Helsinki-NLP/opus-mt-en-de""": """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm"""
    },
    """vocab""": {
        """Helsinki-NLP/opus-mt-en-de""": """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json"""
    },
    """tokenizer_config_file""": {
        """Helsinki-NLP/opus-mt-en-de""": (
            """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json"""
        )
    },
}

UpperCAmelCase__			:  int					     =     {"""Helsinki-NLP/opus-mt-en-de""": 512}
UpperCAmelCase__			:  Union[str, Any]					     =     {}







class 							a__							( UpperCAmelCase							):







  """simple docstring"""




  UpperCAmelCase__   :			Union[str, Any]      							=VOCAB_FILES_NAMES
  UpperCAmelCase__   :			List[Any]      							=PRETRAINED_VOCAB_FILES_MAP
  UpperCAmelCase__   :			int      							=PRETRAINED_INIT_CONFIGURATION
  UpperCAmelCase__   :			int      							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCAmelCase__   :			Tuple      							=["""input_ids""", """attention_mask"""]
  UpperCAmelCase__   :			Tuple      							=re.compile(""">>.+<<"""							)  # type: re.Pattern


  def __init__(      self     :			Dict					,					UpperCAmelCase__     :			str					,					UpperCAmelCase__     :			Optional[Any]					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			Union[str, Any]=None					,					UpperCAmelCase__     :			Optional[Any]=None					,					UpperCAmelCase__     :			Tuple=None					,					UpperCAmelCase__     :			Optional[Any]="<unk>"					,					UpperCAmelCase__     :			List[str]="</s>"					,					UpperCAmelCase__     :			Any="<pad>"					,					UpperCAmelCase__     :			Union[str, Any]=5_1_2					,					UpperCAmelCase__     :			Optional[Dict[str, Any]] = None					,					UpperCAmelCase__     :			Dict=False					,					**UpperCAmelCase__     :			List[str]					,					)       ->None:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[str]	  = {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
            source_lang=UpperCAmelCase__					,					target_lang=UpperCAmelCase__					,					unk_token=UpperCAmelCase__					,					eos_token=UpperCAmelCase__					,					pad_token=UpperCAmelCase__					,					model_max_length=UpperCAmelCase__					,					sp_model_kwargs=self.sp_model_kwargs					,					target_vocab_file=UpperCAmelCase__					,					separate_vocabs=UpperCAmelCase__					,					**UpperCAmelCase__					,					)
        assert Path(UpperCAmelCase__		).exists(), f"cannot find spm source {source_spm}"

        SCREAMING_SNAKE_CASE			:   Any	  = separate_vocabs
        SCREAMING_SNAKE_CASE			:   Optional[int]	  = load_json(UpperCAmelCase__		)
        if self.unk_token not in self.encoder:
              raise KeyError("""<unk> token must be in vocab"""		)
        assert self.pad_token in self.encoder

        if separate_vocabs:
              SCREAMING_SNAKE_CASE			:   str	  = load_json(UpperCAmelCase__		)
              SCREAMING_SNAKE_CASE			:   str	  = {v: k for k, v in self.target_encoder.items()}
              SCREAMING_SNAKE_CASE			:   Tuple	  = []
        else:
              SCREAMING_SNAKE_CASE			:   List[str]	  = {v: k for k, v in self.encoder.items()}
              SCREAMING_SNAKE_CASE			:   list	  = [k for k in self.encoder if k.startswith(""">>"""		) and k.endswith("""<<"""		)]

        SCREAMING_SNAKE_CASE			:   Dict	  = source_lang
        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = target_lang
        SCREAMING_SNAKE_CASE			:   str	  = [source_spm, target_spm]

        # load SentencePiece model for pre-processing
        SCREAMING_SNAKE_CASE			:   int	  = load_spm(UpperCAmelCase__					,					self.sp_model_kwargs		)
        SCREAMING_SNAKE_CASE			:   Dict	  = load_spm(UpperCAmelCase__					,					self.sp_model_kwargs		)
        SCREAMING_SNAKE_CASE			:   Optional[int]	  = self.spm_source
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = self.encoder

        # Multilingual target side: default to using first supported language code.

        self._setup_normalizer()


  def 			_lowercase   (      self     :			Optional[int]		)       ->Union[str, Any]:
        """simple docstring"""







        try:
              from sacremoses import MosesPunctNormalizer

              SCREAMING_SNAKE_CASE			:   Optional[int]	  = MosesPunctNormalizer(self.source_lang		).normalize
        except (ImportError, FileNotFoundError):
              warnings.warn("""Recommended: pip install sacremoses."""		)
              SCREAMING_SNAKE_CASE			:   Optional[int]	  = lambda UpperCAmelCase__		: x


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			str		)       ->str:
        """simple docstring"""







        return self.punc_normalizer(UpperCAmelCase__		) if x else ""


  def 			_lowercase   (      self     :			Union[str, Any]					,					UpperCAmelCase__     :			Tuple		)       ->Dict:
        """simple docstring"""







        return self.current_encoder.get(UpperCAmelCase__					,					self.current_encoder[self.unk_token]		)


  def 			_lowercase   (      self     :			Any					,					UpperCAmelCase__     :			str		)       ->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Dict	  = self.language_code_re.match(UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE			:   list	  = [match.group(0		)] if match else []
        return code, self.language_code_re.sub(""""""					,					UpperCAmelCase__		)


  def 			_lowercase   (      self     :			List[str]					,					UpperCAmelCase__     :			str		)       ->List[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE  ,						SCREAMING_SNAKE_CASE			:   str	  = self.remove_language_code(UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE			:   int	  = self.current_spm.encode(UpperCAmelCase__					,					out_type=UpperCAmelCase__		)
        return code + pieces


  def 			_lowercase   (      self     :			Dict					,					UpperCAmelCase__     :			int		)       ->str:
        """simple docstring"""







        return self.decoder.get(UpperCAmelCase__					,					self.unk_token		)


  def 			_lowercase   (      self     :			List[Any]					,					UpperCAmelCase__     :			List[str]					,					**UpperCAmelCase__     :			List[Any]		)       ->Dict:
        """simple docstring"""







        return super().batch_decode(UpperCAmelCase__					,					**UpperCAmelCase__		)


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			str					,					**UpperCAmelCase__     :			List[Any]		)       ->Optional[int]:
        """simple docstring"""







        return super().decode(UpperCAmelCase__					,					**UpperCAmelCase__		)


  def 			_lowercase   (      self     :			int					,					UpperCAmelCase__     :			List[str]		)       ->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Tuple	  = self.spm_source if self._decode_use_source_tokenizer else self.spm_target
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = []
        SCREAMING_SNAKE_CASE			:   Tuple	  = """"""
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += sp_model.decode_pieces(UpperCAmelCase__		) + token + " "
                    SCREAMING_SNAKE_CASE			:   List[str]	  = []
              else:
                    current_sub_tokens.append(UpperCAmelCase__		)
        out_string += sp_model.decode_pieces(UpperCAmelCase__		)
        return out_string.strip()


  def 			_lowercase   (      self     :			int					,					UpperCAmelCase__     :			Tuple					,					UpperCAmelCase__     :			Union[str, Any]=None		)       ->List[int]:
        """simple docstring"""







        if token_ids_a is None:
              return token_ids_a + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_a + token_ids_a + [self.eos_token_id]


  def 			_lowercase   (      self     :			Tuple		)       ->Tuple:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   int	  = self.spm_source
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = self.encoder


  def 			_lowercase   (      self     :			Optional[int]		)       ->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[Any]	  = self.spm_target
        if self.separate_vocabs:
              SCREAMING_SNAKE_CASE			:   List[Any]	  = self.target_encoder


  @property
  def 			_lowercase   (      self     :			List[Any]		)       ->int:
        """simple docstring"""







        return len(self.encoder		)


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			str					,					UpperCAmelCase__     :			Optional[str] = None		)       ->Tuple[str]:
        """simple docstring"""







        if not os.path.isdir(UpperCAmelCase__		):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
              return
        SCREAMING_SNAKE_CASE			:   Tuple	  = []

        if self.separate_vocabs:
              SCREAMING_SNAKE_CASE			:   Dict	  = os.path.join(
                  UpperCAmelCase__					,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab"""]					,					)
              SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = os.path.join(
                  UpperCAmelCase__					,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""target_vocab_file"""]					,					)
              save_json(self.encoder					,					UpperCAmelCase__		)
              save_json(self.target_encoder					,					UpperCAmelCase__		)
              saved_files.append(UpperCAmelCase__		)
              saved_files.append(UpperCAmelCase__		)
        else:
              SCREAMING_SNAKE_CASE			:   Tuple	  = os.path.join(
                  UpperCAmelCase__					,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab"""]		)
              save_json(self.encoder					,					UpperCAmelCase__		)
              saved_files.append(UpperCAmelCase__		)

        for spm_save_filename, spm_orig_path, spm_model in zip(
            [VOCAB_FILES_NAMES["""source_spm"""], VOCAB_FILES_NAMES["""target_spm"""]]					,					self.spm_files					,					[self.spm_source, self.spm_target]					,					):
              SCREAMING_SNAKE_CASE			:   Any	  = os.path.join(
                  UpperCAmelCase__					,					(filename_prefix + """-""" if filename_prefix else """""") + spm_save_filename		)
              if os.path.abspath(UpperCAmelCase__		) != os.path.abspath(UpperCAmelCase__		) and os.path.isfile(UpperCAmelCase__		):
                    copyfile(UpperCAmelCase__					,					UpperCAmelCase__		)
                    saved_files.append(UpperCAmelCase__		)
              elif not os.path.isfile(UpperCAmelCase__		):
                    with open(UpperCAmelCase__					,					"""wb"""		) as fi:
                          SCREAMING_SNAKE_CASE			:   List[str]	  = spm_model.serialized_model_proto()
                          fi.write(UpperCAmelCase__		)
                    saved_files.append(UpperCAmelCase__		)

        return tuple(UpperCAmelCase__		)


  def 			_lowercase   (      self     :			int		)       ->Dict:
        """simple docstring"""







        return self.get_src_vocab()


  def 			_lowercase   (      self     :			str		)       ->Union[str, Any]:
        """simple docstring"""







        return dict(self.encoder					,					**self.added_tokens_encoder		)


  def 			_lowercase   (      self     :			str		)       ->Tuple:
        """simple docstring"""







        return dict(self.target_encoder					,					**self.added_tokens_decoder		)


  def __getstate__(      self     :			Optional[int]		)       ->Dict:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Dict	  = self.__dict__.copy()
        state.update(
            {k: None for k in ["""spm_source""", """spm_target""", """current_spm""", """punc_normalizer""", """target_vocab_file"""]}		)
        return state


  def __setstate__(      self     :			List[Any]					,					UpperCAmelCase__     :			Dict		)       ->None:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[str]	  = d

        # for backward compatibility
        if not hasattr(self					,					"""sp_model_kwargs"""		):
              SCREAMING_SNAKE_CASE			:   Optional[int]	  = {}

        SCREAMING_SNAKE_CASE  ,						SCREAMING_SNAKE_CASE			:   List[Any]	  = (load_spm(UpperCAmelCase__					,					self.sp_model_kwargs		) for f in self.spm_files)
        SCREAMING_SNAKE_CASE			:   Any	  = self.spm_source
        self._setup_normalizer()


  def 			_lowercase   (      self     :			Dict					,					*UpperCAmelCase__     :			str					,					**UpperCAmelCase__     :			List[str]		)       ->Dict:
        """simple docstring"""







        return 1


  def 			_lowercase   (      self     :			List[Any]					,					UpperCAmelCase__     :			Dict		)       ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = set(self.all_special_ids		)  # call it once instead of inside list comp
        all_special_ids.remove(self.unk_token_id		)  # <unk> is only sometimes special
        return [1 if x in all_special_ids else 0 for x in seq]




  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			List					,					UpperCAmelCase__     :			Optional[List] = None					,					UpperCAmelCase__     :			bool = False		)       ->List[int]:
        """simple docstring"""







        if already_has_special_tokens:
              return self._special_token_mask(UpperCAmelCase__		)
        elif token_ids_a is None:
              return self._special_token_mask(UpperCAmelCase__		) + [1]
        else:
              return self._special_token_mask(token_ids_a + token_ids_a		) + [1]






def        __lowercase   (  _A    ,     _A      )  ->    sentencepiece.SentencePieceProcessor:
      SCREAMING_SNAKE_CASE			:   List[str]	  = sentencepiece.SentencePieceProcessor(**_A      )
      spm.Load(_A      )
      return spm






def        __lowercase   (  _A    ,     _A      )  ->    None:
      with open(_A    ,     """w"""      ) as f:
            json.dump(_A    ,     _A    ,     indent=2      )







def        __lowercase   (  _A      )  ->    Union[Dict, List]:
      with open(_A    ,     """r"""      ) as f:
            return json.load(_A      )







import safs







class 							a__							( safs.SaFileSystem							):







  """simple docstring"""




