
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_mbartaa import MBartaaTokenizer
else:
      a__				:      Tuple    		=     None


a__				:      Optional[Any]    		=     logging.get_logger(__name__)

a__				:      Any    		=     {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

a__				:      List[str]    		=     {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
        ),
    },
}

a__				:      Tuple    		=     {
    '''facebook/mbart-large-50-one-to-many-mmt''': 1_024,
}

# fmt: off
a__				:      Optional[int]    		=     ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']






class     a_       (			a__							):



       """simple docstring"""
       __SCREAMING_SNAKE_CASE       :					Optional[int]						=  VOCAB_FILES_NAMES
       __SCREAMING_SNAKE_CASE       :					List[Any]						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __SCREAMING_SNAKE_CASE       :					List[str]						=  PRETRAINED_VOCAB_FILES_MAP
       __SCREAMING_SNAKE_CASE       :					Union[str, Any]						=  ['input_ids', 'attention_mask']
       __SCREAMING_SNAKE_CASE       :					int						=  MBartaaTokenizer

       __SCREAMING_SNAKE_CASE       :					List[int]						=  []
       __SCREAMING_SNAKE_CASE       :					List[int]						=  []






       def __init__(		self				,       _lowerCamelCase=None				,       _lowerCamelCase=None				,       _lowerCamelCase=None				,       _lowerCamelCase=None				,       _lowerCamelCase="</s>"				,       _lowerCamelCase="</s>"				,       _lowerCamelCase="<s>"				,       _lowerCamelCase="<unk>"				,       _lowerCamelCase="<pad>"				,       _lowerCamelCase="<mask>"				,       **_lowerCamelCase				,       )					->int:
           # Mask token behave like a normal word, i.e. include the space before it
           SCREAMING_SNAKE_CASE				:				List[Any]				      = AddedToken(_lowerCamelCase				,       lstrip=_lowerCamelCase				,       rstrip=_lowerCamelCase   ) if isinstance(_lowerCamelCase				,       _lowerCamelCase   ) else mask_token

           SCREAMING_SNAKE_CASE				:				str				      = kwargs.get('''additional_special_tokens'''				,       []   )
           kwargs["additional_special_tokens"] += [
               code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               _lowerCamelCase				,       src_lang=_lowerCamelCase				,       tgt_lang=_lowerCamelCase				,       tokenizer_file=_lowerCamelCase				,       eos_token=_lowerCamelCase				,       sep_token=_lowerCamelCase				,       cls_token=_lowerCamelCase				,       unk_token=_lowerCamelCase				,       pad_token=_lowerCamelCase				,       mask_token=_lowerCamelCase				,       **_lowerCamelCase				,       )

           SCREAMING_SNAKE_CASE				:				List[str]				      = vocab_file
           SCREAMING_SNAKE_CASE				:				List[str]				      = False if not self.vocab_file else True

           SCREAMING_SNAKE_CASE				:				int				      = {
               lang_code: self.convert_tokens_to_ids(_lowerCamelCase   ) for lang_code in FAIRSEQ_LANGUAGE_CODES
           }

           SCREAMING_SNAKE_CASE				:				str				      = src_lang if src_lang is not None else '''en_XX'''
           SCREAMING_SNAKE_CASE				:				List[Any]				      = tgt_lang
           SCREAMING_SNAKE_CASE				:				List[Any]				      = self.lang_code_to_id[self._src_lang]
           self.set_src_lang_special_tokens(self._src_lang   )






       @property
       def 	__lowerCAmelCase     (		self   )					->str:
           return self._src_lang






       @src_lang.setter
       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->None:
           SCREAMING_SNAKE_CASE				:				Dict				      = new_src_lang
           self.set_src_lang_special_tokens(self._src_lang   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase = None   )					->List[int]:
           if token_ids_a is None:
               return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase = "en_XX"				,       _lowerCamelCase = None				,       _lowerCamelCase = "ro_RO"				,       **_lowerCamelCase				,       )					->BatchEncoding:
           SCREAMING_SNAKE_CASE				:				List[str]				      = src_lang
           SCREAMING_SNAKE_CASE				:				int				      = tgt_lang
           return super().prepare_seqaseq_batch(_lowerCamelCase				,       _lowerCamelCase				,       **_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->Union[str, Any]:
           return self.set_src_lang_special_tokens(self.src_lang   )






       def 	__lowerCAmelCase     (		self   )					->Union[str, Any]:
           return self.set_tgt_lang_special_tokens(self.tgt_lang   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->None:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.convert_tokens_to_ids(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = [self.cur_lang_code_id]
           SCREAMING_SNAKE_CASE				:				List[Any]				      = [self.eos_token_id]

           SCREAMING_SNAKE_CASE				:				List[str]				      = self.convert_ids_to_tokens(self.prefix_tokens   )
           SCREAMING_SNAKE_CASE				:				Dict				      = self.convert_ids_to_tokens(self.suffix_tokens   )

           SCREAMING_SNAKE_CASE				:				Dict				      = processors.TemplateProcessing(
               single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,       self.prefix_tokens + self.suffix_tokens   )   )				,       )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->None:
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = self.convert_tokens_to_ids(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Dict				      = [self.cur_lang_code_id]
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = [self.eos_token_id]

           SCREAMING_SNAKE_CASE				:				List[str]				      = self.convert_ids_to_tokens(self.prefix_tokens   )
           SCREAMING_SNAKE_CASE				:				List[Any]				      = self.convert_ids_to_tokens(self.suffix_tokens   )

           SCREAMING_SNAKE_CASE				:				int				      = processors.TemplateProcessing(
               single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,       self.prefix_tokens + self.suffix_tokens   )   )				,       )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       **_lowerCamelCase   )					->Any:
           if src_lang is None or tgt_lang is None:
               raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''   )
           SCREAMING_SNAKE_CASE				:				Any				      = src_lang
           SCREAMING_SNAKE_CASE				:				int				      = self(_lowerCamelCase				,       add_special_tokens=_lowerCamelCase				,       return_tensors=_lowerCamelCase				,       **_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = self.convert_tokens_to_ids(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = tgt_lang_id
           return inputs






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase = None   )					->Tuple[str]:
           if not self.can_save_slow_tokenizer:
               raise ValueError(
                   '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                   '''tokenizer.'''   )

           if not os.path.isdir(_lowerCamelCase   ):
               logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
               return
           SCREAMING_SNAKE_CASE				:				int				      = os.path.join(
               _lowerCamelCase				,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

           if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowerCamelCase   ):
               copyfile(self.vocab_file				,       _lowerCamelCase   )

           return (out_vocab_file,)


# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
      import tensorflow as tf






class     a_       (			TensorFormatter[Mapping, 'tf.Tensor', Mapping]							):



       """simple docstring"""






       def __init__(		self				,       _lowerCamelCase=None				,       **_lowerCamelCase   )					->int:
           super().__init__(features=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Any				      = tf_tensor_kwargs
           import tensorflow as tf  # noqa: import tf at initialization






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Tuple:
           import tensorflow as tf

           if isinstance(_lowerCamelCase				,       _lowerCamelCase   ) and column:
               if all(
                   isinstance(_lowerCamelCase				,       tf.Tensor   ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column   ):
                   return tf.stack(_lowerCamelCase   )
               elif all(
                   isinstance(_lowerCamelCase				,       (tf.Tensor, tf.RaggedTensor)   ) and x.ndim == 1 and x.dtype == column[0].dtype
                   for x in column   ):
                   # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
                   return tf.ragged.stack(_lowerCamelCase   )

           return column






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Union[str, Any]:
           import tensorflow as tf

           if value is None:
               return value

           SCREAMING_SNAKE_CASE				:				int				      = {}

           if isinstance(_lowerCamelCase				,       (np.number, np.ndarray)   ) and np.issubdtype(value.dtype				,       np.integer   ):
               SCREAMING_SNAKE_CASE				:				List[Any]				      = {'''dtype''': tf.intaa}
           elif isinstance(_lowerCamelCase				,       (np.number, np.ndarray)   ) and np.issubdtype(value.dtype				,       np.floating   ):
               SCREAMING_SNAKE_CASE				:				Optional[int]				      = {'''dtype''': tf.floataa}
           elif config.PIL_AVAILABLE and "PIL" in sys.modules:
               import PIL.Image

               if isinstance(_lowerCamelCase				,       PIL.Image.Image   ):
                   SCREAMING_SNAKE_CASE				:				Any				      = np.asarray(_lowerCamelCase   )

           return tf.convert_to_tensor(_lowerCamelCase				,       **{**default_dtype, **self.tf_tensor_kwargs}   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Tuple:
           import tensorflow as tf

           # support for torch, tf, jax etc.
           if config.TORCH_AVAILABLE and "torch" in sys.modules:
               import torch

               if isinstance(_lowerCamelCase				,       torch.Tensor   ):
                   return self._tensorize(data_struct.detach().cpu().numpy()[()]   )
           if hasattr(_lowerCamelCase				,       '''__array__'''   ) and not isinstance(_lowerCamelCase				,       tf.Tensor   ):
               SCREAMING_SNAKE_CASE				:				int				      = data_struct.__array__()
           # support for nested types like struct of list of struct
           if isinstance(_lowerCamelCase				,       np.ndarray   ):
               if data_struct.dtype == object:  # tf tensors cannot be instantied from an array of objects
                   return self._consolidate([self.recursive_tensorize(_lowerCamelCase   ) for substruct in data_struct]   )
           elif isinstance(_lowerCamelCase				,       (list, tuple)   ):
               return self._consolidate([self.recursive_tensorize(_lowerCamelCase   ) for substruct in data_struct]   )
           return self._tensorize(_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Optional[int]:
           return map_nested(self._recursive_tensorize				,       _lowerCamelCase				,       map_list=_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Mapping:
           SCREAMING_SNAKE_CASE				:				str				      = self.numpy_arrow_extractor().extract_row(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				List[str]				      = self.python_features_decoder.decode_row(_lowerCamelCase   )
           return self.recursive_tensorize(_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->"tf.Tensor":
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = self.numpy_arrow_extractor().extract_column(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				str				      = self.python_features_decoder.decode_column(_lowerCamelCase				,       pa_table.column_names[0]   )
           SCREAMING_SNAKE_CASE				:				str				      = self.recursive_tensorize(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				int				      = self._consolidate(_lowerCamelCase   )
           return column






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Mapping:
           SCREAMING_SNAKE_CASE				:				Tuple				      = self.numpy_arrow_extractor().extract_batch(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Dict				      = self.python_features_decoder.decode_batch(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				int				      = self.recursive_tensorize(_lowerCamelCase   )
           for column_name in batch:
               SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self._consolidate(batch[column_name]   )
           return batch
