



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_mbartaa import MBartaaTokenizer
else:
	_lowercase				:					List[str]     					=None


_lowercase				:					Tuple     					=logging.get_logger(__name__)

_lowercase				:					Tuple     					={"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_lowercase				:					Optional[int]     					={
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

_lowercase				:					Tuple     					={
    """facebook/mbart-large-50-one-to-many-mmt""": 10_24,
}

# fmt: off
_lowercase				:					int     					=["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]
class lowerCAmelCase_       (   __SCREAMING_SNAKE_CASE			):







	'''simple docstring'''




	A_       :     Optional[int]									=		VOCAB_FILES_NAMES
	A_       :     Union[str, Any]									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A_       :     int									=		PRETRAINED_VOCAB_FILES_MAP
	A_       :     str									=		['input_ids', 'attention_mask']
	A_       :     str									=		MBartaaTokenizer

	A_       :     int									=		[]
	A_       :     List[str]									=		[]

	def __init__(	self   ,       lowerCamelCase=None   ,       lowerCamelCase=None   ,       lowerCamelCase=None   ,       lowerCamelCase=None   ,       lowerCamelCase="</s>"   ,       lowerCamelCase="</s>"   ,       lowerCamelCase="<s>"   ,       lowerCamelCase="<unk>"   ,       lowerCamelCase="<pad>"   ,       lowerCamelCase="<mask>"   ,       **lowerCamelCase   ,       ):



				'''simple docstring'''


				a__					=  AddedToken(_A   ,       lstrip=_A   ,       rstrip=_A       ) if isinstance(_A   ,       _A       ) else mask_token

				a__					=  kwargs.get("""additional_special_tokens"""   ,       []       )
				kwargs["additional_special_tokens"] += [
				    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    _A   ,       src_lang=_A   ,       tgt_lang=_A   ,       tokenizer_file=_A   ,       eos_token=_A   ,       sep_token=_A   ,       cls_token=_A   ,       unk_token=_A   ,       pad_token=_A   ,       mask_token=_A   ,       **_A   ,       )

				a__					=  vocab_file
				a__					=  False if not self.vocab_file else True

				a__					=  {
				    lang_code: self.convert_tokens_to_ids(_A       ) for lang_code in FAIRSEQ_LANGUAGE_CODES
				}

				a__					=  src_lang if src_lang is not None else """en_XX"""
				a__					=  tgt_lang
				a__					=  self.lang_code_to_id[self._src_lang]
				self.set_src_lang_special_tokens(self._src_lang       )

	@property
	def 				_A			(	self       ):



				'''simple docstring'''


				return self._src_lang

	@src_lang.setter
	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				a__					=  new_src_lang
				self.set_src_lang_special_tokens(self._src_lang       )

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None       ):



				'''simple docstring'''


				if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = "en_XX"   ,       lowerCamelCase = None   ,       lowerCamelCase = "ro_RO"   ,       **lowerCamelCase   ,       ):



				'''simple docstring'''


				a__					=  src_lang
				a__					=  tgt_lang
				return super().prepare_seqaseq_batch(_A   ,       _A   ,       **_A       )

	def 				_A			(	self       ):



				'''simple docstring'''


				return self.set_src_lang_special_tokens(self.src_lang       )

	def 				_A			(	self       ):



				'''simple docstring'''


				return self.set_tgt_lang_special_tokens(self.tgt_lang       )

	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				a__					=  self.convert_tokens_to_ids(_A       )
				a__					=  [self.cur_lang_code_id]
				a__					=  [self.eos_token_id]

				a__					=  self.convert_ids_to_tokens(self.prefix_tokens       )
				a__					=  self.convert_ids_to_tokens(self.suffix_tokens       )

				a__					=  processors.TemplateProcessing(
				    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str   ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str   ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,       self.prefix_tokens + self.suffix_tokens       )       )   ,       )

	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				a__					=  self.convert_tokens_to_ids(_A       )
				a__					=  [self.cur_lang_code_id]
				a__					=  [self.eos_token_id]

				a__					=  self.convert_ids_to_tokens(self.prefix_tokens       )
				a__					=  self.convert_ids_to_tokens(self.suffix_tokens       )

				a__					=  processors.TemplateProcessing(
				    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str   ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str   ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,       self.prefix_tokens + self.suffix_tokens       )       )   ,       )

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase   ,       lowerCamelCase   ,       lowerCamelCase   ,       **lowerCamelCase       ):



				'''simple docstring'''


				if src_lang is None or tgt_lang is None:
							raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""       )
				a__					=  src_lang
				a__					=  self(_A   ,       add_special_tokens=_A   ,       return_tensors=_A   ,       **_A       )
				a__					=  self.convert_tokens_to_ids(_A       )
				a__					=  tgt_lang_id
				return inputs







	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None       ):



				'''simple docstring'''


				if not self.can_save_slow_tokenizer:
							raise ValueError(
							    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
							    """tokenizer."""       )

				if not os.path.isdir(_A       ):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'       )
							return
				a__					=  os.path.join(
				    _A   ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

				if os.path.abspath(self.vocab_file       ) != os.path.abspath(_A       ):
							copyfile(self.vocab_file   ,       _A       )

				return (out_vocab_file,)



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__UpperCAmelCase   						=     {
    "configuration_donut_swin": ["DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP", "DonutSwinConfig"],
    "processing_donut": ["DonutProcessor"],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __UpperCAmelCase   						=     [
       "DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST",
       "DonutSwinModel",
       "DonutSwinPreTrainedModel",
   ]

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __UpperCAmelCase   						=     ["DonutFeatureExtractor"]
   __UpperCAmelCase   						=     ["DonutImageProcessor"]


if TYPE_CHECKING:
   from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
   from .processing_donut import DonutProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_donut_swin import (
          DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
          DonutSwinModel,
          DonutSwinPreTrainedModel,
      )

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_donut import DonutFeatureExtractor
      from .image_processing_donut import DonutImageProcessor

else:
   import sys

   __UpperCAmelCase   						=     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


