
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_mbartaa import MBartaaTokenizer
else:
	__a							  =						None


__a							  =						logging.get_logger(__name__)

__a							  =						{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

__a							  =						{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

__a							  =						{
    """facebook/mbart-large-50-one-to-many-mmt""": 1_0_2_4,
}

# fmt: off
__a							  =						["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]

class        UpperCamelCase__(    lowerCAmelCase__	):

	"""simple docstring"""

	_A  =		VOCAB_FILES_NAMES
	_A  =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A  =		PRETRAINED_VOCAB_FILES_MAP
	_A  =		["input_ids", "attention_mask"]
	_A  =		MBartaaTokenizer

	_A  =		[]
	_A  =		[]







	def __init__(      self				:			List[Any]							,			snake_case__				:			Optional[Any]=None							,			snake_case__				:			Dict=None							,			snake_case__				:			Tuple=None							,			snake_case__				:			Tuple=None							,			snake_case__				:			List[str]="</s>"							,			snake_case__				:			str="</s>"							,			snake_case__				:			Any="<s>"							,			snake_case__				:			List[Any]="<unk>"							,			snake_case__				:			str="<pad>"							,			snake_case__				:			List[Any]="<mask>"							,			**snake_case__				:			Optional[Any]							,			):
			"""simple docstring"""





			A				=AddedToken(__UpperCAmelCase							,			lstrip=__UpperCAmelCase							,			rstrip=__UpperCAmelCase ) if isinstance(__UpperCAmelCase							,			__UpperCAmelCase ) else mask_token

			A				=kwargs.get("additional_special_tokens"							,			[] )
			kwargs["additional_special_tokens"] += [
			    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
			]

			super().__init__(
			    __UpperCAmelCase							,			src_lang=__UpperCAmelCase							,			tgt_lang=__UpperCAmelCase							,			tokenizer_file=__UpperCAmelCase							,			eos_token=__UpperCAmelCase							,			sep_token=__UpperCAmelCase							,			cls_token=__UpperCAmelCase							,			unk_token=__UpperCAmelCase							,			pad_token=__UpperCAmelCase							,			mask_token=__UpperCAmelCase							,			**__UpperCAmelCase							,			)

			A				=vocab_file
			A				=False if not self.vocab_file else True

			A				={
			    lang_code: self.convert_tokens_to_ids(__UpperCAmelCase ) for lang_code in FAIRSEQ_LANGUAGE_CODES
			}

			A				=src_lang if src_lang is not None else "en_XX"
			A				=tgt_lang
			A				=self.lang_code_to_id[self._src_lang]
			self.set_src_lang_special_tokens(self._src_lang )







	@property
	def        _a					(      self				:			List[str] ):
			"""simple docstring"""





			return self._src_lang







	@src_lang.setter
	def        _a					(      self				:			Optional[int]							,			snake_case__				:			Optional[int] ):
			"""simple docstring"""





			A				=new_src_lang
			self.set_src_lang_special_tokens(self._src_lang )







	def        _a					(      self				:			Optional[Any]							,			snake_case__				:			List[Any]							,			snake_case__				:			Tuple = None ):
			"""simple docstring"""





			if token_ids_a is None:
					return self.prefix_tokens + token_ids_a + self.suffix_tokens
			# We don't expect to process pairs, but leave the pair logic for API consistency
			return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







	def        _a					(      self				:			Any							,			snake_case__				:			Tuple							,			snake_case__				:			Dict = "en_XX"							,			snake_case__				:			str = None							,			snake_case__				:			Any = "ro_RO"							,			**snake_case__				:			Optional[Any]							,			):
			"""simple docstring"""





			A				=src_lang
			A				=tgt_lang
			return super().prepare_seqaseq_batch(__UpperCAmelCase							,			__UpperCAmelCase							,			**__UpperCAmelCase )







	def        _a					(      self				:			List[Any] ):
			"""simple docstring"""





			return self.set_src_lang_special_tokens(self.src_lang )







	def        _a					(      self				:			Dict ):
			"""simple docstring"""





			return self.set_tgt_lang_special_tokens(self.tgt_lang )







	def        _a					(      self				:			List[Any]							,			snake_case__				:			Dict ):
			"""simple docstring"""





			A				=self.convert_tokens_to_ids(__UpperCAmelCase )
			A				=[self.cur_lang_code_id]
			A				=[self.eos_token_id]

			A				=self.convert_ids_to_tokens(self.prefix_tokens )
			A				=self.convert_ids_to_tokens(self.suffix_tokens )

			A				=processors.TemplateProcessing(
			    single=prefix_tokens_str + ["$A"] + suffix_tokens_str							,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str							,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,			self.prefix_tokens + self.suffix_tokens ) )							,			)







	def        _a					(      self				:			Optional[int]							,			snake_case__				:			Union[str, Any] ):
			"""simple docstring"""





			A				=self.convert_tokens_to_ids(__UpperCAmelCase )
			A				=[self.cur_lang_code_id]
			A				=[self.eos_token_id]

			A				=self.convert_ids_to_tokens(self.prefix_tokens )
			A				=self.convert_ids_to_tokens(self.suffix_tokens )

			A				=processors.TemplateProcessing(
			    single=prefix_tokens_str + ["$A"] + suffix_tokens_str							,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str							,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,			self.prefix_tokens + self.suffix_tokens ) )							,			)







	def        _a					(      self				:			Optional[Any]							,			snake_case__				:			Dict							,			snake_case__				:			Dict							,			snake_case__				:			int							,			snake_case__				:			str							,			**snake_case__				:			List[str] ):
			"""simple docstring"""





			if src_lang is None or tgt_lang is None:
					raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model" )
			A				=src_lang
			A				=self(__UpperCAmelCase							,			add_special_tokens=__UpperCAmelCase							,			return_tensors=__UpperCAmelCase							,			**__UpperCAmelCase )
			A				=self.convert_tokens_to_ids(__UpperCAmelCase )
			A				=tgt_lang_id
			return inputs







	def        _a					(      self				:			str							,			snake_case__				:			List[Any]							,			snake_case__				:			Union[str, Any] = None ):
			"""simple docstring"""





			if not self.can_save_slow_tokenizer:
					raise ValueError(
					    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
					    "tokenizer." )

			if not os.path.isdir(__UpperCAmelCase ):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
					return
			A				=os.path.join(
			    __UpperCAmelCase							,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

			if os.path.abspath(self.vocab_file ) != os.path.abspath(__UpperCAmelCase ):
					copyfile(self.vocab_file							,			__UpperCAmelCase )

			return (out_vocab_file,)






"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__A					=			{
    'configuration_donut_swin': ['DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'DonutSwinConfig'],
    'processing_donut': ['DonutProcessor'],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__A					=			[
			    'DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'DonutSwinModel',
			    'DonutSwinPreTrainedModel',
			]

try:
			if not is_vision_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__A					=			['DonutFeatureExtractor']
			__A					=			['DonutImageProcessor']


if TYPE_CHECKING:
			from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
			from .processing_donut import DonutProcessor

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_donut_swin import (
						    DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
						    DonutSwinModel,
						    DonutSwinPreTrainedModel,
						)

			try:
						if not is_vision_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .feature_extraction_donut import DonutFeatureExtractor
						from .image_processing_donut import DonutImageProcessor

else:
			import sys

			__A					=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



