import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_mbartaa import MBartaaTokenizer
else:
						lowerCAmelCase__  	=      None


lowerCAmelCase__  	=      logging.get_logger(__name__)

lowerCAmelCase__  	=      {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase__  	=      {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

lowerCAmelCase__  	=      {
    "facebook/mbart-large-50-one-to-many-mmt": 1_0_2_4,
}

# fmt: off
lowerCAmelCase__  	=      ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]
class 	_a  (							lowerCamelCase_					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					VOCAB_FILES_NAMES
	__SCREAMING_SNAKE_CASE              =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__SCREAMING_SNAKE_CASE              =					PRETRAINED_VOCAB_FILES_MAP
	__SCREAMING_SNAKE_CASE              =					['input_ids', 'attention_mask']
	__SCREAMING_SNAKE_CASE              =					MBartaaTokenizer

	__SCREAMING_SNAKE_CASE              =					[]
	__SCREAMING_SNAKE_CASE              =					[]


	def __init__(   self       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_="</s>"       ,  lowerCAmelCase_="</s>"       ,  lowerCAmelCase_="<s>"       ,  lowerCAmelCase_="<unk>"       ,  lowerCAmelCase_="<pad>"       ,  lowerCAmelCase_="<mask>"       ,  **lowerCAmelCase_       ,  ):
		# Mask token behave like a normal word, i.e. include the space before it
		_lowercase   =AddedToken(lowerCAmelCase_       ,  lstrip=lowerCAmelCase_       ,  rstrip=lowerCAmelCase_       ) if isinstance(lowerCAmelCase_       ,  lowerCAmelCase_       ) else mask_token

		_lowercase   =kwargs.get("additional_special_tokens"       ,  []       )
		kwargs["additional_special_tokens"] += [
		    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
		]

		super().__init__(
		    lowerCAmelCase_       ,  src_lang=lowerCAmelCase_       ,  tgt_lang=lowerCAmelCase_       ,  tokenizer_file=lowerCAmelCase_       ,  eos_token=lowerCAmelCase_       ,  sep_token=lowerCAmelCase_       ,  cls_token=lowerCAmelCase_       ,  unk_token=lowerCAmelCase_       ,  pad_token=lowerCAmelCase_       ,  mask_token=lowerCAmelCase_       ,  **lowerCAmelCase_       ,  )

		_lowercase   =vocab_file
		_lowercase   =False if not self.vocab_file else True

		_lowercase   ={
		    lang_code: self.convert_tokens_to_ids(lowerCAmelCase_       ) for lang_code in FAIRSEQ_LANGUAGE_CODES
		}

		_lowercase   =src_lang if src_lang is not None else "en_XX"
		_lowercase   =tgt_lang
		_lowercase   =self.lang_code_to_id[self._src_lang]
		self.set_src_lang_special_tokens(self._src_lang       )


	@property
	def __lowerCAmelCase      (   self       ):
		return self._src_lang


	@src_lang.setter
	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   =new_src_lang
		self.set_src_lang_special_tokens(self._src_lang       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_ = None       ):
		if token_ids_a is None:
			return self.prefix_tokens + token_ids_a + self.suffix_tokens
		# We don't expect to process pairs, but leave the pair logic for API consistency
		return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_ = "en_XX"       ,  lowerCAmelCase_ = None       ,  lowerCAmelCase_ = "ro_RO"       ,  **lowerCAmelCase_       ,  ):
		_lowercase   =src_lang
		_lowercase   =tgt_lang
		return super().prepare_seqaseq_batch(lowerCAmelCase_       ,  lowerCAmelCase_       ,  **lowerCAmelCase_       )


	def __lowerCAmelCase      (   self       ):
		return self.set_src_lang_special_tokens(self.src_lang       )


	def __lowerCAmelCase      (   self       ):
		return self.set_tgt_lang_special_tokens(self.tgt_lang       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   =self.convert_tokens_to_ids(lowerCAmelCase_       )
		_lowercase   =[self.cur_lang_code_id]
		_lowercase   =[self.eos_token_id]

		_lowercase   =self.convert_ids_to_tokens(self.prefix_tokens       )
		_lowercase   =self.convert_ids_to_tokens(self.suffix_tokens       )

		_lowercase   =processors.TemplateProcessing(
		    single=prefix_tokens_str + ["$A"] + suffix_tokens_str       ,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str       ,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,  self.prefix_tokens + self.suffix_tokens       )       )       ,  )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   =self.convert_tokens_to_ids(lowerCAmelCase_       )
		_lowercase   =[self.cur_lang_code_id]
		_lowercase   =[self.eos_token_id]

		_lowercase   =self.convert_ids_to_tokens(self.prefix_tokens       )
		_lowercase   =self.convert_ids_to_tokens(self.suffix_tokens       )

		_lowercase   =processors.TemplateProcessing(
		    single=prefix_tokens_str + ["$A"] + suffix_tokens_str       ,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str       ,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,  self.prefix_tokens + self.suffix_tokens       )       )       ,  )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  **lowerCAmelCase_       ):
		if src_lang is None or tgt_lang is None:
			raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"       )
		_lowercase   =src_lang
		_lowercase   =self(lowerCAmelCase_       ,  add_special_tokens=lowerCAmelCase_       ,  return_tensors=lowerCAmelCase_       ,  **lowerCAmelCase_       )
		_lowercase   =self.convert_tokens_to_ids(lowerCAmelCase_       )
		_lowercase   =tgt_lang_id
		return inputs


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_ = None       ):
		if not self.can_save_slow_tokenizer:
			raise ValueError(
			    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
			    "tokenizer."       )

		if not os.path.isdir(lowerCAmelCase_       ):
			logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
			return
		_lowercase   =os.path.join(
		    lowerCAmelCase_       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

		if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowerCAmelCase_       ):
			copyfile(self.vocab_file       ,  lowerCAmelCase_       )

		return (out_vocab_file,)










from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


__lowerCamelCase :    Tuple      						=      {
    '''configuration_utils''': ['''GenerationConfig'''],
    '''streamers''': ['''TextIteratorStreamer''', '''TextStreamer'''],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase :    Any      						=      [
	    '''Constraint''',
	    '''ConstraintListState''',
	    '''DisjunctiveConstraint''',
	    '''PhrasalConstraint''',
	]
	__lowerCamelCase :    int      						=      [
	    '''BeamHypotheses''',
	    '''BeamScorer''',
	    '''BeamSearchScorer''',
	    '''ConstrainedBeamSearchScorer''',
	]
	__lowerCamelCase :    Union[str, Any]      						=      [
	    '''EpsilonLogitsWarper''',
	    '''EtaLogitsWarper''',
	    '''ForcedBOSTokenLogitsProcessor''',
	    '''ForcedEOSTokenLogitsProcessor''',
	    '''HammingDiversityLogitsProcessor''',
	    '''InfNanRemoveLogitsProcessor''',
	    '''LogitsProcessor''',
	    '''LogitsProcessorList''',
	    '''LogitsWarper''',
	    '''MinLengthLogitsProcessor''',
	    '''MinNewTokensLengthLogitsProcessor''',
	    '''NoBadWordsLogitsProcessor''',
	    '''NoRepeatNGramLogitsProcessor''',
	    '''PrefixConstrainedLogitsProcessor''',
	    '''RepetitionPenaltyLogitsProcessor''',
	    '''SequenceBiasLogitsProcessor''',
	    '''EncoderRepetitionPenaltyLogitsProcessor''',
	    '''TemperatureLogitsWarper''',
	    '''TopKLogitsWarper''',
	    '''TopPLogitsWarper''',
	    '''TypicalLogitsWarper''',
	    '''EncoderNoRepeatNGramLogitsProcessor''',
	    '''ExponentialDecayLengthPenalty''',
	    '''LogitNormalization''',
	]
	__lowerCamelCase :    Union[str, Any]      						=      [
	    '''MaxNewTokensCriteria''',
	    '''MaxLengthCriteria''',
	    '''MaxTimeCriteria''',
	    '''StoppingCriteria''',
	    '''StoppingCriteriaList''',
	    '''validate_stopping_criteria''',
	]
	__lowerCamelCase :    Union[str, Any]      						=      [
	    '''GenerationMixin''',
	    '''top_k_top_p_filtering''',
	    '''GreedySearchEncoderDecoderOutput''',
	    '''GreedySearchDecoderOnlyOutput''',
	    '''SampleEncoderDecoderOutput''',
	    '''SampleDecoderOnlyOutput''',
	    '''BeamSearchEncoderDecoderOutput''',
	    '''BeamSearchDecoderOnlyOutput''',
	    '''BeamSampleEncoderDecoderOutput''',
	    '''BeamSampleDecoderOnlyOutput''',
	    '''ContrastiveSearchEncoderDecoderOutput''',
	    '''ContrastiveSearchDecoderOnlyOutput''',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase :    Tuple      						=      [
	    '''TFForcedBOSTokenLogitsProcessor''',
	    '''TFForcedEOSTokenLogitsProcessor''',
	    '''TFLogitsProcessor''',
	    '''TFLogitsProcessorList''',
	    '''TFLogitsWarper''',
	    '''TFMinLengthLogitsProcessor''',
	    '''TFNoBadWordsLogitsProcessor''',
	    '''TFNoRepeatNGramLogitsProcessor''',
	    '''TFRepetitionPenaltyLogitsProcessor''',
	    '''TFTemperatureLogitsWarper''',
	    '''TFTopKLogitsWarper''',
	    '''TFTopPLogitsWarper''',
	    '''TFForceTokensLogitsProcessor''',
	    '''TFSuppressTokensAtBeginLogitsProcessor''',
	    '''TFSuppressTokensLogitsProcessor''',
	]
	__lowerCamelCase :    Any      						=      [
	    '''TFGenerationMixin''',
	    '''tf_top_k_top_p_filtering''',
	    '''TFGreedySearchDecoderOnlyOutput''',
	    '''TFGreedySearchEncoderDecoderOutput''',
	    '''TFSampleEncoderDecoderOutput''',
	    '''TFSampleDecoderOnlyOutput''',
	    '''TFBeamSearchEncoderDecoderOutput''',
	    '''TFBeamSearchDecoderOnlyOutput''',
	    '''TFBeamSampleEncoderDecoderOutput''',
	    '''TFBeamSampleDecoderOnlyOutput''',
	    '''TFContrastiveSearchEncoderDecoderOutput''',
	    '''TFContrastiveSearchDecoderOnlyOutput''',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase :    Optional[int]      						=      [
	    '''FlaxForcedBOSTokenLogitsProcessor''',
	    '''FlaxForcedEOSTokenLogitsProcessor''',
	    '''FlaxLogitsProcessor''',
	    '''FlaxLogitsProcessorList''',
	    '''FlaxLogitsWarper''',
	    '''FlaxMinLengthLogitsProcessor''',
	    '''FlaxTemperatureLogitsWarper''',
	    '''FlaxTopKLogitsWarper''',
	    '''FlaxTopPLogitsWarper''',
	]
	__lowerCamelCase :    Union[str, Any]      						=      [
	    '''FlaxGenerationMixin''',
	    '''FlaxGreedySearchOutput''',
	    '''FlaxSampleOutput''',
	    '''FlaxBeamSearchOutput''',
	]

if TYPE_CHECKING:
	from .configuration_utils import GenerationConfig
	from .streamers import TextIteratorStreamer, TextStreamer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
		from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
		from .logits_process import (
		    EncoderNoRepeatNGramLogitsProcessor,
		    EncoderRepetitionPenaltyLogitsProcessor,
		    EpsilonLogitsWarper,
		    EtaLogitsWarper,
		    ExponentialDecayLengthPenalty,
		    ForcedBOSTokenLogitsProcessor,
		    ForcedEOSTokenLogitsProcessor,
		    HammingDiversityLogitsProcessor,
		    InfNanRemoveLogitsProcessor,
		    LogitNormalization,
		    LogitsProcessor,
		    LogitsProcessorList,
		    LogitsWarper,
		    MinLengthLogitsProcessor,
		    MinNewTokensLengthLogitsProcessor,
		    NoBadWordsLogitsProcessor,
		    NoRepeatNGramLogitsProcessor,
		    PrefixConstrainedLogitsProcessor,
		    RepetitionPenaltyLogitsProcessor,
		    SequenceBiasLogitsProcessor,
		    TemperatureLogitsWarper,
		    TopKLogitsWarper,
		    TopPLogitsWarper,
		    TypicalLogitsWarper,
		)
		from .stopping_criteria import (
		    MaxLengthCriteria,
		    MaxNewTokensCriteria,
		    MaxTimeCriteria,
		    StoppingCriteria,
		    StoppingCriteriaList,
		    validate_stopping_criteria,
		)
		from .utils import (
		    BeamSampleDecoderOnlyOutput,
		    BeamSampleEncoderDecoderOutput,
		    BeamSearchDecoderOnlyOutput,
		    BeamSearchEncoderDecoderOutput,
		    ContrastiveSearchDecoderOnlyOutput,
		    ContrastiveSearchEncoderDecoderOutput,
		    GenerationMixin,
		    GreedySearchDecoderOnlyOutput,
		    GreedySearchEncoderDecoderOutput,
		    SampleDecoderOnlyOutput,
		    SampleEncoderDecoderOutput,
		    top_k_top_p_filtering,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tf_logits_process import (
		    TFForcedBOSTokenLogitsProcessor,
		    TFForcedEOSTokenLogitsProcessor,
		    TFForceTokensLogitsProcessor,
		    TFLogitsProcessor,
		    TFLogitsProcessorList,
		    TFLogitsWarper,
		    TFMinLengthLogitsProcessor,
		    TFNoBadWordsLogitsProcessor,
		    TFNoRepeatNGramLogitsProcessor,
		    TFRepetitionPenaltyLogitsProcessor,
		    TFSuppressTokensAtBeginLogitsProcessor,
		    TFSuppressTokensLogitsProcessor,
		    TFTemperatureLogitsWarper,
		    TFTopKLogitsWarper,
		    TFTopPLogitsWarper,
		)
		from .tf_utils import (
		    TFBeamSampleDecoderOnlyOutput,
		    TFBeamSampleEncoderDecoderOutput,
		    TFBeamSearchDecoderOnlyOutput,
		    TFBeamSearchEncoderDecoderOutput,
		    TFContrastiveSearchDecoderOnlyOutput,
		    TFContrastiveSearchEncoderDecoderOutput,
		    TFGenerationMixin,
		    TFGreedySearchDecoderOnlyOutput,
		    TFGreedySearchEncoderDecoderOutput,
		    TFSampleDecoderOnlyOutput,
		    TFSampleEncoderDecoderOutput,
		    tf_top_k_top_p_filtering,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .flax_logits_process import (
		    FlaxForcedBOSTokenLogitsProcessor,
		    FlaxForcedEOSTokenLogitsProcessor,
		    FlaxLogitsProcessor,
		    FlaxLogitsProcessorList,
		    FlaxLogitsWarper,
		    FlaxMinLengthLogitsProcessor,
		    FlaxTemperatureLogitsWarper,
		    FlaxTopKLogitsWarper,
		    FlaxTopPLogitsWarper,
		)
		from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
	import sys

	__lowerCamelCase :    int      						=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



