



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_mbartaa import MBartaaTokenizer
else:
		lowerCAmelCase					:				List[Any]   	=None


lowerCAmelCase					:				List[Any]   	=logging.get_logger(__name__)

lowerCAmelCase					:				Dict   	={'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase					:				str   	={
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

lowerCAmelCase					:				Optional[int]   	={
    'facebook/mbart-large-50-one-to-many-mmt': 1_024,
}

# fmt: off
lowerCAmelCase					:				Dict   	=['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']






class 	_a			(      snake_case_  ):
			_UpperCamelCase:   Dict	     =		VOCAB_FILES_NAMES
			_UpperCamelCase:   List[Any]	     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase:   int	     =		PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase:   List[Any]	     =		["input_ids", "attention_mask"]
			_UpperCamelCase:   List[Any]	     =		MBartaaTokenizer

			_UpperCamelCase:   List[int]	     =		[]
			_UpperCamelCase:   List[int]	     =		[]






			def __init__(    self    ,  lowercase_=None    ,  lowercase_=None    ,  lowercase_=None    ,  lowercase_=None    ,  lowercase_="</s>"    ,  lowercase_="</s>"    ,  lowercase_="<s>"    ,  lowercase_="<unk>"    ,  lowercase_="<pad>"    ,  lowercase_="<mask>"    ,  **lowercase_    ,  )	->					Union[str, Any]:
									# Mask token behave like a normal word, i.e. include the space before it
									lowerCAmelCase	:   Union[str, Any]					     =					AddedToken(lowercase_    ,  lstrip=lowercase_    ,  rstrip=lowercase_ ) if isinstance(lowercase_    ,  lowercase_ ) else mask_token

									lowerCAmelCase	:   Dict					     =					kwargs.get("""additional_special_tokens"""    ,  [] )
									kwargs["additional_special_tokens"] += [
									    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
									]

									super().__init__(
									    lowercase_    ,  src_lang=lowercase_    ,  tgt_lang=lowercase_    ,  tokenizer_file=lowercase_    ,  eos_token=lowercase_    ,  sep_token=lowercase_    ,  cls_token=lowercase_    ,  unk_token=lowercase_    ,  pad_token=lowercase_    ,  mask_token=lowercase_    ,  **lowercase_    ,  )

									lowerCAmelCase	:   List[str]					     =					vocab_file
									lowerCAmelCase	:   Tuple					     =					False if not self.vocab_file else True

									lowerCAmelCase	:   Union[str, Any]					     =					{
									    lang_code: self.convert_tokens_to_ids(lowercase_ ) for lang_code in FAIRSEQ_LANGUAGE_CODES
									}

									lowerCAmelCase	:   Dict					     =					src_lang if src_lang is not None else """en_XX"""
									lowerCAmelCase	:   Union[str, Any]					     =					tgt_lang
									lowerCAmelCase	:   Optional[Any]					     =					self.lang_code_to_id[self._src_lang]
									self.set_src_lang_special_tokens(self._src_lang )






			@property
			def 							_snake_case   (    self )	->					str:
									return self._src_lang






			@src_lang.setter
			def 							_snake_case   (    self    ,  lowercase_ )	->					None:
									lowerCAmelCase	:   Union[str, Any]					     =					new_src_lang
									self.set_src_lang_special_tokens(self._src_lang )






			def 							_snake_case   (    self    ,  lowercase_    ,  lowercase_ = None )	->					List[int]:
									if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






			def 							_snake_case   (    self    ,  lowercase_    ,  lowercase_ = "en_XX"    ,  lowercase_ = None    ,  lowercase_ = "ro_RO"    ,  **lowercase_    ,  )	->					BatchEncoding:
									lowerCAmelCase	:   List[Any]					     =					src_lang
									lowerCAmelCase	:   Dict					     =					tgt_lang
									return super().prepare_seqaseq_batch(lowercase_    ,  lowercase_    ,  **lowercase_ )






			def 							_snake_case   (    self )	->					Tuple:
									return self.set_src_lang_special_tokens(self.src_lang )






			def 							_snake_case   (    self )	->					List[str]:
									return self.set_tgt_lang_special_tokens(self.tgt_lang )






			def 							_snake_case   (    self    ,  lowercase_ )	->					None:
									lowerCAmelCase	:   Optional[Any]					     =					self.convert_tokens_to_ids(lowercase_ )
									lowerCAmelCase	:   str					     =					[self.cur_lang_code_id]
									lowerCAmelCase	:   Optional[Any]					     =					[self.eos_token_id]

									lowerCAmelCase	:   List[str]					     =					self.convert_ids_to_tokens(self.prefix_tokens )
									lowerCAmelCase	:   Any					     =					self.convert_ids_to_tokens(self.suffix_tokens )

									lowerCAmelCase	:   Optional[int]					     =					processors.TemplateProcessing(
									    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,  pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,  self.prefix_tokens + self.suffix_tokens ) )    ,  )






			def 							_snake_case   (    self    ,  lowercase_ )	->					None:
									lowerCAmelCase	:   Union[str, Any]					     =					self.convert_tokens_to_ids(lowercase_ )
									lowerCAmelCase	:   List[str]					     =					[self.cur_lang_code_id]
									lowerCAmelCase	:   str					     =					[self.eos_token_id]

									lowerCAmelCase	:   Any					     =					self.convert_ids_to_tokens(self.prefix_tokens )
									lowerCAmelCase	:   Tuple					     =					self.convert_ids_to_tokens(self.suffix_tokens )

									lowerCAmelCase	:   str					     =					processors.TemplateProcessing(
									    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,  pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,  self.prefix_tokens + self.suffix_tokens ) )    ,  )






			def 							_snake_case   (    self    ,  lowercase_    ,  lowercase_    ,  lowercase_    ,  lowercase_    ,  **lowercase_ )	->					str:
									if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""" )
									lowerCAmelCase	:   List[str]					     =					src_lang
									lowerCAmelCase	:   Tuple					     =					self(lowercase_    ,  add_special_tokens=lowercase_    ,  return_tensors=lowercase_    ,  **lowercase_ )
									lowerCAmelCase	:   str					     =					self.convert_tokens_to_ids(lowercase_ )
									lowerCAmelCase	:   List[Any]					     =					tgt_lang_id
									return inputs






			def 							_snake_case   (    self    ,  lowercase_    ,  lowercase_ = None )	->					Tuple[str]:
									if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer.""" )

									if not os.path.isdir(lowercase_ ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
															return
									lowerCAmelCase	:   Dict					     =					os.path.join(
									    lowercase_    ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

									if os.path.abspath(self.vocab_file ) != os.path.abspath(lowercase_ ):
															copyfile(self.vocab_file    ,  lowercase_ )

									return (out_vocab_file,)






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_torch_available,
)


lowerCAmelCase					:				Optional[int]   	={
    'configuration_speech_to_text_2': ['SPEECH_TO_TEXT_2_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Speech2Text2Config'],
    'processing_speech_to_text_2': ['Speech2Text2Processor'],
    'tokenization_speech_to_text_2': ['Speech2Text2Tokenizer'],
}


try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCAmelCase					:				int   	=[
		    'SPEECH_TO_TEXT_2_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'Speech2Text2ForCausalLM',
		    'Speech2Text2PreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_speech_to_text_a import SPEECH_TO_TEXT_2_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextaConfig
		from .processing_speech_to_text_a import SpeechaTextaProcessor
		from .tokenization_speech_to_text_a import SpeechaTextaTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_speech_to_text_a import (
				    SPEECH_TO_TEXT_2_PRETRAINED_MODEL_ARCHIVE_LIST,
				    SpeechaTextaForCausalLM,
				    SpeechaTextaPreTrainedModel,
				)

else:
		import sys

		lowerCAmelCase					:				Union[str, Any]   	=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

