





'''simple docstring'''

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_nllb import NllbTokenizer
else:
      lowercase_					= None


lowercase_					= logging.get_logger(__name__)


lowercase_					= {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowercase_					= {
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase_					= {
    """facebook/nllb-large-en-ro""": 1_024,
    """facebook/nllb-200-distilled-600M""": 1_024,
}

# fmt: off
lowercase_					= ["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]


class 					a_     (				snake_case_       ):

    '''simple docstring'''
    UpperCamelCase     = VOCAB_FILES_NAMES
    UpperCamelCase     = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCamelCase     = PRETRAINED_VOCAB_FILES_MAP
    UpperCamelCase     = ['''input_ids''', '''attention_mask''']
    UpperCamelCase     = NllbTokenizer

    UpperCamelCase     = []
    UpperCamelCase     = []


    def __init__(		self ,		A=None ,		A=None ,		A="<s>" ,		A="</s>" ,		A="</s>" ,		A="<s>" ,		A="<unk>" ,		A="<pad>" ,		A="<mask>" ,		A=None ,		A=None ,		A=None ,		A=False ,		**A ,		)		->   Dict:
        # Mask token behave like a normal word, i.e. include the space before it
        _SCREAMING_SNAKE_CASE									=			AddedToken(A ,		lstrip=A ,		rstrip=A       ) if isinstance(A ,		A       ) else mask_token
        _SCREAMING_SNAKE_CASE									=			legacy_behaviour
        super().__init__(
            vocab_file=A ,		tokenizer_file=A ,		bos_token=A ,		eos_token=A ,		sep_token=A ,		cls_token=A ,		unk_token=A ,		pad_token=A ,		mask_token=A ,		src_lang=A ,		tgt_lang=A ,		additional_special_tokens=A ,		legacy_behaviour=A ,		**A ,		)

        _SCREAMING_SNAKE_CASE									=			vocab_file
        _SCREAMING_SNAKE_CASE									=			False if not self.vocab_file else True

        _SCREAMING_SNAKE_CASE									=			FAIRSEQ_LANGUAGE_CODES.copy()

        if additional_special_tokens is not None:
            # Only add those special tokens if they are not already there.
            _additional_special_tokens.extend(
                [t for t in additional_special_tokens if t not in _additional_special_tokens]       )

        self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}       )
        _SCREAMING_SNAKE_CASE									=			{
            lang_code: self.convert_tokens_to_ids(A       ) for lang_code in FAIRSEQ_LANGUAGE_CODES
        }

        _SCREAMING_SNAKE_CASE									=			src_lang if src_lang is not None else """eng_Latn"""
        _SCREAMING_SNAKE_CASE									=			self.convert_tokens_to_ids(self._src_lang       )
        _SCREAMING_SNAKE_CASE									=			tgt_lang
        self.set_src_lang_special_tokens(self._src_lang       )


    @property
    def 			snake_case_(		self       )		->   str:
        return self._src_lang


    @src_lang.setter
    def 			snake_case_(		self ,		A       )		->   None:
        _SCREAMING_SNAKE_CASE									=			new_src_lang
        self.set_src_lang_special_tokens(self._src_lang       )


    def 			snake_case_(		self ,		A ,		A = None       )		->   List[int]:
        if token_ids_a is None:
            return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


    def 			snake_case_(		self ,		A ,		A = None       )		->   List[int]:
        _SCREAMING_SNAKE_CASE									=			[self.sep_token_id]
        _SCREAMING_SNAKE_CASE									=			[self.cls_token_id]

        if token_ids_a is None:
            return len(cls + token_ids_a + sep       ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]


    def 			snake_case_(		self ,		A ,		A ,		A ,		A ,		**A       )		->   Dict:
        if src_lang is None or tgt_lang is None:
            raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""       )
        _SCREAMING_SNAKE_CASE									=			src_lang
        _SCREAMING_SNAKE_CASE									=			self(A ,		add_special_tokens=A ,		return_tensors=A ,		**A       )
        _SCREAMING_SNAKE_CASE									=			self.convert_tokens_to_ids(A       )
        _SCREAMING_SNAKE_CASE									=			tgt_lang_id
        return inputs


    def 			snake_case_(		self ,		A ,		A = "eng_Latn" ,		A = None ,		A = "fra_Latn" ,		**A ,		)		->   BatchEncoding:
        _SCREAMING_SNAKE_CASE									=			src_lang
        _SCREAMING_SNAKE_CASE									=			tgt_lang
        return super().prepare_seqaseq_batch(A ,		A ,		**A       )


    def 			snake_case_(		self       )		->   int:
        return self.set_src_lang_special_tokens(self.src_lang       )


    def 			snake_case_(		self       )		->   Any:
        return self.set_tgt_lang_special_tokens(self.tgt_lang       )


    def 			snake_case_(		self ,		A       )		->   None:
        _SCREAMING_SNAKE_CASE									=			self.convert_tokens_to_ids(A       )

        if self.legacy_behaviour:
            _SCREAMING_SNAKE_CASE									=			[]
            _SCREAMING_SNAKE_CASE									=			[self.eos_token_id, self.cur_lang_code]
        else:
            _SCREAMING_SNAKE_CASE									=			[self.cur_lang_code]
            _SCREAMING_SNAKE_CASE									=			[self.eos_token_id]

        _SCREAMING_SNAKE_CASE									=			self.convert_ids_to_tokens(self.prefix_tokens       )
        _SCREAMING_SNAKE_CASE									=			self.convert_ids_to_tokens(self.suffix_tokens       )

        _SCREAMING_SNAKE_CASE									=			processors.TemplateProcessing(
            single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str ,		pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str ,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,		self.prefix_tokens + self.suffix_tokens       )       ) ,		)


    def 			snake_case_(		self ,		A       )		->   None:
        _SCREAMING_SNAKE_CASE									=			self.convert_tokens_to_ids(A       )
        if self.legacy_behaviour:
            _SCREAMING_SNAKE_CASE									=			[]
            _SCREAMING_SNAKE_CASE									=			[self.eos_token_id, self.cur_lang_code]
        else:
            _SCREAMING_SNAKE_CASE									=			[self.cur_lang_code]
            _SCREAMING_SNAKE_CASE									=			[self.eos_token_id]

        _SCREAMING_SNAKE_CASE									=			self.convert_ids_to_tokens(self.prefix_tokens       )
        _SCREAMING_SNAKE_CASE									=			self.convert_ids_to_tokens(self.suffix_tokens       )

        _SCREAMING_SNAKE_CASE									=			processors.TemplateProcessing(
            single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str ,		pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str ,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,		self.prefix_tokens + self.suffix_tokens       )       ) ,		)





    def 			snake_case_(		self ,		A ,		A = None       )		->   Tuple[str]:
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                """tokenizer."""       )

        if not os.path.isdir(A       ):
            logger.error(f'Vocabulary path ({save_directory}) should be a directory.'       )
            return
        _SCREAMING_SNAKE_CASE									=			os.path.join(
            A ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

        if os.path.abspath(self.vocab_file       ) != os.path.abspath(A       ):
            copyfile(self.vocab_file ,		A       )

        return (out_vocab_file,)










'''simple docstring'''

import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_					= logging.get_logger(__name__)

lowercase_					= {
    """BridgeTower/bridgetower-base""": """https://huggingface.co/BridgeTower/bridgetower-base/blob/main/config.json""",
    """BridgeTower/bridgetower-base-itm-mlm""": (
        """https://huggingface.co/BridgeTower/bridgetower-base-itm-mlm/blob/main/config.json"""
    ),
}


class 					a_     (				snake_case_       ):

    '''simple docstring'''
    UpperCamelCase     = '''bridgetower_vision_model'''


    def __init__(		self ,		A=768 ,		A=12 ,		A=3 ,		A=16 ,		A=288 ,		A=1 ,		A=1e-05 ,		A=False ,		A=True ,		A=False ,		**A ,		)		->   Dict:
        super().__init__(**A       )
        _SCREAMING_SNAKE_CASE									=			hidden_size
        _SCREAMING_SNAKE_CASE									=			num_hidden_layers
        _SCREAMING_SNAKE_CASE									=			num_channels
        _SCREAMING_SNAKE_CASE									=			patch_size
        _SCREAMING_SNAKE_CASE									=			image_size
        _SCREAMING_SNAKE_CASE									=			initializer_factor
        _SCREAMING_SNAKE_CASE									=			layer_norm_eps
        _SCREAMING_SNAKE_CASE									=			stop_gradient
        _SCREAMING_SNAKE_CASE									=			share_layernorm
        _SCREAMING_SNAKE_CASE									=			remove_last_layer





    @classmethod
    def 			snake_case_(		cls ,		A ,		**A       )		->   "PretrainedConfig":
        _SCREAMING_SNAKE_CASE				,       _SCREAMING_SNAKE_CASE									=			cls.get_config_dict(A ,		**A       )

        if config_dict.get("""model_type"""       ) == "bridgetower":
            _SCREAMING_SNAKE_CASE									=			config_dict["""text_config"""]

        if "model_type" in config_dict and hasattr(cls ,		"""model_type"""       ) and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
                f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'       )

        return cls.from_dict(A ,		**A       )




class 					a_     (				snake_case_       ):

    '''simple docstring'''
    UpperCamelCase     = '''bridgetower_text_model'''


    def __init__(		self ,		A=5_0265 ,		A=768 ,		A=12 ,		A=12 ,		A=1 ,		A=3072 ,		A="gelu" ,		A=0.1 ,		A=0.1 ,		A=514 ,		A=1 ,		A=1e-05 ,		A=1 ,		A=0 ,		A=2 ,		A="absolute" ,		A=True ,		**A ,		)		->   Union[str, Any]:
        super().__init__(**A       )

        _SCREAMING_SNAKE_CASE									=			vocab_size
        _SCREAMING_SNAKE_CASE									=			hidden_size
        _SCREAMING_SNAKE_CASE									=			num_hidden_layers
        _SCREAMING_SNAKE_CASE									=			num_attention_heads
        _SCREAMING_SNAKE_CASE									=			hidden_act
        _SCREAMING_SNAKE_CASE									=			initializer_factor
        _SCREAMING_SNAKE_CASE									=			intermediate_size
        _SCREAMING_SNAKE_CASE									=			hidden_dropout_prob
        _SCREAMING_SNAKE_CASE									=			attention_probs_dropout_prob
        _SCREAMING_SNAKE_CASE									=			max_position_embeddings
        _SCREAMING_SNAKE_CASE									=			type_vocab_size
        _SCREAMING_SNAKE_CASE									=			layer_norm_eps
        _SCREAMING_SNAKE_CASE									=			position_embedding_type
        _SCREAMING_SNAKE_CASE									=			use_cache
        _SCREAMING_SNAKE_CASE									=			pad_token_id
        _SCREAMING_SNAKE_CASE									=			bos_token_id
        _SCREAMING_SNAKE_CASE									=			eos_token_id





    @classmethod
    def 			snake_case_(		cls ,		A ,		**A       )		->   "PretrainedConfig":
        _SCREAMING_SNAKE_CASE				,       _SCREAMING_SNAKE_CASE									=			cls.get_config_dict(A ,		**A       )

        if config_dict.get("""model_type"""       ) == "bridgetower":
            _SCREAMING_SNAKE_CASE									=			config_dict["""text_config"""]

        if "model_type" in config_dict and hasattr(cls ,		"""model_type"""       ) and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
                f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'       )

        return cls.from_dict(A ,		**A       )




class 					a_     (				snake_case_       ):

    '''simple docstring'''
    UpperCamelCase     = '''bridgetower'''


    def __init__(		self ,		A=True ,		A="gelu" ,		A=768 ,		A=1 ,		A=1e-05 ,		A=False ,		A="add" ,		A=12 ,		A=6 ,		A=False ,		A=False ,		A=None ,		A=None ,		**A ,		)		->   Tuple:
        # TODO: remove this once the Hub files are updated.
        _SCREAMING_SNAKE_CASE									=			kwargs.pop("""text_config_dict""" ,		A       )
        _SCREAMING_SNAKE_CASE									=			kwargs.pop("""vision_config_dict""" ,		A       )

        super().__init__(**A       )
        _SCREAMING_SNAKE_CASE									=			share_cross_modal_transformer_layers
        _SCREAMING_SNAKE_CASE									=			hidden_act
        _SCREAMING_SNAKE_CASE									=			hidden_size
        _SCREAMING_SNAKE_CASE									=			initializer_factor
        _SCREAMING_SNAKE_CASE									=			layer_norm_eps
        _SCREAMING_SNAKE_CASE									=			share_link_tower_layers
        _SCREAMING_SNAKE_CASE									=			link_tower_type
        _SCREAMING_SNAKE_CASE									=			num_attention_heads
        _SCREAMING_SNAKE_CASE									=			num_hidden_layers
        _SCREAMING_SNAKE_CASE									=			tie_word_embeddings
        _SCREAMING_SNAKE_CASE									=			init_layernorm_from_vision_encoder

        if text_config is None:
            _SCREAMING_SNAKE_CASE									=			{}
            logger.info("""`text_config` is `None`. Initializing the `BridgeTowerTextConfig` with default values."""       )

        if vision_config is None:
            _SCREAMING_SNAKE_CASE									=			{}
            logger.info("""`vision_config` is `None`. Initializing the `BridgeTowerVisionConfig` with default values."""       )

        _SCREAMING_SNAKE_CASE									=			BridgeTowerTextConfig(**A       )
        _SCREAMING_SNAKE_CASE									=			BridgeTowerVisionConfig(**A       )


    @classmethod
    def 			snake_case_(		cls ,		A ,		A ,		**A       )		->   int:
        return cls(text_config=text_config.to_dict() ,		vision_config=vision_config.to_dict() ,		**A       )





    def 			snake_case_(		self       )		->   List[Any]:
        _SCREAMING_SNAKE_CASE									=			copy.deepcopy(self.__dict__       )
        _SCREAMING_SNAKE_CASE									=			self.text_config.to_dict()
        _SCREAMING_SNAKE_CASE									=			self.vision_config.to_dict()
        _SCREAMING_SNAKE_CASE									=			self.__class__.model_type
        return output



