


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_xglm import XGLMTokenizer
else:
						_UpperCAmelCase      :	Dict  =None


_UpperCAmelCase      :	Optional[int]  =logging.get_logger(__name__)

_UpperCAmelCase      :	Optional[Any]  ={"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase      :	Tuple  ={
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json""",
    },
}

_UpperCAmelCase      :	int  ={
    """facebook/xglm-564M""": 2048,
}







class 					snake_case__(				__SCREAMING_SNAKE_CASE				):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__   :	List[str]       =				VOCAB_FILES_NAMES
						SCREAMING_SNAKE_CASE__   :	List[str]       =				PRETRAINED_VOCAB_FILES_MAP
						SCREAMING_SNAKE_CASE__   :	Optional[Any]       =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						SCREAMING_SNAKE_CASE__   :	int       =				["""input_ids""", """attention_mask"""]
						SCREAMING_SNAKE_CASE__   :	Optional[int]       =				XGLMTokenizer




						def __init__(					self ,    __lowercase=None ,    __lowercase=None ,    __lowercase="<s>" ,    __lowercase="</s>" ,    __lowercase="</s>" ,    __lowercase="<s>" ,    __lowercase="<unk>" ,    __lowercase="<pad>" ,    **__lowercase ,    )       ->     Union[str, Any]:
										lowerCAmelCase_			:   List[Any]     					=					7
										lowerCAmelCase_			:   List[str]     					=					[f"""<madeupword{i}>""" for i in range(self.num_madeup_words				)]

										lowerCAmelCase_			:   List[str]     					=					kwargs.get('''additional_special_tokens''' ,    []				)
										kwargs["additional_special_tokens"] += [
										    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
										]

										super().__init__(
										    __UpperCamelCase ,    tokenizer_file=__UpperCamelCase ,    bos_token=__UpperCamelCase ,    eos_token=__UpperCamelCase ,    sep_token=__UpperCamelCase ,    cls_token=__UpperCamelCase ,    unk_token=__UpperCamelCase ,    pad_token=__UpperCamelCase ,    **__UpperCamelCase ,    )

										lowerCAmelCase_			:   Optional[int]     					=					vocab_file
										lowerCAmelCase_			:   str     					=					False if not self.vocab_file else True




						def     lowercase_			(					self ,    __lowercase ,    __lowercase = None				)       ->     Tuple:


										if token_ids_a is None:
														return [self.sep_token_id] + token_ids_a
										lowerCAmelCase_			:   Dict     					=					[self.sep_token_id]
										return sep + token_ids_a + sep + sep + token_ids_a




						def     lowercase_			(					self ,    __lowercase ,    __lowercase = None				)       ->     Tuple:
										lowerCAmelCase_			:   List[Any]     					=					[self.sep_token_id]

										if token_ids_a is None:
														return len(sep + token_ids_a				) * [0]
										return len(sep + token_ids_a + sep + sep + token_ids_a				) * [0]




						def     lowercase_			(					self ,    __lowercase ,    __lowercase = None				)       ->     Union[str, Any]:


										if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''				)

										if not os.path.isdir(__UpperCamelCase				):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""				)
														return
										lowerCAmelCase_			:   Optional[int]     					=					os.path.join(
										    __UpperCamelCase ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

										if os.path.abspath(self.vocab_file				) != os.path.abspath(__UpperCamelCase				):
														copyfile(self.vocab_file ,    __UpperCamelCase				)

										return (out_vocab_file,)

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A							=       logging.get_logger(__name__)

_A							=       {
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}

class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):
 A__    :					Dict    =     """opt"""
 A__    :					int    =     ["""past_key_values"""]

 def __init__(   self  ,						__UpperCamelCase=5_0_2_7_2  ,						__UpperCamelCase=7_6_8  ,						__UpperCamelCase=1_2  ,						__UpperCamelCase=3_0_7_2  ,						__UpperCamelCase=2_0_4_8  ,						__UpperCamelCase=True  ,						__UpperCamelCase=False  ,						__UpperCamelCase=None  ,						__UpperCamelCase=0.1  ,						__UpperCamelCase=0.0  ,						__UpperCamelCase=1_2  ,						__UpperCamelCase="relu"  ,						__UpperCamelCase=0.0  ,						__UpperCamelCase=0.02  ,						__UpperCamelCase=True  ,						__UpperCamelCase=1  ,						__UpperCamelCase=2  ,						__UpperCamelCase=2  ,						__UpperCamelCase=True  ,						__UpperCamelCase=True  ,						**__UpperCamelCase  ,						):




        """simple docstring"""


        super().__init__(
            pad_token_id=__UpperCamelCase  ,						bos_token_id=__UpperCamelCase  ,						eos_token_id=__UpperCamelCase  ,						**__UpperCamelCase  ,						)
        UpperCamelCase_											=							vocab_size
        UpperCamelCase_											=							max_position_embeddings
        UpperCamelCase_											=							num_attention_heads
        UpperCamelCase_											=							word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
        UpperCamelCase_											=							ffn_dim
        UpperCamelCase_											=							hidden_size
        UpperCamelCase_											=							num_hidden_layers
        UpperCamelCase_											=							dropout
        UpperCamelCase_											=							attention_dropout
        UpperCamelCase_											=							activation_function
        UpperCamelCase_											=							init_std
        UpperCamelCase_											=							layerdrop
        UpperCamelCase_											=							use_cache
        UpperCamelCase_											=							do_layer_norm_before
        # We keep these variables at `True` for backward compatibility.
        UpperCamelCase_											=							enable_bias
        UpperCamelCase_											=							layer_norm_elementwise_affine

        # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
        # with checkpoints that have been fine-tuned before transformers v4.20.1
        # see https://github.com/facebookresearch/metaseq/pull/164
        UpperCamelCase_											=							_remove_final_layer_norm



