



"""simple docstring"""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_xglm import XGLMTokenizer
else:
						__a           =							None


__a           =							logging.get_logger(__name__)

__a           =							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__a           =							{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
    },
}

__a           =							{
    "facebook/xglm-564M": 20_48,
}
class 							lowerCamelCase    (     _lowerCAmelCase							):






					'''simple docstring'''





					_A						:     int		=	VOCAB_FILES_NAMES
					_A						:     Optional[int]		=	PRETRAINED_VOCAB_FILES_MAP
					_A						:     Optional[Any]		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_A						:     Union[str, Any]		=	["""input_ids""", """attention_mask"""]
					_A						:     Optional[Any]		=	XGLMTokenizer






					def __init__(			self:				str		,	snake_case:				Optional[Any]=None		,	snake_case:				List[str]=None		,	snake_case:				Dict="<s>"		,	snake_case:				Dict="</s>"		,	snake_case:				Optional[Any]="</s>"		,	snake_case:				Optional[int]="<s>"		,	snake_case:				Optional[Any]="<unk>"		,	snake_case:				List[str]="<pad>"		,	**snake_case:				Optional[Any]		,	)  ->      Optional[Any]:
											# Compatibility with the original tokenizer
											snake_case_					:List[str]		=     7
											snake_case_					:int		=     [f"""<madeupword{i}>""" for i in range(self.num_madeup_words		)]

											snake_case_					:List[Any]		=     kwargs.get("""additional_special_tokens"""		,	[]		)
											kwargs["additional_special_tokens"] += [
											    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    snake_case		,	tokenizer_file=snake_case		,	bos_token=snake_case		,	eos_token=snake_case		,	sep_token=snake_case		,	cls_token=snake_case		,	unk_token=snake_case		,	pad_token=snake_case		,	**snake_case		,	)

											snake_case_					:str		=     vocab_file
											snake_case_					:Union[str, Any]		=     False if not self.vocab_file else True






					def   lowerCAmelCase_							(			self:				Tuple		,	snake_case:				List[int]		,	snake_case:				Optional[List[int]] = None		)  ->      List[int]:

											if token_ids_a is None:
																	return [self.sep_token_id] + token_ids_a
											snake_case_					:Any		=     [self.sep_token_id]
											return sep + token_ids_a + sep + sep + token_ids_a






					def   lowerCAmelCase_							(			self:				Tuple		,	snake_case:				List[int]		,	snake_case:				Optional[List[int]] = None		)  ->      List[int]:
											snake_case_					:Tuple		=     [self.sep_token_id]

											if token_ids_a is None:
																	return len(sep + token_ids_a		) * [0]
											return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]






					def   lowerCAmelCase_							(			self:				Tuple		,	snake_case:				str		,	snake_case:				Optional[str] = None		)  ->      Tuple[str]:
											if not self.can_save_slow_tokenizer:
																	raise ValueError(
																	    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																	    """tokenizer."""		)

											if not os.path.isdir(snake_case		):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""		)
																	return
											snake_case_					:Any		=     os.path.join(
											    snake_case		,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(snake_case		):
																	copyfile(self.vocab_file		,	snake_case		)

											return (out_vocab_file,)

"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE	:		str   =   logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE	:		Any   =   {
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}






class      _SCREAMING_SNAKE_CASE							(			UpperCAmelCase				):



   '''simple docstring'''



   __UpperCAmelCase	:							Tuple    =    "opt"
   __UpperCAmelCase	:							Any    =    ["past_key_values"]







   def __init__(	self ,				_SCREAMING_SNAKE_CASE=50272 ,				_SCREAMING_SNAKE_CASE=768 ,				_SCREAMING_SNAKE_CASE=12 ,				_SCREAMING_SNAKE_CASE=3072 ,				_SCREAMING_SNAKE_CASE=2048 ,				_SCREAMING_SNAKE_CASE=True ,				_SCREAMING_SNAKE_CASE=False ,				_SCREAMING_SNAKE_CASE=None ,				_SCREAMING_SNAKE_CASE=0.1 ,				_SCREAMING_SNAKE_CASE=0.0 ,				_SCREAMING_SNAKE_CASE=12 ,				_SCREAMING_SNAKE_CASE="relu" ,				_SCREAMING_SNAKE_CASE=0.0 ,				_SCREAMING_SNAKE_CASE=0.02 ,				_SCREAMING_SNAKE_CASE=True ,				_SCREAMING_SNAKE_CASE=1 ,				_SCREAMING_SNAKE_CASE=2 ,				_SCREAMING_SNAKE_CASE=2 ,				_SCREAMING_SNAKE_CASE=True ,				_SCREAMING_SNAKE_CASE=True ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      super().__init__(
          pad_token_id=_SCREAMING_SNAKE_CASE ,				bos_token_id=_SCREAMING_SNAKE_CASE ,				eos_token_id=_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE ,				)
      __lowerCamelCase    		=					vocab_size
      __lowerCamelCase    		=					max_position_embeddings
      __lowerCamelCase    		=					num_attention_heads
      __lowerCamelCase    		=					word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
      __lowerCamelCase    		=					ffn_dim
      __lowerCamelCase    		=					hidden_size
      __lowerCamelCase    		=					num_hidden_layers
      __lowerCamelCase    		=					dropout
      __lowerCamelCase    		=					attention_dropout
      __lowerCamelCase    		=					activation_function
      __lowerCamelCase    		=					init_std
      __lowerCamelCase    		=					layerdrop
      __lowerCamelCase    		=					use_cache
      __lowerCamelCase    		=					do_layer_norm_before
      # We keep these variables at `True` for backward compatibility.
      __lowerCamelCase    		=					enable_bias
      __lowerCamelCase    		=					layer_norm_elementwise_affine

      # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
      # with checkpoints that have been fine-tuned before transformers v4.20.1
      # see https://github.com/facebookresearch/metaseq/pull/164
      __lowerCamelCase    		=					_remove_final_layer_norm






