






"""simple docstring"""



from functools import partial
from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload

import torch
import torch.nn as nn
import torch.types




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->torch.Tensor:
 # The first operation in a checkpoint can't be in-place, but it's
 # nice to have in-place addition during inference. Thus...
 if not inplace:
  UpperCAmelCase__ 						=				ma + ma
 else:
  ma += ma

 return ma




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->torch.Tensor:
 UpperCAmelCase__ 						=				-1 * len(_SCREAMING_SNAKE_CASE			)
 UpperCAmelCase__ 						=				list(range(len(tensor.shape[:zero_index]			)			)			)
 return tensor.permute(first_inds + [zero_index + i for i in inds]			)




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->torch.Tensor:
 return t.reshape(t.shape[:-no_dims] + (-1,)			)




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = 1E-4			)    ->torch.Tensor:
 UpperCAmelCase__ 						=				mask.expand(*value.shape			)
 return torch.sum(mask * value       ,	dim=_SCREAMING_SNAKE_CASE			) / (eps + torch.sum(_SCREAMING_SNAKE_CASE       ,	dim=_SCREAMING_SNAKE_CASE			))




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = 2.3125       ,	_SCREAMING_SNAKE_CASE = 21.6875       ,	_SCREAMING_SNAKE_CASE = 6_4			)    ->torch.Tensor:
 UpperCAmelCase__ 						=				torch.linspace(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	no_bins - 1       ,	device=pts.device			)
 UpperCAmelCase__ 						=				torch.sqrt(torch.sum((pts.unsqueeze(-2			) - pts.unsqueeze(-3			)) ** 2       ,	dim=-1			)			)
 return torch.bucketize(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->dict:
 UpperCAmelCase__ 						=				dicts[0]
 UpperCAmelCase__ 						=				{}
 for k, v in first.items():
  UpperCAmelCase__ 						=				[d[k] for d in dicts]
  if isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			):
   UpperCAmelCase__ 						=				dict_multimap(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  else:
   UpperCAmelCase__ 						=				fn(_SCREAMING_SNAKE_CASE			)

 return new_dict




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->torch.Tensor:
 UpperCAmelCase__ 						=				v_bins.view(((1,) * len(x.shape			)) + (len(_SCREAMING_SNAKE_CASE			),)			)
 UpperCAmelCase__ 						=				x[..., None] - reshaped_bins
 UpperCAmelCase__ 						=				torch.argmin(torch.abs(_SCREAMING_SNAKE_CASE			)       ,	dim=-1			)
 return nn.functional.one_hot(_SCREAMING_SNAKE_CASE       ,	num_classes=len(_SCREAMING_SNAKE_CASE			)			).float()




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = 0       ,	_SCREAMING_SNAKE_CASE = 0			)    ->torch.Tensor:
 UpperCAmelCase__ 						=				[]
 for i, s in enumerate(data.shape[:no_batch_dims]			):
  UpperCAmelCase__ 						=				torch.arange(_SCREAMING_SNAKE_CASE			)
  UpperCAmelCase__ 						=				r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape			) - i - 1)))			)
  ranges.append(_SCREAMING_SNAKE_CASE			)

 UpperCAmelCase__ 						=				[slice(_SCREAMING_SNAKE_CASE			) for _ in range(len(data.shape			) - no_batch_dims			)]
 UpperCAmelCase__ 						=				inds
 ranges.extend(_SCREAMING_SNAKE_CASE			)
 # Matt note: Editing this to get around the behaviour of using a list as an array index changing
 # in recent Numpy versions
 return data[tuple(_SCREAMING_SNAKE_CASE			)]


a  :						Optional[int]					       =							TypeVar('''T''')




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->Dict[Any, Union[dict, list, tuple, Any]]:
 UpperCAmelCase__ 						=				{}
 for k, v in dic.items():
  if isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			):
   UpperCAmelCase__ 						=				dict_map(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  else:
   UpperCAmelCase__ 						=				tree_map(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)

 return new_dict




@overload
def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->Any:
 ...




@overload
def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->dict:
 ...




@overload
def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->list:
 ...




@overload
def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->tuple:
 ...




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)    ->Any:
 if isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			):
  return dict_map(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
 elif isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			):
  return [tree_map(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			) for x in tree]
 elif isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			):
  return tuple(tree_map(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			) for x in tree			)
 elif isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			):
  return fn(_SCREAMING_SNAKE_CASE			)
 else:
  print(type(_SCREAMING_SNAKE_CASE			)			)
  raise ValueError("""Not supported"""			)
a  :						Optional[Any]					       =							partial(tree_map, leaf_type=torch.Tensor)











"""simple docstring"""



from ...configuration_utils import PretrainedConfig
from ...utils import logging


a  :						Optional[Any]					       =							logging.get_logger(__name__)

a  :						int					       =							{
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}


class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   Optional[Any]											=						'opt'
    __lowercase    :   str											=						['past_key_values']


    def __init__(      self   ,			__lowercase=50272   ,			__lowercase=768   ,			__lowercase=12   ,			__lowercase=3072   ,			__lowercase=2048   ,			__lowercase=True   ,			__lowercase=False   ,			__lowercase=None   ,			__lowercase=0.1   ,			__lowercase=0.0   ,			__lowercase=12   ,			__lowercase="relu"   ,			__lowercase=0.0   ,			__lowercase=0.02   ,			__lowercase=True   ,			__lowercase=1   ,			__lowercase=2   ,			__lowercase=2   ,			__lowercase=True   ,			__lowercase=True   ,			**__lowercase   ,			):
     super().__init__(
         pad_token_id=__lowercase   ,			bos_token_id=__lowercase   ,			eos_token_id=__lowercase   ,			**__lowercase   ,			)
     UpperCAmelCase__ 						=				vocab_size
     UpperCAmelCase__ 						=				max_position_embeddings
     UpperCAmelCase__ 						=				num_attention_heads
     UpperCAmelCase__ 						=				word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
     UpperCAmelCase__ 						=				ffn_dim
     UpperCAmelCase__ 						=				hidden_size
     UpperCAmelCase__ 						=				num_hidden_layers
     UpperCAmelCase__ 						=				dropout
     UpperCAmelCase__ 						=				attention_dropout
     UpperCAmelCase__ 						=				activation_function
     UpperCAmelCase__ 						=				init_std
     UpperCAmelCase__ 						=				layerdrop
     UpperCAmelCase__ 						=				use_cache
     UpperCAmelCase__ 						=				do_layer_norm_before
     # We keep these variables at `True` for backward compatibility.
     UpperCAmelCase__ 						=				enable_bias
     UpperCAmelCase__ 						=				layer_norm_elementwise_affine

     # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
     # with checkpoints that have been fine-tuned before transformers v4.20.1
     # see https://github.com/facebookresearch/metaseq/pull/164
     UpperCAmelCase__ 						=				_remove_final_layer_norm



