from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
    from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


__lowercase							 =					logging.get_logger(__name__)

__lowercase							 =					{
    '''bigscience/bloom''': '''https://huggingface.co/bigscience/bloom/resolve/main/config.json''',
    '''bigscience/bloom-560m''': '''https://huggingface.co/bigscience/bloom-560m/blob/main/config.json''',
    '''bigscience/bloom-1b1''': '''https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json''',
    '''bigscience/bloom-1b7''': '''https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json''',
    '''bigscience/bloom-3b''': '''https://huggingface.co/bigscience/bloom-3b/blob/main/config.json''',
    '''bigscience/bloom-7b1''': '''https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json''',
}


class 	lowerCamelCase_     (   UpperCAmelCase_ ):






    '''simple docstring'''






    a__	:						List[Any]						=      """bloom"""
    a__	:						Dict						=      ["""past_key_values"""]
    a__	:						Dict						=      {
        """num_hidden_layers""": """n_layer""",
        """num_attention_heads""": """n_head""",
    }







    def __init__(	self , __lowercase=250_880 , __lowercase=64 , __lowercase=2 , __lowercase=8 , __lowercase=1E-5 , __lowercase=0.02 , __lowercase=True , __lowercase=1 , __lowercase=2 , __lowercase=False , __lowercase=0.0 , __lowercase=0.0 , __lowercase=1 , __lowercase=False , **__lowercase , )       ->	Optional[Any]:
        __UpperCamelCase     :Tuple			  =		vocab_size
        # Backward compatibility with n_embed kwarg
        __UpperCamelCase     :Union[str, Any]			  =		kwargs.pop('''n_embed''' , __lowercase)
        __UpperCamelCase     :Any			  =		hidden_size if n_embed is None else n_embed
        __UpperCamelCase     :List[str]			  =		n_layer
        __UpperCamelCase     :Optional[int]			  =		n_head
        __UpperCamelCase     :Optional[Any]			  =		layer_norm_epsilon
        __UpperCamelCase     :Optional[Any]			  =		initializer_range
        __UpperCamelCase     :str			  =		use_cache
        __UpperCamelCase     :List[Any]			  =		pretraining_tp
        __UpperCamelCase     :Tuple			  =		apply_residual_connection_post_layernorm
        __UpperCamelCase     :int			  =		hidden_dropout
        __UpperCamelCase     :Optional[int]			  =		attention_dropout

        __UpperCamelCase     :Dict			  =		bos_token_id
        __UpperCamelCase     :List[Any]			  =		eos_token_id
        __UpperCamelCase     :int			  =		slow_but_exact

        super().__init__(bos_token_id=__lowercase , eos_token_id=__lowercase , **__lowercase)


class 	lowerCamelCase_     (   UpperCAmelCase_ ):






    '''simple docstring'''






    a__	:						Optional[int]						=      version.parse("""1.12""" )







    def __init__(	self , __lowercase , __lowercase = "default" , __lowercase = None , __lowercase = False , )       ->	List[Any]:
        super().__init__(__lowercase , task=__lowercase , patching_specs=__lowercase , use_past=__lowercase)
        if not getattr(self._config , '''pad_token_id''' , __lowercase):
            # TODO: how to do that better?
            __UpperCamelCase     :List[Any]			  =		0







    @property
    def 			UpperCamelCase__   (	self)       ->	Mapping[str, Mapping[int, str]]:
        __UpperCamelCase     :Union[str, Any]			  =		OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}})
        if self.use_past:
            # BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
            self.fill_with_past_key_values_(__lowercase , direction='''inputs''' , inverted_values_shape=__lowercase)
            __UpperCamelCase     :List[str]			  =		{0: '''batch''', 1: '''past_sequence + sequence'''}
        else:
            __UpperCamelCase     :Union[str, Any]			  =		{0: '''batch''', 1: '''sequence'''}

        return common_inputs







    @property
    def 			UpperCamelCase__   (	self)       ->	int:
        return self._config.n_layer







    @property
    def 			UpperCamelCase__   (	self)       ->	int:
        return self._config.n_head







    @property
    def 			UpperCamelCase__   (	self)       ->	float:
        return 1E-3







    def 			UpperCamelCase__   (	self , __lowercase , __lowercase = -1 , __lowercase = -1 , __lowercase = False , __lowercase = None , )       ->	Mapping[str, Any]:
        __UpperCamelCase     :str			  =		super(__lowercase , self).generate_dummy_inputs(
            __lowercase , batch_size=__lowercase , seq_length=__lowercase , is_pair=__lowercase , framework=__lowercase)

        # We need to order the input in the way they appears in the forward()
        __UpperCamelCase     :Optional[Any]			  =		OrderedDict({'''input_ids''': common_inputs['''input_ids''']})

        # Need to add the past_keys
        if self.use_past:
            if not is_torch_available():
                raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.''')
            else:
                import torch

                __UpperCamelCase  ,       __UpperCamelCase     :Tuple			  =		common_inputs['''input_ids'''].shape
                # Not using the same length for past_key_values
                __UpperCamelCase     :List[str]			  =		seqlen + 2
                __UpperCamelCase     :Optional[int]			  =		self._config.hidden_size // self.num_attention_heads
                __UpperCamelCase     :Union[str, Any]			  =		(
                    batch * self.num_attention_heads,
                    head_dim,
                    past_key_values_length,
                )
                __UpperCamelCase     :Optional[Any]			  =		(
                    batch * self.num_attention_heads,
                    past_key_values_length,
                    head_dim,
                )
                __UpperCamelCase     :Tuple			  =		[
                    (torch.zeros(__lowercase), torch.zeros(__lowercase)) for _ in range(self.num_layers)
                ]

        __UpperCamelCase     :str			  =		common_inputs['''attention_mask''']
        if self.use_past:
            __UpperCamelCase     :int			  =		ordered_inputs['''attention_mask'''].dtype
            __UpperCamelCase     :Any			  =		torch.cat(
                [ordered_inputs['''attention_mask'''], torch.ones(__lowercase , __lowercase , dtype=__lowercase)] , dim=1)

        return ordered_inputs







    @property
    def 			UpperCamelCase__   (	self)       ->	int:
        return 13






import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


__lowercase							 =					logging.get_logger(__name__)



def lowerCamelCase  (   SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :Tuple			  =		R'''\w+[.]\d+'''
    __UpperCamelCase     :List[str]			  =		re.findall(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)
    for pat in pats:
        __UpperCamelCase     :int			  =		key.replace(SCREAMING_SNAKE_CASE  ,			'''_'''.join(pat.split('''.'''	)	)	)
    return key



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :Tuple			  =		pt_tuple_key[:-1] + ('''scale''',)
    if (
        any('''norm''' in str_ for str_ in pt_tuple_key	)
        and (pt_tuple_key[-1] == "bias")
        and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
        and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
    ):
        __UpperCamelCase     :str			  =		pt_tuple_key[:-1] + ('''scale''',)
        return renamed_pt_tuple_key, pt_tensor
    elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
        __UpperCamelCase     :Any			  =		pt_tuple_key[:-1] + ('''scale''',)
        return renamed_pt_tuple_key, pt_tensor

    # embedding
    if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
        __UpperCamelCase     :str			  =		pt_tuple_key[:-1] + ('''embedding''',)
        return renamed_pt_tuple_key, pt_tensor

    # conv layer
    __UpperCamelCase     :List[str]			  =		pt_tuple_key[:-1] + ('''kernel''',)
    if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
        __UpperCamelCase     :List[Any]			  =		pt_tensor.transpose(2  ,			3  ,			1  ,			0	)
        return renamed_pt_tuple_key, pt_tensor

    # linear layer
    __UpperCamelCase     :List[str]			  =		pt_tuple_key[:-1] + ('''kernel''',)
    if pt_tuple_key[-1] == "weight":
        __UpperCamelCase     :Any			  =		pt_tensor.T
        return renamed_pt_tuple_key, pt_tensor

    # old PyTorch layer norm weight
    __UpperCamelCase     :int			  =		pt_tuple_key[:-1] + ('''weight''',)
    if pt_tuple_key[-1] == "gamma":
        return renamed_pt_tuple_key, pt_tensor

    # old PyTorch layer norm bias
    __UpperCamelCase     :int			  =		pt_tuple_key[:-1] + ('''bias''',)
    if pt_tuple_key[-1] == "beta":
        return renamed_pt_tuple_key, pt_tensor

    return pt_tuple_key, pt_tensor



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=42	):
    '''simple docstring'''







    __UpperCamelCase     :Union[str, Any]			  =		{k: v.numpy() for k, v in pt_state_dict.items()}

    # Step 2: Since the model is stateless, get random Flax params
    __UpperCamelCase     :str			  =		flax_model.init_weights(PRNGKey(SCREAMING_SNAKE_CASE	)	)

    __UpperCamelCase     :int			  =		flatten_dict(SCREAMING_SNAKE_CASE	)
    __UpperCamelCase     :List[Any]			  =		{}

    # Need to change some parameters name to match Flax names
    for pt_key, pt_tensor in pt_state_dict.items():
        __UpperCamelCase     :List[Any]			  =		rename_key(SCREAMING_SNAKE_CASE	)
        __UpperCamelCase     :List[Any]			  =		tuple(renamed_pt_key.split('''.'''	)	)

        # Correctly rename weight parameters
        __UpperCamelCase  ,       __UpperCamelCase     :Any			  =		rename_key_and_reshape_tensor(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)

        if flax_key in random_flax_state_dict:
            if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                raise ValueError(
                    f"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
                    f"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)

        # also add unexpected weight so that warning is thrown
        __UpperCamelCase     :str			  =		jnp.asarray(SCREAMING_SNAKE_CASE	)

    return unflatten_dict(SCREAMING_SNAKE_CASE	)





