
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase							=       logging.get_logger(__name__)

__lowerCamelCase							=       {
    'microsoft/unispeech-large-1500h-cv': (
        'https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}






class   UpperCAmelCase		(						_lowerCamelCase ):
   A__					:		Dict						     =     "unispeech"



   def __init__(self		:					int    ,						snake_case__		:					Any=32    ,						snake_case__		:					Optional[int]=7_68    ,						snake_case__		:					Union[str, Any]=12    ,						snake_case__		:					Optional[int]=12    ,						snake_case__		:					List[Any]=30_72    ,						snake_case__		:					Tuple="gelu"    ,						snake_case__		:					Tuple=0.1    ,						snake_case__		:					Optional[Any]=0.1    ,						snake_case__		:					Tuple=0.1    ,						snake_case__		:					List[str]=0.0    ,						snake_case__		:					List[str]=0.0    ,						snake_case__		:					Tuple=0.1    ,						snake_case__		:					Tuple=0.1    ,						snake_case__		:					List[str]=0.02    ,						snake_case__		:					str=1e-5    ,						snake_case__		:					Optional[int]="group"    ,						snake_case__		:					Union[str, Any]="gelu"    ,						snake_case__		:					List[Any]=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)    ,						snake_case__		:					List[str]=(5, 2, 2, 2, 2, 2, 2)    ,						snake_case__		:					Dict=(10, 3, 3, 3, 3, 2, 2)    ,						snake_case__		:					Optional[Any]=False    ,						snake_case__		:					Optional[Any]=1_28    ,						snake_case__		:					Any=16    ,						snake_case__		:					Optional[int]=False    ,						snake_case__		:					List[str]=True    ,						snake_case__		:					Optional[int]=0.05    ,						snake_case__		:					Dict=10    ,						snake_case__		:					int=2    ,						snake_case__		:					Optional[int]=0.0    ,						snake_case__		:					Optional[Any]=10    ,						snake_case__		:					Dict=0    ,						snake_case__		:					str=3_20    ,						snake_case__		:					Dict=2    ,						snake_case__		:					Tuple=0.1    ,						snake_case__		:					Tuple=1_00    ,						snake_case__		:					Tuple=2_56    ,						snake_case__		:					str=2_56    ,						snake_case__		:					Union[str, Any]=0.1    ,						snake_case__		:					Any="mean"    ,						snake_case__		:					Union[str, Any]=False    ,						snake_case__		:					Tuple=False    ,						snake_case__		:					Tuple=2_56    ,						snake_case__		:					Tuple=80    ,						snake_case__		:					int=0    ,						snake_case__		:					List[str]=1    ,						snake_case__		:					Optional[int]=2    ,						snake_case__		:					Optional[Any]=0.5    ,						**snake_case__		:					Dict    ,						)		->	Tuple:


          '''simple docstring'''


          super().__init__(**lowercase_    ,						pad_token_id=lowercase_    ,						bos_token_id=lowercase_    ,						eos_token_id=lowercase_ )
          snake_case       :   int							=      hidden_size
          snake_case       :   int							=      feat_extract_norm
          snake_case       :   Optional[int]							=      feat_extract_activation
          snake_case       :   str							=      list(lowercase_ )
          snake_case       :   Any							=      list(lowercase_ )
          snake_case       :   Optional[Any]							=      list(lowercase_ )
          snake_case       :   str							=      conv_bias
          snake_case       :   Optional[Any]							=      num_conv_pos_embeddings
          snake_case       :   List[str]							=      num_conv_pos_embedding_groups
          snake_case       :   Tuple							=      len(self.conv_dim )
          snake_case       :   Any							=      num_hidden_layers
          snake_case       :   List[str]							=      intermediate_size
          snake_case       :   Optional[int]							=      hidden_act
          snake_case       :   Tuple							=      num_attention_heads
          snake_case       :   Any							=      hidden_dropout
          snake_case       :   Dict							=      attention_dropout
          snake_case       :   int							=      activation_dropout
          snake_case       :   Dict							=      feat_proj_dropout
          snake_case       :   Optional[int]							=      final_dropout
          snake_case       :   Any							=      layerdrop
          snake_case       :   List[Any]							=      layer_norm_eps
          snake_case       :   int							=      initializer_range
          snake_case       :   List[str]							=      num_ctc_classes
          snake_case       :   Tuple							=      vocab_size
          snake_case       :   Dict							=      do_stable_layer_norm
          snake_case       :   List[str]							=      use_weighted_layer_sum
          snake_case       :   Dict							=      classifier_proj_size

          if (
              (len(self.conv_stride ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel ) != self.num_feat_extract_layers)
              or (len(self.conv_dim ) != self.num_feat_extract_layers)
          ):
                 raise ValueError(
                     "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                     " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                     f""" {len(self.conv_dim )}`, `len(config.conv_stride) = {len(self.conv_stride )}`,"""
                     f""" `len(config.conv_kernel) = {len(self.conv_kernel )}`.""" )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          snake_case       :   Optional[int]							=      apply_spec_augment
          snake_case       :   Tuple							=      mask_time_prob
          snake_case       :   Optional[int]							=      mask_time_length
          snake_case       :   Optional[int]							=      mask_time_min_masks
          snake_case       :   List[str]							=      mask_feature_prob
          snake_case       :   int							=      mask_feature_length
          snake_case       :   Optional[int]							=      mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          snake_case       :   Tuple							=      num_codevectors_per_group
          snake_case       :   List[Any]							=      num_codevector_groups
          snake_case       :   List[str]							=      contrastive_logits_temperature
          snake_case       :   int							=      feat_quantizer_dropout
          snake_case       :   Optional[int]							=      num_negatives
          snake_case       :   Optional[Any]							=      codevector_dim
          snake_case       :   int							=      proj_codevector_dim
          snake_case       :   Optional[Any]							=      diversity_loss_weight

          # ctc loss
          snake_case       :   int							=      ctc_loss_reduction
          snake_case       :   int							=      ctc_zero_infinity

          # pretraining loss
          snake_case       :   Optional[int]							=      replace_prob



   @property
   def 				_SCREAMING_SNAKE_CASE   (self		:					str )		->	Optional[Any]:


          '''simple docstring'''


          return functools.reduce(operator.mul    ,						self.conv_stride    ,						1 )











'''simple docstring'''

from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxCrossAttnUpBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
    FlaxUpBlockaD,
)






@flax.struct.dataclass
class 						a				(   _lowerCamelCase     ):
				snake_case_      =		42







@flax_register_to_config
class 						a				(   nn.Module   ,     _lowerCamelCase   ,     _lowerCamelCase     ):
				snake_case_      =		32
				snake_case_      =		4
				snake_case_      =		4
				snake_case_      =		(
				    "CrossAttnDownBlock2D",
				    "CrossAttnDownBlock2D",
				    "CrossAttnDownBlock2D",
				    "DownBlock2D",
				)
				snake_case_      =		("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")
				snake_case_      =		False
				snake_case_      =		(320, 640, 1_280, 1_280)
				snake_case_      =		2
				snake_case_      =		8
				snake_case_      =		None
				snake_case_      =		1_280
				snake_case_      =		0.0
				snake_case_      =		False
				snake_case_      =		jnp.floataa
				snake_case_      =		True
				snake_case_      =		0
				snake_case_      =		False


				def   A_ (		self    :					Optional[int]       ,				lowercase_    :					jax.random.KeyArray					):
									# init input tensors
									snake_case_									=				(1, self.in_channels, self.sample_size, self.sample_size)
									snake_case_									=				jnp.zeros(lowercase_       ,				dtype=jnp.floataa					)
									snake_case_									=				jnp.ones((1,)       ,				dtype=jnp.intaa					)
									snake_case_									=				jnp.zeros((1, 1, self.cross_attention_dim)       ,				dtype=jnp.floataa					)

									snake_case_			,snake_case_									=				jax.random.split(lowercase_					)
									snake_case_									=				{'''params''': params_rng, '''dropout''': dropout_rng}

									return self.init(lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)["params"]


				def   A_ (		self    :					List[str]					):
									snake_case_									=				self.block_out_channels
									snake_case_									=				block_out_channels[0] * 4

									if self.num_attention_heads is not None:
														raise ValueError(
														    '''At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19.'''					)

									# If `num_attention_heads` is not defined (which is the case for most models)
									# it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
									# The reason for this behavior is to correct for incorrectly named variables that were introduced
									# when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
									# Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
									# which is why we correct for the naming here.
									snake_case_									=				self.num_attention_heads or self.attention_head_dim

									# input
									snake_case_									=				nn.Conv(
									    block_out_channels[0]       ,				kernel_size=(3, 3)       ,				strides=(1, 1)       ,				padding=((1, 1), (1, 1))       ,				dtype=self.dtype       ,				)

									# time
									snake_case_									=				FlaxTimesteps(
									    block_out_channels[0]       ,				flip_sin_to_cos=self.flip_sin_to_cos       ,				freq_shift=self.config.freq_shift					)
									snake_case_									=				FlaxTimestepEmbedding(lowercase_       ,				dtype=self.dtype					)

									snake_case_									=				self.only_cross_attention
									if isinstance(lowercase_       ,				lowercase_					):
														snake_case_									=				(only_cross_attention,) * len(self.down_block_types					)

									if isinstance(lowercase_       ,				lowercase_					):
														snake_case_									=				(num_attention_heads,) * len(self.down_block_types					)

									# down
									snake_case_									=				[]
									snake_case_									=				block_out_channels[0]
									for i, down_block_type in enumerate(self.down_block_types					):
														snake_case_									=				output_channel
														snake_case_									=				block_out_channels[i]
														snake_case_									=				i == len(lowercase_					) - 1

														if down_block_type == "CrossAttnDownBlock2D":
																			snake_case_									=				FlaxCrossAttnDownBlockaD(
																			    in_channels=lowercase_       ,				out_channels=lowercase_       ,				dropout=self.dropout       ,				num_layers=self.layers_per_block       ,				num_attention_heads=num_attention_heads[i]       ,				add_downsample=not is_final_block       ,				use_linear_projection=self.use_linear_projection       ,				only_cross_attention=only_cross_attention[i]       ,				use_memory_efficient_attention=self.use_memory_efficient_attention       ,				dtype=self.dtype       ,				)
														else:
																			snake_case_									=				FlaxDownBlockaD(
																			    in_channels=lowercase_       ,				out_channels=lowercase_       ,				dropout=self.dropout       ,				num_layers=self.layers_per_block       ,				add_downsample=not is_final_block       ,				dtype=self.dtype       ,				)

														down_blocks.append(lowercase_					)
									snake_case_									=				down_blocks

									# mid
									snake_case_									=				FlaxUNetMidBlockaDCrossAttn(
									    in_channels=block_out_channels[-1]       ,				dropout=self.dropout       ,				num_attention_heads=num_attention_heads[-1]       ,				use_linear_projection=self.use_linear_projection       ,				use_memory_efficient_attention=self.use_memory_efficient_attention       ,				dtype=self.dtype       ,				)

									# up
									snake_case_									=				[]
									snake_case_									=				list(reversed(lowercase_					)					)
									snake_case_									=				list(reversed(lowercase_					)					)
									snake_case_									=				list(reversed(lowercase_					)					)
									snake_case_									=				reversed_block_out_channels[0]
									for i, up_block_type in enumerate(self.up_block_types					):
														snake_case_									=				output_channel
														snake_case_									=				reversed_block_out_channels[i]
														snake_case_									=				reversed_block_out_channels[min(i + 1       ,				len(lowercase_					) - 1					)]

														snake_case_									=				i == len(lowercase_					) - 1

														if up_block_type == "CrossAttnUpBlock2D":
																			snake_case_									=				FlaxCrossAttnUpBlockaD(
																			    in_channels=lowercase_       ,				out_channels=lowercase_       ,				prev_output_channel=lowercase_       ,				num_layers=self.layers_per_block + 1       ,				num_attention_heads=reversed_num_attention_heads[i]       ,				add_upsample=not is_final_block       ,				dropout=self.dropout       ,				use_linear_projection=self.use_linear_projection       ,				only_cross_attention=only_cross_attention[i]       ,				use_memory_efficient_attention=self.use_memory_efficient_attention       ,				dtype=self.dtype       ,				)
														else:
																			snake_case_									=				FlaxUpBlockaD(
																			    in_channels=lowercase_       ,				out_channels=lowercase_       ,				prev_output_channel=lowercase_       ,				num_layers=self.layers_per_block + 1       ,				add_upsample=not is_final_block       ,				dropout=self.dropout       ,				dtype=self.dtype       ,				)

														up_blocks.append(lowercase_					)
														snake_case_									=				output_channel
									snake_case_									=				up_blocks

									# out
									snake_case_									=				nn.GroupNorm(num_groups=32       ,				epsilon=1e-5					)
									snake_case_									=				nn.Conv(
									    self.out_channels       ,				kernel_size=(3, 3)       ,				strides=(1, 1)       ,				padding=((1, 1), (1, 1))       ,				dtype=self.dtype       ,				)


				def __call__(		self    :					Union[str, Any]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					Any       ,				lowercase_    :					int=None       ,				lowercase_    :					Any=None       ,				lowercase_    :					bool = True       ,				lowercase_    :					bool = False       ,				):
									# 1. time
									if not isinstance(lowercase_       ,				jnp.ndarray					):
														snake_case_									=				jnp.array([timesteps]       ,				dtype=jnp.intaa					)
									elif isinstance(lowercase_       ,				jnp.ndarray					) and len(timesteps.shape					) == 0:
														snake_case_									=				timesteps.astype(dtype=jnp.floataa					)
														snake_case_									=				jnp.expand_dims(lowercase_       ,				0					)

									snake_case_									=				self.time_proj(lowercase_					)
									snake_case_									=				self.time_embedding(lowercase_					)

									# 2. pre-process
									snake_case_									=				jnp.transpose(lowercase_       ,				(0, 2, 3, 1)					)
									snake_case_									=				self.conv_in(lowercase_					)

									# 3. down
									snake_case_									=				(sample,)
									for down_block in self.down_blocks:
														if isinstance(lowercase_       ,				lowercase_					):
																			snake_case_			,snake_case_									=				down_block(lowercase_       ,				lowercase_       ,				lowercase_       ,				deterministic=not train					)
														else:
																			snake_case_			,snake_case_									=				down_block(lowercase_       ,				lowercase_       ,				deterministic=not train					)
														down_block_res_samples += res_samples

									if down_block_additional_residuals is not None:
														snake_case_									=				()

														for down_block_res_sample, down_block_additional_residual in zip(
														    lowercase_       ,				lowercase_					):
																			down_block_res_sample += down_block_additional_residual
																			new_down_block_res_samples += (down_block_res_sample,)

														snake_case_									=				new_down_block_res_samples

									# 4. mid
									snake_case_									=				self.mid_block(lowercase_       ,				lowercase_       ,				lowercase_       ,				deterministic=not train					)

									if mid_block_additional_residual is not None:
														sample += mid_block_additional_residual

									# 5. up
									for up_block in self.up_blocks:
														snake_case_									=				down_block_res_samples[-(self.layers_per_block + 1) :]
														snake_case_									=				down_block_res_samples[: -(self.layers_per_block + 1)]
														if isinstance(lowercase_       ,				lowercase_					):
																			snake_case_									=				up_block(
																			    lowercase_       ,				temb=lowercase_       ,				encoder_hidden_states=lowercase_       ,				res_hidden_states_tuple=lowercase_       ,				deterministic=not train       ,				)
														else:
																			snake_case_									=				up_block(lowercase_       ,				temb=lowercase_       ,				res_hidden_states_tuple=lowercase_       ,				deterministic=not train					)

        # 6. post-process
									snake_case_									=				self.conv_norm_out(lowercase_					)
									snake_case_									=				nn.silu(lowercase_					)
									snake_case_									=				self.conv_out(lowercase_					)
									snake_case_									=				jnp.transpose(lowercase_       ,				(0, 3, 1, 2)					)

									if not return_dict:
														return (sample,)

									return FlaxUNetaDConditionOutput(sample=lowercase_					)

