


from dataclasses import dataclass
from typing import Optional

import numpy as np
import torch
import torch.nn as nn

from ..utils import BaseOutput, is_torch_version, randn_tensor
from .attention_processor import SpatialNorm
from .unet_ad_blocks import UNetMidBlockaD, get_down_block, get_up_block







@dataclass
class     __lowercase    (				SCREAMING_SNAKE_CASE_   ):



		"""simple docstring"""

		UpperCamelCase			: torch.FloatTensor







class     __lowercase    (				nn.Module   ):



		"""simple docstring"""

		def __init__(      self						,	A=3						,	A=3						,	A=("DownEncoderBlock2D",)						,	A=(64,)						,	A=2						,	A=32						,	A="silu"						,	A=True						,	)   ->      Optional[int]:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase  						=   layers_per_block

					lowerCamelCase  						=   torch.nn.Convad(
					    __UpperCAmelCase						,	block_out_channels[0]						,	kernel_size=3						,	stride=1						,	padding=1						,	)

					lowerCamelCase  						=   None
					lowerCamelCase  						=   nn.ModuleList([]     )

					# down
					lowerCamelCase  						=   block_out_channels[0]
					for i, down_block_type in enumerate(__UpperCAmelCase     ):
								lowerCamelCase  						=   output_channel
								lowerCamelCase  						=   block_out_channels[i]
								lowerCamelCase  						=   i == len(__UpperCAmelCase     ) - 1

								lowerCamelCase  						=   get_down_block(
								    __UpperCAmelCase						,	num_layers=self.layers_per_block						,	in_channels=__UpperCAmelCase						,	out_channels=__UpperCAmelCase						,	add_downsample=not is_final_block						,	resnet_eps=1e-6						,	downsample_padding=0						,	resnet_act_fn=__UpperCAmelCase						,	resnet_groups=__UpperCAmelCase						,	attention_head_dim=__UpperCAmelCase						,	temb_channels=__UpperCAmelCase						,	)
								self.down_blocks.append(__UpperCAmelCase     )

					# mid
					lowerCamelCase  						=   UNetMidBlockaD(
					    in_channels=block_out_channels[-1]						,	resnet_eps=1e-6						,	resnet_act_fn=__UpperCAmelCase						,	output_scale_factor=1						,	resnet_time_scale_shift="""default"""						,	attention_head_dim=block_out_channels[-1]						,	resnet_groups=__UpperCAmelCase						,	temb_channels=__UpperCAmelCase						,	)

					# out
					lowerCamelCase  						=   nn.GroupNorm(num_channels=block_out_channels[-1]						,	num_groups=__UpperCAmelCase						,	eps=1e-6     )
					lowerCamelCase  						=   nn.SiLU()

					lowerCamelCase  						=   2 * out_channels if double_z else out_channels
					lowerCamelCase  						=   nn.Convad(block_out_channels[-1]						,	__UpperCAmelCase						,	3						,	padding=1     )

					lowerCamelCase  						=   False







		def  __A					(      self						,	A     )   ->      Optional[Any]:







					'''simple docstring'''







					lowerCamelCase  						=   x
					lowerCamelCase  						=   self.conv_in(__UpperCAmelCase     )

					if self.training and self.gradient_checkpointing:

								def create_custom_forward(A     ):
											def custom_forward(*A     ):
														return module(*__UpperCAmelCase     )

											return custom_forward

								# down
								if is_torch_version(""">="""						,	"""1.11.0"""     ):
											for down_block in self.down_blocks:
														lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(
														    create_custom_forward(__UpperCAmelCase     )						,	__UpperCAmelCase						,	use_reentrant=__UpperCAmelCase     )
											# middle
											lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(
											    create_custom_forward(self.mid_block     )						,	__UpperCAmelCase						,	use_reentrant=__UpperCAmelCase     )
								else:
											for down_block in self.down_blocks:
														lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(create_custom_forward(__UpperCAmelCase     )						,	__UpperCAmelCase     )
											# middle
											lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(create_custom_forward(self.mid_block     )						,	__UpperCAmelCase     )

					else:
								# down
								for down_block in self.down_blocks:
											lowerCamelCase  						=   down_block(__UpperCAmelCase     )

								# middle
								lowerCamelCase  						=   self.mid_block(__UpperCAmelCase     )

					# post-process
					lowerCamelCase  						=   self.conv_norm_out(__UpperCAmelCase     )
					lowerCamelCase  						=   self.conv_act(__UpperCAmelCase     )
					lowerCamelCase  						=   self.conv_out(__UpperCAmelCase     )

					return sample







class     __lowercase    (				nn.Module   ):



		"""simple docstring"""

		def __init__(      self						,	A=3						,	A=3						,	A=("UpDecoderBlock2D",)						,	A=(64,)						,	A=2						,	A=32						,	A="silu"						,	A="group"						,	)   ->      List[Any]:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase  						=   layers_per_block

					lowerCamelCase  						=   nn.Convad(
					    __UpperCAmelCase						,	block_out_channels[-1]						,	kernel_size=3						,	stride=1						,	padding=1						,	)

					lowerCamelCase  						=   None
					lowerCamelCase  						=   nn.ModuleList([]     )

					lowerCamelCase  						=   in_channels if norm_type == """spatial""" else None

					# mid
					lowerCamelCase  						=   UNetMidBlockaD(
					    in_channels=block_out_channels[-1]						,	resnet_eps=1e-6						,	resnet_act_fn=__UpperCAmelCase						,	output_scale_factor=1						,	resnet_time_scale_shift="""default""" if norm_type == """group""" else norm_type						,	attention_head_dim=block_out_channels[-1]						,	resnet_groups=__UpperCAmelCase						,	temb_channels=__UpperCAmelCase						,	)

					# up
					lowerCamelCase  						=   list(reversed(__UpperCAmelCase     )     )
					lowerCamelCase  						=   reversed_block_out_channels[0]
					for i, up_block_type in enumerate(__UpperCAmelCase     ):
								lowerCamelCase  						=   output_channel
								lowerCamelCase  						=   reversed_block_out_channels[i]

								lowerCamelCase  						=   i == len(__UpperCAmelCase     ) - 1

								lowerCamelCase  						=   get_up_block(
								    __UpperCAmelCase						,	num_layers=self.layers_per_block + 1						,	in_channels=__UpperCAmelCase						,	out_channels=__UpperCAmelCase						,	prev_output_channel=__UpperCAmelCase						,	add_upsample=not is_final_block						,	resnet_eps=1e-6						,	resnet_act_fn=__UpperCAmelCase						,	resnet_groups=__UpperCAmelCase						,	attention_head_dim=__UpperCAmelCase						,	temb_channels=__UpperCAmelCase						,	resnet_time_scale_shift=__UpperCAmelCase						,	)
								self.up_blocks.append(__UpperCAmelCase     )
								lowerCamelCase  						=   output_channel

					# out
					if norm_type == "spatial":
								lowerCamelCase  						=   SpatialNorm(block_out_channels[0]						,	__UpperCAmelCase     )
					else:
								lowerCamelCase  						=   nn.GroupNorm(num_channels=block_out_channels[0]						,	num_groups=__UpperCAmelCase						,	eps=1e-6     )
					lowerCamelCase  						=   nn.SiLU()
					lowerCamelCase  						=   nn.Convad(block_out_channels[0]						,	__UpperCAmelCase						,	3						,	padding=1     )

					lowerCamelCase  						=   False







		def  __A					(      self						,	A						,	A=None     )   ->      Optional[Any]:







					'''simple docstring'''







					lowerCamelCase  						=   z
					lowerCamelCase  						=   self.conv_in(__UpperCAmelCase     )

					lowerCamelCase  						=   next(iter(self.up_blocks.parameters()     )     ).dtype
					if self.training and self.gradient_checkpointing:

								def create_custom_forward(A     ):
											def custom_forward(*A     ):
														return module(*__UpperCAmelCase     )

											return custom_forward

								if is_torch_version(""">="""						,	"""1.11.0"""     ):
											# middle
											lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(
											    create_custom_forward(self.mid_block     )						,	__UpperCAmelCase						,	__UpperCAmelCase						,	use_reentrant=__UpperCAmelCase     )
											lowerCamelCase  						=   sample.to(__UpperCAmelCase     )

											# up
											for up_block in self.up_blocks:
														lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(
														    create_custom_forward(__UpperCAmelCase     )						,	__UpperCAmelCase						,	__UpperCAmelCase						,	use_reentrant=__UpperCAmelCase     )
								else:
											# middle
											lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(
											    create_custom_forward(self.mid_block     )						,	__UpperCAmelCase						,	__UpperCAmelCase     )
											lowerCamelCase  						=   sample.to(__UpperCAmelCase     )

											# up
											for up_block in self.up_blocks:
														lowerCamelCase  						=   torch.utils.checkpoint.checkpoint(create_custom_forward(__UpperCAmelCase     )						,	__UpperCAmelCase						,	__UpperCAmelCase     )
					else:
								# middle
								lowerCamelCase  						=   self.mid_block(__UpperCAmelCase						,	__UpperCAmelCase     )
								lowerCamelCase  						=   sample.to(__UpperCAmelCase     )

								# up
								for up_block in self.up_blocks:
											lowerCamelCase  						=   up_block(__UpperCAmelCase						,	__UpperCAmelCase     )

        # post-process
					if latent_embeds is None:
								lowerCamelCase  						=   self.conv_norm_out(__UpperCAmelCase     )
					else:
								lowerCamelCase  						=   self.conv_norm_out(__UpperCAmelCase						,	__UpperCAmelCase     )
					lowerCamelCase  						=   self.conv_act(__UpperCAmelCase     )
					lowerCamelCase  						=   self.conv_out(__UpperCAmelCase     )

					return sample







class     __lowercase    (				nn.Module   ):



		"""simple docstring"""

		def __init__(      self						,	A						,	A						,	A						,	A=None						,	A="random"						,	A=False						,	A=True     )   ->      List[Any]:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase  						=   n_e
					lowerCamelCase  						=   vq_embed_dim
					lowerCamelCase  						=   beta
					lowerCamelCase  						=   legacy

					lowerCamelCase  						=   nn.Embedding(self.n_e						,	self.vq_embed_dim     )
					self.embedding.weight.data.uniform_(-1.0 / self.n_e						,	1.0 / self.n_e     )

					lowerCamelCase  						=   remap
					if self.remap is not None:
								self.register_buffer("""used"""						,	torch.tensor(np.load(self.remap     )     )     )
								lowerCamelCase  						=   self.used.shape[0]
								lowerCamelCase  						=   unknown_index  # "random" or "extra" or integer
								if self.unknown_index == "extra":
											lowerCamelCase  						=   self.re_embed
											lowerCamelCase  						=   self.re_embed + 1
								print(
								    F'Remapping {self.n_e} indices to {self.re_embed} indices. '
								    F'Using {self.unknown_index} for unknown indices.'     )
					else:
								lowerCamelCase  						=   n_e

					lowerCamelCase  						=   sane_index_shape







		def  __A					(      self						,	A     )   ->      List[str]:







					'''simple docstring'''







					lowerCamelCase  						=   inds.shape
					assert len(__UpperCAmelCase     ) > 1
					lowerCamelCase  						=   inds.reshape(ishape[0]						,	-1     )
					lowerCamelCase  						=   self.used.to(__UpperCAmelCase     )
					lowerCamelCase  						=   (inds[:, :, None] == used[None, None, ...]).long()
					lowerCamelCase  						=   match.argmax(-1     )
					lowerCamelCase  						=   match.sum(2     ) < 1
					if self.unknown_index == "random":
								lowerCamelCase  						=   torch.randint(0						,	self.re_embed						,	size=new[unknown].shape     ).to(device=new.device     )
					else:
								lowerCamelCase  						=   self.unknown_index
					return new.reshape(__UpperCAmelCase     )







		def  __A					(      self						,	A     )   ->      str:







					'''simple docstring'''







					lowerCamelCase  						=   inds.shape
					assert len(__UpperCAmelCase     ) > 1
					lowerCamelCase  						=   inds.reshape(ishape[0]						,	-1     )
					lowerCamelCase  						=   self.used.to(__UpperCAmelCase     )
					if self.re_embed > self.used.shape[0]:  # extra token
								lowerCamelCase  						=   0  # simply set to zero
					lowerCamelCase  						=   torch.gather(used[None, :][inds.shape[0] * [0], :]						,	1						,	__UpperCAmelCase     )
					return back.reshape(__UpperCAmelCase     )







		def  __A					(      self						,	A     )   ->      Union[str, Any]:







					'''simple docstring'''







					lowerCamelCase  						=   z.permute(0						,	2						,	3						,	1     ).contiguous()
					lowerCamelCase  						=   z.view(-1						,	self.vq_embed_dim     )

					# distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
					lowerCamelCase  						=   torch.argmin(torch.cdist(__UpperCAmelCase						,	self.embedding.weight     )						,	dim=1     )

					lowerCamelCase  						=   self.embedding(__UpperCAmelCase     ).view(z.shape     )
					lowerCamelCase  						=   None
					lowerCamelCase  						=   None

					# compute loss for embedding
					if not self.legacy:
								lowerCamelCase  						=   self.beta * torch.mean((z_q.detach() - z) ** 2     ) + torch.mean((z_q - z.detach()) ** 2     )
					else:
								lowerCamelCase  						=   torch.mean((z_q.detach() - z) ** 2     ) + self.beta * torch.mean((z_q - z.detach()) ** 2     )

					# preserve gradients
					lowerCamelCase  						=   z + (z_q - z).detach()

					# reshape back to match original input shape
					lowerCamelCase  						=   z_q.permute(0						,	3						,	1						,	2     ).contiguous()

					if self.remap is not None:
								lowerCamelCase  						=   min_encoding_indices.reshape(z.shape[0]						,	-1     )  # add batch axis
								lowerCamelCase  						=   self.remap_to_used(__UpperCAmelCase     )
								lowerCamelCase  						=   min_encoding_indices.reshape(-1						,	1     )  # flatten

					if self.sane_index_shape:
								lowerCamelCase  						=   min_encoding_indices.reshape(z_q.shape[0]						,	z_q.shape[2]						,	z_q.shape[3]     )

					return z_q, loss, (perplexity, min_encodings, min_encoding_indices)







		def  __A					(      self						,	A						,	A     )   ->      Tuple:







					'''simple docstring'''







					if self.remap is not None:
								lowerCamelCase  						=   indices.reshape(shape[0]						,	-1     )  # add batch axis
								lowerCamelCase  						=   self.unmap_to_all(__UpperCAmelCase     )
								lowerCamelCase  						=   indices.reshape(-1     )  # flatten again

					# get quantized latent vectors
					lowerCamelCase  						=   self.embedding(__UpperCAmelCase     )

					if shape is not None:
								lowerCamelCase  						=   z_q.view(__UpperCAmelCase     )
								# reshape back to match original input shape
								lowerCamelCase  						=   z_q.permute(0						,	3						,	1						,	2     ).contiguous()

					return z_q







class     __lowercase    (				SCREAMING_SNAKE_CASE_   ):



		"""simple docstring"""

		def __init__(      self						,	A						,	A=False     )   ->      Dict:







					'''simple docstring'''







					lowerCamelCase  						=   parameters
					lowerCamelCase  						=   torch.chunk(__UpperCAmelCase						,	2						,	dim=1     )
					lowerCamelCase  						=   torch.clamp(self.logvar						,	-30.0						,	20.0     )
					lowerCamelCase  						=   deterministic
					lowerCamelCase  						=   torch.exp(0.5 * self.logvar     )
					lowerCamelCase  						=   torch.exp(self.logvar     )
					if self.deterministic:
								lowerCamelCase  						=   torch.zeros_like(
								    self.mean						,	device=self.parameters.device						,	dtype=self.parameters.dtype     )







		def  __A					(      self						,	A = None     )   ->      torch.FloatTensor:







					'''simple docstring'''







					lowerCamelCase  						=   randn_tensor(
					    self.mean.shape						,	generator=__UpperCAmelCase						,	device=self.parameters.device						,	dtype=self.parameters.dtype     )
					lowerCamelCase  						=   self.mean + self.std * sample
					return x







		def  __A					(      self						,	A=None     )   ->      str:







					'''simple docstring'''







					if self.deterministic:
								return torch.Tensor([0.0]     )
					else:
								if other is None:
											return 0.5 * torch.sum(torch.pow(self.mean						,	2     ) + self.var - 1.0 - self.logvar						,	dim=[1, 2, 3]     )
								else:
											return 0.5 * torch.sum(
											    torch.pow(self.mean - other.mean						,	2     ) / other.var
											    + self.var / other.var
											    - 1.0
											    - self.logvar
											    + other.logvar						,	dim=[1, 2, 3]						,	)







		def  __A					(      self						,	A						,	A=[1, 2, 3]     )   ->      Optional[Any]:







					'''simple docstring'''







					if self.deterministic:
								return torch.Tensor([0.0]     )
					lowerCamelCase  						=   np.log(2.0 * np.pi     )
					return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean						,	2     ) / self.var						,	dim=__UpperCAmelCase     )







		def  __A					(      self     )   ->      Optional[int]:







					'''simple docstring'''







					return self.mean






'''simple docstring'''





from __future__ import annotations

import math
from collections import Counter
from string import ascii_lowercase
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase			):



    """simple docstring"""





    lowerCAmelCase__      ,   lowerCAmelCase__ :  List[str]							      =					analyze_text(UpperCamelCase			)
    lowerCAmelCase__ :  Optional[int]							      =					list(""" """ + ascii_lowercase			)
    # what is our total sum of probabilities.
    lowerCAmelCase__ :  List[Any]							      =					sum(single_char_strings.values()			)

    # one length string
    lowerCAmelCase__ :  Optional[int]							      =					0
    # for each alpha we go in our dict and if it is in it we calculate entropy
    for ch in my_alphas:
        if ch in single_char_strings:
            lowerCAmelCase__ :  List[Any]							      =					single_char_strings[ch]
            lowerCAmelCase__ :  List[Any]							      =					my_str / all_sum
            my_fir_sum += prob * math.loga(UpperCamelCase			)  # entropy formula.

    # print entropy
    print(f"""{round(-1 * my_fir_sum			):.1f}"""			)

    # two len string
    lowerCAmelCase__ :  Dict							      =					sum(two_char_strings.values()			)
    lowerCAmelCase__ :  int							      =					0
    # for each alpha (two in size) calculate entropy.
    for cha in my_alphas:
        for cha in my_alphas:
            lowerCAmelCase__ :  Union[str, Any]							      =					cha + cha
            if sequence in two_char_strings:
                lowerCAmelCase__ :  Dict							      =					two_char_strings[sequence]
                lowerCAmelCase__ :  Tuple							      =					int(UpperCamelCase			) / all_sum
                my_sec_sum += prob * math.loga(UpperCamelCase			)

    # print second entropy
    print(f"""{round(-1 * my_sec_sum			):.1f}"""			)

    # print the difference between them
    print(f"""{round((-1 * my_sec_sum) - (-1 * my_fir_sum)			):.1f}"""			)
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase			):



    """simple docstring"""





    lowerCAmelCase__ :  Optional[Any]							      =					Counter()  # type: ignore
    lowerCAmelCase__ :  Tuple							      =					Counter()  # type: ignore
    single_char_strings[text[-1]] += 1

    # first case when we have space at start.
    two_char_strings[" " + text[0]] += 1
    for i in range(0							,    len(UpperCamelCase			) - 1			):
        single_char_strings[text[i]] += 1
        two_char_strings[text[i : i + 2]] += 1
    return single_char_strings, two_char_strings
def    _SCREAMING_SNAKE_CASE  (       ):



    """simple docstring"""





    import doctest

    doctest.testmod()
    # text = (
    #     "Had repulsive dashwoods suspicion sincerity but advantage now him. Remark "
    #     "easily garret nor nay. Civil those mrs enjoy shy fat merry. You greatest "
    #     "jointure saw horrible. He private he on be imagine suppose. Fertile "
    #     "beloved evident through no service elderly is. Blind there if every no so "
    #     "at. Own neglected you preferred way sincerity delivered his attempted. To "
    #     "of message cottage windows do besides against uncivil.  Delightful "
    #     "unreserved impossible few estimating men favourable see entreaties. She "
    #     "propriety immediate was improving. He or entrance humoured likewise "
    #     "moderate. Much nor game son say feel. Fat make met can must form into "
    #     "gate. Me we offending prevailed discovery. "
    # )

    # calculate_prob(text)


if __name__ == "__main__":
 main()


