


'''simple docstring'''






from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings






@maybe_allow_in_graph
class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  int    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  Dict=0.0    ,						lowerCamelCase    :  Optional[int] = None    ,						lowerCamelCase    :  str = "geglu"    ,						lowerCamelCase    :  Optional[int] = None    ,						lowerCamelCase    :  bool = False    ,						lowerCamelCase    :  bool = False    ,						lowerCamelCase    :  bool = False    ,						lowerCamelCase    :  bool = False    ,						lowerCamelCase    :  bool = True    ,						lowerCamelCase    :  str = "layer_norm"    ,						lowerCamelCase    :  bool = False    ,						)   ->  Dict:
      super().__init__()
      lowerCAmelCase_			:      Dict			=   only_cross_attention

      lowerCAmelCase_			:      Optional[int]			=   (num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
      lowerCAmelCase_			:      int			=   (num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

      if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
        raise ValueError(
            F'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
            F' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'		)

      # Define 3 blocks. Each block has its own normalization layer.
      # 1. Self-Attn
      if self.use_ada_layer_norm:
        lowerCAmelCase_			:      Any			=   AdaLayerNorm(a__    ,						a__		)
      elif self.use_ada_layer_norm_zero:
        lowerCAmelCase_			:      List[str]			=   AdaLayerNormZero(a__    ,						a__		)
      else:
        lowerCAmelCase_			:      str			=   nn.LayerNorm(a__    ,						elementwise_affine=a__		)
      lowerCAmelCase_			:      Union[str, Any]			=   Attention(
          query_dim=a__    ,						heads=a__    ,						dim_head=a__    ,						dropout=a__    ,						bias=a__    ,						cross_attention_dim=cross_attention_dim if only_cross_attention else None    ,						upcast_attention=a__    ,						)

      # 2. Cross-Attn
      if cross_attention_dim is not None or double_self_attention:
        # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
        # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
        # the second cross attention block.
        lowerCAmelCase_			:      Any			=   (
            AdaLayerNorm(a__    ,						a__		)
            if self.use_ada_layer_norm
            else nn.LayerNorm(a__    ,						elementwise_affine=a__		)
        )
        lowerCAmelCase_			:      Dict			=   Attention(
            query_dim=a__    ,						cross_attention_dim=cross_attention_dim if not double_self_attention else None    ,						heads=a__    ,						dim_head=a__    ,						dropout=a__    ,						bias=a__    ,						upcast_attention=a__    ,						)  # is self-attn if encoder_hidden_states is none
      else:
        lowerCAmelCase_			:      Dict			=   None
        lowerCAmelCase_			:      List[Any]			=   None

      # 3. Feed-forward
      lowerCAmelCase_			:      Optional[Any]			=   nn.LayerNorm(a__    ,						elementwise_affine=a__		)
      lowerCAmelCase_			:      Union[str, Any]			=   FeedForward(a__    ,						dropout=a__    ,						activation_fn=a__    ,						final_dropout=a__		)

      # let chunk size default to None
      lowerCAmelCase_			:      List[str]			=   None
      lowerCAmelCase_			:      Tuple			=   0

    def __lowercase   (   self    :  int    ,						lowerCamelCase    :  Optional[int]    ,						lowerCamelCase    :  int		)   ->  List[Any]:
      lowerCAmelCase_			:      Union[str, Any]			=   chunk_size
      lowerCAmelCase_			:      Optional[Any]			=   dim





    def __lowercase   (   self    :  Dict    ,						lowerCamelCase    :  torch.FloatTensor    ,						lowerCamelCase    :  Optional[torch.FloatTensor] = None    ,						lowerCamelCase    :  Optional[torch.FloatTensor] = None    ,						lowerCamelCase    :  Optional[torch.FloatTensor] = None    ,						lowerCamelCase    :  Optional[torch.LongTensor] = None    ,						lowerCamelCase    :  Dict[str, Any] = None    ,						lowerCamelCase    :  Optional[torch.LongTensor] = None    ,						)   ->  Union[str, Any]:



      if self.use_ada_layer_norm:
        lowerCAmelCase_			:      Optional[Any]			=   self.norma(a__    ,						a__		)
      elif self.use_ada_layer_norm_zero:
        lowerCAmelCase_,							lowerCAmelCase_,							lowerCAmelCase_,							lowerCAmelCase_,							lowerCAmelCase_			:      Union[str, Any]			=   self.norma(
            a__    ,						a__    ,						a__    ,						hidden_dtype=hidden_states.dtype		)
      else:
        lowerCAmelCase_			:      List[Any]			=   self.norma(a__		)

      lowerCAmelCase_			:      Any			=   cross_attention_kwargs if cross_attention_kwargs is not None else {}

      lowerCAmelCase_			:      Dict			=   self.attna(
          a__    ,						encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    ,						attention_mask=a__    ,						**a__    ,						)
      if self.use_ada_layer_norm_zero:
        lowerCAmelCase_			:      int			=   gate_msa.unsqueeze(1		) * attn_output
      lowerCAmelCase_			:      int			=   attn_output + hidden_states

      # 2. Cross-Attention
      if self.attna is not None:
        lowerCAmelCase_			:      Any			=   (
            self.norma(a__    ,						a__		) if self.use_ada_layer_norm else self.norma(a__		)
        )

        lowerCAmelCase_			:      Tuple			=   self.attna(
            a__    ,						encoder_hidden_states=a__    ,						attention_mask=a__    ,						**a__    ,						)
        lowerCAmelCase_			:      Dict			=   attn_output + hidden_states

      # 3. Feed-forward
      lowerCAmelCase_			:      str			=   self.norma(a__		)

      if self.use_ada_layer_norm_zero:
        lowerCAmelCase_			:      Optional[int]			=   norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

      if self._chunk_size is not None:
        # "feed_forward_chunk_size" can be used to save memory
        if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
          raise ValueError(
              F'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'		)

        lowerCAmelCase_			:      Tuple			=   norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
        lowerCAmelCase_			:      int			=   torch.cat(
            [self.ff(a__		) for hid_slice in norm_hidden_states.chunk(a__    ,						dim=self._chunk_dim		)]    ,						dim=self._chunk_dim    ,						)
      else:
        lowerCAmelCase_			:      str			=   self.ff(a__		)

      if self.use_ada_layer_norm_zero:
        lowerCAmelCase_			:      Optional[int]			=   gate_mlp.unsqueeze(1		) * ff_output

      lowerCAmelCase_			:      Optional[Any]			=   ff_output + hidden_states

      return hidden_states







class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  Any    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  Optional[int] = None    ,						lowerCamelCase    :  int = 4    ,						lowerCamelCase    :  float = 0.0    ,						lowerCamelCase    :  str = "geglu"    ,						lowerCamelCase    :  bool = False    ,						)   ->  Tuple:
      super().__init__()
      lowerCAmelCase_			:      str			=   int(dim * mult		)
      lowerCAmelCase_			:      Any			=   dim_out if dim_out is not None else dim

      if activation_fn == "gelu":
        lowerCAmelCase_			:      Union[str, Any]			=   GELU(a__    ,						a__		)
      if activation_fn == "gelu-approximate":
        lowerCAmelCase_			:      str			=   GELU(a__    ,						a__    ,						approximate="""tanh"""		)
      elif activation_fn == "geglu":
        lowerCAmelCase_			:      Optional[int]			=   GEGLU(a__    ,						a__		)
      elif activation_fn == "geglu-approximate":
        lowerCAmelCase_			:      Tuple			=   ApproximateGELU(a__    ,						a__		)

      lowerCAmelCase_			:      Optional[Any]			=   nn.ModuleList([]		)
      # project in
      self.net.append(a__		)
      # project dropout
      self.net.append(nn.Dropout(a__		)		)
      # project out
      self.net.append(nn.Linear(a__    ,						a__		)		)
      # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
      if final_dropout:
        self.net.append(nn.Dropout(a__		)		)





    def __lowercase   (   self    :  Union[str, Any]    ,						lowerCamelCase    :  Optional[int]		)   ->  Union[str, Any]:



      for module in self.net:
        lowerCAmelCase_			:      int			=   module(a__		)
      return hidden_states







class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  Tuple    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  str = "none"		)   ->  Optional[Any]:
      super().__init__()
      lowerCAmelCase_			:      Optional[Any]			=   nn.Linear(a__    ,						a__		)
      lowerCAmelCase_			:      Optional[Any]			=   approximate

    def __lowercase   (   self    :  Any    ,						lowerCamelCase    :  str		)   ->  Any:



      if gate.device.type != "mps":
        return F.gelu(a__    ,						approximate=self.approximate		)
      # mps: gelu is not implemented for float16
      return F.gelu(gate.to(dtype=torch.floataa		)    ,						approximate=self.approximate		).to(dtype=gate.dtype		)





    def __lowercase   (   self    :  Any    ,						lowerCamelCase    :  List[str]		)   ->  Tuple:
      lowerCAmelCase_			:      Union[str, Any]			=   self.proj(a__		)
      lowerCAmelCase_			:      Any			=   self.gelu(a__		)
      return hidden_states







class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  Optional[int]    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int		)   ->  List[str]:
      super().__init__()
      lowerCAmelCase_			:      int			=   nn.Linear(a__    ,						dim_out * 2		)

    def __lowercase   (   self    :  Dict    ,						lowerCamelCase    :  Optional[int]		)   ->  str:



      if gate.device.type != "mps":
        return F.gelu(a__		)
      # mps: gelu is not implemented for float16
      return F.gelu(gate.to(dtype=torch.floataa		)		).to(dtype=gate.dtype		)





    def __lowercase   (   self    :  str    ,						lowerCamelCase    :  List[Any]		)   ->  Dict:
      lowerCAmelCase_,							lowerCAmelCase_			:      Optional[Any]			=   self.proj(a__		).chunk(2    ,						dim=-1		)
      return hidden_states * self.gelu(a__		)







class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  Dict    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int		)   ->  Union[str, Any]:
      super().__init__()
      lowerCAmelCase_			:      str			=   nn.Linear(a__    ,						a__		)





    def __lowercase   (   self    :  str    ,						lowerCamelCase    :  Tuple		)   ->  Union[str, Any]:
      lowerCAmelCase_			:      Optional[int]			=   self.proj(a__		)
      return x * torch.sigmoid(1.702 * x		)







class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  Dict    ,						lowerCamelCase    :  Optional[Any]    ,						lowerCamelCase    :  Union[str, Any]		)   ->  List[str]:
      super().__init__()
      lowerCAmelCase_			:      List[Any]			=   nn.Embedding(a__    ,						a__		)
      lowerCAmelCase_			:      Any			=   nn.SiLU()
      lowerCAmelCase_			:      Tuple			=   nn.Linear(a__    ,						embedding_dim * 2		)
      lowerCAmelCase_			:      Tuple			=   nn.LayerNorm(a__    ,						elementwise_affine=a__		)





    def __lowercase   (   self    :  Any    ,						lowerCamelCase    :  Optional[int]    ,						lowerCamelCase    :  Union[str, Any]		)   ->  List[str]:
      lowerCAmelCase_			:      Union[str, Any]			=   self.linear(self.silu(self.emb(a__		)		)		)
      lowerCAmelCase_,							lowerCAmelCase_			:      Optional[Any]			=   torch.chunk(a__    ,						2		)
      lowerCAmelCase_			:      List[Any]			=   self.norm(a__		) * (1 + scale) + shift
      return x







class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  Dict    ,						lowerCamelCase    :  Dict    ,						lowerCamelCase    :  str		)   ->  Any:
      super().__init__()

      lowerCAmelCase_			:      Dict			=   CombinedTimestepLabelEmbeddings(a__    ,						a__		)

      lowerCAmelCase_			:      Any			=   nn.SiLU()
      lowerCAmelCase_			:      Dict			=   nn.Linear(a__    ,						6 * embedding_dim    ,						bias=a__		)
      lowerCAmelCase_			:      Optional[Any]			=   nn.LayerNorm(a__    ,						elementwise_affine=a__    ,						eps=1E-6		)





    def __lowercase   (   self    :  Optional[Any]    ,						lowerCamelCase    :  List[Any]    ,						lowerCamelCase    :  Union[str, Any]    ,						lowerCamelCase    :  Optional[Any]    ,						lowerCamelCase    :  Union[str, Any]=None		)   ->  Any:
      lowerCAmelCase_			:      Any			=   self.linear(self.silu(self.emb(a__    ,						a__    ,						hidden_dtype=a__		)		)		)
      lowerCAmelCase_,							lowerCAmelCase_,							lowerCAmelCase_,							lowerCAmelCase_,							lowerCAmelCase_,							lowerCAmelCase_			:      Tuple			=   emb.chunk(6    ,						dim=1		)
      lowerCAmelCase_			:      Optional[int]			=   self.norm(a__		) * (1 + scale_msa[:, None]) + shift_msa[:, None]
      return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class    __snake_case       (   nn.Module):







    """simple docstring"""



    def __init__(   self    :  Optional[int]    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  Optional[str] = None    ,						lowerCamelCase    :  float = 1E-5		)   ->  List[str]:
      super().__init__()
      lowerCAmelCase_			:      Any			=   num_groups
      lowerCAmelCase_			:      Union[str, Any]			=   eps

      if act_fn is None:
        lowerCAmelCase_			:      Dict			=   None
      else:
        lowerCAmelCase_			:      Tuple			=   get_activation(a__		)

      lowerCAmelCase_			:      Optional[int]			=   nn.Linear(a__    ,						out_dim * 2		)





    def __lowercase   (   self    :  List[str]    ,						lowerCamelCase    :  Dict    ,						lowerCamelCase    :  List[str]		)   ->  int:



      if self.act:
        lowerCAmelCase_			:      Tuple			=   self.act(a__		)
      lowerCAmelCase_			:      Any			=   self.linear(a__		)
      lowerCAmelCase_			:      List[str]			=   emb[:, :, None, None]
      lowerCAmelCase_,							lowerCAmelCase_			:      Union[str, Any]			=   emb.chunk(2    ,						dim=1		)

      lowerCAmelCase_			:      Any			=   F.group_norm(a__    ,						self.num_groups    ,						eps=self.eps		)
      lowerCAmelCase_			:      Any			=   x * (1 + scale) + shift
      return x










import socket






def lowerCamelCase__    (							)      ->				Any:
   __snake_case							       =   socket.socket(socket.AF_INET       ,      socket.SOCK_STREAM       )
   __snake_case							       =   socket.gethostname()
   __snake_case							       =   1_2312

   sock.connect((host, port)       )
   sock.send(B'''Hello server!'''       )

   with open('''Received_file'''       ,      '''wb'''       ) as out_file:
      print('''File opened'''       )
      print('''Receiving data...'''       )
      while True:
         __snake_case							       =   sock.recv(1024       )
         if not data:
            break
         out_file.write(snake_case_       )

   print('''Successfully received the file'''       )
   sock.close()
   print('''Connection closed'''       )


if __name__ == "__main__":
    main()

