Update app.py

app.py

@@ -6,6 +6,9 @@ from einops import rearrange
 import gradio as gr
 from stable_audio_tools import get_pretrained_model
 from stable_audio_tools.inference.generation import generate_diffusion_cond
+from diffusers import DiffusionPipeline
+from PIL import Image
+from moviepy.editor import AudioFileClip, ImageClip
 
 # Authenticate
 token = os.getenv("HUGGINGFACE_TOKEN")
@@ -13,19 +16,25 @@ if not token:
     raise RuntimeError("HUGGINGFACE_TOKEN not set")
 login(token=token, add_to_git_credential=False)
 
-# Load model
+# Load audio model
 device = "cuda" if torch.cuda.is_available() else "cpu"
-model, config = get_pretrained_model("stabilityai/stable-audio-open-small")
-model = model.to(device)
-sample_rate = config["sample_rate"]
-sample_size = config["sample_size"]
+audio_model, audio_config = get_pretrained_model("stabilityai/stable-audio-open-small")
+audio_model = audio_model.to(device)
+sample_rate = audio_config["sample_rate"]
+sample_size = audio_config["sample_size"]
 
-# Inference function
+# Load image model (Kandinsky)
+image_pipe = DiffusionPipeline.from_pretrained(
+    "kandinsky-community/kandinsky-3",
+    torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
+).to(device)
+
+# Generate audio
 def generate_audio(prompt):
     conditioning = [{"prompt": prompt, "seconds_total": 11}]
     with torch.no_grad():
         output = generate_diffusion_cond(
-            model,
+            audio_model,
             steps=8,
             conditioning=conditioning,
             sample_size=sample_size,
@@ -37,24 +46,45 @@ def generate_audio(prompt):
     torchaudio.save(path, output, sample_rate)
     return path
 
-# 🌀 Hot Prompt Club UI
-gr.Interface(
-    fn=generate_audio,
+# Generate image
+def generate_image(prompt):
+    image = image_pipe(prompt=prompt, height=500, width=500).images[0]
+    image_path = "output.png"
+    image.save(image_path)
+    return image_path
+
+# Combine audio + image into mp4
+def combine_to_video(image_path, audio_path, output_path="output.mp4"):
+    clip = ImageClip(image_path).set_duration(12).set_audio(AudioFileClip(audio_path))
+    clip = clip.set_fps(1)
+    clip.write_videofile(output_path, codec="libx264", audio_codec="aac", fps=1)
+    return output_path
+
+# Unified generation
+def generate_av(prompt):
+    audio_path = generate_audio(prompt)
+    image_path = generate_image(prompt)
+    video_path = combine_to_video(image_path, audio_path)
+    return video_path
+
+# UI
+interface = gr.Interface(
+    fn=generate_av,
     inputs=gr.Textbox(
         label="🎤 Prompt your sonic art here",
         placeholder="e.g. 'drunk driving with mario and yung lean'"
     ),
-    outputs=gr.Audio(
-        type="filepath",
-        label="🧠 Generated Audio"
+    outputs=gr.Video(
+        label="🧠 Generated Audiovisual Clip"
     ),
     title='🌐 Hot Prompts in Your Area: "My Husband Is Dead"',
-    description="Enter a fun sound idea for music art.",
+    description="Enter a fun sound idea for music art. Returns a synced image + audio mp4.",
     examples=[
         "ghosts peeing in a server room",
         "tech startup boss villain entrance music",
         "AI doing acid in a technofeudalist dystopia"
     ],
     css="style.css"
-).launch()
+)
 
+interface.launch()