app.py

import gradio as gr
import torch
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
import time
import os
import numpy as np
import soundfile as sf
import librosa

# --- Configuration ---
# Device selection (GPU if available, else CPU)
device = "cuda:0" if torch.cuda.is_available() else "cpu"
torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
print(f"Using device: {device}")

# STT Model (Use smaller model for lower latency)
stt_model_id = "openai/whisper-tiny" # Or "openai/whisper-base". Avoid larger models for streaming.

# Summarization Model
summarizer_model_id = "sshleifer/distilbart-cnn-6-6" # Use a distilled/smaller model for speed

# Summarization Interval (seconds) - How often to regenerate the summary
SUMMARY_INTERVAL = 30.0 # Summarize every 30 seconds

# --- Load Models ---
# (Keep the model loading code exactly the same as before)
print("Loading STT model...")
stt_model = AutoModelForSpeechSeq2Seq.from_pretrained(
    stt_model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True
)
stt_model.to(device)
processor = AutoProcessor.from_pretrained(stt_model_id)
stt_pipeline = pipeline(
    "automatic-speech-recognition",
    model=stt_model,
    tokenizer=processor.tokenizer,
    feature_extractor=processor.feature_extractor,
    max_new_tokens=128,
    chunk_length_s=30,
    batch_size=16,
    torch_dtype=torch_dtype,
    device=device,
)
print("STT model loaded.")

print("Loading Summarization pipeline...")
summarizer = pipeline(
    "summarization",
    model=summarizer_model_id,
    device=device
)
print("Summarization pipeline loaded.")


# --- Helper Functions ---
# (Keep the format_summary_as_bullets function exactly the same)
def format_summary_as_bullets(summary_text):
    """Attempts to format a summary text block into bullet points."""
    if not summary_text:
        return ""
    # Simple approach: split by sentences and add bullets.
    # More advanced NLP could be used here.
    sentences = summary_text.replace(". ", ".\n- ").split('\n')
    bullet_summary = "- " + "\n".join(sentences).strip()
    # Remove potential empty bullets
    bullet_summary = "\n".join([line for line in bullet_summary.split('\n') if line.strip() not in ['-', '']])
    return bullet_summary


# --- Processing Function for Streaming ---
# (Keep the process_audio_stream function exactly the same)
# This function ONLY processes audio, it doesn't interact with the webcam video
def process_audio_stream(
    new_chunk_tuple, # Gradio streaming yields (sample_rate, numpy_data)
    accumulated_transcript_state, # gr.State holding the full text
    last_summary_time_state,    # gr.State holding the timestamp of the last summary
    current_summary_state       # gr.State holding the last generated summary
    ):

    if new_chunk_tuple is None:
        # Initial call or stream ended, return current state
        return accumulated_transcript_state, current_summary_state, accumulated_transcript_state, last_summary_time_state, current_summary_state

    sample_rate, audio_chunk = new_chunk_tuple

    if audio_chunk is None or sample_rate is None or audio_chunk.size == 0:
        # Handle potential empty chunks gracefully
        return accumulated_transcript_state, current_summary_state, accumulated_transcript_state, last_summary_time_state, current_summary_state

    print(f"Received chunk: {audio_chunk.shape}, Sample Rate: {sample_rate}, Duration: {len(audio_chunk)/sample_rate:.2f}s")

    # Ensure audio is float32 and mono, as Whisper expects
    if audio_chunk.dtype != np.float32:
        # Normalize assuming input is int16
        # Adjust if your microphone provides different integer types
        audio_chunk = audio_chunk.astype(np.float32) / 32768.0 # Max value for int16 is 32767

    # --- 1. Transcribe the new chunk ---
    new_text = ""
    try:
        result = stt_pipeline({"sampling_rate": sample_rate, "raw": audio_chunk.copy()})
        new_text = result["text"].strip() if result["text"] else ""
        print(f"Transcription chunk: '{new_text}'")

    except Exception as e:
        print(f"Error during transcription chunk: {e}")
        new_text = f"[Transcription Error: {e}]"

    # --- 2. Update Accumulated Transcript ---
    if accumulated_transcript_state and not accumulated_transcript_state.endswith((" ", "\n")) and new_text:
         updated_transcript = accumulated_transcript_state + " " + new_text
    else:
         updated_transcript = accumulated_transcript_state + new_text

    # --- 3. Periodic Summarization ---
    current_time = time.time()
    new_summary = current_summary_state # Keep the old summary by default
    updated_last_summary_time = last_summary_time_state

    # Check transcript length to avoid summarizing tiny bits of text too early
    if updated_transcript and len(updated_transcript) > 50 and (current_time - last_summary_time_state > SUMMARY_INTERVAL):
        print(f"Summarizing transcript (length: {len(updated_transcript)})...")
        try:
            # Summarize the *entire* transcript up to this point
            summary_result = summarizer(updated_transcript, max_length=150, min_length=30, do_sample=False)
            if summary_result and isinstance(summary_result, list):
                 raw_summary = summary_result[0]['summary_text']
                 new_summary = format_summary_as_bullets(raw_summary)
                 updated_last_summary_time = current_time # Update time only on successful summary
                 print("Summary updated.")
            else:
                 print("Summarization did not produce expected output.")

        except Exception as e:
            print(f"Error during summarization: {e}")
            # Display error in summary box but keep the last known good summary in state
            # To avoid overwriting a potentially useful summary with just an error message
            # We return the error message for display, but not update summary_state with it
            error_display_summary = f"[Summarization Error]\n\nLast good summary:\n{current_summary_state}"
            return updated_transcript, error_display_summary, updated_transcript, last_summary_time_state, current_summary_state


    # --- 4. Return Updated State and Outputs ---
    return updated_transcript, new_summary, updated_transcript, updated_last_summary_time, new_summary


# --- Gradio Interface ---
print("Creating Gradio interface...")
with gr.Blocks() as demo:
    gr.Markdown("# Real-Time Meeting Notes with Webcam View")
    gr.Markdown("Speak into your microphone. Transcription appears below. Summary updates periodically.")

    # State variables to store data between stream calls
    transcript_state = gr.State("") # Holds the full transcript
    last_summary_time = gr.State(0.0) # Holds the time the summary was last generated
    summary_state = gr.State("")    # Holds the current bullet point summary

    with gr.Row():
        with gr.Column(scale=1):
            # Input: Microphone stream
            audio_stream = gr.Audio(sources=["microphone"], streaming=True, label="Live Microphone Input", type="numpy")

            # NEW: Webcam Display
            # Use gr.Image which is simpler for just displaying webcam feed
            # live=True makes it update continuously
            webcam_view = gr.Image(sources=["webcam"], label="Your Webcam", streaming=True) # Use streaming=True for live view

        with gr.Column(scale=2):
             transcription_output = gr.Textbox(label="Full Transcription", lines=15, interactive=False) # Display only
             summary_output = gr.Textbox(label=f"Bullet Point Summary (Updates ~every {SUMMARY_INTERVAL}s)", lines=10, interactive=False) # Display only


    # Connect the streaming audio input to the processing function
    # Note: The webcam component runs independently in the browser, it doesn't feed data here
    audio_stream.stream(
        fn=process_audio_stream,
        inputs=[audio_stream, transcript_state, last_summary_time, summary_state],
        outputs=[transcription_output, summary_output, transcript_state, last_summary_time, summary_state],
    )

    # Add a button to clear the state if needed
    def clear_state_values():
        print("Clearing state.")
        return "", "", 0.0, "" # Clear transcript display, summary display, reset time state, clear summary state
    # Need separate function to clear states vs displays if they differ
    def clear_state():
        return "", 0.0, "" # Clear transcript_state, last_summary_time, summary_state

    clear_button = gr.Button("Clear Transcript & Summary")
    # This button clears the display textboxes AND resets the internal states
    clear_button.click(
        fn=lambda: ("", "", "", 0.0, ""), # Return empty values for all outputs/states
        inputs=[],
        outputs=[transcription_output, summary_output, transcript_state, last_summary_time, summary_state]
    )


print("Launching Gradio interface...")
demo.queue() # Enable queue for handling multiple requests/stream chunks
demo.launch(debug=True, share=True) # share=True for Colab public link