fix image q

app.py

@@ -15,6 +15,8 @@ from transformers import (
     SamProcessor,
     VitsModel,
     pipeline,
+    BlipForQuestionAnswering,
+    BlipProcessor,
 )
 
 
@@ -94,6 +96,17 @@ def get_zero_shot_audio_pipeline():
     return MODEL_STORE["audio_zero_shot_clap"]
 
 
+def get_blip_vqa_components() -> Tuple[BlipForQuestionAnswering, BlipProcessor]:
+    if "blip_vqa_model" not in MODEL_STORE or "blip_vqa_processor" not in MODEL_STORE:
+        blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+        blip_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
+        MODEL_STORE["blip_vqa_model"] = blip_model
+        MODEL_STORE["blip_vqa_processor"] = blip_processor
+
+    blip_model = MODEL_STORE["blip_vqa_model"]
+    blip_processor = MODEL_STORE["blip_vqa_processor"]
+    return blip_model, blip_processor
+
 def get_vision_pipeline(model_key: str):
     if model_key in MODEL_STORE:
         return MODEL_STORE[model_key]
@@ -185,16 +198,15 @@ def get_silero_tts_model():
     return MODEL_STORE["silero_tts_model"]
 
 
-def get_mms_tts_components() -> Tuple[VitsModel, AutoTokenizer]:
-    if "mms_tts_model" not in MODEL_STORE or "mms_tts_tokenizer" not in MODEL_STORE:
-        vits_model = VitsModel.from_pretrained("kakao-enterprise/vits-ljs")
-        vits_tokenizer = AutoTokenizer.from_pretrained("kakao-enterprise/vits-ljs")
-        MODEL_STORE["mms_tts_model"] = vits_model
-        MODEL_STORE["mms_tts_tokenizer"] = vits_tokenizer
+def get_mms_tts_components():
+    if "mms_tts_pipeline" not in MODEL_STORE:
+        tts_pipeline = pipeline(
+            task="text-to-speech",
+            model="kakao-enterprise/vits-ljs",
+        )
+        MODEL_STORE["mms_tts_pipeline"] = tts_pipeline
 
-    vits_model = MODEL_STORE["mms_tts_model"]
-    vits_tokenizer = MODEL_STORE["mms_tts_tokenizer"]
-    return vits_model, vits_tokenizer
+    return MODEL_STORE["mms_tts_pipeline"]
 
 
 def get_sam_components() -> Tuple[SamModel, SamProcessor]:
@@ -262,18 +274,6 @@ def recognize_speech(audio_path: str, model_key: str) -> str:
 
 
 def synthesize_speech(text_value: str, model_key: str):
-    if model_key == "silero":
-        silero_model = get_silero_tts_model()
-
-        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as file_object:
-            silero_model.save_wav(
-                text=text_value,
-                speaker="aidar",
-                sample_rate=48000,
-                audio_path=file_object.name,
-            )
-            return file_object.name
-
     if model_key == "Google TTS":
         with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as file_object:
             text_to_speech_engine = gTTS(text=text_value, lang="ru")
@@ -281,18 +281,18 @@ def synthesize_speech(text_value: str, model_key: str):
             return file_object.name
 
     if model_key == "vits-ljs":
-        vits_model, vits_tokenizer = get_mms_tts_components()
-        tokenized_input = vits_tokenizer(text_value, return_tensors="pt")
+        tts_pipeline = get_mms_tts_components()
 
-        with torch.no_grad():
-            waveform_tensor = vits_model(**tokenized_input).waveform
+        tts_output = tts_pipeline(text_value)
+
+        audio_array = tts_output["audio"]
+        sampling_rate_value = tts_output["sampling_rate"]
 
         with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as file_object:
-            waveform_array = waveform_tensor.numpy().squeeze()
             soundfile_module.write(
                 file_object.name,
-                waveform_array,
-                vits_model.config.sampling_rate,
+                audio_array,
+                sampling_rate_value,
             )
             return file_object.name
 
@@ -381,6 +381,26 @@ def answer_visual_question(image_object, question_text: str, model_key: str) ->
     if not question_text.strip():
         return "Пожалуйста, введите вопрос об изображении."
 
+    if model_key == "vqa_blip_base":
+        blip_model, blip_processor = get_blip_vqa_components()
+
+        inputs = blip_processor(
+            images=image_object,
+            text=question_text,
+            return_tensors="pt",
+        )
+
+        with torch.no_grad():
+            output_ids = blip_model.generate(**inputs)
+
+        decoded_answers = blip_processor.batch_decode(
+            output_ids,
+            skip_special_tokens=True,
+        )
+        answer_text = decoded_answers[0] if decoded_answers else ""
+
+        return answer_text or "Модель не смогла сгенерировать ответ."
+
     vqa_pipeline = get_vision_pipeline(model_key)
 
     vqa_result = vqa_pipeline(
@@ -680,9 +700,9 @@ def build_interface():
                         lines=3,
                     )
                     tts_model_selector = gr.Dropdown(
-                        choices=["vits-ljs", "Google TTS",],
+                        choices=["vits-ljs", "Google TTS"],
                         label="Выберите модель",
-                        value="silero",
+                        value="vits-ljs",
                         info=(
                             "kakao-enterprise/vits-ljs"
                             "Google TTS"