Update app.py

app.py

@@ -5,8 +5,8 @@ import os
 import json
 import logging
 import pandas as pd
-import numpy as np # Added for mean calculation
-import matplotlib.pyplot as plt # Added for plotting
+import numpy as np
+import matplotlib.pyplot as plt
 from typing import Optional, List, Dict, Any
 
 # ------------------------
@@ -41,7 +41,16 @@ body {
 }
 .plot-container {
     min-height: 400px; /* Ensure plot area is visible */
+    display: flex;
+    justify-content: center; /* Center the plot */
+    align-items: center; /* Center vertically if needed */
 }
+/* Specific style for the plot title to potentially reduce overlap */
+.plot-container .gradio-html-title {
+    text-align: center;
+    width: 100%; /* Ensure title centers */
+}
+
 """
 
 theme = gr.themes.Soft(
@@ -79,38 +88,56 @@ def fetch_content_from_url(url: str, timeout: int = 15) -> str:
 
              # Limit the amount of data read to avoid excessive memory usage
              max_bytes_to_read = 2 * 1024 * 1024 # 2MB limit for initial read
-             content = response.content[:max_bytes_to_read]
-             if len(response.content) > max_bytes_to_read:
-                 logger.warning(f"Content for {url} is larger than {max_bytes_to_read} bytes, reading truncated content.")
+             # Read only up to max_bytes_to_read
+             content_bytes = b''
+             for chunk in response.iter_content(chunk_size=8192):
+                 if not chunk:
+                     break
+                 content_bytes += chunk
+                 if len(content_bytes) >= max_bytes_to_read:
+                     logger.warning(f"Content for {url} exceeded {max_bytes_to_read} bytes, stopped reading.")
+                     break
+
+        # Use detect_encoding if possible, fallback to utf-8
+        try:
+            # Attempt to get encoding from headers or detect it
+            encoding = requests.utils.get_encoding_from_headers(response.headers) or requests.utils.guess_json_utf(content_bytes)
+            content = content_bytes.decode(encoding, errors='replace')
+        except Exception as e:
+            logger.warning(f"Could not detect encoding for {url}, falling back to utf-8: {e}")
+            content = content_bytes.decode('utf-8', errors='replace')
+
 
         soup = BeautifulSoup(content, 'html.parser')
 
         # Attempt to find main content block
         # Prioritize more specific semantic tags
-        main_content = soup.find('article') or soup.find('main') or soup.find(class_=lambda x: x and ('content' in x.lower() or 'article' in x.lower()))
+        # Added some common class names as fallback
+        main_content = soup.find('article') or soup.find('main') or soup.find(class_=lambda x: x and ('content' in x.lower() or 'article' in x.lower() or 'post' in x.lower() or 'body' in x.lower()))
 
 
         if main_content:
              # Extract text from common text-containing tags within the main block
-            text_elements = main_content.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'figcaption'])
+            text_elements = main_content.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'figcaption', 'pre', 'code'])
             text = ' '.join([elem.get_text() for elem in text_elements])
         else:
             # Fallback to extracting text from body if no main block found
-            text_elements = soup.body.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'figcaption'])
+            text_elements = soup.body.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'figcaption', 'pre', 'code'])
             text = ' '.join([elem.get_text() for elem in text_elements])
-            logger.warning(f"No specific content tags (<article>, <main>, etc.) found for {url}, extracting from body.")
+            logger.warning(f"No specific content tags (<article>, <main>, etc.) or common class names found for {url}, extracting from body.")
 
         # Clean up extra whitespace
         text = ' '.join(text.split())
 
         # Limit text length *after* extraction and cleaning
         # Adjust based on API limits/cost. WordLift's typical text APIs handle up to ~1M chars.
-        max_text_length = 1000000
+        max_text_length = 1000000 # 1 Million characters
         if len(text) > max_text_length:
             logger.warning(f"Extracted text for {url} is too long ({len(text)} chars), truncating to {max_text_length} chars.")
             text = text[:max_text_length]
 
-        return text.strip() if text else None # Return None if text is empty after processing
+        return text.strip() if text and text.strip() else None # Return None if text is empty after processing
+
 
     except requests.exceptions.RequestException as e:
         logger.error(f"Failed to fetch content from {url}: {e}")
@@ -172,13 +199,14 @@ def call_wordlift_api(text: str, keywords: Optional[List[str]] = None) -> Option
 # Plotting Logic
 # ------------------------
 
-def plot_average_radar(average_scores: Dict[str, float], avg_overall: Optional[float]) -> Any:
+def plot_average_radar(average_scores: Dict[str, Optional[float]], avg_overall: Optional[float]) -> Any:
     """Return a radar (spider) plot as a Matplotlib figure showing average scores."""
 
-    if not average_scores or all(v is None for v in average_scores.values()):
+    # Check if we have any valid scores to plot
+    if not average_scores or all(v is None or pd.isna(v) for v in average_scores.values()):
         # Return a placeholder figure if no valid data is available
         fig, ax = plt.subplots(figsize=(6, 6))
-        ax.text(0.5, 0.5, "No successful evaluations to plot.", horizontalalignment='center', verticalalignment='center', transform=ax.transAxes, fontsize=12)
+        ax.text(0.5, 0.5, "No successful evaluations to plot\naverage scores.", horizontalalignment='center', verticalalignment='center', transform=ax.transAxes, fontsize=12)
         ax.axis('off') # Hide axes
         plt.title("Average Content Quality Scores", size=16, y=1.05)
         plt.tight_layout()
@@ -186,16 +214,16 @@ def plot_average_radar(average_scores: Dict[str, float], avg_overall: Optional[f
 
 
     categories = list(average_scores.keys())
-    values = [average_scores[cat] for cat in categories]
+    # Convert None/NaN values to 0 for plotting, but keep track of original for annotation
+    values_raw = [average_scores[cat] for cat in categories]
+    values_for_plot = [float(v) if v is not None and pd.notna(v) else 0 for v in values_raw]
 
-    # Ensure values are floats, replace None with 0 for plotting
-    values = [float(v) if v is not None else 0 for v in values]
 
     num_vars = len(categories)
     # Calculate angles for the radar chart
     angles = [n / float(num_vars) * 2 * np.pi for n in range(num_vars)]
     angles += angles[:1] # Complete the circle
-    values += values[:1] # Complete the circle for values
+    values_for_plot += values_for_plot[:1] # Complete the circle for values
 
     fig, ax = plt.subplots(figsize=(6, 6), subplot_kw=dict(projection='polar'))
 
@@ -205,8 +233,8 @@ def plot_average_radar(average_scores: Dict[str, float], avg_overall: Optional[f
     annotation_color = '#191919'
 
     # Plot data
-    ax.plot(angles, values, 'o-', linewidth=2, color=line_color, label='Average Scores')
-    ax.fill(angles, values, alpha=0.4, color=fill_color)
+    ax.plot(angles, values_for_plot, 'o-', linewidth=2, color=line_color, label='Average Scores')
+    ax.fill(angles, values_for_plot, alpha=0.4, color=fill_color)
 
     # Set tick locations and labels
     ax.set_xticks(angles[:-1])
@@ -214,22 +242,27 @@ def plot_average_radar(average_scores: Dict[str, float], avg_overall: Optional[f
 
     # Set y-axis limits. Max score is 100.
     ax.set_ylim(0, 100)
+    ax.set_yticks([0, 20, 40, 60, 80, 100]) # Explicitly set y-ticks
 
     # Draw grid lines and axes
     ax.grid(True, alpha=0.5, color=fill_color)
     ax.set_facecolor(background_color)
 
-    # Add score annotations next to points
-    for angle, value, category in zip(angles[:-1], values[:-1], categories):
-        # Adjust position slightly so text doesn't overlap the point/line
-        # Radius adjustment can be tricky; let's just add text at the point for simplicity
-        ax.text(angle, value + 5, f'{value:.1f}', color=annotation_color,
-                horizontalalignment='center', verticalalignment='bottom' if value > 50 else 'top', fontsize=9)
+    # Add score annotations next to points - Use raw values if not None/NaN
+    for angle, value_raw, value_plotted in zip(angles[:-1], values_raw, values_for_plot[:-1]):
+        if value_raw is not None and pd.notna(value_raw):
+            # Adjust position slightly based on angle and value
+            # More sophisticated positioning needed for perfect placement, simple offset below
+            # Let's just add text slightly outside the point along the radial line
+            radius = value_plotted + 5 # Offset outward
+            # Ensure annotation stays within limits if needed, but 105 should be fine for ylim 100
+            ax.text(angle, radius, f'{value_raw:.1f}', color=annotation_color,
+                    horizontalalignment='center', verticalalignment='center', fontsize=9)
 
 
-    # Add title
-    overall_title = f'Average Content Quality Scores\nOverall: {avg_overall:.1f}/100' if avg_overall is not None else 'Average Content Quality Scores'
-    plt.title(overall_title, size=16, y=1.1, color=annotation_color)
+    # Add title - Only "Overall: XX.X/100" part
+    overall_title_text = f'Overall: {avg_overall:.1f}/100' if avg_overall is not None and pd.notna(avg_overall) else 'Overall: -'
+    plt.title(overall_title_text, size=16, y=1.1, color=annotation_color) # y=1.1 places it above the plot area
 
     plt.tight_layout()
     return fig
@@ -260,40 +293,45 @@ def evaluate_urls_batch(url_data: pd.DataFrame):
              'Content Accuracy', 'Content Depth', 'Readability Score (API)',
              'Readability Grade Level', 'SEO Score', 'Word Count', 'Error/Details'
         ])
-        return empty_summary_df, {}, plot_average_radar(None, None) # Pass None to plotting function
+        return empty_summary_df, {}, plot_average_radar(None, None) # Pass None, None to plotting function
 
     summary_results = []
     full_results = {}
 
     # Lists to store scores for calculating averages
+    # Initialize with None/NaN to correctly handle empty inputs or failures
     purpose_scores = []
     accuracy_scores = []
     depth_scores = []
-    readability_scores = []
+    readability_scores = [] # Note: API returns float like 2.5
     seo_scores = []
     overall_scores = []
 
-    # Ensure columns exist, add them if not (though Dataframe component should enforce this)
-    # Using .get() with default None is safer if columns might sometimes be missing
-    urls = url_data.get('URL', pd.Series(dtype=str))
-    keywords_col = url_data.get('Target Keywords (comma-separated)', pd.Series(dtype=str))
+    # Ensure columns exist and handle potential NaNs from the DataFrame input
+    urls = url_data.get('URL', pd.Series(dtype=str)).fillna('') # Replace NaN URLs with empty strings
+    keywords_col = url_data.get('Target Keywords (comma-separated)', pd.Series(dtype=str)).fillna('') # Replace NaN keywords with empty strings
 
 
     for index, url in enumerate(urls):
-        url = url.strip() if pd.notna(url) else ""
-        keywords_str = keywords_col.iloc[index].strip() if pd.notna(keywords_col.iloc[index]) else ""
+        url = url.strip()
+        keywords_str = keywords_col.iloc[index].strip()
         keywords = [kw.strip() for kw in keywords_str.split(',') if kw.strip()]
 
-        # Generate a unique key for full_results, especially if URL is empty or duplicate
-        result_key = url if url else f"Row_{index}"
-        # Ensure unique key in case of duplicate empty URLs, maybe use index always?
-        result_key = f"Row_{index}_{url}" if url else f"Row_{index}"
+        # Generate a unique key for full_results based on index and URL/placeholder
+        result_key = f"Row_{index}" + (f": {url}" if url else "")
 
 
         if not url:
             summary_results.append(["", "Skipped", "-", "-", "-", "-", "-", "-", "-", "-", "Empty URL"])
             full_results[result_key] = {"status": "Skipped", "error": "Empty URL input."}
             logger.warning(f"Skipping evaluation for row {index}: Empty URL")
+            # Append None to scores lists for skipped/failed rows
+            purpose_scores.append(np.nan)
+            accuracy_scores.append(np.nan)
+            depth_scores.append(np.nan)
+            readability_scores.append(np.nan)
+            seo_scores.append(np.nan)
+            overall_scores.append(np.nan)
             continue # Move to next URL
 
         logger.info(f"Processing URL: {url} (Row {index}) with keywords: {keywords}")
@@ -307,6 +345,13 @@ def evaluate_urls_batch(url_data: pd.DataFrame):
             summary_results.append([url, status, "-", "-", "-", "-", "-", "-", "-", "-", error_msg])
             full_results[result_key] = {"status": status, "error": error_msg}
             logger.error(f"Processing failed for {url} (Row {index}): {error_msg}")
+            # Append None to scores lists for skipped/failed rows
+            purpose_scores.append(np.nan)
+            accuracy_scores.append(np.nan)
+            depth_scores.append(np.nan)
+            readability_scores.append(np.nan)
+            seo_scores.append(np.nan)
+            overall_scores.append(np.nan)
             continue # Move to next URL
 
         # 2. Call WordLift API
@@ -324,25 +369,27 @@ def evaluate_urls_batch(url_data: pd.DataFrame):
             metadata = api_result.get('metadata', {})
 
             # Append scores for average calculation (only for successful calls)
-            purpose_scores.append(content_breakdown.get('purpose'))
-            accuracy_scores.append(content_breakdown.get('accuracy'))
-            depth_scores.append(content_breakdown.get('depth'))
-            readability_scores.append(readability_breakdown.get('score')) # API's readability score (e.g. 2.5)
-            seo_scores.append(seo_breakdown.get('score'))
-            overall_scores.append(qs.get('overall'))
+            # Use .get() with None default, then convert to float, allowing NaN
+            purpose_scores.append(float(content_breakdown.get('purpose')) if content_breakdown.get('purpose') is not None else np.nan)
+            accuracy_scores.append(float(content_breakdown.get('accuracy')) if content_breakdown.get('accuracy') is not None else np.nan)
+            depth_scores.append(float(content_breakdown.get('depth')) if content_breakdown.get('depth') is not None else np.nan)
+            readability_scores.append(float(readability_breakdown.get('score')) if readability_breakdown.get('score') is not None else np.nan)
+            seo_scores.append(float(seo_breakdown.get('score')) if seo_breakdown.get('score') is not None else np.nan)
+            overall_scores.append(float(qs.get('overall')) if qs.get('overall') is not None else np.nan)
 
 
             # Append data for the summary table row
+            # Use .get() with '-' default for display
             summary_row.extend([
                 status,
-                f'{qs.get("overall", "-"): .1f}',
-                f'{content_breakdown.get("purpose", "-"): .0f}', # Assuming integer scores
-                f'{content_breakdown.get("accuracy", "-"): .0f}', # Assuming integer scores
-                f'{content_breakdown.get("depth", "-"): .0f}', # Assuming integer scores
-                f'{readability_breakdown.get("score", "-"): .1f}',
-                f'{readability_breakdown.get("grade_level", "-"): .0f}', # Assuming integer grade
-                f'{seo_breakdown.get("score", "-"): .1f}',
-                f'{metadata.get("word_count", "-"): .0f}', # Assuming integer word count
+                f'{qs.get("overall", "-"): .1f}' if qs.get('overall') is not None else "-",
+                f'{content_breakdown.get("purpose", "-"): .0f}' if content_breakdown.get('purpose') is not None else "-",
+                f'{content_breakdown.get("accuracy", "-"): .0f}' if content_breakdown.get('accuracy') is not None else "-",
+                f'{content_breakdown.get("depth", "-"): .0f}' if content_breakdown.get('depth') is not None else "-",
+                f'{readability_breakdown.get("score", "-"): .1f}' if readability_breakdown.get('score') is not None else "-",
+                f'{readability_breakdown.get("grade_level", "-"): .0f}' if readability_breakdown.get('grade_level') is not None else "-",
+                f'{seo_breakdown.get("score", "-"): .1f}' if seo_breakdown.get('score') is not None else "-",
+                f'{metadata.get("word_count", "-"): .0f}' if metadata.get('word_count') is not None else "-",
                 None # No error
             ])
             full_results[result_key] = api_result # Store full API result
@@ -359,15 +406,33 @@ def evaluate_urls_batch(url_data: pd.DataFrame):
             full_results[result_key] = {"status": status, "error": error_msg, "details": details}
             logger.error(f"API call failed for {url} (Row {index}): {error_msg} {details}")
 
+            # Append None/NaN to scores lists for failed rows
+            purpose_scores.append(np.nan)
+            accuracy_scores.append(np.nan)
+            depth_scores.append(np.nan)
+            readability_scores.append(np.nan)
+            seo_scores.append(np.nan)
+            overall_scores.append(np.nan)
+
+
         summary_results.append(summary_row)
 
-    # Calculate Averages *after* processing all URLs
-    avg_purpose = np.nanmean(purpose_scores) if purpose_scores else None # Use nanmean to ignore None/NaN
-    avg_accuracy = np.nanmean(accuracy_scores) if accuracy_scores else None
-    avg_depth = np.nanmean(depth_scores) if depth_scores else None
-    avg_readability = np.nanmean(readability_scores) if readability_scores else None
-    avg_seo = np.nanmean(seo_scores) if seo_scores else None
-    avg_overall = np.nanmean(overall_scores) if overall_scores else None
+    # Calculate Averages *after* processing all URLs, ignoring NaNs
+    avg_purpose = np.nanmean(purpose_scores)
+    avg_accuracy = np.nanmean(accuracy_scores)
+    avg_depth = np.nanmean(depth_scores)
+    avg_readability = np.nanmean(readability_scores)
+    avg_seo = np.nanmean(seo_scores)
+    avg_overall = np.nanmean(overall_scores)
+
+    # Convert potentially NaN averages to None if there were no valid scores
+    avg_purpose = avg_purpose if pd.notna(avg_purpose) else None
+    avg_accuracy = avg_accuracy if pd.notna(avg_accuracy) else None
+    avg_depth = avg_depth if pd.notna(avg_depth) else None
+    avg_readability = avg_readability if pd.notna(avg_readability) else None
+    avg_seo = avg_seo if pd.notna(avg_seo) else None
+    avg_overall = avg_overall if pd.notna(avg_overall) else None
+
 
     # Prepare scores for the radar plot function
     average_scores_dict = {
@@ -393,7 +458,6 @@ def evaluate_urls_batch(url_data: pd.DataFrame):
     # using f-strings like f'{value: .1f}' or f'{value: .0f}', and setting '-' for None.
     # This ensures that pandas DataFrame displays formatted strings directly.
 
-
     return summary_df, full_results, average_radar_fig # Return the plot too
 
 # ------------------------
@@ -416,11 +480,14 @@ with gr.Blocks(css=css, theme=theme) as demo:
                 row_count=(1, 30), # Allow adding rows up to 30
                 col_count=(2, "fixed"),
                 value=[
-                    ["https://www.wordlift.io/blog/google-helpful-content-update-2023/", "helpful content, google update"],
-                    ["https://www.wordlift.io/blog/what-is-a-knowledge-graph/", "knowledge graph, semantic web"],
+                    ["https://wordlift.io/blog/en/query-fan-out-ai-search/", "query fan out, ai search, google, ai"], # Added first URL
+                    ["https://wordlift.io/blog/en/entity/google-knowledge-graph/", "google knowledge graph, entity, semantic web, seo"], # Added second URL
                     ["https://www.example.com/non-existent-page", ""], # Example of a failing URL
-                    ["", ""] # Example of an empty row
-                ], # Default examples
+                    ["", ""], # Example of an empty row
+                    ["", ""], # Add some extra empty rows for easier input
+                    ["", ""],
+                    ["", ""],
+                ],
                 label="URLs and Keywords"
             )
             submit_button = gr.Button("Evaluate All URLs", elem_classes=["primary-btn"])
@@ -467,6 +534,9 @@ if __name__ == "__main__":
         logger.error("  # in your script before getting the key.")
         logger.error("----------------------------------------------------------\n")
         # You might want to sys.exit(1) here if the API key is mandatory
+        # import sys
+        # sys.exit(1)
+
 
     logger.info("Launching Gradio app...")
     # Consider using share=True for easy sharing, but be mindful of security/costs