First commit

app.py

@@ -0,0 +1,5 @@
+import evaluate
+from gradio_tst import launch_gradio_widget2
+
+module = evaluate.load("regression_evaluator.py")
+launch_gradio_widget2(module)

gradio_tst.py

@@ -0,0 +1,130 @@
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+import numpy as np
+from datasets import Value
+
+import logging
+
+
+
+REGEX_YAML_BLOCK = re.compile(r"---[\n\r]+([\S\s]*?)[\n\r]+---[\n\r]")
+
+
+def infer_gradio_input_types(feature_types):
+    """
+    Maps metric feature types to input types for gradio Dataframes:
+        - float/int -> numbers
+        - string -> strings
+        - any other -> json
+    Note that json is not a native gradio type but will be treated as string that
+    is then parsed as a json.
+    """
+    input_types = []
+    for feature_type in feature_types:
+        input_type = "json"
+        if isinstance(feature_type, Value):
+            if feature_type.dtype.startswith("int") or feature_type.dtype.startswith("float"):
+                input_type = "number"
+            elif feature_type.dtype == "string":
+                input_type = "str"
+        input_types.append(input_type)
+    return input_types
+
+
+def json_to_string_type(input_types):
+    """Maps json input type to str."""
+    return ["str" if i == "json" else i for i in input_types]
+
+
+def parse_readme(filepath):
+    """Parses a repositories README and removes"""
+    if not os.path.exists(filepath):
+        return "No README.md found."
+    with open(filepath, "r") as f:
+        text = f.read()
+        match = REGEX_YAML_BLOCK.search(text)
+        if match:
+            text = text[match.end() :]
+    return text
+
+
+def parse_gradio_data(data, input_types):
+    """Parses data from gradio Dataframe for use in metric."""
+    metric_inputs = {}
+    data.replace("", np.nan, inplace=True)
+    data.dropna(inplace=True)
+    for feature_name, input_type in zip(data, input_types):
+        if input_type == "json":
+            metric_inputs[feature_name] = [json.loads(d) for d in data[feature_name].to_list()]
+        elif input_type == "str":
+            metric_inputs[feature_name] = [d.strip('"') for d in data[feature_name].to_list()]
+        else:
+            metric_inputs[feature_name] = data[feature_name]
+    return metric_inputs
+
+
+def parse_test_cases(test_cases, feature_names, input_types):
+    """
+    Parses test cases to be used in gradio Dataframe. Note that an apostrophe is added
+    to strings to follow the format in json.
+    """
+    if len(test_cases) == 0:
+        return None
+    examples = []
+    for test_case in test_cases:
+        parsed_cases = []
+        for feat, input_type in zip(feature_names, input_types):
+            if input_type == "json":
+                parsed_cases.append([str(element) for element in test_case[feat]])
+            elif input_type == "str":
+                parsed_cases.append(['"' + element + '"' for element in test_case[feat]])
+            else:
+                parsed_cases.append(test_case[feat])
+        examples.append([list(i) for i in zip(*parsed_cases)])
+    return examples
+
+
+def launch_gradio_widget2(metric):
+    """Launches `metric` widget with Gradio."""
+
+    try:
+        import gradio as gr
+    except ImportError as error:
+        logging.error("To create a metric widget with Gradio make sure gradio is installed.")
+        raise error
+
+    local_path = Path(sys.path[0])
+    # if there are several input types, use first as default.
+    if isinstance(metric.features, list):
+        (feature_names, feature_types) = zip(*metric.features[0].items())
+    else:
+        (feature_names, feature_types) = zip(*metric.features.items())
+    gradio_input_types = infer_gradio_input_types(feature_types)
+
+    def compute(data):
+        return metric.compute(**parse_gradio_data(data, gradio_input_types))
+
+    iface = gr.Interface(
+        fn=compute,
+        inputs=gr.Dataframe(
+            headers=feature_names,
+            col_count=len(feature_names),
+            row_count=1,
+            datatype=json_to_string_type(gradio_input_types),
+        ),
+        outputs=gr.Textbox(label=metric.name),
+        description=(
+            metric.info.description + "\nIf this is a text-based metric, make sure to wrap you input in double quotes."
+            " Alternatively you can use a JSON-formatted list as input."
+        ),
+        title=f"Metric: {metric.name}",
+        article=parse_readme(local_path / "README.md"),
+        # TODO: load test cases and use them to populate examples
+        # examples=[parse_test_cases(test_cases, feature_names, gradio_input_types)]
+    )
+
+    iface.launch(share=True)

regression_evaluator.py

@@ -0,0 +1,108 @@
+import evaluate
+from datasets import Features, Value
+from scipy.stats import kendalltau, pearsonr, spearmanr
+from sklearn.metrics import (
+    max_error,
+    mean_absolute_error,
+    mean_absolute_percentage_error,
+    mean_squared_error,
+    r2_score,
+)
+
+_CITATION = """
+@article{scikit-learn,
+  title={Scikit-learn: Machine Learning in {P}ython},
+  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
+         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
+         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
+         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
+  journal={Journal of Machine Learning Research},
+  volume={12},
+  pages={2825--2830},
+  year={2011}
+}
+
+@article{2020SciPy-NMeth,
+  author  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and
+            Haberland, Matt and Reddy, Tyler and Cournapeau, David and
+            Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and
+            Bright, Jonathan and {van der Walt}, St{\'e}fan J. and
+            Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and
+            Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and
+            Kern, Robert and Larson, Eric and Carey, C J and
+            Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and
+            {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and
+            Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and
+            Harris, Charles R. and Archibald, Anne M. and
+            Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and
+            {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},
+  title   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific
+            Computing in Python}},
+  journal = {Nature Methods},
+  year    = {2020},
+  volume  = {17},
+  pages   = {261--272},
+  adsurl  = {https://rdcu.be/b08Wh},
+  doi     = {10.1038/s41592-019-0686-2},
+}
+"""
+
+
+_DESCRIPTION = """
+This evaluator computes multiple regression metrics to assess the performance of a model. Metrics calculated include: mean absolute error (MAE),
+mean absolute percentage error (MAPE), mean squared error (MSE), R-squared (R2), max error (ME), Pearson, Spearman and Kendall Tau correlation measures.
+"""
+
+_KWARGS_DESCRIPTION = """
+Args:
+    predictions (`list` of `float`): Predicted values.
+    references (`list` of `float`): Ground truth values.
+Returns:
+    Returns: a dict containing:
+        mean_absolute_error (float): https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html
+        mean_absolute_performance_error (float): https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_percentage_error.html
+        mean_squared_error (float): https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html
+        r2_score (float): https://scikit-learn.org/stable/modules/generated/sklearn.metrics.r2_score.html
+        max_error (float): https://scikit-learn.org/stable/modules/generated/sklearn.metrics.max_error.html
+        pearson_correlation (Tuple[float, float]): the first value being the score and the second one the p-value
+                                                   (https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html)
+        spearman_correlation (Tuple[float, float]): the first value being the score and the second one the p-value
+                                                    (https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html)
+        kendall_tau_correlation (Tuple[float, float]): the first value being the score and the second one the p-value
+                                                       (https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.kendalltau.html)
+"""
+
+
+class RegressionEvaluator(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=Features(
+                {"predictions": Value("float"), "references": Value("float")}
+            ),
+        )
+
+    def _compute(self, predictions, references):
+        error_fns = [
+            mean_absolute_error,
+            mean_absolute_percentage_error,
+            mean_squared_error,
+            max_error,
+            r2_score,
+        ]
+        correlation_fns = [pearsonr, spearmanr, kendalltau]
+        results = {}
+
+        # Compute error functions
+        for fn in error_fns:
+            results[fn.__name__] = fn(references, predictions)
+
+        # Compute statistical measures with p-values
+        for fn in correlation_fns:
+            output = fn(references, predictions)
+            score, p_value = output.statistic, output.pvalue
+            results[fn.__name__] = (float(score), float(p_value))
+
+        return results

requirements.txt

@@ -0,0 +1,5 @@
+evaluate
+datasets
+scikit-learn
+scipy
+gradio