diff --git a/notebooks/code_sharing/multiclass_classification_tests.ipynb b/notebooks/code_sharing/multiclass_classification_tests.ipynb new file mode 100644 index 000000000..5e05c122e --- /dev/null +++ b/notebooks/code_sharing/multiclass_classification_tests.ipynb @@ -0,0 +1,369 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "325f27e9", + "metadata": {}, + "source": [ + "# Multiclass support for sklearn classification tests\n", + "\n", + "This sample exercises four ValidMind sklearn model-validation tests on both a **binary**\n", + "and a **multiclass** classification model:\n", + "\n", + "- `ROCCurve`\n", + "- `PrecisionRecallCurve`\n", + "- `ConfusionMatrix`\n", + "- `PopulationStabilityIndex`\n", + "\n", + "Each test keeps its original behavior for binary targets and gains explicit multiclass\n", + "handling:\n", + "\n", + "- **ROCCurve / PrecisionRecallCurve** — render **one-vs-rest** curves for multiclass\n", + " targets (one curve per class plus a micro-average), using the model's per-class\n", + " `predict_proba`.\n", + "- **ConfusionMatrix** — renders the full N×N matrix. The `threshold` parameter only\n", + " applies to binary targets; for multiclass the model's argmax class predictions are used.\n", + "- **PopulationStabilityIndex** — computes **one-vs-rest** PSI (one table/plot per class)\n", + " from the model's per-class `predict_proba`.\n", + "\n", + "Where a multiclass model can't produce a full per-class probability matrix (e.g.\n", + "metadata-only / Foundation models, or predictions supplied as a single precomputed\n", + "probability column), the ROC/PR/PSI tests **skip gracefully** rather than crashing.\n", + "\n", + "> Runs fully offline — no ValidMind platform connection required (`generate_description=False`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94ce80a2", + "metadata": {}, + "outputs": [], + "source": [ + "# This notebook exercises UNRELEASED multiclass changes on this branch, so install\n", + "# THIS local checkout (editable) into the kernel — not the PyPI release:\n", + "%pip install -q -e ../..\n", + "\n", + "# To test the published release instead (note: no multiclass ROC/PR/PSI support yet), use:\n", + "# %pip install -q validmind" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d3c54c0", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.datasets import make_classification\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "import validmind as vm\n", + "from validmind.tests import run_test\n", + "\n", + "ROC = \"validmind.model_validation.sklearn.ROCCurve\"\n", + "PR = \"validmind.model_validation.sklearn.PrecisionRecallCurve\"\n", + "CM = \"validmind.model_validation.sklearn.ConfusionMatrix\"\n", + "PSI = \"validmind.model_validation.sklearn.PopulationStabilityIndex\"\n", + "\n", + "\n", + "def build_inputs(n_classes, prefix):\n", + " \"\"\"Fit a RandomForest and return (model, train_dataset, test_dataset).\n", + "\n", + " PopulationStabilityIndex compares two datasets, so we keep the train split\n", + " around too; ROC/PR/ConfusionMatrix only use the test dataset.\n", + " \"\"\"\n", + " X, y = make_classification(\n", + " n_samples=1500,\n", + " n_features=8,\n", + " n_informative=5,\n", + " n_classes=n_classes,\n", + " n_clusters_per_class=1,\n", + " random_state=42,\n", + " )\n", + " cols = [f\"f{i}\" for i in range(X.shape[1])]\n", + " X_train, X_test, y_train, y_test = train_test_split(\n", + " X, y, test_size=0.25, random_state=42\n", + " )\n", + "\n", + " train_df = pd.DataFrame(X_train, columns=cols)\n", + " train_df[\"target\"] = y_train\n", + " test_df = pd.DataFrame(X_test, columns=cols)\n", + " test_df[\"target\"] = y_test\n", + "\n", + " train_ds = vm.init_dataset(\n", + " input_id=f\"{prefix}_train\", dataset=train_df, target_column=\"target\", __log=False\n", + " )\n", + " test_ds = vm.init_dataset(\n", + " input_id=f\"{prefix}_test\", dataset=test_df, target_column=\"target\", __log=False\n", + " )\n", + "\n", + " clf = RandomForestClassifier(n_estimators=100, random_state=42).fit(X_train, y_train)\n", + " model = vm.init_model(input_id=f\"{prefix}_model\", model=clf, __log=False)\n", + " train_ds.assign_predictions(model)\n", + " test_ds.assign_predictions(model)\n", + " return model, train_ds, test_ds" + ] + }, + { + "cell_type": "markdown", + "id": "fff08866", + "metadata": {}, + "source": [ + "## 1. ROC & Precision-Recall curves\n", + "\n", + "### 1a. Binary classification\n", + "\n", + "Expect a single ROC curve (with AUC) and a single Precision-Recall curve — the original behavior." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b0ea48a", + "metadata": {}, + "outputs": [], + "source": [ + "bin_model, bin_train_ds, bin_ds = build_inputs(n_classes=2, prefix=\"binary\")\n", + "\n", + "run_test(ROC, inputs={\"model\": bin_model, \"dataset\": bin_ds}, generate_description=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57830ec5", + "metadata": {}, + "outputs": [], + "source": [ + "run_test(PR, inputs={\"model\": bin_model, \"dataset\": bin_ds}, generate_description=False)" + ] + }, + { + "cell_type": "markdown", + "id": "6a5394b7", + "metadata": {}, + "source": [ + "### 1b. Multiclass classification (one-vs-rest)\n", + "\n", + "Expect **one curve per class plus a micro-average**. `ROCCurve` also draws the random\n", + "baseline; `PrecisionRecallCurve` reports average precision (AP) per class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a47072c5", + "metadata": {}, + "outputs": [], + "source": [ + "mc_model, mc_train_ds, mc_ds = build_inputs(n_classes=4, prefix=\"multiclass\")\n", + "\n", + "run_test(ROC, inputs={\"model\": mc_model, \"dataset\": mc_ds}, generate_description=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "898061e9", + "metadata": {}, + "outputs": [], + "source": [ + "run_test(PR, inputs={\"model\": mc_model, \"dataset\": mc_ds}, generate_description=False)" + ] + }, + { + "cell_type": "markdown", + "id": "c72442b1", + "metadata": {}, + "source": [ + "## 2. Confusion matrix\n", + "\n", + "The confusion matrix renders for any number of classes. For **binary** targets the cells\n", + "are labeled TN / FP / FN / TP and the `threshold` parameter splits the positive-class\n", + "probability. For **multiclass** targets it renders the full N×N matrix from the model's\n", + "argmax class predictions (`threshold` is ignored)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7494b029", + "metadata": {}, + "outputs": [], + "source": [ + "run_test(CM, inputs={\"model\": bin_model, \"dataset\": bin_ds}, generate_description=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a7191b8", + "metadata": {}, + "outputs": [], + "source": [ + "run_test(CM, inputs={\"model\": mc_model, \"dataset\": mc_ds}, generate_description=False)" + ] + }, + { + "cell_type": "markdown", + "id": "8df7f354", + "metadata": {}, + "source": [ + "## 3. Population Stability Index\n", + "\n", + "PSI compares the model's predicted-probability distribution between two datasets (here\n", + "train vs test). For **binary** targets it produces a single PSI table/plot on the\n", + "positive-class probability. For **multiclass** targets it produces **one-vs-rest** PSI —\n", + "one table and one subplot per class — from the model's per-class `predict_proba`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa437289", + "metadata": {}, + "outputs": [], + "source": [ + "run_test(PSI, inputs={\"model\": bin_model, \"datasets\": [bin_train_ds, bin_ds]}, generate_description=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3dafafc", + "metadata": {}, + "outputs": [], + "source": [ + "run_test(PSI, inputs={\"model\": mc_model, \"datasets\": [mc_train_ds, mc_ds]}, generate_description=False)" + ] + }, + { + "cell_type": "markdown", + "id": "7ceb0272", + "metadata": {}, + "source": [ + "## 4. Inspect the raw values\n", + "\n", + "For multiclass, `RawData` is keyed per class (by label). ROC/PR also include a `micro`\n", + "entry. This confirms the one-vs-rest breakdown behind the plots." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96ebd0fa", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.tests.model_validation.sklearn.ROCCurve import ROCCurve\n", + "from validmind.tests.model_validation.sklearn.PrecisionRecallCurve import (\n", + " PrecisionRecallCurve,\n", + ")\n", + "from validmind.tests.model_validation.sklearn.PopulationStabilityIndex import (\n", + " PopulationStabilityIndex,\n", + ")\n", + "\n", + "_, roc_raw = ROCCurve(mc_model, mc_ds)\n", + "print(\"ROC per-class + micro AUC:\")\n", + "print({k: round(float(v), 3) for k, v in roc_raw.auc.items()})\n", + "\n", + "_, pr_raw = PrecisionRecallCurve(mc_model, mc_ds)\n", + "print(\"\\nPR per-class + micro average precision:\")\n", + "print({k: round(float(v), 3) for k, v in pr_raw.average_precision.items()})\n", + "\n", + "_, _, psi_raw = PopulationStabilityIndex([mc_train_ds, mc_ds], mc_model)\n", + "print(\"\\nPSI one-vs-rest total per class:\")\n", + "print({cls: round(rows[-1][\"psi\"], 4) for cls, rows in psi_raw.psi_raw.items()})" + ] + }, + { + "cell_type": "markdown", + "id": "d1ff9f16", + "metadata": {}, + "source": [ + "## 5. Graceful skip\n", + "\n", + "When a multiclass model cannot provide a per-class probability matrix, the ROC/PR/PSI\n", + "tests raise `SkipTestError` instead of crashing. Below we simulate that with a model\n", + "whose `predict_proba` returns a single precomputed probability column." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94e6b3a9", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.errors import SkipTestError\n", + "\n", + "\n", + "# A model whose predict_proba can't yield a per-class matrix for these classes.\n", + "class SingleColumnProbaModel:\n", + " def predict(self, X):\n", + " return np.zeros(len(X), dtype=int)\n", + "\n", + " def predict_proba(self, X):\n", + " return np.full(len(X), 0.5) # 1-D: not a per-class matrix\n", + "\n", + "\n", + "skip_model = vm.init_model(\n", + " input_id=\"skip_model\", model=SingleColumnProbaModel(), __log=False\n", + ")\n", + "\n", + "# The multiclass paths read the underlying model's predict_proba directly, so no\n", + "# assign_predictions is needed here — the missing per-class matrix triggers the skip.\n", + "try:\n", + " ROCCurve(skip_model, mc_ds)\n", + "except SkipTestError as e:\n", + " print(\"ROCCurve skipped as expected:\", e)\n", + "\n", + "try:\n", + " PopulationStabilityIndex([mc_train_ds, mc_ds], skip_model)\n", + "except SkipTestError as e:\n", + " print(\"PopulationStabilityIndex skipped as expected:\", e)" + ] + }, + { + "cell_type": "markdown", + "id": "copyright-2ccf7188bf3e4f0c93be0902ec5c20d0", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/unit_tests/model_validation/sklearn/test_PrecisionRecallCurve.py b/tests/unit_tests/model_validation/sklearn/test_PrecisionRecallCurve.py new file mode 100644 index 000000000..efc2d7cc9 --- /dev/null +++ b/tests/unit_tests/model_validation/sklearn/test_PrecisionRecallCurve.py @@ -0,0 +1,87 @@ +import unittest + +import numpy as np +import pandas as pd +import plotly.graph_objects as go +from sklearn.model_selection import train_test_split + +import validmind as vm +from validmind.tests.model_validation.sklearn.PrecisionRecallCurve import ( + PrecisionRecallCurve, +) + +try: + from xgboost import XGBClassifier +except ImportError: + XGBClassifier = None # type: ignore[misc,assignment] + + +@unittest.skipUnless(XGBClassifier is not None, "xgboost optional extra required") +class TestPrecisionRecallCurveBinary(unittest.TestCase): + def setUp(self): + np.random.seed(42) + n = 800 + X = np.random.randn(n, 2) + y = (X[:, 0] + X[:, 1] + np.random.randn(n) * 0.1 > 0).astype(int) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.25, random_state=42 + ) + test_df = pd.DataFrame( + {"f1": X_test[:, 0], "f2": X_test[:, 1], "target": y_test} + ) + self.ds = vm.init_dataset( + input_id="bin_test", dataset=test_df, target_column="target", __log=False + ) + model = XGBClassifier().fit(X_train, y_train) + self.model = vm.init_model(input_id="bin_model", model=model, __log=False) + self.ds.assign_predictions(self.model) + + def test_binary_single_curve(self): + fig, raw = PrecisionRecallCurve(self.model, self.ds) + self.assertIsInstance(fig, go.Figure) + self.assertIsInstance(raw, vm.RawData) + self.assertEqual(len(fig.data), 1) + self.assertEqual(fig.data[0].name, "Precision-Recall Curve") + + +@unittest.skipUnless(XGBClassifier is not None, "xgboost optional extra required") +class TestPrecisionRecallCurveMulticlass(unittest.TestCase): + def setUp(self): + np.random.seed(0) + n = 900 + X = np.random.randn(n, 3) + y = ( + np.stack([X[:, 0], X[:, 1], X[:, 2]], axis=1) + np.random.randn(n, 3) * 0.3 + ).argmax(axis=1) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.25, random_state=0 + ) + test_df = pd.DataFrame( + { + "f1": X_test[:, 0], + "f2": X_test[:, 1], + "f3": X_test[:, 2], + "target": y_test, + } + ) + self.ds = vm.init_dataset( + input_id="mc_pr_test", dataset=test_df, target_column="target", __log=False + ) + model = XGBClassifier().fit(X_train, y_train) + self.model = vm.init_model(input_id="mc_pr_model", model=model, __log=False) + self.ds.assign_predictions(self.model) + + def test_multiclass_one_vs_rest_traces(self): + fig, raw = PrecisionRecallCurve(self.model, self.ds) + self.assertIsInstance(fig, go.Figure) + self.assertIsInstance(raw, vm.RawData) + + names = [t.name for t in fig.data] + # 3 per-class curves + micro-average = 4 traces (no random baseline for PR). + self.assertEqual(len(fig.data), 4) + self.assertEqual(sum(n.startswith("Class ") for n in names), 3) + self.assertTrue(any(n.startswith("Micro-average") for n in names)) + + self.assertEqual(set(raw.average_precision) - {"micro"}, {"0", "1", "2"}) + for key in ("0", "1", "2", "micro"): + self.assertGreater(raw.average_precision[key], 0.5) diff --git a/tests/unit_tests/model_validation/sklearn/test_ROCCurve.py b/tests/unit_tests/model_validation/sklearn/test_ROCCurve.py index 72bf19c96..8c474a741 100644 --- a/tests/unit_tests/model_validation/sklearn/test_ROCCurve.py +++ b/tests/unit_tests/model_validation/sklearn/test_ROCCurve.py @@ -148,3 +148,53 @@ def test_perfect_separation(self): # Check AUC score (should be very close to 1.0) auc = float(fig.data[0].name.split("=")[1].strip().rstrip(")")) self.assertGreater(auc, 0.95) + + +@unittest.skipUnless(XGBClassifier is not None, "xgboost optional extra required") +class TestROCCurveMulticlass(unittest.TestCase): + def setUp(self): + np.random.seed(0) + n_samples = 900 + X = np.random.randn(n_samples, 3) + # 3-class target with real signal so per-class AUCs beat random. + score = np.stack([X[:, 0], X[:, 1], X[:, 2]], axis=1) + y = (score + np.random.randn(n_samples, 3) * 0.3).argmax(axis=1) + + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.25, random_state=0 + ) + + test_df = pd.DataFrame( + { + "f1": X_test[:, 0], + "f2": X_test[:, 1], + "f3": X_test[:, 2], + "target": y_test, + } + ) + + self.vm_test_ds = vm.init_dataset( + input_id="mc_test", dataset=test_df, target_column="target", __log=False + ) + model = XGBClassifier() + model.fit(X_train, y_train) + self.vm_model = vm.init_model(input_id="mc_model", model=model, __log=False) + self.vm_test_ds.assign_predictions(self.vm_model) + + def test_multiclass_one_vs_rest_traces(self): + fig, raw = ROCCurve(self.vm_model, self.vm_test_ds) + + self.assertIsInstance(fig, go.Figure) + self.assertIsInstance(raw, vm.RawData) + + names = [t.name for t in fig.data] + # 3 per-class curves + micro-average + random baseline = 5 traces. + self.assertEqual(len(fig.data), 5) + self.assertEqual(sum(n.startswith("Class ") for n in names), 3) + self.assertTrue(any(n.startswith("Micro-average") for n in names)) + self.assertTrue(any(n == "Random (AUC = 0.5)" for n in names)) + + # Per-class RawData keyed by class label + micro; AUCs beat random. + self.assertEqual(set(raw.auc) - {"micro"}, {"0", "1", "2"}) + for key in ("0", "1", "2", "micro"): + self.assertGreater(raw.auc[key], 0.5) diff --git a/uv.lock b/uv.lock index 01f2a671f..a74e875a4 100644 --- a/uv.lock +++ b/uv.lock @@ -11340,7 +11340,7 @@ wheels = [ [[package]] name = "validmind" -version = "2.13.5" +version = "2.13.6" source = { editable = "." } dependencies = [ { name = "aiohttp", extra = ["speedups"] }, diff --git a/validmind/tests/model_validation/sklearn/ConfusionMatrix.py b/validmind/tests/model_validation/sklearn/ConfusionMatrix.py index fd57ae9e0..70d6a569a 100644 --- a/validmind/tests/model_validation/sklearn/ConfusionMatrix.py +++ b/validmind/tests/model_validation/sklearn/ConfusionMatrix.py @@ -72,15 +72,20 @@ def ConfusionMatrix( - It mainly serves as a descriptive tool and does not offer the capability for statistical hypothesis testing. - Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data. These metrics have to be computed separately. + - The `threshold` parameter only applies to binary classification (it splits a single positive-class probability + into two classes). For multiclass targets the model's argmax class predictions are used and `threshold` is ignored. """ - # Get predictions using threshold for binary classification if possible - if hasattr(model.model, "predict_proba"): + # The `threshold` only has a meaning for binary classification (it splits a + # single positive-class probability into two classes). For multiclass we use + # the model's argmax class predictions directly, since thresholding a + # single probability column would silently produce wrong labels. + n_classes = len(np.unique(dataset.y)) + if n_classes == 2 and hasattr(model.model, "predict_proba"): y_prob = dataset.y_prob(model) # Handle both 1D and 2D probability arrays if y_prob.ndim == 2: - y_pred = (y_prob[:, 1] > threshold).astype(int) - else: - y_pred = (y_prob > threshold).astype(int) + y_prob = y_prob[:, 1] + y_pred = (y_prob > threshold).astype(int) else: y_pred = dataset.y_pred(model) diff --git a/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py b/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py index 352db3e5d..8815cddb4 100644 --- a/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +++ b/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd import plotly.graph_objects as go +from plotly.subplots import make_subplots from validmind import RawData, tags, tasks from validmind.errors import SkipTestError @@ -15,6 +16,8 @@ logger = get_logger(__name__) +_PSI_PALETTE = ["#DE257E", "#1F77B4", "#2CA02C", "#FF7F0E", "#9467BD", "#8C564B"] + def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"): """ @@ -76,6 +79,152 @@ def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"): return psi_df.to_dict(orient="records") +def _psi_table_rows(psi_results): + """Append the summed 'Total' row and format PSI records as table rows.""" + total_psi = { + key: sum(d.get(key, 0) for d in psi_results) + for key in psi_results[0].keys() + if isinstance(psi_results[0][key], (int, float)) + } + rows_with_total = psi_results + [total_psi] + + table_rows = [ + { + "Bin": ( + i if i < (len(rows_with_total) - 1) else "Total" + ), # The last bin is the "Total" bin + "Count Initial": values["initial"], + "Percent Initial (%)": values["percent_initial"] * 100, + "Count New": values["new"], + "Percent New (%)": values["percent_new"] * 100, + "PSI": values["psi"], + } + for i, values in enumerate(rows_with_total) + ] + return rows_with_total, table_rows + + +def _multiclass_psi(datasets, model, classes, num_bins, mode): + """One-vs-rest PSI for a multiclass model. + + PSI needs a 1-D score distribution to compare across the two datasets. The + stored single probability column cannot represent every class, so we ask the + underlying estimator for the full per-class probability matrix (mirroring the + ROC/PR curve tests). Models without a usable ``predict_proba`` (metadata-only + / precomputed single-column predictions) are skipped rather than crashed. + """ + raw_model = getattr(model, "model", None) + proba_fn = getattr(raw_model, "predict_proba", None) + if not callable(proba_fn): + raise SkipTestError( + "Multiclass Population Stability Index requires per-class " + "probabilities from the underlying model's predict_proba, which is " + "not available for this model (e.g. metadata-only / precomputed " + "predictions). Skipping." + ) + try: + prob_initial = np.asarray(proba_fn(datasets[0].x_df())) + prob_new = np.asarray(proba_fn(datasets[1].x_df())) + except Exception as e: + raise SkipTestError( + "Multiclass Population Stability Index could not compute per-class " + f"probabilities ({type(e).__name__}). Skipping." + ) from e + + n_classes = len(classes) + for prob in (prob_initial, prob_new): + if prob.ndim != 2 or prob.shape[1] != n_classes: + raise SkipTestError( + "Multiclass Population Stability Index requires a per-class " + f"probability matrix with one column per class (got shape " + f"{getattr(prob, 'shape', None)} for {n_classes} classes). Skipping." + ) + + # predict_proba columns are ordered by sorted class label == np.unique. + fig = make_subplots( + rows=n_classes, + cols=1, + specs=[[{"secondary_y": True}] for _ in range(n_classes)], + subplot_titles=[f"Class {cls}" for cls in classes], + vertical_spacing=0.08, + ) + + tables = {} + raw_per_class = {} + for i, cls in enumerate(classes): + psi_results = calculate_psi( + prob_initial[:, i].copy(), + prob_new[:, i].copy(), + num_bins=num_bins, + mode=mode, + ) + x = list(range(len(psi_results))) + color = _PSI_PALETTE[i % len(_PSI_PALETTE)] + fig.add_trace( + go.Bar( + x=x, + y=[d["percent_initial"] for d in psi_results], + name="Initial", + marker=dict(color="#DE257E"), + showlegend=i == 0, + legendgroup="initial", + ), + row=i + 1, + col=1, + secondary_y=False, + ) + fig.add_trace( + go.Bar( + x=x, + y=[d["percent_new"] for d in psi_results], + name="New", + marker=dict(color="#E8B1F8"), + showlegend=i == 0, + legendgroup="new", + ), + row=i + 1, + col=1, + secondary_y=False, + ) + fig.add_trace( + go.Scatter( + x=x, + y=[d["psi"] for d in psi_results], + name="PSI", + line=dict(color=color), + showlegend=i == 0, + legendgroup="psi", + ), + row=i + 1, + col=1, + secondary_y=True, + ) + + rows_with_total, table_rows = _psi_table_rows(psi_results) + table_title = ( + f"Population Stability Index for Class {cls} " + f"({datasets[0].input_id} vs {datasets[1].input_id})" + ) + tables[table_title] = table_rows + raw_per_class[str(cls)] = rows_with_total + + fig.update_layout( + title="Population Stability Index (PSI) — one-vs-rest per class", + barmode="group", + height=300 * n_classes, + ) + + return ( + tables, + fig, + RawData( + psi_raw=raw_per_class, + model=model.input_id, + datasets=[datasets[0].input_id, datasets[1].input_id], + ), + ) + + @tags( "sklearn", "binary_classification", "multiclass_classification", "model_performance" ) @@ -132,10 +281,18 @@ def PopulationStabilityIndex( lead to misinterpretations. Any changes in PSI could be due to shifts in the model (model drift), changes in the relationships between features and the target variable (concept drift), or both. However, distinguishing between these causes is non-trivial. + - For multiclass models the PSI is computed one-vs-rest (one table/plot per class), which requires per-class + probabilities from the model's `predict_proba`. Models that cannot produce a full per-class probability matrix + (e.g. metadata-only models, or predictions supplied as a single precomputed probability column) are skipped for + the multiclass case. """ if model.library in ["statsmodels", "pytorch", "catboost"]: raise SkipTestError(f"Skiping PSI for {model.library} models") + classes = np.unique(datasets[0].y) + if len(classes) > 2: + return _multiclass_psi(datasets, model, classes, num_bins, mode) + psi_results = calculate_psi( datasets[0].y_prob(model).copy(), datasets[1].y_prob(model).copy(), @@ -182,35 +339,16 @@ def PopulationStabilityIndex( ), ) - # sum up the PSI values to get the total values - total_psi = { - key: sum(d.get(key, 0) for d in psi_results) - for key in psi_results[0].keys() - if isinstance(psi_results[0][key], (int, float)) - } - psi_results.append(total_psi) + # sum up the PSI values to get the total values and format the table rows + rows_with_total, table_rows = _psi_table_rows(psi_results) table_title = f"Population Stability Index for {datasets[0].input_id} and {datasets[1].input_id} Datasets" return ( - { - table_title: [ - { - "Bin": ( - i if i < (len(psi_results) - 1) else "Total" - ), # The last bin is the "Total" bin - "Count Initial": values["initial"], - "Percent Initial (%)": values["percent_initial"] * 100, - "Count New": values["new"], - "Percent New (%)": values["percent_new"] * 100, - "PSI": values["psi"], - } - for i, values in enumerate(psi_results) - ], - }, + {table_title: table_rows}, fig, RawData( - psi_raw=psi_results, + psi_raw=rows_with_total, model=model.input_id, datasets=[datasets[0].input_id, datasets[1].input_id], ), diff --git a/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py b/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py index c6b2bd76a..1843d92b7 100644 --- a/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +++ b/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py @@ -6,7 +6,8 @@ import numpy as np import plotly.graph_objects as go -from sklearn.metrics import precision_recall_curve +from sklearn.metrics import average_precision_score, precision_recall_curve +from sklearn.preprocessing import label_binarize from validmind import RawData, tags, tasks from validmind.errors import SkipTestError @@ -14,7 +15,13 @@ from validmind.vm_models import VMDataset, VMModel -@tags("sklearn", "binary_classification", "model_performance", "visualization") +@tags( + "sklearn", + "binary_classification", + "multiclass_classification", + "model_performance", + "visualization", +) @tasks("classification", "text_classification") def PrecisionRecallCurve( model: VMModel, dataset: VMDataset @@ -54,8 +61,10 @@ def PrecisionRecallCurve( ### Limitations - - This metric is only applicable to binary classification models - it raises errors for multiclass classification - models or Foundation models. + - For multiclass models the curve is computed one-vs-rest (one curve per class plus a micro-average), which + requires per-class probabilities from the model's `predict_proba`. Models that cannot produce a full per-class + probability matrix (e.g. Foundation/metadata-only models, or predictions supplied as a single precomputed + probability column) are skipped for the multiclass case. - It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives are extremely different, or if the dataset is heavily imbalanced. """ @@ -63,11 +72,10 @@ def PrecisionRecallCurve( raise SkipTestError("Skipping PrecisionRecallCurve for Foundation models") y_true = dataset.y - # Binary-only by design: multiclass is skipped, not handled (unlike MinimumF1Score). - if len(np.unique(y_true)) > 2: - raise SkipTestError( - "Precision Recall Curve is only supported for binary classification models" - ) + classes = np.unique(y_true) + + if len(classes) > 2: + return _multiclass_pr_curve(model, dataset, classes) precision, recall, _ = precision_recall_curve(y_true, dataset.y_prob(model)) @@ -94,3 +102,103 @@ def PrecisionRecallCurve( model=model.input_id, dataset=dataset.input_id, ) + + +def _multiclass_pr_curve( + model: VMModel, dataset: VMDataset, classes: np.ndarray +) -> Tuple[go.Figure, RawData]: + """One-vs-rest precision-recall curves for a multiclass model. + + Needs the full per-class probability matrix, which the stored single + probability column cannot provide, so we ask the model for it directly. + Models without a usable ``predict_proba`` (Foundation/metadata-only, + precomputed single-column probabilities) are skipped rather than crashed. + """ + # The VMModel wrapper's predict_proba is binary-only (it returns just the + # positive-class column), so reach the underlying estimator for the full + # per-class probability matrix. + raw_model = getattr(model, "model", None) + proba_fn = getattr(raw_model, "predict_proba", None) + if not callable(proba_fn): + raise SkipTestError( + "Multiclass Precision-Recall Curve requires per-class probabilities " + "from the underlying model's predict_proba, which is not available " + "for this model (e.g. Foundation / metadata-only / precomputed " + "predictions). Skipping." + ) + try: + y_prob = np.asarray(proba_fn(dataset.x_df())) + except Exception as e: + raise SkipTestError( + "Multiclass Precision-Recall Curve could not compute per-class " + f"probabilities ({type(e).__name__}). Skipping." + ) from e + + n_classes = len(classes) + if y_prob.ndim != 2 or y_prob.shape[1] != n_classes: + raise SkipTestError( + "Multiclass Precision-Recall Curve requires a per-class probability " + f"matrix with one column per class (got shape " + f"{getattr(y_prob, 'shape', None)} for {n_classes} classes). Skipping." + ) + + # One-hot the true labels in the same class order predict_proba columns use + # (sklearn orders predict_proba columns by sorted class label == np.unique). + y_bin = label_binarize(dataset.y.flatten(), classes=classes) + + traces = [] + raw_precision = {} + raw_recall = {} + raw_ap = {} + palette = ["#DE257E", "#1F77B4", "#2CA02C", "#FF7F0E", "#9467BD", "#8C564B"] + for i, cls in enumerate(classes): + precision, recall, _ = precision_recall_curve(y_bin[:, i], y_prob[:, i]) + ap = average_precision_score(y_bin[:, i], y_prob[:, i]) + key = str(cls) + raw_precision[key] = precision + raw_recall[key] = recall + raw_ap[key] = ap + traces.append( + go.Scatter( + x=recall, + y=precision, + mode="lines", + name=f"Class {key} (AP = {ap:.2f})", + line=dict(color=palette[i % len(palette)]), + ) + ) + + # Micro-average across all one-vs-rest decisions. + micro_precision, micro_recall, _ = precision_recall_curve( + y_bin.ravel(), y_prob.ravel() + ) + micro_ap = average_precision_score(y_bin, y_prob, average="micro") + raw_precision["micro"] = micro_precision + raw_recall["micro"] = micro_recall + raw_ap["micro"] = micro_ap + traces.append( + go.Scatter( + x=micro_recall, + y=micro_precision, + mode="lines", + name=f"Micro-average (AP = {micro_ap:.2f})", + line=dict(color="black", dash="dot"), + ) + ) + + fig = go.Figure( + data=traces, + layout=go.Layout( + title=f"Precision-Recall Curve (one-vs-rest) for {model.input_id} on {dataset.input_id}", + xaxis=dict(title="Recall"), + yaxis=dict(title="Precision"), + ), + ) + + return fig, RawData( + precision=raw_precision, + recall=raw_recall, + average_precision=raw_ap, + model=model.input_id, + dataset=dataset.input_id, + ) diff --git a/validmind/tests/model_validation/sklearn/ROCCurve.py b/validmind/tests/model_validation/sklearn/ROCCurve.py index 6109ca1d8..ab2086a7f 100644 --- a/validmind/tests/model_validation/sklearn/ROCCurve.py +++ b/validmind/tests/model_validation/sklearn/ROCCurve.py @@ -7,6 +7,7 @@ import numpy as np import plotly.graph_objects as go from sklearn.metrics import roc_auc_score, roc_curve +from sklearn.preprocessing import label_binarize from validmind import RawData, tags, tasks from validmind.errors import SkipTestError @@ -23,26 +24,28 @@ @tasks("classification", "text_classification") def ROCCurve(model: VMModel, dataset: VMDataset) -> Tuple[go.Figure, RawData]: """ - Evaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic - (ROC) curve and calculating the Area Under Curve (AUC) score. + Evaluates classification model performance by generating and plotting the Receiver Operating Characteristic + (ROC) curve and calculating the Area Under Curve (AUC) score, for both binary and multiclass models. ### Purpose - The Receiver Operating Characteristic (ROC) curve is designed to evaluate the performance of binary classification - models. This curve illustrates the balance between the True Positive Rate (TPR) and False Positive Rate (FPR) - across various threshold levels. In combination with the Area Under the Curve (AUC), the ROC curve aims to measure - the model's discrimination ability between the two defined classes in a binary classification problem (e.g., - default vs non-default). Ideally, a higher AUC score signifies superior model performance in accurately - distinguishing between the positive and negative classes. + The Receiver Operating Characteristic (ROC) curve evaluates the performance of classification models. This curve + illustrates the balance between the True Positive Rate (TPR) and False Positive Rate (FPR) across various threshold + levels. In combination with the Area Under the Curve (AUC), the ROC curve measures the model's discrimination + ability between classes. For binary problems (e.g., default vs non-default) a single curve is drawn for the + positive class. For multiclass problems the curve is computed one-vs-rest — one curve and AUC per class, plus a + micro-average across all classes — so the model's discrimination ability can be assessed for every class. Ideally, + a higher AUC score signifies superior model performance in accurately distinguishing between classes. ### Test Mechanism - First, this script selects the target model and datasets that require binary classification. It then calculates the - predicted probabilities for the test set, and uses this data, along with the true outcomes, to generate and plot - the ROC curve. Additionally, it includes a line signifying randomness (AUC of 0.5). The AUC score for the model's - ROC curve is also computed, presenting a numerical estimation of the model's performance. If any Infinite values - are detected in the ROC threshold, these are effectively eliminated. The resulting ROC curve, AUC score, and - thresholds are consequently saved for future reference. + This test selects the target model and dataset and determines the number of classes from the true labels. For + binary targets it computes the predicted probabilities for the positive class and, together with the true + outcomes, generates and plots a single ROC curve. For multiclass targets it obtains the full per-class probability + matrix from the model and plots a one-vs-rest curve for each class along with a micro-average curve. In both cases + a line signifying randomness (AUC of 0.5) is included, and the AUC score(s) are computed as a numerical estimation + of performance. If any Infinite values are detected in the ROC threshold, these are effectively eliminated. The + resulting ROC curves, AUC scores, and thresholds are consequently saved for future reference. ### Signs of High Risk @@ -61,18 +64,19 @@ def ROCCurve(model: VMModel, dataset: VMDataset) -> Tuple[go.Figure, RawData]: ### Limitations - - The primary limitation is that this test is exclusively structured for binary classification tasks, thus limiting - its application towards other model types. + - For multiclass models the curve is computed one-vs-rest (one curve per class plus a micro-average), which + requires per-class probabilities from the model's `predict_proba`. Models that cannot produce a full per-class + probability matrix (e.g. metadata-only models, or predictions supplied as a single precomputed probability column) + are skipped for the multiclass case. - Furthermore, its performance might be subpar with models that output probabilities highly skewed towards 0 or 1. - At the extreme, the ROC curve could reflect high performance even when the majority of classifications are incorrect, provided that the model's ranking format is retained. This phenomenon is commonly termed the "Class Imbalance Problem". """ - # Binary-only by design: multiclass is skipped, not handled (unlike MinimumF1Score). - if len(np.unique(dataset.y)) > 2: - raise SkipTestError( - "ROC Curve is only supported for binary classification models" - ) + classes = np.unique(dataset.y) + + if len(classes) > 2: + return _multiclass_roc_curve(model, dataset, classes) y_prob = dataset.y_prob(model) y_true = dataset.y.astype(y_prob.dtype).flatten() @@ -110,3 +114,114 @@ def ROCCurve(model: VMModel, dataset: VMDataset) -> Tuple[go.Figure, RawData]: fpr=fpr, tpr=tpr, auc=auc, model=model.input_id, dataset=dataset.input_id ), ) + + +def _multiclass_roc_curve( + model: VMModel, dataset: VMDataset, classes: np.ndarray +) -> Tuple[go.Figure, RawData]: + """One-vs-rest ROC curves for a multiclass model. + + Needs the full per-class probability matrix, which the stored single + probability column cannot provide, so we ask the model for it directly. + Models without a usable ``predict_proba`` (metadata-only, precomputed + single-column probabilities) are skipped rather than crashed. + """ + # The VMModel wrapper's predict_proba is binary-only (it returns just the + # positive-class column), so reach the underlying estimator for the full + # per-class probability matrix. + raw_model = getattr(model, "model", None) + proba_fn = getattr(raw_model, "predict_proba", None) + if not callable(proba_fn): + raise SkipTestError( + "Multiclass ROC Curve requires per-class probabilities from the " + "underlying model's predict_proba, which is not available for this " + "model (e.g. metadata-only / precomputed predictions). Skipping." + ) + try: + y_prob = np.asarray(proba_fn(dataset.x_df())) + except Exception as e: + raise SkipTestError( + "Multiclass ROC Curve could not compute per-class probabilities " + f"({type(e).__name__}). Skipping." + ) from e + + n_classes = len(classes) + if y_prob.ndim != 2 or y_prob.shape[1] != n_classes: + raise SkipTestError( + "Multiclass ROC Curve requires a per-class probability matrix with " + f"one column per class (got shape {getattr(y_prob, 'shape', None)} " + f"for {n_classes} classes). Skipping." + ) + + # One-hot the true labels in the same class order predict_proba columns use + # (sklearn orders predict_proba columns by sorted class label == np.unique). + y_true = dataset.y.flatten() + y_bin = label_binarize(y_true, classes=classes) + + traces = [] + raw_fpr = {} + raw_tpr = {} + raw_auc = {} + palette = ["#DE257E", "#1F77B4", "#2CA02C", "#FF7F0E", "#9467BD", "#8C564B"] + for i, cls in enumerate(classes): + fpr, tpr, _ = roc_curve(y_bin[:, i], y_prob[:, i], drop_intermediate=False) + auc = roc_auc_score(y_bin[:, i], y_prob[:, i]) + key = str(cls) + raw_fpr[key] = fpr + raw_tpr[key] = tpr + raw_auc[key] = auc + traces.append( + go.Scatter( + x=fpr, + y=tpr, + mode="lines", + name=f"Class {key} (AUC = {auc:.2f})", + line=dict(color=palette[i % len(palette)]), + ) + ) + + # Micro-average across all one-vs-rest decisions. + micro_fpr, micro_tpr, _ = roc_curve(y_bin.ravel(), y_prob.ravel()) + micro_auc = roc_auc_score(y_bin, y_prob, average="micro", multi_class="ovr") + raw_fpr["micro"] = micro_fpr + raw_tpr["micro"] = micro_tpr + raw_auc["micro"] = micro_auc + traces.append( + go.Scatter( + x=micro_fpr, + y=micro_tpr, + mode="lines", + name=f"Micro-average (AUC = {micro_auc:.2f})", + line=dict(color="black", dash="dot"), + ) + ) + + traces.append( + go.Scatter( + x=[0, 1], + y=[0, 1], + mode="lines", + name="Random (AUC = 0.5)", + line=dict(color="grey", dash="dash"), + ) + ) + + return ( + go.Figure( + data=traces, + layout=go.Layout( + title=f"ROC Curve (one-vs-rest) for {model.input_id} on {dataset.input_id}", + xaxis=dict(title="False Positive Rate"), + yaxis=dict(title="True Positive Rate"), + width=700, + height=500, + ), + ), + RawData( + fpr=raw_fpr, + tpr=raw_tpr, + auc=raw_auc, + model=model.input_id, + dataset=dataset.input_id, + ), + )