Learning Curve generation for: RPR × T × MFML#

In this example we will demonstrate a full set of learning curves for the following cases of single fidelity and multifidelity ML models. We will be using the KRR basic architecture.

Single-fidelity: MFML: 1. Isotropic 5. MFML Isotropic 2. RPR + Isotropic 6. MFML RPR + Isotropic 3. Coupled T 7. MFML Coupled T 4. RPR + Coupled T 8. MFML RPR + Coupled T

X-axis: number of training samples at the highest fidelity. MFML uses all 5 QeMFi fidelities with 2x scaling per level down.

Imports#

[1]:
import copy
import numpy as np
import matplotlib.pyplot as plt

from mfml_qc.datasets import load_nitrophenol_data
from mfml_qc.representations import compute_flat_coulomb
from mfml_qc.utils import top_down_subsetting
from mfml_qc.vector_krr import VectorKRR, CoupledVectorKRR
from mfml_qc.rpr import RPRVectorKRR

Load Data#

We load the QeMFi nitrophenol dataset and split into a fixed test set of 1000 geometries and a training pool of the remaining 14000. The test set uses the highest-fidelity (def2-TZVP) electronic dipole moments as reference.

[2]:
dataset = load_nitrophenol_data()
R = dataset["R"]
Z = dataset["Z"]
X_CM = dataset["X_CM"]
DPe = dataset["DPe"]

NUM_FIDS = 5
N_TEST = 1000

rng = np.random.RandomState(42)
perm = rng.permutation(len(R))
test_idx = perm[:N_TEST]
pool_idx = perm[N_TEST:]

DPe_test_ref = DPe[test_idx, -1, :]
X_test_flat = R[test_idx].reshape(N_TEST, -1)
CM_test = X_CM[test_idx]
CM_pool = X_CM[pool_idx]
R_pool = R[pool_idx]
DPe_pool = DPe[pool_idx]

Configuration#

We sweep over highest-fidelity training sizes from 16 to 256. Each model variant is defined by a tuple of (name, use_rpr, use_T, is_mfml). The eval_preds helper computes overall MAE, per-component MAE, cosine similarity, and mean angular error.

[3]:
N_HF_VALUES = [16, 32, 64, 128, 256]
KERNEL = "gaussian"
SIGMA = 250.0
REG = 1e-10


def eval_preds(pred, true):
    mae = np.mean(np.abs(pred - true))
    comp_mae = np.mean(np.abs(pred - true), axis=0)
    mag_p, mag_t = np.linalg.norm(pred, axis=-1), np.linalg.norm(true, axis=-1)
    dot = np.sum(pred * true, axis=-1)
    cos_sim = np.clip(dot / np.maximum(mag_p * mag_t, 1e-12), -1.0, 1.0)
    angle = np.mean(np.degrees(np.arccos(cos_sim)))
    return {
        "mae": mae,
        "comp_mae": comp_mae,
        "cos_sim": np.mean(cos_sim),
        "angle": angle,
        "pred": pred,
    }


def make_model(use_rpr, use_T):
    if use_T:
        inner = CoupledVectorKRR(
            kernel_type=KERNEL,
            sigma=SIGMA,
            reg=REG,
            val_fraction=0.15,
            max_opt_iter=200,
        )
    else:
        inner = VectorKRR(kernel_type=KERNEL, sigma=SIGMA, reg=REG)
    if use_rpr:
        return RPRVectorKRR(inner, Z, compute_flat_coulomb)
    return inner


# recall that the variants are labelled by (name, use_rpr, use_T, is_mfml)
VARIANTS = [
    ("Iso", False, False, False),
    ("RPR+Iso", True, False, False),
    ("T", False, True, False),
    ("RPR+T", True, True, False),
    ("MFML Iso", False, False, True),
    ("MFML RPR+Iso", True, False, True),
    ("MFML T", False, True, True),
    ("MFML RPR+T", True, True, True),
]

results = {v[0]: [] for v in VARIANTS}

Single-Fidelity Models#

Trains a single model on the n_hf highest-fidelity samples. Non-RPR models receive pre-computed Coulomb matrices; RPR models receive flattened Cartesian coordinates and compute canonicalized descriptors internally.

[4]:
def run_single_fidelity(n_hf, use_rpr, use_T):
    Y_train = DPe_pool[:n_hf, -1, :]
    Y_mean = np.mean(Y_train, axis=0)
    model = make_model(use_rpr, use_T)
    if use_rpr:
        model.train(R_pool[:n_hf].reshape(n_hf, -1), Y_train - Y_mean)
        pred = model.predict(X_test_flat) + Y_mean
    else:
        model.train(CM_pool[:n_hf], Y_train - Y_mean)
        pred = model.predict(CM_test) + Y_mean
    return eval_preds(pred, DPe_test_ref)

MFML Models#

Trains 2N-1 = 9 sub-models following the SGCT scheme. The hierarchy uses n_hf samples at the highest fidelity and doubles at each lower level. Upper sub-models learn fidelity-i targets on the fidelity-i subset; lower sub-models learn fidelity-i targets on the fidelity-(i+1) subset.

[5]:
def run_mfml(n_hf, use_rpr, use_T):
    n_trains = [
        min(n_hf * 2 ** (NUM_FIDS - 1 - i), len(R_pool)) for i in range(NUM_FIDS)
    ]

    y_trains_full = np.zeros(NUM_FIDS, dtype=object)
    indexes_full = np.zeros(NUM_FIDS, dtype=object)
    for i in range(NUM_FIDS):
        vals = DPe_pool[:, i, :]
        y_trains_full[i] = vals - np.mean(vals, axis=0)
        indexes_full[i] = np.column_stack(
            [np.arange(len(R_pool)), np.arange(len(R_pool))]
        )

    subset_y, subset_idx = top_down_subsetting(
        y_trains_full, indexes_full, n_trains, seed=42
    )

    n_sub = 2 * NUM_FIDS - 1
    sub_X, sub_Y, sub_Y_mean = [], [], []

    for i in range(NUM_FIDS):
        idx_i = subset_idx[i][:, 0]
        Y_i = DPe_pool[idx_i, i, :]
        m_i = np.mean(Y_i, axis=0)
        sub_Y.append(Y_i - m_i)
        sub_Y_mean.append(m_i)
        sub_X.append(
            R_pool[idx_i].reshape(len(idx_i), -1) if use_rpr else CM_pool[idx_i]
        )

    for i in range(NUM_FIDS - 1):
        idx_ip1 = subset_idx[i + 1][:, 0]
        Y_lower = DPe_pool[idx_ip1, i, :]
        m_i = np.mean(Y_lower, axis=0)
        sub_Y.append(Y_lower - m_i)
        sub_Y_mean.append(m_i)
        sub_X.append(
            R_pool[idx_ip1].reshape(len(idx_ip1), -1) if use_rpr else CM_pool[idx_ip1]
        )

    sub_models = []
    for s in range(n_sub):
        model_s = make_model(use_rpr, use_T)
        model_s.train(sub_X[s], sub_Y[s])
        sub_models.append(model_s)

    X_pred = X_test_flat if use_rpr else CM_test
    pred = np.zeros((N_TEST, 3))
    for i in range(NUM_FIDS):
        pred += sub_models[i].predict(X_pred) + sub_Y_mean[i]
    for i in range(NUM_FIDS - 1):
        pred -= sub_models[NUM_FIDS + i].predict(X_pred) + sub_Y_mean[NUM_FIDS + i]

    return eval_preds(pred, DPe_test_ref)

Run All Variants#

For each highest-fidelity training size, all 8 variants are trained and evaluated. Results are stored in a dictionary keyed by variant name.

[6]:
for n_hf in N_HF_VALUES:
    for name, use_rpr, use_T, is_mfml in VARIANTS:
        r = (
            run_mfml(n_hf, use_rpr, use_T)
            if is_mfml
            else run_single_fidelity(n_hf, use_rpr, use_T)
        )
        results[name].append(r)

Overall MAE and Angular Error Learning Curves#

Dashed lines represent single-fidelity models; solid lines represent MFML. Colour encodes the (RPR, T) combination. The left panel shows overall MAE on a log-log scale; the right panel shows mean angular error.

[7]:
combo_colors = {
    "Iso": plt.cm.viridis(0.1),
    "RPR+Iso": plt.cm.viridis(0.4),
    "T": plt.cm.viridis(0.65),
    "RPR+T": plt.cm.viridis(0.9),
}


def _base_key(use_rpr, use_T):
    return ("RPR+" if use_rpr else "") + ("T" if use_T else "Iso")


fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))

for name, use_rpr, use_T, is_mfml in VARIANTS:
    maes = [results[name][i]["mae"] for i in range(len(N_HF_VALUES))]
    angles = [results[name][i]["angle"] for i in range(len(N_HF_VALUES))]
    color = combo_colors[_base_key(use_rpr, use_T)]
    ls = "-" if is_mfml else "--"
    marker = "s" if is_mfml else "o"
    ax1.loglog(
        N_HF_VALUES,
        maes,
        marker=marker,
        linestyle=ls,
        color=color,
        lw=2,
        ms=6,
        label=name,
    )
    ax2.semilogx(
        N_HF_VALUES,
        angles,
        marker=marker,
        linestyle=ls,
        color=color,
        lw=2,
        ms=6,
        label=name,
    )

for ax, ylabel, title in [
    (ax1, "Overall MAE (Debye)", "Learning Curve: MAE"),
    (ax2, "Mean Angular Error (°)", "Learning Curve: Angular Error"),
]:
    ax.set_xlabel("N (highest fidelity training samples)")
    ax.set_ylabel(ylabel)
    ax.set_title(title)
    ax.legend(fontsize=7, ncol=2)
    ax.grid(True, alpha=0.3, which="both")
    ax.set_xticks(N_HF_VALUES)
    ax.set_xticklabels([str(n) for n in N_HF_VALUES])

plt.tight_layout()
plt.show()
../_images/auto_examples_13_vector_mfml_learning_curves_14_0.png

Per-Component Learning Curves#

The same learning curves broken down by dipole component (μ_x, μ_y, μ_z). This reveals which components benefit most from RPR, coupled T, or MFML. Unlike the last example, we DO NOT train individual component-wise MFML models but only look at the separate components post hoc.

[8]:
comp_labels = ["μ_x", "μ_y", "μ_z"]
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

for c in range(3):
    for name, use_rpr, use_T, is_mfml in VARIANTS:
        vals = [results[name][i]["comp_mae"][c] for i in range(len(N_HF_VALUES))]
        color = combo_colors[_base_key(use_rpr, use_T)]
        ls = "-" if is_mfml else "--"
        marker = "s" if is_mfml else "o"
        axes[c].loglog(
            N_HF_VALUES,
            vals,
            marker=marker,
            linestyle=ls,
            color=color,
            lw=2,
            ms=5,
            label=name,
        )
    axes[c].set_xlabel("N (highest fidelity)")
    axes[c].set_ylabel("MAE (Debye)")
    axes[c].set_title(f"Learning Curve: {comp_labels[c]}")
    axes[c].grid(True, alpha=0.3, which="both")
    axes[c].set_xticks(N_HF_VALUES)
    axes[c].set_xticklabels([str(n) for n in N_HF_VALUES])
    if c == 0:
        axes[c].legend(fontsize=6, ncol=2)

plt.tight_layout()
plt.show()
../_images/auto_examples_13_vector_mfml_learning_curves_16_0.png

Parity Plots at Largest N_hf#

Predicted vs. reference dipole components for all 8 variants at the largest training size. Points are coloured by the true dipole magnitude. Each row corresponds to one model variant; columns are the three Cartesian components.

[9]:
n_hf_last = N_HF_VALUES[-1]
mag_true = np.linalg.norm(DPe_test_ref, axis=-1)
cmap = plt.cm.viridis
vmin, vmax = mag_true.min(), mag_true.max()

fig, axes = plt.subplots(
    len(VARIANTS),
    3,
    figsize=(5 * 3, 4 * len(VARIANTS)),
    gridspec_kw={"hspace": 0.4, "wspace": 0.3},
)

for row, (name, _, _, _) in enumerate(VARIANTS):
    pred = results[name][-1]["pred"]
    comp_mae = np.mean(np.abs(pred - DPe_test_ref), axis=0)
    for c in range(3):
        sc = axes[row, c].scatter(
            DPe_test_ref[:, c],
            pred[:, c],
            s=2,
            alpha=0.4,
            c=mag_true,
            cmap=cmap,
            vmin=vmin,
            vmax=vmax,
        )
        lo = min(DPe_test_ref[:, c].min(), pred[:, c].min())
        hi = max(DPe_test_ref[:, c].max(), pred[:, c].max())
        axes[row, c].plot([lo, hi], [lo, hi], "r--", lw=1.5)
        axes[row, c].set_xlabel(f"True {comp_labels[c]} (Debye)")
        axes[row, c].set_ylabel(f"Predicted {comp_labels[c]}")
        axes[row, c].set_title(f"{name} — {comp_labels[c]} (MAE: {comp_mae[c]:.4f})")

fig.subplots_adjust(right=0.88)
cbar_ax = fig.add_axes([0.90, 0.05, 0.015, 0.9])
fig.colorbar(sc, cax=cbar_ax).set_label("|μ| (Debye)")
plt.show()
../_images/auto_examples_13_vector_mfml_learning_curves_18_0.png