Quick Start#

This example compares two algorithms across three datasets.

import numpy as np

from compml.bayesian.tests import BayesianSignedRankTest
from compml.evaluation_scores import EvaluationScores
from compml.frequentist.tests import SignedRankTest

ds1_alg_a = np.array([0.87, 0.85, 0.86, 0.88, 0.84])
ds1_alg_b = np.array([0.84, 0.83, 0.82, 0.85, 0.81])

ds2_alg_a = np.array([0.79, 0.80, 0.78, 0.81, 0.80])
ds2_alg_b = np.array([0.77, 0.76, 0.78, 0.75, 0.77])

ds3_alg_a = np.array([0.91, 0.90, 0.92, 0.89, 0.90])
ds3_alg_b = np.array([0.88, 0.87, 0.89, 0.86, 0.88])

scores = np.array(
    [
        [ds1_alg_a, ds1_alg_b],
        [ds2_alg_a, ds2_alg_b],
        [ds3_alg_a, ds3_alg_b],
    ],
    dtype=float,
)

evaluation_scores = EvaluationScores(
    scores=scores,
    metric="accuracy",
    dataset_names=("ds_1", "ds_2", "ds_3"),
    algorithm_names=("alg_a", "alg_b"),
)

signed_rank_result = SignedRankTest(alpha=0.05).run(evaluation_scores)
print(signed_rank_result.summary())

bayes_signed_rank_result = BayesianSignedRankTest(
    rope=0.01,
    n_posterior_samples=1000,
).run(evaluation_scores)
print(bayes_signed_rank_result.summary())

Next steps#