Skip to content

Evaluator

torchloop.evaluator.Evaluator

Classification model evaluator.

Parameters:

Name Type Description Default
model

Trained nn.Module.

 required
device

'cuda', 'cpu', or 'mps'. Auto-detects if None.

 required
Source code in src/torchloop/evaluator.py 
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
class Evaluator:
    """
    Classification model evaluator.

    Args:
        model       : Trained nn.Module.
        device      : 'cuda', 'cpu', or 'mps'. Auto-detects if None.
    """

    def __init__(self, model: nn.Module, device: Optional[str] = None):
        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
        self.model = model.to(self.device)

    # ------------------------------------------------------------------
    # Public API
    # ------------------------------------------------------------------

    def report(
        self,
        loader: DataLoader,
        class_names: Optional[list[str]] = None,
    ) -> dict:
        """
        Print full sklearn classification report.

        Returns:
            dict with keys: accuracy, macro_f1, weighted_f1, per_class_f1
        """
        preds, targets = self._infer(loader)
        report = classification_report(
            targets, preds, target_names=class_names, zero_division=0
        )
        print(report)
        per_class = f1_score(targets, preds, average=None, zero_division=0).tolist()
        return {
            "accuracy": float((np.array(preds) == np.array(targets)).mean()),
            "macro_f1": float(
                f1_score(targets, preds, average="macro", zero_division=0)
            ),
            "weighted_f1": float(
                f1_score(targets, preds, average="weighted", zero_division=0)
            ),
            "per_class_f1": {
                (class_names[i] if class_names else str(i)): round(v, 4)
                for i, v in enumerate(per_class)
            },
        }

    def confusion_matrix(
        self,
        loader: DataLoader,
        class_names: Optional[list[str]] = None,
        normalize: Optional[str] = "true",   # 'true' | 'pred' | 'all' | None
        figsize: tuple = (8, 6),
    ) -> plt.Figure:
        """
        Plot and return confusion matrix figure.
        """
        preds, targets = self._infer(loader)
        cm = confusion_matrix(targets, preds, normalize=normalize)
        fig, ax = plt.subplots(figsize=figsize)
        disp = ConfusionMatrixDisplay(cm, display_labels=class_names)
        disp.plot(ax=ax, colorbar=True, cmap="Blues")
        ax.set_title("Confusion Matrix")
        plt.tight_layout()
        return fig

    def f1_per_class(
        self,
        loader: DataLoader,
        class_names: Optional[list[str]] = None,
    ) -> dict[str, float]:
        """
        Returns per-class F1 as a dict. Clean for logging to W&B or MLflow.
        """
        preds, targets = self._infer(loader)
        scores = f1_score(targets, preds, average=None, zero_division=0)
        return {
            (class_names[i] if class_names else str(i)): round(float(s), 4)
            for i, s in enumerate(scores)
        }

    # ------------------------------------------------------------------
    # Internal
    # ------------------------------------------------------------------

    def _infer(self, loader: DataLoader) -> tuple[list, list]:
        self.model.eval()
        all_preds, all_targets = [], []
        with torch.no_grad():
            for inputs, targets in loader:
                inputs = inputs.to(self.device)
                outputs = self.model(inputs)
                preds = outputs.argmax(dim=1).cpu().tolist()
                all_preds.extend(preds)
                all_targets.extend(targets.tolist())
        return all_preds, all_targets

confusion_matrix(loader, class_names=None, normalize='true', figsize=(8, 6))

Plot and return confusion matrix figure.

Source code in src/torchloop/evaluator.py 
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
def confusion_matrix(
    self,
    loader: DataLoader,
    class_names: Optional[list[str]] = None,
    normalize: Optional[str] = "true",   # 'true' | 'pred' | 'all' | None
    figsize: tuple = (8, 6),
) -> plt.Figure:
    """
    Plot and return confusion matrix figure.
    """
    preds, targets = self._infer(loader)
    cm = confusion_matrix(targets, preds, normalize=normalize)
    fig, ax = plt.subplots(figsize=figsize)
    disp = ConfusionMatrixDisplay(cm, display_labels=class_names)
    disp.plot(ax=ax, colorbar=True, cmap="Blues")
    ax.set_title("Confusion Matrix")
    plt.tight_layout()
    return fig

f1_per_class(loader, class_names=None)

Returns per-class F1 as a dict. Clean for logging to W&B or MLflow.

Source code in src/torchloop/evaluator.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
def f1_per_class(
    self,
    loader: DataLoader,
    class_names: Optional[list[str]] = None,
) -> dict[str, float]:
    """
    Returns per-class F1 as a dict. Clean for logging to W&B or MLflow.
    """
    preds, targets = self._infer(loader)
    scores = f1_score(targets, preds, average=None, zero_division=0)
    return {
        (class_names[i] if class_names else str(i)): round(float(s), 4)
        for i, s in enumerate(scores)
    }

report(loader, class_names=None)

Print full sklearn classification report.

Returns:

Type Description
dict

dict with keys: accuracy, macro_f1, weighted_f1, per_class_f1
Source code in src/torchloop/evaluator.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def report(
    self,
    loader: DataLoader,
    class_names: Optional[list[str]] = None,
) -> dict:
    """
    Print full sklearn classification report.

    Returns:
        dict with keys: accuracy, macro_f1, weighted_f1, per_class_f1
    """
    preds, targets = self._infer(loader)
    report = classification_report(
        targets, preds, target_names=class_names, zero_division=0
    )
    print(report)
    per_class = f1_score(targets, preds, average=None, zero_division=0).tolist()
    return {
        "accuracy": float((np.array(preds) == np.array(targets)).mean()),
        "macro_f1": float(
            f1_score(targets, preds, average="macro", zero_division=0)
        ),
        "weighted_f1": float(
            f1_score(targets, preds, average="weighted", zero_division=0)
        ),
        "per_class_f1": {
            (class_names[i] if class_names else str(i)): round(v, 4)
            for i, v in enumerate(per_class)
        },
    }