Skip to content

confusion

ConfusionMatrixNovelty

Bases: ConfusionMatrix

Confusion Matrix for novelty detection in data streams.

Parameters:

Name Type Description Default
known_classes list of int

List of known labels, the labels the algorithm knows prior to the online phase

 required

Attributes:

Name Type Description
novel_cm ConfusionMatrix

Binary confusion matrix representing the problem in a binary manner, including class 0 (known) and class 1 (novelty)
nc_samples int

Number of samples representing a novelty
fe int

Known samples that have been classified as a known class other than its ground truth
Source code in streamndr/metrics/confusion.py 
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
class ConfusionMatrixNovelty(metrics.confusion.ConfusionMatrix):
    """Confusion Matrix for novelty detection in data streams.

    Parameters
    ----------
    known_classes : list of int
        List of known labels, the labels the algorithm knows prior to the online phase

    Attributes
    ----------
    novel_cm : river.metrics.Confusion.ConfusionMatrix
        Binary confusion matrix representing the problem in a binary manner, including class 0 (known) and class 1 (novelty)
    nc_samples : int
        Number of samples representing a novelty
    fe : int
        Known samples that have been classified as a known class other than its ground truth
    """
    def __init__(self, known_classes):
        super().__init__(known_classes)
        self.novel_cm = metrics.confusion.ConfusionMatrix()
        self.nc_samples = 0
        self.fe = 0

    def get_associated_classes(self):
        """Computes the associated known class to each novelty pattern discovered, as described in [1], by using the real class most represented in each novelty pattern.
        Ignores the unknown samples (label -1).

        [1] E. R. Faria, I. J. C. R. Gonçalves, J. Gama and A. C. P. L. F. Carvalho, "Evaluation Methodology for Multiclass Novelty Detection Algorithms," 
        2013 Brazilian Conference on Intelligent Systems, Fortaleza, Brazil, 2013, pp. 19-25, doi: 10.1109/BRACIS.2013.12.

        Returns
        -------
        ConfusionMatrixNovelty
            The confusion matrix using the most represented known class for each of the novelty pattern reported
        """
        associated_classes_conf_matrix = ConfusionMatrixNovelty(self._init_classes)
        unknown_classes = [x for x in self.classes if x not in self._init_classes.union({-1})]

        for cl in self.classes:
            col = [int(self.data[row][cl]) for row in self.classes]

            #If the class is a novelty pattern, we select the real class most represented within the novelty pattern
            if cl in unknown_classes:
                index_max = col.index(max(col))
                pred = self.classes[index_max]

                for row in self.classes:
                    associated_classes_conf_matrix.update(row, pred, self.data[row][cl])
            elif cl != -1:
                for row in self.classes:
                    associated_classes_conf_matrix.update(row, cl, self.data[row][cl])

        return associated_classes_conf_matrix


    def update(self, y_true, y_pred, w=1.0):
        super().update(y_true, y_pred, w)

        known_class = int(y_true in self._init_classes)
        pred_known_class = int(y_pred in self._init_classes)

        if known_class == 0:
            self.nc_samples += 1
        elif known_class == pred_known_class == 1 and y_true != y_pred: #If the prediction is not a novelty, but we predicted the wrong class
            self.fe += 1

        self.novel_cm.update(1-known_class, 1-pred_known_class)

    def revert(self, y_true, y_pred, w=1.0):
        super.revert(self, y_true, y_pred, w)

        known_class = int(y_true in self._init_classes)
        pred_known_class = int(y_pred in self._init_classes)

        if known_class == 1:
            self.nc_samples -= 1
            if pred_known_class == 1 and y_true != y_pred:
                self.fe -= 1

        self.novel_cm.revert(1-known_class, 1-pred_known_class)

    def true_positives_novelty(self):
        return self.novel_cm.true_positives(1)

    def true_negatives_novelty(self):
        return self.novel_cm.true_negatives(1)

    def false_positives_novelty(self):
        return self.novel_cm.false_positives(1)

    def false_negatives_novelty(self):
        return self.novel_cm.false_negatives(1)

get_associated_classes()

Computes the associated known class to each novelty pattern discovered, as described in [1], by using the real class most represented in each novelty pattern. Ignores the unknown samples (label -1).

[1] E. R. Faria, I. J. C. R. Gonçalves, J. Gama and A. C. P. L. F. Carvalho, "Evaluation Methodology for Multiclass Novelty Detection Algorithms," 2013 Brazilian Conference on Intelligent Systems, Fortaleza, Brazil, 2013, pp. 19-25, doi: 10.1109/BRACIS.2013.12.

Returns:

Type Description
ConfusionMatrixNovelty

The confusion matrix using the most represented known class for each of the novelty pattern reported
Source code in streamndr/metrics/confusion.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def get_associated_classes(self):
    """Computes the associated known class to each novelty pattern discovered, as described in [1], by using the real class most represented in each novelty pattern.
    Ignores the unknown samples (label -1).

    [1] E. R. Faria, I. J. C. R. Gonçalves, J. Gama and A. C. P. L. F. Carvalho, "Evaluation Methodology for Multiclass Novelty Detection Algorithms," 
    2013 Brazilian Conference on Intelligent Systems, Fortaleza, Brazil, 2013, pp. 19-25, doi: 10.1109/BRACIS.2013.12.

    Returns
    -------
    ConfusionMatrixNovelty
        The confusion matrix using the most represented known class for each of the novelty pattern reported
    """
    associated_classes_conf_matrix = ConfusionMatrixNovelty(self._init_classes)
    unknown_classes = [x for x in self.classes if x not in self._init_classes.union({-1})]

    for cl in self.classes:
        col = [int(self.data[row][cl]) for row in self.classes]

        #If the class is a novelty pattern, we select the real class most represented within the novelty pattern
        if cl in unknown_classes:
            index_max = col.index(max(col))
            pred = self.classes[index_max]

            for row in self.classes:
                associated_classes_conf_matrix.update(row, pred, self.data[row][cl])
        elif cl != -1:
            for row in self.classes:
                associated_classes_conf_matrix.update(row, cl, self.data[row][cl])

    return associated_classes_conf_matrix