Skip to content

Configs

AccuracyConfig dataclass

Configuration class for completeness settings. Methods: post_init(): Validates the provided algorithms and ensemble flag.

Source code in iot_dqa/utils/configs.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
@dataclass
class AccuracyConfig:
    """
    Configuration class for completeness settings.
    Methods:
        __post_init__(): Validates the provided algorithms and ensemble flag.
    """

    ensemble: bool = True
    """Flag to indicate if ensemble methods should be used. Default is True."""
    mad_threshold: int = 3
    """ Threshold for Median Absolute Deviation (MAD). Default is 3. Using 3 * STD as decribed in the literature."""
    optimize_iqr_with_optuna: bool = True
    """Flag to indicate if IQR optimization should be performed using optuna. Default is True."""
    iqr_optuna_trials: Union[int, None] = 10
    """10 trials when optimizing the IQR"""
    iqr_optuna_q1_min: Union[int, None] = 0
    """Minimum value for the first quartile (Q1) in IQR optimization. Default is 0.0."""
    iqr_optuna_q1_max: Union[float, None] = 0.5
    """Maximum value for the first quartile (Q1) in IQR optimization. Default is 0.5."""
    iqr_optuna_q3_min: Union[float, None] = 0.5
    """Minimum value for the third quartile (Q3) in IQR optimization. Default is 0.5."""
    iqr_optuna_q3_max: Union[int, None] = 1
    """Maximum value for the third quartile (Q3) in IQR optimization. Default is 1.0."""

    algorithms: list[OutlierDetectionAlgorithm] = field(
        default_factory=lambda: [x.value for x in OutlierDetectionAlgorithm]
    )
    """List of outlier detection algorithms to be used. Default is all values of OutlierDetectionAlgorithm."""
    strategy: AccuracyStrategy = AccuracyStrategy.NONE.value
    """Determine the approach to use for the accuracy computation."""
    isolation_forest: IsolationForestConfig = field(
        default_factory=IsolationForestConfig
    )
    """Configuration for Isolation Forest settings."""

    def __post_init__(self):
        if not all(
            algo in OutlierDetectionAlgorithm._value2member_map_
            for algo in self.algorithms
        ):
            raise ValueError(
                f"All algorithms must be valid values of OutlierDetectionAlgorithm. Provided: {self.algorithms}"
            )
        if not isinstance(self.ensemble, bool):
            raise ValueError(
                f"Ensemble must be valid boolean. Provided: {self.ensemble}"
            )

        if not isinstance(self.optimize_iqr_with_optuna, bool):
            raise ValueError(
                f"IQR optimization must be valid boolean. Provided: {self.optimize_iqr_with_optuna}"
            )
        if not isinstance(self.mad_threshold, int):
            raise ValueError(
                f"MAD threshold must be valid boolean. Provided: {self.mad_threshold}"
            )
        if not isinstance(self.iqr_optuna_trials, int):
            raise ValueError(
                f"IQR optuna trial must be valid integer. Provided: {self.iqr_optuna_trials}"
            )

        if not isinstance(self.iqr_optuna_q1_min, int):
            raise ValueError(
                f"IQR optuna Q1 min must be valid integer. Provided: {self.iqr_optuna_q1_min}"
            )

        if not isinstance(self.iqr_optuna_q1_max, float):
            raise ValueError(
                f"IQR optuna Q1 max must be valid float. Provided: {self.iqr_optuna_q1_max}"
            )

        if not isinstance(self.iqr_optuna_q3_min, float):
            raise ValueError(
                f"IQR optuna Q1 min must be valid float. Provided: {self.iqr_optuna_q3_min}"
            )

        if not isinstance(self.iqr_optuna_q3_max, int):
            raise ValueError(
                f"IQR optuna Q1 max must be valid integer. Provided: {self.iqr_optuna_q3_max}"
            )

        if self.iqr_optuna_q3_max > 1:
            raise ValueError(
                f"IQR Q3 max must be less than or equal to 1. Provided: {self.iqr_optuna_q3_max}"
            )
        if self.iqr_optuna_q1_min < 0:
            raise ValueError(
                f"IQR Q1 min must be greater than or equal to 0. Provided: {self.iqr_optuna_q1_min}"
            )

algorithms: list[OutlierDetectionAlgorithm] = field(default_factory=lambda: [x.value for x in OutlierDetectionAlgorithm]) class-attribute instance-attribute

List of outlier detection algorithms to be used. Default is all values of OutlierDetectionAlgorithm.

ensemble: bool = True class-attribute instance-attribute

Flag to indicate if ensemble methods should be used. Default is True.

iqr_optuna_q1_max: Union[float, None] = 0.5 class-attribute instance-attribute

Maximum value for the first quartile (Q1) in IQR optimization. Default is 0.5.

iqr_optuna_q1_min: Union[int, None] = 0 class-attribute instance-attribute

Minimum value for the first quartile (Q1) in IQR optimization. Default is 0.0.

iqr_optuna_q3_max: Union[int, None] = 1 class-attribute instance-attribute

Maximum value for the third quartile (Q3) in IQR optimization. Default is 1.0.

iqr_optuna_q3_min: Union[float, None] = 0.5 class-attribute instance-attribute

Minimum value for the third quartile (Q3) in IQR optimization. Default is 0.5.

iqr_optuna_trials: Union[int, None] = 10 class-attribute instance-attribute

10 trials when optimizing the IQR

isolation_forest: IsolationForestConfig = field(default_factory=IsolationForestConfig) class-attribute instance-attribute

Configuration for Isolation Forest settings.

mad_threshold: int = 3 class-attribute instance-attribute

Threshold for Median Absolute Deviation (MAD). Default is 3. Using 3 * STD as decribed in the literature.

optimize_iqr_with_optuna: bool = True class-attribute instance-attribute

Flag to indicate if IQR optimization should be performed using optuna. Default is True.

strategy: AccuracyStrategy = AccuracyStrategy.NONE.value class-attribute instance-attribute

Determine the approach to use for the accuracy computation.

IsolationForestConfig dataclass

Configuration class for Isolation Forest settings. Attributes: n_estimators (int): Number of trees in the forest. Default is 100. max_samples (float): Number of samples to draw to train each base estimator. Default is 0.8. contamination (float): Proportion of outliers in the data set. Default is 0.1. max_features (int): Number of features to draw to train each base estimator. Default is 1.

Source code in iot_dqa/utils/configs.py 
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
@dataclass
class IsolationForestConfig:
    """
    Configuration class for Isolation Forest settings.
    Attributes:
        n_estimators (int): Number of trees in the forest. Default is 100.
        max_samples (float): Number of samples to draw to train each base estimator. Default is 0.8.
        contamination (float): Proportion of outliers in the data set. Default is 0.1.
        max_features (int): Number of features to draw to train each base estimator. Default is 1.
    """

    n_estimators: int = 100
    max_samples: float = 0.8
    contamination: Union[float, str] = 0.1
    max_features: int = 1
    random_state: int = 42

TimelinessConfig dataclass

Configuration class for timeliness settings. Attributes: frequency (str): Frequency of the data. Default is "1H". iat_method (str): Method to calculate inter-arrival time. Default is "mean".

Source code in iot_dqa/utils/configs.py 
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
@dataclass
class TimelinessConfig:
    """
    Configuration class for timeliness settings.
    Attributes:
        frequency (str): Frequency of the data. Default is "1H".
        iat_method (str): Method to calculate inter-arrival time. Default is "mean".
    """

    iat_method: FrequencyCalculationMethod = FrequencyCalculationMethod.MIN.value
    """Method to calculate inter-arrival time. Default is 'min'."""

    def __post_init__(self):
        if not (self.iat_method):
            raise ValueError(
                "At least one of 'frequency_unit' or 'frequency_unit' or 'iat_method' must be provided."
            )

        if not isinstance(self.iat_method, str):
            raise ValueError(
                f"IAT method must be valid string. Provided: {self.iat_method}"
            )

iat_method: FrequencyCalculationMethod = FrequencyCalculationMethod.MIN.value class-attribute instance-attribute

Method to calculate inter-arrival time. Default is 'min'.