Adding implementations of change point detection methods

treeherder/perf/alerts.py

@@ -11,7 +11,12 @@
 from django.db.models import Exists, OuterRef, Subquery
 
 from treeherder.perf.email import AlertNotificationWriter
-from treeherder.perf.methods import StudentDetector
+from treeherder.perf.methods.CramerVonMisesDetector import CramerVonMisesDetector
+from treeherder.perf.methods.KolmogorovSmirnovDetector import KolmogorovSmirnovDetector
+from treeherder.perf.methods.LeveneDetector import LeveneDetector
+from treeherder.perf.methods.MannWhitneyUDetector import MannWhitneyUDetector
+from treeherder.perf.methods.StudentDetector import StudentDetector
+from treeherder.perf.methods.WelchDetector import WelchDetector
 from treeherder.perf.models import (
     PerformanceAlert,
     PerformanceAlertSummary,
@@ -214,7 +219,8 @@ def generate_new_alerts_in_series(signature):
 
 
 def build_cpd_methods():
-    student = StudentDetector.StudentDetector(
+    student = StudentDetector(
+        name="student",
         min_back_window=12,
         max_back_window=24,
         fore_window=12,
@@ -223,7 +229,64 @@ def build_cpd_methods():
         mag_check=True,
         above_threshold_is_anomaly=True,
     )
-    methods = {"student": student}
+    cvm = CramerVonMisesDetector(
+        name="cvm",
+        min_back_window=12,
+        max_back_window=24,
+        fore_window=12,
+        alert_threshold=2.0,
+        confidence_threshold=0.05,
+        mag_check=False,
+        above_threshold_is_anomaly=False,
+    )
+    ks = KolmogorovSmirnovDetector(
+        name="ks",
+        min_back_window=12,
+        max_back_window=24,
+        fore_window=12,
+        alert_threshold=2.0,
+        confidence_threshold=0.05,
+        mag_check=False,
+        above_threshold_is_anomaly=False,
+    )
+    welch = WelchDetector(
+        name="welch",
+        min_back_window=12,
+        max_back_window=24,
+        fore_window=12,
+        alert_threshold=2.0,
+        confidence_threshold=0.05,
+        mag_check=False,
+        above_threshold_is_anomaly=False,
+    )
+    levene = LeveneDetector(
+        name="levene",
+        min_back_window=12,
+        max_back_window=24,
+        fore_window=12,
+        alert_threshold=2.0,
+        confidence_threshold=0.05,
+        mag_check=False,
+        above_threshold_is_anomaly=False,
+    )
+    mwu = MannWhitneyUDetector(
+        name="mwu",
+        min_back_window=12,
+        max_back_window=24,
+        fore_window=12,
+        alert_threshold=2.0,
+        confidence_threshold=0.05,
+        mag_check=False,
+        above_threshold_is_anomaly=False,
+    )
+    methods = {
+        "student": student,
+        "cvm": cvm,
+        "ks": ks,
+        "welch": welch,
+        "levene": levene,
+        "mwu": mwu,
+    }
     return methods
 
 
@@ -313,7 +376,7 @@ def generate_new_test_alerts_in_series(signature):
     # get series data starting from either:
     # (1) the last alert, if there is one
     # (2) the alerts max age
-    # (use whichever is newer)
+    # use whichever is newer
     max_alert_age = alert_after_ts = datetime.now() - settings.PERFHERDER_ALERTS_MAX_AGE
     series = PerformanceDatum.objects.filter(signature=signature, push_timestamp__gte=max_alert_age)
     latest_alert_timestamp = (

treeherder/perf/methods/CramerVonMisesDetector.py

@@ -0,0 +1,32 @@
+from scipy import stats
+
+from treeherder.perf.methods.BaseDetector import BaseDetector
+
+
+class CramerVonMisesDetector(BaseDetector):
+    """
+    Detector using Cramér-von Mises test.
+    """
+
+    def calc_confidence(self, jw, kw, confidence_threshold, confidence):
+        """
+        Calculate Cramér-von Mises test statistic and p-value.
+        """
+        jw_values = [v for datum in jw for v in datum.values]
+        kw_values = [v for datum in kw for v in datum.values]
+
+        if len(jw_values) < 2 or len(kw_values) < 2:
+            return 1.0, confidence + 1
+
+        try:
+            result = stats.cramervonmises_2samp(jw_values, kw_values)
+            p = result.pvalue
+        except Exception:
+            p = 1.0
+
+        if p < confidence_threshold:
+            confidence = 0
+        else:
+            confidence += 1
+
+        return p, confidence

treeherder/perf/methods/KolmogorovSmirnovDetector.py

@@ -0,0 +1,32 @@
+from scipy import stats
+
+from treeherder.perf.methods.BaseDetector import BaseDetector
+
+
+class KolmogorovSmirnovDetector(BaseDetector):
+    """
+    Detector using Kolmogorov-Smirnov test.
+    """
+
+    def calc_confidence(self, jw, kw, confidence_threshold, confidence):
+        """
+        Calculate Kolmogorov-Smirnov test statistic and p-value.
+        """
+        jw_values = [v for datum in jw for v in datum.values]
+        kw_values = [v for datum in kw for v in datum.values]
+
+        if len(jw_values) < 2 or len(kw_values) < 2:
+            return 1.0, confidence + 1
+
+        try:
+            result = stats.ks_2samp(jw_values, kw_values)
+            p = result.pvalue
+        except Exception:
+            p = 0, 1.0
+
+        if p < confidence_threshold:
+            confidence = 0
+        else:
+            confidence += 1
+
+        return p, confidence

treeherder/perf/methods/LeveneDetector.py

@@ -0,0 +1,32 @@
+from scipy import stats
+
+from treeherder.perf.methods.BaseDetector import BaseDetector
+
+
+class LeveneDetector(BaseDetector):
+    """
+    Detector using Levene's test (tests for equal variances).
+    """
+
+    def calc_confidence(self, jw, kw, confidence_threshold, last_seen_regression):
+        """
+        Calculate Levene's test statistic and p-value.
+        """
+        jw_values = [v for datum in jw for v in datum.values]
+        kw_values = [v for datum in kw for v in datum.values]
+
+        if len(jw_values) < 2 or len(kw_values) < 2:
+            return 1.0, last_seen_regression + 1
+
+        try:
+            result = stats.levene(jw_values, kw_values)
+            p = result.pvalue
+        except Exception:
+            p = 1.0
+
+        if p < confidence_threshold:
+            last_seen_regression = 0
+        else:
+            last_seen_regression += 1
+
+        return p, last_seen_regression

treeherder/perf/methods/MannWhitneyUDetector.py

@@ -0,0 +1,32 @@
+from scipy import stats
+
+from treeherder.perf.methods.BaseDetector import BaseDetector
+
+
+class MannWhitneyUDetector(BaseDetector):
+    """
+    Detector using Mann-Whitney U test (non-parametric).
+    """
+
+    def calc_confidence(self, jw, kw, confidence_threshold, last_seen_regression):
+        """
+        Calculate Mann-Whitney U test statistic and p-value.
+        """
+        jw_values = [v for datum in jw for v in datum.values]
+        kw_values = [v for datum in kw for v in datum.values]
+
+        if len(jw_values) < 2 or len(kw_values) < 2:
+            return 1.0, last_seen_regression + 1
+
+        try:
+            result = stats.mannwhitneyu(jw_values, kw_values, alternative="two-sided")
+            p = result.pvalue
+        except Exception:
+            p = 1.0
+
+        if p < confidence_threshold:
+            last_seen_regression = 0
+        else:
+            last_seen_regression += 1
+
+        return p, last_seen_regression

treeherder/perf/methods/StudentDetector.py

@@ -2,28 +2,6 @@
 
 
 class StudentDetector(BaseDetector):
-    def __init__(
-        self,
-        name="student",
-        min_back_window=12,
-        max_back_window=24,
-        fore_window=12,
-        alert_threshold=2.0,
-        confidence_threshold=7,
-        mag_check=True,
-        above_threshold_is_anomaly=True,
-    ):
-        super().__init__(
-            name=name,
-            min_back_window=min_back_window,
-            max_back_window=max_back_window,
-            fore_window=fore_window,
-            alert_threshold=alert_threshold,
-            confidence_threshold=confidence_threshold,
-            mag_check=mag_check,
-            above_threshold_is_anomaly=above_threshold_is_anomaly,
-        )
-
     def calc_confidence(self, w1, w2, confidence_threshold, last_seen_regression):
         # replaces calc_t function
         """Perform a Students t-test on the two sets of revision data.

treeherder/perf/methods/WelchDetector.py

@@ -0,0 +1,32 @@
+from scipy import stats
+
+from treeherder.perf.methods.BaseDetector import BaseDetector
+
+
+class WelchDetector(BaseDetector):
+    """
+    Detector using Welch's t-test (unequal variances t-test).
+    """
+
+    def calc_confidence(self, jw, kw, confidence_threshold, last_seen_regression):
+        """
+        Calculate Welch's t-test statistic and p-value.
+        """
+        jw_values = [v for datum in jw for v in datum.values]
+        kw_values = [v for datum in kw for v in datum.values]
+
+        if len(jw_values) < 2 or len(kw_values) < 2:
+            return 1.0, last_seen_regression + 1  # p-value of 1.0 (no significance)
+
+        try:
+            result = stats.ttest_ind(jw_values, kw_values, equal_var=False)
+            p = result.pvalue
+        except Exception:
+            p = 1.0
+
+        if p < confidence_threshold:
+            last_seen_regression = 0
+        else:
+            last_seen_regression += 1
+
+        return p, last_seen_regression