Alert for ingestion processing

.github/workflows/alerts.yaml

@@ -74,6 +74,17 @@ jobs:
             notification_processor_replicas=2
           github_token: ${{ secrets.GIT_ACCESS_TOKEN }}
 
+      - name: Render and test ingestion producer
+        uses: scality/action-prom-render-test@1.0.3
+        with:
+          alert_file_path: monitoring/ingestion/ingestion-producer-alert.yaml
+          test_file_path: monitoring/ingestion/ingestion-producer-alert.test.yaml
+          alert_inputs: |
+            namespace=zenko
+            job_ingestion_producer=artesca-data-backbeat-ingestion-producer-headless
+            replicas=1
+          github_token: ${{ secrets.GIT_ACCESS_TOKEN }}
+
       - name: Render and test oplog populator 
         uses: scality/action-prom-render-test@1.0.3
         with:

monitoring/ingestion/ingestion-producer-alert.test.yaml

@@ -0,0 +1,84 @@
+evaluation_interval: 1m
+rule_files:
+  - ingestion-producer-alert.rendered.yaml
+
+tests:
+
+  # getRaftId errors stay at 100% on that op even when getRaftLog volume is high.
+  # The warning and critical rules share one name, so Alertmanager raises the
+  # severity instead of sending two alerts.
+  - name: Ingestion Producer Source Setup Failing
+    interval: 1m
+    input_series:
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"}
+        values: 0+12x20
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="success"}
+        values: 0+12000x20
+    alert_rule_test:
+      - alertname: IngestionProducerSourceErrorRate3Percent
+        eval_time: 5m
+        exp_alerts: []
+      - alertname: IngestionProducerSourceErrorRate3Percent
+        eval_time: 12m
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              op: getRaftId
+            exp_annotations:
+              zenko_service: backbeat-ingestion-producer
+              description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes"
+              summary: "Ingestion producer source error rate has increased"
+          - exp_labels:
+              severity: critical
+              op: getRaftId
+            exp_annotations:
+              zenko_service: backbeat-ingestion-producer
+              description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes"
+              summary: "Ingestion producer source error rate has increased"
+
+  # A source that fails to set up while its location restarts, then recovers,
+  # must not alert.
+  - name: Ingestion Producer Source Setup Recovering
+    interval: 1m
+    input_series:
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"}
+        values: 0 12 24 24x17
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"}
+        values: 0 0 0 1+1x17
+    alert_rule_test:
+      - alertname: IngestionProducerSourceErrorRate3Percent
+        eval_time: 8m
+        exp_alerts: []
+      - alertname: IngestionProducerSourceErrorRate3Percent
+        eval_time: 15m
+        exp_alerts: []
+
+  # A later getRaftLog failure is the same 3% / 5% alert on that op.
+  - name: Ingestion Producer Source Read Failing
+    interval: 1m
+    input_series:
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"}
+        values: 0+12x20
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="error"}
+        values: 0+12x20
+    alert_rule_test:
+      - alertname: IngestionProducerSourceErrorRate3Percent
+        eval_time: 5m
+        exp_alerts: []
+      - alertname: IngestionProducerSourceErrorRate3Percent
+        eval_time: 12m
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              op: getRaftLog
+            exp_annotations:
+              zenko_service: backbeat-ingestion-producer
+              description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes"
+              summary: "Ingestion producer source error rate has increased"
+          - exp_labels:
+              severity: critical
+              op: getRaftLog
+            exp_annotations:
+              zenko_service: backbeat-ingestion-producer
+              description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes"
+              summary: "Ingestion producer source error rate has increased"

monitoring/ingestion/ingestion-producer-alert.yaml

@@ -70,11 +70,13 @@ groups:
       description: "Ingestion producer has not queued any messages in the last 5 minutes"
       summary: "Ingestion producer has not queued messages"
 
+  # Ratio per source op so a stuck getRaftId is not hidden by getRaftLog
+  # volume, and a later getRaftLog failure is the same alert on that op.
   - alert: IngestionProducerSourceErrorRate3Percent
     Expr: |
-      sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
+      sum by (op) (rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
         /
-      sum(rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
+      sum by (op) (rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
       > 0.03
     For:   "5m"
     Labels:
@@ -86,9 +88,9 @@ groups:
 
   - alert: IngestionProducerSourceErrorRate3Percent
     Expr: |
-      sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
+      sum by (op) (rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
         /
-      sum(rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
+      sum by (op) (rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
       > 0.05
     For:   "5m"
     Labels: