diff --git a/.github/workflows/alerts.yaml b/.github/workflows/alerts.yaml index 4ebbb7445c..3e3ba043bc 100644 --- a/.github/workflows/alerts.yaml +++ b/.github/workflows/alerts.yaml @@ -74,6 +74,17 @@ jobs: notification_processor_replicas=2 github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test ingestion producer + uses: scality/action-prom-render-test@1.0.3 + with: + alert_file_path: monitoring/ingestion/ingestion-producer-alert.yaml + test_file_path: monitoring/ingestion/ingestion-producer-alert.test.yaml + alert_inputs: | + namespace=zenko + job_ingestion_producer=artesca-data-backbeat-ingestion-producer-headless + replicas=1 + github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test oplog populator uses: scality/action-prom-render-test@1.0.3 with: diff --git a/monitoring/ingestion/ingestion-producer-alert.test.yaml b/monitoring/ingestion/ingestion-producer-alert.test.yaml new file mode 100644 index 0000000000..c400e3126e --- /dev/null +++ b/monitoring/ingestion/ingestion-producer-alert.test.yaml @@ -0,0 +1,84 @@ +evaluation_interval: 1m +rule_files: + - ingestion-producer-alert.rendered.yaml + +tests: + + # getRaftId errors stay at 100% on that op even when getRaftLog volume is high. + # The warning and critical rules share one name, so Alertmanager raises the + # severity instead of sending two alerts. + - name: Ingestion Producer Source Setup Failing + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0+12x20 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="success"} + values: 0+12000x20 + alert_rule_test: + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 5m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 12m + exp_alerts: + - exp_labels: + severity: warning + op: getRaftId + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased" + - exp_labels: + severity: critical + op: getRaftId + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased" + + # A source that fails to set up while its location restarts, then recovers, + # must not alert. + - name: Ingestion Producer Source Setup Recovering + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0 12 24 24x17 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"} + values: 0 0 0 1+1x17 + alert_rule_test: + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 8m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 15m + exp_alerts: [] + + # A later getRaftLog failure is the same 3% / 5% alert on that op. + - name: Ingestion Producer Source Read Failing + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"} + values: 0+12x20 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="error"} + values: 0+12x20 + alert_rule_test: + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 5m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 12m + exp_alerts: + - exp_labels: + severity: warning + op: getRaftLog + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased" + - exp_labels: + severity: critical + op: getRaftLog + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased" diff --git a/monitoring/ingestion/ingestion-producer-alert.yaml b/monitoring/ingestion/ingestion-producer-alert.yaml index 73ec1073b4..6d32a24f5b 100644 --- a/monitoring/ingestion/ingestion-producer-alert.yaml +++ b/monitoring/ingestion/ingestion-producer-alert.yaml @@ -70,11 +70,13 @@ groups: description: "Ingestion producer has not queued any messages in the last 5 minutes" summary: "Ingestion producer has not queued messages" + # Ratio per source op so a stuck getRaftId is not hidden by getRaftLog + # volume, and a later getRaftLog failure is the same alert on that op. - alert: IngestionProducerSourceErrorRate3Percent Expr: | - sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) / - sum(rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) > 0.03 For: "5m" Labels: @@ -86,9 +88,9 @@ groups: - alert: IngestionProducerSourceErrorRate3Percent Expr: | - sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) / - sum(rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) > 0.05 For: "5m" Labels: