Platform and deployment alerts

1
- alert: ManyUnhealthySchedulers
2
  expr: count(rate(airflow_scheduler_heartbeat{}[1m]) <= 0) > 5
3
  for: 5m
4
  labels:
5
    tier: platform
6
    severity: critical
7
  annotations:
8
    summary: "{{ $value }} airflow schedulers are not heartbeating"
9
    description: "More than 5 Airflow schedulers have not emitted a heartbeat for over 5 minutes."

1
alertmanager:
2
  receivers:
3
    platform:
4
      email_configs:
5
        - smarthost: smtp.example.com:587
6
          from: alerts@example.com
7
          to: ops-team@example.com
8
          auth_username: alerts@example.com
9
          auth_password: ${SMTP_PASSWORD}
10
          send_resolved: true

1
alertmanager:
2
  receivers:
3
    platformCritical:
4
      slack_configs:
5
        - api_url: https://hooks.slack.com/services/xxx/yyy/zzz
6
          channel: '#platform-alerts'
7
          title: '{{ .CommonAnnotations.summary }}'
8
          text: |-
9
            {{ range .Alerts }}{{ .Annotations.description }}
10
            {{ end }}

1
alertmanager:
2
  receivers:
3
    platformCritical:
4
      pagerduty_configs:
5
        - service_key: ${PAGERDUTY_SERVICE_KEY}
6
          severity: '{{ .CommonLabels.severity }}'
7
          description: '{{ .CommonAnnotations.summary }}'

1
alertmanager:
2
  receivers:
3
    platformCritical:
4
      opsgenie_configs:
5
        - api_key: ${OPSGENIE_API_KEY}
6
          message: '{{ .CommonAnnotations.summary }}'
7
          priority: '{{ if eq .CommonLabels.severity "critical" }}P1{{ else }}P3{{ end }}'

1
alertmanager:
2
  customRoutes:
3
    - receiver: deployment-high-receiver
4
      match_re:
5
        tier: airflow
6
        severity: high
7
    - receiver: deployment-warning-receiver
8
      match_re:
9
        tier: airflow
10
        severity: warning

1
alertmanager:
2
  customReceiver:
3
    - name: sns-receiver
4
      sns_configs:
5
        - api_url: <SNS_ENDPOINT>
6
          topic_arn: <SNS_TOPIC_ARN>
7
          subject: '[Alert: {{ .GroupLabels.alertname }}]'
8
          sigv4:
9
            region: <AWS_REGION>
10
            role_arn: <SNS_ROLE_ARN>
11
  customRoutes:
12
    - receiver: sns-receiver
13
      match_re:
14
        tier: platform
15
        severity: critical

1
prometheus:
2
  additionalAlerts:
3
    platform: |
4
      - alert: MultipleSchedulersUnhealthy
5
        expr: count(rate(airflow_scheduler_heartbeat{}[1m]) <= 0) > 2
6
        for: 5m
7
        labels:
8
          tier: platform
9
          severity: critical
10
        annotations:
11
          summary: "{{ $value }} schedulers are not heartbeating"
12
          description: "More than 2 Airflow schedulers are unhealthy for over 5 minutes."

1
prometheus:
2
  additionalAlerts:
3
    airflow: |
4
      - alert: HighTaskFailureRate
5
        expr: |
6
          (
7
            sum(increase(airflow_ti_failures{}[1h])) by (deployment)
8
            /
9
            sum(increase(airflow_ti_successes{}[1h]) + increase(airflow_ti_failures{}[1h])) by (deployment)
10
          ) > 0.1
11
        for: 15m
12
        labels:
13
          tier: airflow
14
          severity: warning
15
        annotations:
16
          summary: "High task failure rate in {{ $labels.deployment }}"
17
          description: "Task failure rate exceeds 10% for the past 15 minutes."