{
    "annotations": {
      "list": [
        {
          "builtIn": 1,
          "datasource": {
            "type": "datasource",
            "uid": "grafana"
          },
          "enable": true,
          "hide": true,
          "iconColor": "rgba(0, 211, 255, 1)",
          "name": "Annotations & Alerts",
          "target": {
            "limit": 100,
            "matchAny": false,
            "tags": [],
            "type": "dashboard"
          },
          "type": "dashboard"
        },
        {
          "datasource": {
            "type": "datasource",
            "uid": "grafana"
          },
          "enable": true,
          "hide": false,
          "iconColor": "red",
          "name": "flux events",
          "target": {
            "limit": 100,
            "matchAny": false,
            "tags": [
              "flux"
            ],
            "type": "tags"
          }
        }
      ]
    },
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 0,
    "id": 115,
    "links": [],
    "panels": [
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "decimals": 0,
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "blue",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 100
                }
              ]
            },
            "unit": "short"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 5,
          "w": 6,
          "x": 0,
          "y": 0
        },
        "id": 24,
        "options": {
          "colorMode": "value",
          "graphMode": "none",
          "justifyMode": "auto",
          "orientation": "auto",
          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [
              "last"
            ],
            "fields": "",
            "values": false
          },
          "showPercentChange": false,
          "text": {},
          "textMode": "value",
          "wideLayout": true
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "editorMode": "code",
            "expr": "sum(go_info{pod=~\".*-controller-.*\"})",
            "interval": "",
            "legendFormat": "pods",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Controllers",
        "type": "stat"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "blue",
                  "value": null
                },
                {
                  "color": "#EAB839",
                  "value": 50
                },
                {
                  "color": "red",
                  "value": 100
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 5,
          "w": 6,
          "x": 6,
          "y": 0
        },
        "id": 23,
        "options": {
          "colorMode": "value",
          "graphMode": "area",
          "justifyMode": "auto",
          "orientation": "auto",
          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [
              "lastNotNull"
            ],
            "fields": "",
            "values": false
          },
          "showPercentChange": false,
          "text": {},
          "textMode": "auto",
          "wideLayout": true
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "max(workqueue_longest_running_processor_seconds{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
            "hide": false,
            "interval": "",
            "legendFormat": "seconds",
            "refId": "B"
          }
        ],
        "title": "Max Work Queue",
        "type": "stat"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "blue",
                  "value": null
                },
                {
                  "color": "#EAB839",
                  "value": 500000000
                },
                {
                  "color": "red",
                  "value": 900000000
                }
              ]
            },
            "unit": "decbits"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 5,
          "w": 6,
          "x": 12,
          "y": 0
        },
        "id": 25,
        "options": {
          "minVizHeight": 75,
          "minVizWidth": 75,
          "orientation": "auto",
          "reduceOptions": {
            "calcs": [
              "lastNotNull"
            ],
            "fields": "",
            "values": false
          },
          "showThresholdLabels": false,
          "showThresholdMarkers": true,
          "sizing": "auto",
          "text": {}
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(go_memstats_alloc_bytes{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
            "interval": "",
            "legendFormat": "",
            "refId": "A"
          }
        ],
        "title": "Memory",
        "type": "gauge"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "blue",
                  "value": null
                },
                {
                  "color": "#EAB839",
                  "value": 50
                },
                {
                  "color": "red",
                  "value": 100
                }
              ]
            }
          },
          "overrides": []
        },
        "gridPos": {
          "h": 5,
          "w": 6,
          "x": 18,
          "y": 0
        },
        "id": 26,
        "options": {
          "colorMode": "value",
          "graphMode": "area",
          "justifyMode": "auto",
          "orientation": "auto",
          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [
              "mean"
            ],
            "fields": "",
            "values": false
          },
          "showPercentChange": false,
          "text": {},
          "textMode": "auto",
          "wideLayout": true
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m]))",
            "interval": "",
            "legendFormat": "requests",
            "refId": "A"
          }
        ],
        "title": "API Requests",
        "type": "stat"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "line",
              "fillOpacity": 10,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "short"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 24,
          "x": 0,
          "y": 5
        },
        "id": 21,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\"}[1m]))",
            "hide": false,
            "interval": "",
            "legendFormat": "total",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",code!~\"2..\"}[1m]))",
            "hide": false,
            "interval": "",
            "legendFormat": "errors",
            "refId": "B"
          }
        ],
        "title": "Kubernetes API Requests",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 13
        },
        "id": 15,
        "panels": [],
        "title": "Resource Usage",
        "type": "row"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "line",
              "fillOpacity": 10,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "normal"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 11,
          "w": 12,
          "x": 0,
          "y": 14
        },
        "id": 11,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "rate(process_cpu_seconds_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m])",
            "interval": "",
            "legendFormat": "{{pod}}",
            "refId": "A"
          }
        ],
        "title": "CPU Usage",
        "type": "timeseries"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "line",
              "fillOpacity": 10,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "normal"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "decimals": 0,
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "bytes"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 11,
          "w": 12,
          "x": 12,
          "y": 14
        },
        "id": 13,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
            "hide": false,
            "interval": "",
            "legendFormat": "{{pod}}",
            "refId": "A"
          }
        ],
        "title": "Memory Usage",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 25
        },
        "id": 17,
        "panels": [],
        "title": "Reconciliation Stats",
        "type": "row"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "line",
              "fillOpacity": 10,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 24,
          "x": 0,
          "y": 26
        },
        "id": 27,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "workqueue_longest_running_processor_seconds{name=\"kustomization\"}",
            "hide": false,
            "interval": "",
            "legendFormat": "kustomizations",
            "refId": "B"
          }
        ],
        "title": "Cluster Reconciliation Duration",
        "type": "timeseries"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "bars",
              "fillOpacity": 100,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "stepAfter",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "opm"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 24,
          "x": 0,
          "y": 34
        },
        "id": 2,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result!=\"error\"}[1m])) by (controller)",
            "format": "time_series",
            "interval": "",
            "legendFormat": "successful reconciliations ",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result=\"error\"}[1m])) by (controller)",
            "format": "time_series",
            "interval": "",
            "legendFormat": "failed reconciliations ",
            "refId": "B"
          }
        ],
        "title": "Cluster Reconciliations ops/min",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 42
        },
        "id": 29,
        "panels": [],
        "title": "Sources Stats",
        "type": "row"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "bars",
              "fillOpacity": 100,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "stepAfter",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "opm"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 0,
          "y": 43
        },
        "id": 4,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result!=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "successful git pulls",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "failed git pulls",
            "refId": "B"
          }
        ],
        "title": "Git Repos ops/min",
        "type": "timeseries"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "bars",
              "fillOpacity": 100,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "stepAfter",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "opm"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 12,
          "y": 43
        },
        "id": 30,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result!=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "successful oci pulls",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "failed oci pulls",
            "refId": "B"
          }
        ],
        "title": "OCI Repos ops/min",
        "type": "timeseries"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "bars",
              "fillOpacity": 100,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "stepAfter",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "opm"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 0,
          "y": 52
        },
        "id": 31,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result!=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "successful helm pulls",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "failed helm pulls",
            "refId": "B"
          }
        ],
        "title": "Helm Repos ops/min",
        "type": "timeseries"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "bars",
              "fillOpacity": 100,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "stepAfter",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "opm"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 12,
          "y": 52
        },
        "id": 32,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result!=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "successful bucket pulls",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result=\"error\"}[1m]))",
            "format": "time_series",
            "interval": "",
            "legendFormat": "failed bucket pulls",
            "refId": "B"
          }
        ],
        "title": "Buckets ops/min",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 61
        },
        "id": 19,
        "panels": [],
        "title": "Helm Stats",
        "type": "row"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "line",
              "fillOpacity": 10,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 24,
          "x": 0,
          "y": 62
        },
        "id": 9,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "list",
            "placement": "right",
            "showLegend": false
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
            "hide": true,
            "interval": "",
            "legendFormat": "P50",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
            "hide": true,
            "interval": "",
            "legendFormat": "P90",
            "refId": "B"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
            "hide": false,
            "interval": "",
            "legendFormat": "P99",
            "refId": "C"
          }
        ],
        "title": "Helm Release Duration",
        "type": "timeseries"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "bars",
              "fillOpacity": 100,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "stepAfter",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "opm"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 0,
          "y": 70
        },
        "id": 5,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result!=\"error\"}[1m])) by (controller)",
            "format": "time_series",
            "interval": "",
            "legendFormat": "successful reconciliations ",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result=\"error\"}[1m])) by (controller)",
            "format": "time_series",
            "interval": "",
            "legendFormat": "failed reconciliations ",
            "refId": "B"
          }
        ],
        "title": "Helm Releases ops/min",
        "type": "timeseries"
      },
      {
        "datasource": {
          "uid": "${DS_PROMETHEUS}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "bars",
              "fillOpacity": 100,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "stepAfter",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "opm"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 12,
          "y": 70
        },
        "id": 6,
        "options": {
          "legend": {
            "calcs": [
              "mean",
              "lastNotNull"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "none"
          }
        },
        "pluginVersion": "11.4.0",
        "targets": [
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result!=\"error\"}[1m])) by (controller)",
            "format": "time_series",
            "interval": "",
            "legendFormat": "successful chart pulls",
            "refId": "A"
          },
          {
            "datasource": {
              "uid": "${DS_PROMETHEUS}"
            },
            "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result=\"error\"}[1m])) by (controller)",
            "format": "time_series",
            "interval": "",
            "legendFormat": "failed chart pulls",
            "refId": "B"
          }
        ],
        "title": "Helm Charts ops/min",
        "type": "timeseries"
      }
    ],
    "preload": false,
    "refresh": "10s",
    "schemaVersion": 40,
    "tags": [
      "flux"
    ],
    "templating": {
      "list": [
        {
          "current": {
            "text": "vm-shortterm",
            "value": "24b0d575-0cea-474c-a4eb-c225d250858d"
          },
          "includeAll": false,
          "label": "DS_PROMETHEUS",
          "name": "DS_PROMETHEUS",
          "options": [],
          "query": "prometheus",
          "refresh": 1,
          "regex": "",
          "type": "datasource"
        },
        {
          "current": {
            "text": "cozy-mariadb-operator",
            "value": "cozy-mariadb-operator"
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "definition": "workqueue_work_duration_seconds_count",
          "includeAll": false,
          "name": "namespace",
          "options": [],
          "query": {
            "query": "workqueue_work_duration_seconds_count",
            "refId": "Prometheus-namespace-Variable-Query"
          },
          "refresh": 2,
          "regex": "/.*namespace=\"([^\"]*).*/",
          "type": "query"
        }
      ]
    },
    "time": {
      "from": "now-15m",
      "to": "now"
    },
    "timepicker": {
      "refresh_intervals": [
        "10s",
        "30s",
        "1m",
        "5m",
        "15m",
        "30m",
        "1h",
        "2h",
        "1d"
      ]
    },
    "timezone": "",
    "title": "Flux Control Plane",
    "uid": "flux-control-plane",
    "version": 1,
    "weekStart": ""
  }
