diff --git a/.github/workflows/generate-dashboards.yaml b/.github/workflows/generate-dashboards.yaml new file mode 100644 index 000000000..5de7d2aec --- /dev/null +++ b/.github/workflows/generate-dashboards.yaml @@ -0,0 +1,18 @@ +name: Grafana Dashboard Apply +on: + pull_request: + paths: + - 'prombench/manifests/cluster-infra/dashboards_noparse.yaml' + - 'prombench/manifests/cluster-infra/dashboards/**' + push: + paths: + - 'prombench/manifests/cluster-infra/dashboards_noparse.yaml' + - 'prombench/manifests/cluster-infra/dashboards/**' +jobs: + verify_dashboard_generation: + name: Verify Dashboard Generation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Verify dashboard generation + run: make --always-make generate-dashboards-cm && git diff --exit-code diff --git a/Makefile b/Makefile index f0c616417..44885734b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ DOCKER_REPO ?= prominfra .PHONY: all -all: precheck style check_license lint build test unused docs-check +all: precheck style check_license lint build test unused docs-check generate-dashboards-cm .PHONY: docker docker: @@ -24,4 +24,8 @@ docs: docs-check: ./scripts/genflagdocs.sh check +.PHONY: generate-dashboards-cm +generate-dashboards-cm: + ./scripts/sync-dashboards-to-configmap.sh + include Makefile.common diff --git a/prombench/manifests/cluster-infra/dashboards/node-metrics.json b/prombench/manifests/cluster-infra/dashboards/node-metrics.json new file mode 100644 index 000000000..0cc5ddb3a --- /dev/null +++ b/prombench/manifests/cluster-infra/dashboards/node-metrics.json @@ -0,0 +1,1480 @@ +{ + "__requires": [ + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "id": null, + "title": "Node Metrics", + "description": "Dashboard to view Node metrics", + "tags": [ + "nodes", + "node-metrics" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "25px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 20, + "interval": null, + "links": [], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "repeat": "node", + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "count(node_cpu_seconds_total{namespace=\"$namespace\",node=\"$node\", mode=\"system\"}) or count(node_cpu{namespace=\"$namespace\",node=\"$node\", mode=\"system\"})", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400, + "target": "" + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "decimals": 3, + "editable": true, + "error": false, + "fill": 10, + "description": "(user: The time spent in userland) -- (system: The time spent in the kernel) -- (iowait: Time spent waiting for I/O) -- (idle: Time the CPU had nothing to do -- (irq&softirq: Time servicing interrupts) -- (guest: If you are running VMs, the CPU they use) -- (steal: If you are a VM, time other VMs \"stole\" from your CPUs)", + "grid": { + "threshold1": 0, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"system\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mode}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"user\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "user", + "refId": "B", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"nice\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "nice", + "refId": "C", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"iowait\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "iowait", + "refId": "E", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"steal\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "steal", + "refId": "H", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"idle\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "idle", + "refId": "D", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"irq\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "irq", + "refId": "F", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"softirq\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "softirq", + "refId": "G", + "step": 1200 + }, + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"guest\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"guest\",namespace=\"$namespace\",node=\"$node\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "guest", + "refId": "I", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "%", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "CPU" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Slab": "#E5A8E2", + "Swap": "#E24D42" + }, + "bars": false, + "datasource": "prometheus-meta", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/", + "fill": 5, + "stack": true + }, + { + "alias": "Swap", + "fill": 5, + "stack": true + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "( node_memory_MemTotal_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_MemFree_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_Buffers_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_Cached_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapCached_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_Slab_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_PageTables_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_VmallocUsed_bytes{namespace=\"$namespace\",node=\"$node\"} ) or ( node_memory_MemTotal{namespace=\"$namespace\",node=\"$node\"} - node_memory_MemFree{namespace=\"$namespace\",node=\"$node\"} - node_memory_Buffers{namespace=\"$namespace\",node=\"$node\"} - node_memory_Cached{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapCached{namespace=\"$namespace\",node=\"$node\"} - node_memory_Slab{namespace=\"$namespace\",node=\"$node\"} - node_memory_PageTables{namespace=\"$namespace\",node=\"$node\"} - node_memory_VmallocUsed{namespace=\"$namespace\",node=\"$node\"} )", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Apps", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "node_memory_Buffers_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Buffers{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 1200 + }, + { + "expr": "node_memory_Cached_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Cached{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 1200 + }, + { + "expr": "node_memory_MemFree_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_MemFree{namespace=\"$namespace\",node=\"$node\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "E", + "step": 1200 + }, + { + "expr": "node_memory_Slab_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Slab{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Slab", + "refId": "F", + "step": 1200 + }, + { + "expr": "node_memory_SwapCached_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_SwapCached{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SwapCached", + "refId": "G", + "step": 1200 + }, + { + "expr": "node_memory_PageTables_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_PageTables{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PageTables", + "refId": "H", + "step": 1200 + }, + { + "expr": "node_memory_VmallocUsed_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_VmallocUsed{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "VmallocUsed", + "metric": "", + "refId": "I", + "step": 1200 + }, + { + "expr": "(node_memory_SwapTotal_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapFree{namespace=\"$namespace\",node=\"$node\"}) or (node_memory_SwapTotal{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapFree{namespace=\"$namespace\",node=\"$node\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Swap", + "metric": "", + "refId": "C", + "step": 1200 + }, + { + "expr": "node_memory_Committed_AS_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Committed_AS{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Committed", + "metric": "", + "refId": "J", + "step": 1200 + }, + { + "expr": "node_memory_Mapped_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Mapped{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Mapped", + "refId": "K", + "step": 1200 + }, + { + "expr": "node_memory_Active_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Active{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "metric": "", + "refId": "L", + "step": 1200 + }, + { + "expr": "node_memory_Inactive_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Inactive{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "metric": "", + "refId": "M", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": "GB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Memory" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{namespace=\"$namespace\",node=\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Load" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "decimals": 3, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100.0 - 100 * (node_filesystem_avail_bytes{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size_bytes{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'}) or 100.0 - 100 * (node_filesystem_avail{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Used", + "tooltip": { + "msResolution": true, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Disk Used" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{namespace=\"$namespace\",node=\"$node\"}[5m])/10 or irate(node_disk_io_time_ms{namespace=\"$namespace\",node=\"$node\"}[5m])/10", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Utilization per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Disk Utilization" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{namespace=\"$namespace\",node=\"$node\"}[5m]) or irate(node_disk_reads_completed{namespace=\"$namespace\",node=\"$node\"}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + }, + { + "expr": "irate(node_disk_writes_completed_total{namespace=\"$namespace\",node=\"$node\"}[5m]) or irate(node_disk_writes_completed{namespace=\"$namespace\",node=\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "B", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOs per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "IO/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Disk IOs per device" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*_read/", + "transform": "negative-Y" + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512 or irate(node_disk_sectors_read{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "refId": "B", + "step": 2400 + }, + { + "expr": "irate(node_disk_written_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512 or irate(node_disk_sectors_written{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Throughput per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Disk Throughput per device" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches_total{namespace=\"$namespace\",node=\"$node\"}[5m]) or irate(node_context_switches{namespace=\"$namespace\",node=\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "context switches", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Context Switches", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Network Traffic" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m])*8 or irate(node_network_receive_bytes{namespace=\"$namespace\",node=\"$node\"}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_in", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_network_transmit_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m])*8 or irate(node_network_transmit_bytes{namespace=\"$namespace\",node=\"$node\"}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_out", + "refId": "B", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{namespace=\"$namespace\",node=\"$node\"}", + "intervalFactor": 2, + "legendFormat": "established", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Netstat", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "Udp_NoPorts", + "yaxis": 2 + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{namespace=\"$namespace\",node=\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InDatagrams", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_netstat_Udp_InErrors{namespace=\"$namespace\",node=\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InErrors", + "refId": "B", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{namespace=\"$namespace\",node=\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Udp_OutDatagrams", + "refId": "C", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{namespace=\"$namespace\",node=\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_NoPorts", + "refId": "D", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "UDP Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_nf_conntrack_entries_limit{namespace=\"$namespace\",node=\"$node\"} - node_nf_conntrack_entries{namespace=\"$namespace\",node=\"$node\"}", + "intervalFactor": 2, + "legendFormat": "free", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Conntrack", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "New row" + } + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "prometheus-meta", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(namespace)", + "refresh": 2, + "regex": "default|prombench-\\d+", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "prometheus-meta", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "node", + "options": [], + "query": "label_values({namespace=\"$namespace\"}, node)", + "refresh": 2, + "regex": ".*", + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 12, + "version": 30, + "links": [], + "gnetId": 405 +} \ No newline at end of file diff --git a/prombench/manifests/cluster-infra/dashboards/prombench.json b/prombench/manifests/cluster-infra/dashboards/prombench.json new file mode 100644 index 000000000..eb1ce25e7 --- /dev/null +++ b/prombench/manifests/cluster-infra/dashboards/prombench.json @@ -0,0 +1,4092 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics useful for benchmarking and loadtesting Prometheus itself. Designed primarily for Prometheus 2.3.x.", + "tags": [ + "benchmark", + "node-metrics" + ], + "editable": true, + "gnetId": 6725, + "graphTooltip": 1, + "id": null, + "iteration": 1532083926170, + "links": [], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 47, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{node}} - Total memory", + "refId": "E" + }, + { + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"} - node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"} - node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"} - node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Free", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\",device!~'^(md\\\\\\\\d+$|dm-)'}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - {{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O Utilisation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 57, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches_total{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Context Switches", + "refId": "A" + }, + { + "expr": "irate(node_intr_total{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Interrupts", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Context Switches / Interrupts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Disk space used of all filesystems mounted", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{node=~'(test+).*',device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - {{mountpoint}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Used Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Load 1m", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Load 5m", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - Load 15m", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "System Load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - node_filesystem_free_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\",fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} - {{mountpoint}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filesystem Fullness", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Node Metrics", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 45, + "panels": [], + "title": "Prometheus Benchmark", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_build_info{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{prometheus}} - {{version}} - {{revision}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Version", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "time() - process_start_time_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{prometheus}} - Uptime", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Uptime", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Max": "#e24d42", + "Open": "#508642" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Total number open and maximum file descriptors of both release and PR", + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 2 + }, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_max_fds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{prometheus}} - Max", + "refId": "A" + }, + { + "expr": "process_open_fds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{prometheus}} - Open", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Time series": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "This graph indicates how many time series each instance holds", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Time series", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Time series", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "This graph shows the total number of active appender transactions", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Head Appenders", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Active Appenders", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "samples/s": "#e5a8e2" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "This graph shows the sample ingested per second", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - samples/s", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Samples Appended/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "To persist": "#9AC48A" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Chunks are the collection of series of samples. The active series is known as the head chunk. Total of 120 samples are present in a chunk. This graph shows the total head chunks", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Max.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Chunks", + "metric": "prometheus_local_storage_memory_chunks", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "This graph shows chunks created per second", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Created", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks Created", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Removed": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Head chunk is converted to chunk when it is filled with 120 samples. This graph shows the total number of head chunks created", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Removed", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks Removed", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max": "#447ebc", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Min": "#447ebc", + "Now": "#7eb26d" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Plots of minimum and maximum timestamp of the head block in reference to current time", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max", + "fillBelowTo": "Min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Min", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "time() * 1000", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Now", + "refId": "C" + }, + { + "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Max", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Time Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Average runtime of garbage collection in the head block", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_head_gc_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - GC Time/s", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Avg Head GC Time/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Total Number of currently loaded data blocks", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Queue length", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Blocks Loaded", + "metric": "prometheus_local_storage_indexing_batch_sizes_sum", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Blocks Loaded", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ] + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Total number of bytes allocated, even if freed", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 30 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Allocated Bytes/s", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Allocations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Failed Reloads": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "This graph shows the total number of times the database reloaded block data from disk", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 30 + }, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Reloads", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Reloads/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "This graph shows various tsdb related checks", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 30 + }, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - WAL Corruptions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Reload Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Head Series Not Found", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "C", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Compaction Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "D", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Retention Cutoff Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "E", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Problems/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Total number of times the database cut off block data from disk", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_retention_cutoffs_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Retention Cutoffs", + "metric": "last", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Retention Cutoffs/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "The latency distributions of fsync called by wal. High fsync duration (fsync_durations_seconds) indicates disk issues and might cause the cluster to be unstable", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 37 + }, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Fsync Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "WAL Fsync Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "WAL is used to get the backup when the prometheus server crashes. WAL stores the last 2hrs of data in memory. After that it has to truncate the data. So, this graph shows the latency of truncation", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 37 + }, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Truncate Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "WAL Truncate Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 44 + }, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_size_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Bytes/Sample", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "First Compaction, Avg Bytes/Sample", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 44 + }, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_range_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Chunk Time Range", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "First Compaction, Avg Chunk Time Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 44 + }, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Chunk Samples", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "First Compaction, Avg Chunk Samples", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Total number of compactions that were executed for the partition", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 51 + }, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Compactions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Compactions/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Average compactions that were executed for the partition", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 51 + }, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{prometheus}}", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Avg Compaction Time/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 51 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Irate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - 5m rate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Allocated bytes": "#7EB26D", + "Allocated bytes - 1m max": "#BF1B00", + "Allocated bytes - 1m min": "#BF1B00", + "Allocated bytes - 5m max": "#BF1B00", + "Allocated bytes - 5m min": "#BF1B00", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#447EBC" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 58 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/-/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - RSS", + "metric": "process_resident_memory_bytes", + "refId": "B", + "step": 10 + }, + { + "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Target heap size", + "metric": "go_memstats_alloc_bytes", + "refId": "D", + "step": 10 + }, + { + "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Next GC", + "metric": "go_memstats_next_gc_bytes", + "refId": "C", + "step": 10 + }, + { + "expr": "go_memstats_alloc_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 58 + }, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - {{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "HTTP requests/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 58 + }, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - {{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Avg HTTP request latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Time spent in prepare_time mode, per second", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 65 + }, + "id": 64, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{prometheus}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Avg query engine timings/s - prepare_time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Average Time spent in inner_eval mode, per second", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 65 + }, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{prometheus}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Avg query engine timings/s - inner_eval", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Time spent in queue_time mode, per second", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 65 + }, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{prometheus}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Avg query engine timings/s - queue_time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Average Time spent in result_sort mode, per second", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 62, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{prometheus}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Avg query engine timings/s - result_sort", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 72 + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Rule group missed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Rule evals failed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rule group evaluation problems/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 72 + }, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Rule evaluation duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Evaluation time of rule groups/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Interval": "#890f02", + "Last Duration": "#f9934e", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus-meta", + "description": "Rule group describe the Prometheus alerts. These graph consist of two rules:\n- prometheus_rule_group_interval_seconds: When the first notification was sent, wait for 'group_interval' to send a batch of new alerts that started firing for that group.\n- prometheus_rule_group_last_duration_seconds: It shows the last duration of each one of your groups in seconds\n", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 79 + }, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "RuleGroup", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_rule_group_interval_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Interval", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + }, + { + "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{prometheus}} - Last Duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rule Group: $RuleGroup", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "This graph shows the highest and lowest TSDB watermarks. The highest watermark represents the latest appender and the lowest watermark represents the oldest appender", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 79 + }, + "id": 67, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_isolation_low_watermark{namespace='prombench-${prNumber}'}", + "legendFormat": "{{prometheus}} - low watermark", + "refId": "A" + }, + { + "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-${prNumber}'}", + "interval": "", + "legendFormat": "{{prometheus}} - high watermark", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TSDB Isolation Watermarks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "This graph the difference of highest and lowest of TSDB watermark and also shows the size of isolation linked-chain", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 79 + }, + "id": 68, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-${prNumber}'} - prometheus_tsdb_isolation_low_watermark{namespace='prombench-${prNumber}'}", + "legendFormat": "{{prometheus}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TSDB Isolation Watermarks Difference", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "prometheus-meta", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "RuleGroup", + "options": [], + "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", + "refresh": 2, + "regex": ".*rule_group=\"(.*?)\".*", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "prometheus-meta", + "definition": "label_values(namespace)", + "description": "PR number", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "prNumber", + "options": [], + "query": { + "query": "label_values(namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "prombench-(\\d+)", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Prombench", + "uid": "7gmLoNDmz", + "version": 3 +} \ No newline at end of file diff --git a/prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml b/prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml index bf83eeca4..62745bf05 100644 --- a/prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml +++ b/prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml @@ -1573,7 +1573,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1581,28 +1581,28 @@ data: "refId": "E" }, { - "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"} - node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"} - node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"} - node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Used", "refId": "A" }, { - "expr": "node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Buffers", "refId": "B" }, { - "expr": "node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Cached", "refId": "C" }, { - "expr": "node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Free", @@ -1682,7 +1682,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "irate(node_disk_io_time_seconds_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",device!~'^(md\\\\\\\\d+$|dm-)'}[5m])", + "expr": "irate(node_disk_io_time_seconds_total{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\",device!~'^(md\\\\\\\\d+$|dm-)'}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - {{device}}", @@ -1762,14 +1762,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "irate(node_context_switches_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}[5m])", + "expr": "irate(node_context_switches_total{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Context Switches", "refId": "A" }, { - "expr": "irate(node_intr_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}[5m])", + "expr": "irate(node_intr_total{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Interrupts", @@ -1851,7 +1851,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "100 - ((node_filesystem_avail_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{node=~'(test+).*',device!~'rootfs'})", + "expr": "100 - ((node_filesystem_avail_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{node=~'(test+).*',device!~'rootfs'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - {{mountpoint}}", @@ -1933,21 +1933,21 @@ data: "steppedLine": false, "targets": [ { - "expr": "node_load1{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_load1{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Load 1m", "refId": "A" }, { - "expr": "node_load5{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_load5{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Load 5m", "refId": "B" }, { - "expr": "node_load15{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "node_load15{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - Load 15m", @@ -2027,7 +2027,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "1 - node_filesystem_free_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}", + "expr": "1 - node_filesystem_free_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\",fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size_bytes{job=\"node-exporter\",namespace=\"prombench-${prNumber}\",node=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{node}} - {{mountpoint}}", @@ -2124,7 +2124,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_build_info{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_build_info{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{prometheus}} - {{version}} - {{revision}}", @@ -2204,7 +2204,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "time() - process_start_time_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "time() - process_start_time_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{prometheus}} - Uptime", @@ -2256,6 +2256,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Total number open and maximum file descriptors of both release and PR", "fill": 0, "gridPos": { "h": 7, @@ -2287,14 +2288,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "process_max_fds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "process_max_fds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{prometheus}} - Max", "refId": "A" }, { - "expr": "process_open_fds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "process_open_fds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{prometheus}} - Open", @@ -2349,6 +2350,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "This graph indicates how many time series each instance holds", "editable": true, "error": false, "fill": 1, @@ -2382,7 +2384,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_head_series{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_tsdb_head_series{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Time series", @@ -2439,6 +2441,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "This graph shows the total number of active appender transactions", "editable": true, "error": false, "fill": 1, @@ -2472,7 +2475,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Head Appenders", @@ -2526,6 +2529,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "This graph shows the sample ingested per second", "editable": true, "error": false, "fill": 1, @@ -2559,7 +2563,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - samples/s", @@ -2617,6 +2621,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Chunks are the collection of series of samples. The active series is known as the head chunk. Total of 120 samples are present in a chunk. This graph shows the total head chunks", "editable": true, "error": false, "fill": 0, @@ -2655,7 +2660,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Chunks", @@ -2712,6 +2717,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "This graph shows chunks created per second", "editable": true, "error": false, "fill": 1, @@ -2745,7 +2751,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Created", @@ -2803,6 +2809,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Head chunk is converted to chunk when it is filled with 120 samples. This graph shows the total number of head chunks created", "editable": true, "error": false, "fill": 1, @@ -2836,7 +2843,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Removed", @@ -2896,6 +2903,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Plots of minimum and maximum timestamp of the head block in reference to current time", "editable": true, "error": false, "fill": 0, @@ -2935,7 +2943,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Min", @@ -2952,7 +2960,7 @@ data: "refId": "C" }, { - "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Max", @@ -3010,6 +3018,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Average runtime of garbage collection in the head block", "editable": true, "error": false, "fill": 1, @@ -3043,7 +3052,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_head_gc_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_head_gc_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - GC Time/s", @@ -3100,6 +3109,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Total Number of currently loaded data blocks", "editable": true, "error": false, "fill": 1, @@ -3138,7 +3148,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Blocks Loaded", @@ -3197,6 +3207,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Total number of bytes allocated, even if freed", "editable": true, "error": false, "fill": 1, @@ -3230,7 +3241,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Allocated Bytes/s", "metric": "go_memstats_alloc_bytes", @@ -3288,6 +3299,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "This graph shows the total number of times the database reloaded block data from disk", "editable": true, "error": false, "fill": 1, @@ -3321,7 +3333,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3381,6 +3393,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "This graph shows various tsdb related checks", "editable": true, "error": false, "fill": 1, @@ -3414,7 +3427,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3424,7 +3437,7 @@ data: "step": 10 }, { - "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3434,7 +3447,7 @@ data: "step": 10 }, { - "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3444,7 +3457,7 @@ data: "step": 10 }, { - "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3454,7 +3467,7 @@ data: "step": 10 }, { - "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3514,6 +3527,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Total number of times the database cut off block data from disk", "editable": true, "error": false, "fill": 1, @@ -3547,7 +3561,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_retention_cutoffs_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_retention_cutoffs_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Retention Cutoffs", @@ -3605,6 +3619,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "The latency distributions of fsync called by wal. High fsync duration (fsync_durations_seconds) indicates disk issues and might cause the cluster to be unstable", "editable": true, "error": false, "fill": 1, @@ -3638,7 +3653,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3697,6 +3712,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "WAL is used to get the backup when the prometheus server crashes. WAL stores the last 2hrs of data in memory. After that it has to truncate the data. So, this graph shows the latency of truncation", "editable": true, "error": false, "fill": 1, @@ -3730,7 +3746,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3821,7 +3837,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_compaction_chunk_size_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_compaction_chunk_size_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Bytes/Sample", @@ -3912,7 +3928,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_compaction_chunk_range_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_compaction_chunk_range_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Chunk Time Range", @@ -4003,7 +4019,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Chunk Samples", @@ -4061,6 +4077,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Total number of compactions that were executed for the partition", "editable": true, "error": false, "fill": 1, @@ -4094,7 +4111,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -4152,6 +4169,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Average compactions that were executed for the partition", "editable": true, "error": false, "fill": 1, @@ -4185,7 +4203,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])", + "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[10m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -4275,7 +4293,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Irate", "metric": "prometheus_local_storage_ingested_samples_total", @@ -4283,7 +4301,7 @@ data: "step": 10 }, { - "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[5m])", + "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[5m])", "intervalFactor": 2, "legendFormat": "{{prometheus}} - 5m rate", "metric": "prometheus_local_storage_ingested_samples_total", @@ -4386,7 +4404,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "process_resident_memory_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "intervalFactor": 2, "legendFormat": "{{prometheus}} - RSS", "metric": "process_resident_memory_bytes", @@ -4394,7 +4412,7 @@ data: "step": 10 }, { - "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Target heap size", "metric": "go_memstats_alloc_bytes", @@ -4402,7 +4420,7 @@ data: "step": 10 }, { - "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Next GC", "metric": "go_memstats_next_gc_bytes", @@ -4410,7 +4428,7 @@ data: "step": 10 }, { - "expr": "go_memstats_alloc_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "expr": "go_memstats_alloc_bytes{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Allocated", "metric": "go_memstats_alloc_bytes", @@ -4500,7 +4518,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - {{handler}}", @@ -4591,7 +4609,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - {{handler}}", @@ -4682,7 +4700,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -4774,7 +4792,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -4866,7 +4884,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -4958,7 +4976,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -5049,7 +5067,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) ", + "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m]) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Rule group missed", @@ -5058,7 +5076,7 @@ data: "step": 10 }, { - "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Rule evals failed", @@ -5148,7 +5166,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])", + "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Rule evaluation duration", @@ -5207,6 +5225,7 @@ data: "dashLength": 10, "dashes": false, "datasource": "prometheus-meta", + "description": "Rule group describe the Prometheus alerts. These graph consist of two rules:\n- prometheus_rule_group_interval_seconds: When the first notification was sent, wait for 'group_interval' to send a batch of new alerts that started firing for that group.\n- prometheus_rule_group_last_duration_seconds: It shows the last duration of each one of your groups in seconds\n", "editable": true, "error": false, "fill": 0, @@ -5242,7 +5261,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_rule_group_interval_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n", + "expr": "prometheus_rule_group_interval_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Interval", @@ -5251,7 +5270,7 @@ data: "step": 10 }, { - "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n", + "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{prometheus}} - Last Duration", @@ -5302,6 +5321,7 @@ data: "bars": false, "dashLength": 10, "dashes": false, + "description": "This graph shows the highest and lowest TSDB watermarks. The highest watermark represents the latest appender and the lowest watermark represents the oldest appender", "fill": 0, "fillGradient": 0, "gridPos": { @@ -5336,12 +5356,12 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_isolation_low_watermark{namespace='prombench-[[pr-number]]'}", + "expr": "prometheus_tsdb_isolation_low_watermark{namespace='prombench-${prNumber}'}", "legendFormat": "{{prometheus}} - low watermark", "refId": "A" }, { - "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-[[pr-number]]'}", + "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-${prNumber}'}", "interval": "", "legendFormat": "{{prometheus}} - high watermark", "refId": "B" @@ -5393,6 +5413,7 @@ data: "bars": false, "dashLength": 10, "dashes": false, + "description": "This graph the difference of highest and lowest of TSDB watermark and also shows the size of isolation linked-chain", "fill": 0, "fillGradient": 0, "gridPos": { @@ -5427,7 +5448,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-[[pr-number]]'} - prometheus_tsdb_isolation_low_watermark{namespace='prombench-[[pr-number]]'}", + "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-${prNumber}'} - prometheus_tsdb_isolation_low_watermark{namespace='prombench-${prNumber}'}", "legendFormat": "{{prometheus}}", "refId": "A" } @@ -5489,7 +5510,7 @@ data: "multi": false, "name": "RuleGroup", "options": [], - "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}", + "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-${prNumber}\",prometheus=~\"(test+).*\"}", "refresh": 2, "regex": ".*rule_group=\"(.*?)\".*", "sort": 1, @@ -5503,21 +5524,22 @@ data: "allValue": null, "current": {}, "datasource": "prometheus-meta", + "definition": "label_values(namespace)", + "description": "PR number", "hide": 0, "includeAll": false, - "label": null, "multi": false, - "name": "pr-number", + "name": "prNumber", "options": [], - "query": "label_values(namespace)", - "refresh": 2, + "query": { + "query": "label_values(namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, "regex": "prombench-(\\d+)", + "skipUrlSync": false, "sort": 0, - "tagValuesQuery": null, - "tags": [], - "tagsQuery": null, - "type": "query", - "useTags": false + "type": "query" } ] }, diff --git a/scripts/sync-dashboards-to-configmap.sh b/scripts/sync-dashboards-to-configmap.sh new file mode 100755 index 000000000..868893204 --- /dev/null +++ b/scripts/sync-dashboards-to-configmap.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo 'apiVersion: v1' > prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml +echo 'kind: ConfigMap' >> prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml +echo 'metadata:' >> prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml +echo ' name: grafana-dashboards' >> prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml +echo 'data:' >> prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml + +# Loop over files in prombench/manifests/cluster-infra/dashboards. +for file in $(ls prombench/manifests/cluster-infra/dashboards); do + # Read the file content. + content=$(cat prombench/manifests/cluster-infra/dashboards/$file) + echo " $file: |" >> prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml + echo "$content" | sed 's/^/ /' >> prombench/manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml +done \ No newline at end of file