{ "__inputs": [], "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.4.1" }, { "type": "panel", "id": "logs", "name": "Logs", "version": "" }, { "type": "datasource", "id": "loki", "name": "Loki", "version": "1.0.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Monitor Promtail metrics and logs", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [ { "asDropdown": false, "icon": "doc", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "Github Source", "tooltip": "Github Source", "type": "link", "url": "https://github.com/voidquark/grafana-dashboards" } ], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 3, "panels": [], "title": "Promtail - Stats", "type": "row" }, { "datasource": { "type": "loki", "uid": "Loki" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 0, "y": 1 }, "id": 7, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"debug\" | __error__=\"\" [$__range]))", "queryType": "instant", "refId": "A" } ], "title": "DEBUG Log Lines", "type": "stat" }, { "datasource": { "type": "loki", "uid": "Loki" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 2, "y": 1 }, "id": 6, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"info\" | __error__=\"\" [$__range]))", "queryType": "instant", "refId": "A" } ], "title": "INFO Log Lines", "type": "stat" }, { "datasource": { "type": "loki", "uid": "Loki" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 4, "y": 1 }, "id": 5, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"warn\" | __error__=\"\" [$__range]))", "queryType": "instant", "refId": "A" } ], "title": "WARN Log Lines", "type": "stat" }, { "datasource": { "type": "loki", "uid": "Loki" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 6, "y": 1 }, "id": 4, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"error\" | __error__=\"\" [$__range]))", "queryType": "instant", "refId": "A" } ], "title": "ERROR Log Lines", "type": "stat" }, { "datasource": { "type": "loki", "uid": "Loki" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 8, "y": 1 }, "id": 1, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum(count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | __error__=\"\" [$__range]))", "queryType": "instant", "refId": "A" } ], "title": "Promtail Log Lines", "type": "stat" }, { "datasource": { "type": "loki", "uid": "Loki" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 11, "y": 1 }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum(bytes_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | __error__=\"\" [$__range]))", "queryType": "instant", "refId": "A" } ], "title": "Promtail Log in bytes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "Deployed Promtail Version", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 14, "y": 1 }, "id": 9, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [], "fields": "", "values": false }, "showPercentChange": false, "textMode": "name", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "promtail_build_info{job=~\"$job\", instance=~\"$instance.*\"} ", "format": "time_series", "instant": true, "interval": "", "legendFormat": "{{version}}", "range": false, "refId": "A" } ], "title": "Promtail Version", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "Number of active files.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#77a0ed", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 17, "y": 1 }, "id": 16, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "promtail_files_active_total{job=~\"$job\", instance=~\"$instance.*\"} ", "format": "time_series", "instant": true, "interval": "", "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Active Files", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "Current number of bytes sent. ( Real Time )", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#77a0ed", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 19, "y": 1 }, "id": 30, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_sent_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "instant": true, "interval": "", "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Current Sent Bytes", "type": "stat" }, { "datasource": { "type": "loki", "uid": "Loki" }, "description": "Use LogQL Filter to filter information.", "gridPos": { "h": 12, "w": 22, "x": 0, "y": 4 }, "id": 8, "options": { "dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": false }, "targets": [ { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"debug\" |= \"$filter\" | line_format `➡️ 🔵 {{regexReplaceAll \"level=debug|ts=[^\\\\s]+\\\\s\" __line__ \"$1\" | trim }}`", "queryType": "range", "refId": "A" }, { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"info\" |= \"$filter\" | line_format `➡️ 🟢 {{regexReplaceAll \"level=info|ts=[^\\\\s]+\\\\s\" __line__ \"$1\" | trim }}`", "hide": false, "queryType": "range", "refId": "B" }, { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"warn\" |= \"$filter\" | line_format `➡️ 🟡 {{regexReplaceAll \"level=warn|ts=[^\\\\s]+\\\\s\" __line__ \"$1\" | trim }}`", "hide": false, "queryType": "range", "refId": "C" }, { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"error\" |= \"$filter\" | line_format `➡️ 🔴 {{regexReplaceAll \"level=error|ts=[^\\\\s]+\\\\s\" __line__ \"$1\" | trim }}`", "hide": false, "queryType": "range", "refId": "D" } ], "title": "Promtail Recent Logs", "type": "logs" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "description": "", "fieldConfig": { "defaults": { "color": { "fixedColor": "light-purple", "mode": "fixed" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "series", "axisLabel": "", "axisPlacement": "left", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 6, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "Count" }, { "id": "custom.drawStyle", "value": "points" }, { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }, { "id": "custom.axisPlacement", "value": "left" } ] }, { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "Count" }, { "id": "custom.drawStyle", "value": "points" }, { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }, { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 8, "w": 22, "x": 0, "y": 16 }, "id": 33, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "100 * sum(rate(promtail_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance.*\",status_code=~\"5.*\"}[$interval])) by (job, instance) / sum(rate(promtail_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance.*\"}[$interval])) by (job, instance)", "hide": false, "instant": false, "legendFormat": "Error Requests", "range": true, "refId": "A" }, { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum by (level) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"warn\" | __error__=\"\"[$__auto]))", "hide": false, "legendFormat": "Warning Logs Lines", "queryType": "range", "refId": "B" }, { "datasource": { "type": "loki", "uid": "Loki" }, "editorMode": "code", "expr": "sum by (level) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} | logfmt | level=\"error\" | __error__=\"\"[$__auto]))", "hide": false, "legendFormat": "Error Logs Lines", "queryType": "range", "refId": "C" } ], "title": "Error Requests and Logs Count - Overtime", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 19, "panels": [], "title": "Promtail Details", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "Number of bytes read. - Overtime", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byName", "options": "promtail_read_bytes_total" }, "properties": [ { "id": "color" } ] } ] }, "gridPos": { "h": 12, "w": 12, "x": 0, "y": 25 }, "id": 18, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true, "width": 350 }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_read_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{path}}", "range": true, "refId": "A" } ], "title": "Read Bytes - Overtime", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [ { "id": "unit", "value": "Lines" } ] }, { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [ { "id": "unit", "value": "Bps" } ] } ] }, "gridPos": { "h": 12, "w": 10, "x": 12, "y": 25 }, "id": 28, "options": { "cellHeight": "md", "footer": { "countRows": false, "enablePagination": false, "fields": ["Value"], "reducer": ["lastNotNull"], "show": false }, "showHeader": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "promtail_read_bytes_total{job=~\"$job\", instance=~\"$instance.*\"} ", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_read_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "format": "table", "hide": false, "instant": false, "legendFormat": "", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "expr": "rate(promtail_read_lines_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "format": "table", "hide": false, "instant": false, "legendFormat": "", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "expr": "increase(promtail_read_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[24h])", "format": "table", "hide": false, "instant": false, "interval": "24h", "legendFormat": "", "range": true, "refId": "D" } ], "title": "Read Bytes - Details", "transformations": [ { "id": "joinByField", "options": { "byField": "path", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Time 1": true, "__name__": true, "instance": true, "job": true, "job 1": false }, "includeByName": {}, "indexByName": {}, "renameByName": { "Value": "Read Bytes Total", "Value #A": "Total Read", "Value #B": "Current Read", "Value #C": "Current Lines Read", "Value #D": "Total Read in 24h", "instance": "", "job": "", "path": "Path" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "- Encoded: Number of bytes encoded and ready to send.\n- Send: Number of bytes sent. ", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, "id": 27, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_encoded_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Encoded Bytes", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "expr": "rate(promtail_sent_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "hide": false, "instant": false, "legendFormat": "Send Bytes", "range": true, "refId": "B" } ], "title": "Encoded and Send Bytes - Overtime", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [ { "id": "unit", "value": "Lines" } ] }, { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [ { "id": "unit", "value": "decbytes" } ] }, { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [ { "id": "unit", "value": "Bps" } ] }, { "matcher": { "id": "byName", "options": "Value #A" }, "properties": [ { "id": "unit", "value": "Bps" } ] } ] }, "gridPos": { "h": 8, "w": 10, "x": 12, "y": 37 }, "id": 29, "options": { "cellHeight": "md", "footer": { "countRows": false, "enablePagination": false, "fields": ["Value"], "reducer": ["lastNotNull"], "show": false }, "showHeader": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_sent_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "format": "table", "hide": false, "instant": false, "legendFormat": "", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "expr": "increase(promtail_sent_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[24h])", "format": "table", "hide": false, "instant": false, "interval": "24h", "legendFormat": "", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_encoded_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "format": "table", "hide": false, "instant": false, "legendFormat": "", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "expr": "increase(promtail_encoded_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[24h])", "format": "table", "hide": false, "instant": false, "interval": "24h", "legendFormat": "", "range": true, "refId": "D" } ], "title": "Encoded and Send - Details", "transformations": [ { "id": "joinByField", "options": { "byField": "host", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Time 1": true, "__name__": true, "instance": true, "job": true, "job 1": false }, "includeByName": {}, "indexByName": {}, "renameByName": { "Value": "Read Bytes Total", "Value #A": "Current Send Bytes", "Value #B": "Send Bytes 24h", "Value #C": "Current Encoded Bytes", "Value #D": "Encoded Bytes 24h", "host": "Loki Endpoint", "instance": "", "job": "", "path": "Path" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "- Dropped Bytes: Number of bytes dropped because failed to be sent to the ingester after all retries.\n- Dropped Entries: Number of log entries dropped because failed to be sent to the ingester after all retries.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "Entries" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, "id": 22, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_dropped_bytes_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "hide": false, "instant": false, "legendFormat": "Dropped Bytes - {{reason}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "rate(promtail_dropped_entries_total{job=~\"$job\", instance=~\"$instance.*\"}[$interval])", "hide": false, "instant": false, "legendFormat": "Dropped Entries - {{reason}}", "range": true, "refId": "B" } ], "title": "Dropped Bytes - Overtime", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "description": "99th percentile latency", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 10, "x": 12, "y": 45 }, "id": 23, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "editorMode": "code", "exemplar": false, "expr": "histogram_quantile(0.99, sum(rate(promtail_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance.*\"}[5m])) by (le))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "Request Latency", "range": true, "refId": "A" } ], "title": "Request Latency - 99th Percentile Latency", "type": "timeseries" } ], "refresh": "", "schemaVersion": 39, "tags": ["promtail"], "templating": { "list": [ { "current": { "selected": false, "text": "loki", "value": "P982945308D3682D1" }, "hide": 0, "includeAll": false, "label": "Datasource Loki", "multi": false, "name": "datasource", "options": [], "query": "loki", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": {}, "datasource": { "type": "loki", "uid": "Loki" }, "definition": "", "hide": 0, "includeAll": false, "label": "Label Name", "multi": false, "name": "label_name", "options": [], "query": { "label": "", "refId": "LokiVariableQueryEditor-VariableQuery", "stream": "", "type": 0 }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": {}, "datasource": { "type": "loki", "uid": "Loki" }, "definition": "", "hide": 0, "includeAll": false, "label": "Label Value", "multi": false, "name": "label_value", "options": [], "query": { "label": "$label_name", "refId": "LokiVariableQueryEditor-VariableQuery", "stream": "", "type": 1 }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "Prometheus" }, "definition": "label_values(promtail_build_info,job)", "description": "Job label from Prometheus datasource.", "hide": 0, "includeAll": false, "label": "Job", "multi": false, "name": "job", "options": [], "query": { "qryType": 1, "query": "label_values(promtail_build_info,job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": {}, "datasource": { "type": "loki", "uid": "Loki" }, "definition": "", "description": "Instance label from Loki datasource.", "hide": 0, "includeAll": false, "label": "Instance", "multi": false, "name": "instance", "options": [], "query": { "label": "instance", "refId": "LokiVariableQueryEditor-VariableQuery", "stream": "", "type": 1 }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": false, "text": "prometheus", "value": "P1809F7CD0C75ACF3" }, "hide": 0, "includeAll": false, "label": "Datasource Prometheus", "multi": false, "name": "datasourceProm", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "text": "5m", "value": "5m" }, "hide": 0, "label": "Interval", "name": "interval", "options": [ { "selected": false, "text": "30s", "value": "30s" }, { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "2m", "value": "2m" }, { "selected": false, "text": "3m", "value": "3m" }, { "selected": true, "text": "5m", "value": "5m" }, { "selected": false, "text": "10m", "value": "10m" }, { "selected": false, "text": "30m", "value": "30m" } ], "query": "30s,1m,2m,3m,5m,10m,30m", "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "interval" }, { "current": { "selected": false, "text": "", "value": "" }, "hide": 0, "label": "LogQL Filter", "name": "filter", "options": [ { "selected": true, "text": "", "value": "" } ], "query": "", "skipUrlSync": false, "type": "textbox" } ] }, "time": { "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Promtail Monitoring - Metrics and Logs", "uid": "promtailntfc0a", "version": 1, "weekStart": "", "gnetId": 20881 }