feat(dashboards): [experiment] Building Grafana Dashboards from Loki

[owen-d]

[WIP] Generate grafana compatible dashboard spec from within loki itself, hosted on /grafana/dashboards/{dashboard} endpoints

• [ ] fully functional
• [x] Consumes prometheus' client_golang to extract metrics from within source code and generates dashboards from them.
• [x] dashboards generated from code + some RED method helpers
• [x] generates dashboard spec from foundation sdk
• [x] compiler guarantees we generate correct specifications
• [x] coupling with loki source code itself for no-nonsense upgrading (refactor metric names and have dashboards update accordingly)
• [x] couple source code, operational expertise, and metrics together for compatibility

Example of generated output

{
  "title": "Loki Reads (generated)",
  "tags": [
    "generated",
    "from",
    "go"
  ],
  "timezone": "utc",
  "editable": true,
  "graphTooltip": 0,
  "time": {
    "from": "now-30m",
    "to": "now"
  },
  "fiscalYearStartMonth": 0,
  "refresh": "1m",
  "schemaVersion": 39,
  "panels": [
    {
      "type": "row",
      "collapsed": false,
      "title": "Distributor",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 0,
      "panels": [
        {
          "type": "timeseries",
          "targets": [
            {
              "expr": "\n\tsum(\n\t\trate(\n\t\t\t\"loki_request_duration_seconds\"_sum{ namespace=~\".*loki.*\" }\n\t\t\t[$__rate_interval]\n\t\t)\n\t) by (container=\"distributor\")\n\t/\n\tsum(\n\t\trate(\n\t\t\t\"loki_request_duration_seconds\"_count{ namespace=~\".*loki.*\" }\n\t\t\t[$__rate_interval]\n\t\t)\n\t) by (container=\"distributor\")\n\t",
              "legendFormat": "{{ container=\"distributor\" }}",
              "refId": ""
            }
          ],
          "title": "qps",
          "transparent": false,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 1
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": false,
              "calcs": []
            },
            "tooltip": {
              "mode": "",
              "sort": ""
            }
          },
          "fieldConfig": {
            "defaults": {
              "unit": "seconds",
              "min": 0
            },
            "overrides": null
          }
        },
        {
          "type": "timeseries",
          "targets": [
            {
              "expr": "\n\thistogram_quantile(\n\t  0.50,90,99,\n\t\tsum(\n\t\t\trate(\n\t\t\t  \"loki_request_duration_seconds\"_bucket{ namespace=~\".*loki.*\" }\n\t\t\t\t[$__rate_interval]\n\t\t\t)\n\t\t) by (container=\"distributor\", le)\n\t)\n\t",
              "legendFormat": "{{ container=\"distributor\", p50,90,99 }}",
              "refId": ""
            }
          ],
          "title": "latency",
          "transparent": false,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 12,
            "y": 1
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": false,
              "calcs": []
            },
            "tooltip": {
              "mode": "",
              "sort": ""
            }
          },
          "fieldConfig": {
            "defaults": {
              "unit": "seconds",
              "min": 0
            },
            "overrides": null
          }
        },
        {
          "type": "timeseries",
          "targets": [
            {
              "expr": "\n\thistogram_quantile(\n\t  0.50,90,99,\n\t\tsum(\n\t\t\trate(\n\t\t\t  \"loki_request_duration_seconds\"_bucket{ namespace=~\".*loki.*\" }\n\t\t\t\t[$__rate_interval]\n\t\t\t)\n\t\t) by (container=\"distributor\", pod, le)\n\t)\n\t",
              "legendFormat": "{{ container=\"distributor\", pod, p50,90,99 }}",
              "refId": ""
            }
          ],
          "title": "latency",
          "transparent": false,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 10
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": false,
              "calcs": []
            },
            "tooltip": {
              "mode": "",
              "sort": ""
            }
          },
          "fieldConfig": {
            "defaults": {
              "unit": "seconds",
              "min": 0
            },
            "overrides": null
          }
        }
      ]
    },
    {
      "type": "row",
      "collapsed": false,
      "title": "Ingester",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 0,
      "panels": [
        {
          "type": "timeseries",
          "targets": [
            {
              "expr": "\n\tsum(\n\t\trate(\n\t\t\t\"loki_request_duration_seconds\"_sum{ namespace=~\".*loki.*\" }\n\t\t\t[$__rate_interval]\n\t\t)\n\t) by (container=\"ingester\")\n\t/\n\tsum(\n\t\trate(\n\t\t\t\"loki_request_duration_seconds\"_count{ namespace=~\".*loki.*\" }\n\t\t\t[$__rate_interval]\n\t\t)\n\t) by (container=\"ingester\")\n\t",
              "legendFormat": "{{ container=\"ingester\" }}",
              "refId": ""
            }
          ],
          "title": "qps",
          "transparent": false,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 20
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": false,
              "calcs": []
            },
            "tooltip": {
              "mode": "",
              "sort": ""
            }
          },
          "fieldConfig": {
            "defaults": {
              "unit": "seconds",
              "min": 0
            },
            "overrides": null
          }
        },
        {
          "type": "timeseries",
          "targets": [
            {
              "expr": "\n\thistogram_quantile(\n\t  0.50,90,99,\n\t\tsum(\n\t\t\trate(\n\t\t\t  \"loki_request_duration_seconds\"_bucket{ namespace=~\".*loki.*\" }\n\t\t\t\t[$__rate_interval]\n\t\t\t)\n\t\t) by (container=\"ingester\", le)\n\t)\n\t",
              "legendFormat": "{{ container=\"ingester\", p50,90,99 }}",
              "refId": ""
            }
          ],
          "title": "latency",
          "transparent": false,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 12,
            "y": 20
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": false,
              "calcs": []
            },
            "tooltip": {
              "mode": "",
              "sort": ""
            }
          },
          "fieldConfig": {
            "defaults": {
              "unit": "seconds",
              "min": 0
            },
            "overrides": null
          }
        },
        {
          "type": "timeseries",
          "targets": [
            {
              "expr": "\n\thistogram_quantile(\n\t  0.50,90,99,\n\t\tsum(\n\t\t\trate(\n\t\t\t  \"loki_request_duration_seconds\"_bucket{ namespace=~\".*loki.*\" }\n\t\t\t\t[$__rate_interval]\n\t\t\t)\n\t\t) by (container=\"ingester\", pod, le)\n\t)\n\t",
              "legendFormat": "{{ container=\"ingester\", pod, p50,90,99 }}",
              "refId": ""
            }
          ],
          "title": "latency",
          "transparent": false,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 29
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": false,
              "calcs": []
            },
            "tooltip": {
              "mode": "",
              "sort": ""
            }
          },
          "fieldConfig": {
            "defaults": {
              "unit": "seconds",
              "min": 0
            },
            "overrides": null
          }
        }
      ]
    }
  ],
  "templating": {},
  "annotations": {}
}