Merge pull request #7878 from jayunit100/prometheus

Prometheus dashboard add-on kube application ( #3118 )

contrib/prometheus/Dockerfile

@@ -0,0 +1,7 @@
+FROM       prom/prometheus
+MAINTAINER Marcin Wielgus <mwielgus@google.com>
+
+COPY ./run_prometheus.sh /prometheus/run_prometheus.sh
+
+ENTRYPOINT ["/bin/sh", "/prometheus/run_prometheus.sh"]
+CMD []

contrib/prometheus/README.md

@@ -0,0 +1,71 @@
+# Prometheus in Kubernetes
+
+This is an experimental [Prometheus](http://prometheus.io/) setup for monitoring
+Kubernetes services that expose prometheus-friendly metrics through address
+http://service_address:service_port/metrics.
+
+# Purpose
+The purpose of the setup is to gather performance-related metrics during load
+tests and analyze them to find and fix bottlenecks.
+
+# Quick start
+
+## Promdash/Prometheus
+
+1. Pick a local directory for promdash.  It can be any directory, preferably one which is stable and which you don't mind keeping around.  Then (in our case, we use */mnt/promdash*, just run this docker command `docker run -v /mnt/promdash:/mnt/promdash -e DATABASE_URL=sqlite3:/mnt/promdash/file.sqlite3 prom/promdash ./bin/rake db:migrate`.  In the future, we might use mysql as the promdash database, however, in any case, this 1 time db setup step is required.
+
+Now quickly confirm that /mnt/promdash/file.sqlite3 exists, and has a non-zero size, and make sure its permissions are open so that containers can read from it.  For example:
+```
+    [jay@rhbd kubernetes]$ ls -altrh /mnt/promdash/
+    total 20K
+    drwxr-xr-x. 6 root root 4.0K May  6 23:12 ..
+    -rwxrwxrwx  1 root root  12K May  6 23:33 file.sqlite3
+```
+Looks open enough :).  
+
+1. Now, you can start this pod, like so `kubectl create -f cluster/add-ons/prometheus/prometheusB3.yaml`.  This pod will start both prometheus, the server, as well as promdash, the visualization tool.  You can then configure promdash, and next time you restart the pod - you're configuration will be remain (since the promdash directory was mounted as a local docker volume).
+
+1. Finally, you can simply access localhost:3000, which will have promdash running.  Then, add the prometheus server (locahost:9090)to as a promdash server, and create a dashboard according to the promdash directions.
+
+## Prometheus 
+
+You can launch prometheus easily, by simply running.
+
+`kubectl create -f cluster/addons/prometheus/prometheus.yaml`
+
+This will bind to port 9090 locally.  You can see the prometheus database at that URL.
+
+# How it works
+
+This is a v1beta1 based, containerized prometheus pod, which scrapes endpoints which are readable on the KUBERNETES_RO service (the internal kubernetes service running in the default namespace, which is visible to all pods).
+
+1. The KUBERNETES_RO service is already running : providing read access to the API metrics.
+
+1. The list of services to be monitored is passed as a command line aguments in
+the yaml file.
+
+1. The startup scripts assumes that each service T will have
+2 environment variables set ```T_SERVICE_HOST``` and ```T_SERVICE_PORT``` 
+
+1. Each can be configured manually in yaml file if you want to monitor something
+that is not a regular Kubernetes service.  For example, you can add comma delimted
+endpoints which can be scraped like so...
+```
+- -t
+- KUBERNETES_RO,MY_OTHER_METRIC_SERVICE
+```
+
+# Other notes
+
+For regular Kubernetes services the env variables are set up automatically and injected at runtime. 
+
+By default the metrics are written to a temporary location (that can be changed
+in the the volumes section of the yaml file). Prometheus' UI is available 
+at port 9090.
+
+# TODO
+
+- We should publish this image into the kube/ namespace.
+- Possibly use postgre or mysql as a promdash database.
+- push gateway (https://github.com/prometheus/pushgateway) setup.
+- Setup high availability via NFS

contrib/prometheus/prometheusB3.yaml

@@ -0,0 +1,83 @@
+apiVersion: v1beta3
+kind: Pod
+metadata:
+  creationTimestamp: null
+  labels:
+    name: kube-prometheus
+  name: kube-prometheus
+spec:
+  containers:
+  - capabilities: {}
+    env:
+    - name: DATABASE_URL
+      value: sqlite3:/promdash/file.sqlite3 #see volume comment below.
+    image: prom/promdash
+    imagePullPolicy: IfNotPresent
+    name: kube-promdash
+    ports:
+    - containerPort: 3000
+      hostPort: 3000
+      protocol: TCP
+    resources: {}
+    securityContext:
+      capabilities: {}
+      privileged: false
+    terminationMessagePath: /dev/termination-log
+    volumeMounts:
+    - mountPath: /promdash
+      name: promdashdb
+  - args:
+    - -t
+    - KUBERNETES_RO
+    - -d
+    - /var/prometheus/
+    capabilities: {}
+    image: jayunit100/kube-prometheus
+    imagePullPolicy: IfNotPresent
+    name: kube-prometheus
+    ports:
+    - containerPort: 9090
+      hostPort: 9090
+      protocol: TCP
+    resources: {}
+    securityContext:
+      capabilities: {}
+      privileged: false
+    terminationMessagePath: /dev/termination-log 
+    volumeMounts:
+    - mountPath: /var/prometheus/
+      name: data
+  dnsPolicy: ClusterFirst
+  restartPolicy: Always
+  volumes:
+  # There are many ways to create these volumes.
+  # for example, gcePersistentDisk:, glusterfs:, and so on...
+  # for the shared data volume (which we may not need going forward)
+  # we just use the undefined shared volume type.
+  - awsElasticBlockStore: null
+    emptyDir:
+      medium: ""
+    gcePersistentDisk: null
+    gitRepo: null
+    glusterfs: null
+    hostPath: null
+    iscsi: null
+    name: data
+    nfs: null
+    secret: null
+  # Again, many ways to create the promdash mount.  We are just using local
+  # disk for now.  Later maybe just replace with pure RDBMS rather than file
+  # based sqlite db.  The reason we have a volume is so that its persistent between
+  # pod restarts.
+  - awsElasticBlockStore: null
+    emptyDir: null
+    gcePersistentDisk: null
+    gitRepo: null
+    glusterfs: null
+    hostPath:
+      path: /mnt/promdash
+    iscsi: null
+    name: promdashdb
+    nfs: null
+    secret: null
+status: {}

contrib/prometheus/run_prometheus.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script builds a configuration for Prometheus based on command line
+# arguments and environment variables and starts the Prometheus server.
+#
+# Sample usage (to be run inside Kubernetes-created docker container).
+#  ./run_prometheus -t KUBERNETES_RO -d /tmp/prometheus
+#
+
+show_usage() {
+  echo "usage: ./run_prometheus -t TARGET_1,TARGET_2 -d data_directory"
+  echo "where"
+  echo " -t List of services to be monitored. Each service T should be described by"
+  echo "    the T_SERVICE_HOST and T_SERVICE_PORT env variables."
+  echo "-d Prometheus' root directory (i.e. where config file/metrics data will be stored)."
+}
+
+build_config() {
+  echo >$1 'global: { scrape_interval: "10s" evaluation_interval: "10s"}'
+  local target
+
+  ### Allow prometheus to scrape multiple targets
+  ### i.e. -t KUBERNETES_RO,MY_METRICS_ABC
+  for target in ${2//,/ }; do
+    local host_variable=$target"_SERVICE_HOST"
+    local port_variable=$target"_SERVICE_PORT"
+    local host=`eval echo '$'$host_variable`
+    local port=`eval echo '$'$port_variable`
+    echo "Checking $target"
+    if [ -z $host ]; then
+      echo "No env variable for $host_variable."
+      exit 3
+    fi
+    if [ -z $port ]; then
+      echo "No env variable for $port_variable."
+      exit 3
+    fi
+    local target_address="http://"$host":"$port"/metrics"
+    echo >>$1 "job: { name: \"${target}\" target_group: { target: \"${target_address}\" } }"
+  done
+}
+
+while getopts :t:d: flag; do
+  case $flag in
+  t) # targets.
+    targets=$OPTARG
+    ;;
+  d) # data location
+    location=$OPTARG
+    ;;
+  \?)
+    echo "Unknown parameter: $flag"
+    show_usage
+    exit 2
+    ;;
+  esac
+done
+
+if [ -z $targets ] || [ -z $location ]; then
+  echo "Missing parameters."
+  show_usage
+  exit 2
+fi
+
+echo "------------------"
+echo "Using $location as the root for prometheus configs and data."
+mkdir -p $location
+config="$location/config.pb"
+storage="$location/storage"
+
+echo "-------------------"
+echo "Starting Prometheus with:"
+echo "targets: $targets"
+echo "config: $config"
+echo "storage: $storage"
+
+build_config $config $targets
+echo "-------------------"
+echo "config file:"
+cat $config
+echo "-------------------"
+
+exec /bin/prometheus \
+  "-logtostderr" \
+  "-config.file=$config" \
+  "-storage.local.path=$storage" \
+  "-web.console.libraries=/go/src/github.com/prometheus/prometheus/console_libraries" \
+  "-web.console.templates=/go/src/github.com/prometheus/prometheus/consoles"