diff --git a/cluster/addons/prometheus/README.md b/cluster/addons/prometheus/README.md deleted file mode 100644 index 62c89ce1d58..00000000000 --- a/cluster/addons/prometheus/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Prometheus in Kubernetes - -This is an experimental [Prometheus](http://prometheus.io/) setup for monitoring -Kubernetes services that expose prometheus-friendly metrics through address -http://service_address:service_port/metrics. - -The purpose of the setup is to gather performance-related metrics during load -tests and analyze them to find and fix bottlenecks. - -The list of services to be monitored is passed as a command line aguments in -the yaml file. The startup scripts assumes that each service T will have -2 environment variables set ```T_SERVICE_HOST``` and ```T_SERVICE_PORT``` which -can be configured manually in yaml file if you want to monitor something -that is not a regular Kubernetes service. For regular Kubernetes services -the env variables are set up automatically. - -By default the metrics are written to a temporary location (that can be changed -in the the volumes section of the yaml file). Prometheus' UI is available -at port 9090. diff --git a/cluster/addons/prometheus/prometheus.yaml b/cluster/addons/prometheus/prometheus.yaml deleted file mode 100644 index 412e9aa51e7..00000000000 --- a/cluster/addons/prometheus/prometheus.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v1beta1 -kind: Pod -id: kube-prometheus -desiredState: - manifest: - version: v1beta1 - containers: - - name: kube-prometheus - image: kube/prometheus - command: - - -t - - KUBERNETES_RO - - -d - - /var/prometheus/ - ports: - - containerPort: 9090 - hostPort: 9090 - volumeMounts: - - name: data - mountPath: /var/prometheus/ - volumes: - - name: data - source: - emptyDir: -labels: - name: kube-prometheus diff --git a/cluster/addons/prometheus/Dockerfile b/contrib/prometheus/Dockerfile similarity index 100% rename from cluster/addons/prometheus/Dockerfile rename to contrib/prometheus/Dockerfile diff --git a/contrib/prometheus/README.md b/contrib/prometheus/README.md new file mode 100644 index 00000000000..39d206aeb9e --- /dev/null +++ b/contrib/prometheus/README.md @@ -0,0 +1,71 @@ +# Prometheus in Kubernetes + +This is an experimental [Prometheus](http://prometheus.io/) setup for monitoring +Kubernetes services that expose prometheus-friendly metrics through address +http://service_address:service_port/metrics. + +# Purpose +The purpose of the setup is to gather performance-related metrics during load +tests and analyze them to find and fix bottlenecks. + +# Quick start + +## Promdash/Prometheus + +1. Pick a local directory for promdash. It can be any directory, preferably one which is stable and which you don't mind keeping around. Then (in our case, we use */mnt/promdash*, just run this docker command `docker run -v /mnt/promdash:/mnt/promdash -e DATABASE_URL=sqlite3:/mnt/promdash/file.sqlite3 prom/promdash ./bin/rake db:migrate`. In the future, we might use mysql as the promdash database, however, in any case, this 1 time db setup step is required. + +Now quickly confirm that /mnt/promdash/file.sqlite3 exists, and has a non-zero size, and make sure its permissions are open so that containers can read from it. For example: +``` + [jay@rhbd kubernetes]$ ls -altrh /mnt/promdash/ + total 20K + drwxr-xr-x. 6 root root 4.0K May 6 23:12 .. + -rwxrwxrwx 1 root root 12K May 6 23:33 file.sqlite3 +``` +Looks open enough :). + +1. Now, you can start this pod, like so `kubectl create -f cluster/add-ons/prometheus/prometheusB3.yaml`. This pod will start both prometheus, the server, as well as promdash, the visualization tool. You can then configure promdash, and next time you restart the pod - you're configuration will be remain (since the promdash directory was mounted as a local docker volume). + +1. Finally, you can simply access localhost:3000, which will have promdash running. Then, add the prometheus server (locahost:9090)to as a promdash server, and create a dashboard according to the promdash directions. + +## Prometheus + +You can launch prometheus easily, by simply running. + +`kubectl create -f cluster/addons/prometheus/prometheus.yaml` + +This will bind to port 9090 locally. You can see the prometheus database at that URL. + +# How it works + +This is a v1beta1 based, containerized prometheus pod, which scrapes endpoints which are readable on the KUBERNETES_RO service (the internal kubernetes service running in the default namespace, which is visible to all pods). + +1. The KUBERNETES_RO service is already running : providing read access to the API metrics. + +1. The list of services to be monitored is passed as a command line aguments in +the yaml file. + +1. The startup scripts assumes that each service T will have +2 environment variables set ```T_SERVICE_HOST``` and ```T_SERVICE_PORT``` + +1. Each can be configured manually in yaml file if you want to monitor something +that is not a regular Kubernetes service. For example, you can add comma delimted +endpoints which can be scraped like so... +``` +- -t +- KUBERNETES_RO,MY_OTHER_METRIC_SERVICE +``` + +# Other notes + +For regular Kubernetes services the env variables are set up automatically and injected at runtime. + +By default the metrics are written to a temporary location (that can be changed +in the the volumes section of the yaml file). Prometheus' UI is available +at port 9090. + +# TODO + +- We should publish this image into the kube/ namespace. +- Possibly use postgre or mysql as a promdash database. +- push gateway (https://github.com/prometheus/pushgateway) setup. +- Setup high availability via NFS diff --git a/contrib/prometheus/prometheusB3.yaml b/contrib/prometheus/prometheusB3.yaml new file mode 100644 index 00000000000..74a341202c5 --- /dev/null +++ b/contrib/prometheus/prometheusB3.yaml @@ -0,0 +1,83 @@ +apiVersion: v1beta3 +kind: Pod +metadata: + creationTimestamp: null + labels: + name: kube-prometheus + name: kube-prometheus +spec: + containers: + - capabilities: {} + env: + - name: DATABASE_URL + value: sqlite3:/promdash/file.sqlite3 #see volume comment below. + image: prom/promdash + imagePullPolicy: IfNotPresent + name: kube-promdash + ports: + - containerPort: 3000 + hostPort: 3000 + protocol: TCP + resources: {} + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /promdash + name: promdashdb + - args: + - -t + - KUBERNETES_RO + - -d + - /var/prometheus/ + capabilities: {} + image: jayunit100/kube-prometheus + imagePullPolicy: IfNotPresent + name: kube-prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + resources: {} + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/prometheus/ + name: data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + # There are many ways to create these volumes. + # for example, gcePersistentDisk:, glusterfs:, and so on... + # for the shared data volume (which we may not need going forward) + # we just use the undefined shared volume type. + - awsElasticBlockStore: null + emptyDir: + medium: "" + gcePersistentDisk: null + gitRepo: null + glusterfs: null + hostPath: null + iscsi: null + name: data + nfs: null + secret: null + # Again, many ways to create the promdash mount. We are just using local + # disk for now. Later maybe just replace with pure RDBMS rather than file + # based sqlite db. The reason we have a volume is so that its persistent between + # pod restarts. + - awsElasticBlockStore: null + emptyDir: null + gcePersistentDisk: null + gitRepo: null + glusterfs: null + hostPath: + path: /mnt/promdash + iscsi: null + name: promdashdb + nfs: null + secret: null +status: {} diff --git a/cluster/addons/prometheus/run_prometheus.sh b/contrib/prometheus/run_prometheus.sh similarity index 88% rename from cluster/addons/prometheus/run_prometheus.sh rename to contrib/prometheus/run_prometheus.sh index 6abea96c6fd..c28d8c9711a 100755 --- a/cluster/addons/prometheus/run_prometheus.sh +++ b/contrib/prometheus/run_prometheus.sh @@ -1,5 +1,6 @@ -#!/bin/sh -# Copyright 2014 Google Inc. All rights reserved. +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,12 +27,15 @@ show_usage() { echo "where" echo " -t List of services to be monitored. Each service T should be described by" echo " the T_SERVICE_HOST and T_SERVICE_PORT env variables." - echo " -d Location where the config file and metrics will be written." + echo "-d Prometheus' root directory (i.e. where config file/metrics data will be stored)." } build_config() { echo >$1 'global: { scrape_interval: "10s" evaluation_interval: "10s"}' local target + + ### Allow prometheus to scrape multiple targets + ### i.e. -t KUBERNETES_RO,MY_METRICS_ABC for target in ${2//,/ }; do local host_variable=$target"_SERVICE_HOST" local port_variable=$target"_SERVICE_PORT" @@ -73,6 +77,8 @@ if [ -z $targets ] || [ -z $location ]; then exit 2 fi +echo "------------------" +echo "Using $location as the root for prometheus configs and data." mkdir -p $location config="$location/config.pb" storage="$location/storage"