plugin/health: implement dyn health checks (#1214)

Implement health.Healther in erratic and kubernetes plugin. The
kubernetes' healtcheck is only performed on startup - i.e. turn
healthy after the initial loading.

Erratic follow the drop count: every query%drop turns the healthcheck
unhealthy.

Fixes: #985

7 files changed, 49 insertions, 8 deletions

diff --git a/plugin/erratic/README.md b/plugin/erratic/README.md
index f606eef42..3f3f75d3d 100644
--- a/plugin/erratic/README.md
+++ b/plugin/erratic/README.md
@@ -25,6 +25,10 @@ erratic {
 * `delay`: delay 1 per **AMOUNT** of queries for **DURATION**, the default for **AMOUNT** is 2 and
   the default for **DURATION** is 100ms.
 
+## Health
+
+This plugin implements dynamic health checking. For every dropped query it turns unhealthy.
+
 ## Examples
 
 ~~~ corefile
diff --git a/plugin/erratic/health.go b/plugin/erratic/health.go
new file mode 100644
index 000000000..1d9625e16
--- /dev/null
+++ b/plugin/erratic/health.go
@@ -0,0 +1,14 @@
+package erratic
+
+import (
+	"sync/atomic"
+)
+
+// Health implements the health.Healther interface.
+func (e *Erratic) Health() bool {
+	q := atomic.LoadUint64(&e.q)
+	if e.drop > 0 && q%e.drop == 0 {
+		return false
+	}
+	return true
+}
diff --git a/plugin/health/README.md b/plugin/health/README.md
index f8395d8cd..bde7c22cb 100644
--- a/plugin/health/README.md
+++ b/plugin/health/README.md
@@ -16,6 +16,13 @@ a 503. *health* periodically (1s) polls plugin that exports health information.
 plugin signals that it is unhealthy, the server will go unhealthy too. Each plugin that
 supports health checks has a section "Health" in their README.
 
+## Plugins
+
+The following plugins report health to the health plugin:
+
+* erratic
+* kubernetes
+
 ## Examples
 
 Run another health endpoint on http://localhost:8091.
diff --git a/plugin/health/health_test.go b/plugin/health/health_test.go
index f05f53073..276997329 100644
--- a/plugin/health/health_test.go
+++ b/plugin/health/health_test.go
@@ -1,7 +1,14 @@
 package health
 
-// TODO(miek): enable again if plugin gets health check.
-/*
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/coredns/coredns/plugin/erratic"
+)
+
 func TestHealth(t *testing.T) {
 	h := health{Addr: ":0"}
 	h.h = append(h.h, &erratic.Erratic{})
@@ -14,7 +21,7 @@ func TestHealth(t *testing.T) {
 	// Reconstruct the http address based on the port allocated by operating system.
 	address := fmt.Sprintf("http://%s%s", h.ln.Addr().String(), path)
 
-	// Norhing set should be unhealthy
+	// Nothing set should return unhealthy
 	response, err := http.Get(address)
 	if err != nil {
 		t.Fatalf("Unable to query %s: %v", address, err)
@@ -24,8 +31,7 @@ func TestHealth(t *testing.T) {
 	}
 	response.Body.Close()
 
-	// Make healthy
-	h.Poll()
+	h.poll()
 
 	response, err = http.Get(address)
 	if err != nil {
@@ -44,4 +50,3 @@ func TestHealth(t *testing.T) {
 		t.Errorf("Invalid response body: expecting 'OK', got '%s'", string(content))
 	}
 }
-*/
diff --git a/plugin/health/healther.go b/plugin/health/healther.go
index fa776222a..4cc33a44b 100644
--- a/plugin/health/healther.go
+++ b/plugin/health/healther.go
@@ -38,5 +38,7 @@ func (h *health) poll() {
 }
 
 // Plugins that implements the Healther interface.
-// TODO(miek): none yet.
-var healthers = map[string]bool{}
+var healthers = map[string]bool{
+	"erratic":    true,
+	"kubernetes": true,
+}
diff --git a/plugin/kubernetes/README.md b/plugin/kubernetes/README.md
index 890aad9cd..097f0e65e 100644
--- a/plugin/kubernetes/README.md
+++ b/plugin/kubernetes/README.md
@@ -85,6 +85,11 @@ kubernetes [ZONES...] {
   what the response will be. However, if you specify this option, the query will instead be passed
   on down the plugin chain, which can include another plugin to handle the query.
 
+## Health
+
+This plugin implements dynamic health checking. Currently this is limited to reporting healthy when
+the API has synced.
+
 ## Examples
 
 Handle all queries in the `cluster.local` zone. Connect to Kubernetes in-cluster. Also handle all
diff --git a/plugin/kubernetes/health.go b/plugin/kubernetes/health.go
new file mode 100644
index 000000000..243749210
--- /dev/null
+++ b/plugin/kubernetes/health.go
@@ -0,0 +1,4 @@
+package kubernetes
+
+// Health implements the health.Healther interface.
+func (k *Kubernetes) Health() bool { return k.APIConn.HasSynced() }