middleware/proxy: healthchecks fixes (#183)

* middleware/proxy: add spray keyword

When spray is used, the proxy will, when all backend are down, spray to
each target. When not used, default to the old defaults: max 1 failure
and no spray. These defaults are also used when forwarding queries to
another CoreDNS instance.

Update the README with the new keyword.

* typos

* Make MaxFail = 1 again

* more reversals

6 files changed, 34 insertions, 24 deletions

diff --git a/middleware/etcd/stub_test.go b/middleware/etcd/stub_test.go
index 2ae813099..d36d36e87 100644
--- a/middleware/etcd/stub_test.go
+++ b/middleware/etcd/stub_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/miekg/coredns/middleware"
 	"github.com/miekg/coredns/middleware/etcd/msg"
 	"github.com/miekg/coredns/middleware/test"
+
 	"github.com/miekg/dns"
 )
 
diff --git a/middleware/proxy/README.md b/middleware/proxy/README.md
index d1ffaaf6d..cf6706907 100644
--- a/middleware/proxy/README.md
+++ b/middleware/proxy/README.md
@@ -2,7 +2,8 @@
 
 `proxy` facilitates both a basic reverse proxy and a robust load balancer. The proxy has support for
 multiple backends and adding custom headers. The load balancing features include multiple policies,
-health checks, and failovers.
+health checks, and failovers. If all hosts fails their health check the proxy middleware will fail
+back to randomly selecting a target and sending packets to it.
 
 ## Syntax
 
@@ -24,7 +25,7 @@ proxy from to... {
     max_fails integer
     health_check path:port [duration]
     except ignored_names...
-    ecs [v4 address/mask] [v6 address/mask] (TODO)
+    spray
 }
 ~~~
 
@@ -35,8 +36,7 @@ proxy from to... {
 * `max_fails` is the number of failures within fail_timeout that are needed before considering a backend to be down. If 0, the backend will never be marked as down. Default is 1.
 * `health_check` will check path (on port) on each backend. If a backend returns a status code of 200-399, then that backend is healthy. If it doesn't, the backend is marked as unhealthy for duration and no requests are routed to it. If this option is not provided then health checks are disabled. The default duration is 10 seconds ("10s").
 * `ignored_names...` is a space-separated list of paths to exclude from proxying. Requests that match any of these paths will be passed thru.
-* `ecs` add EDNS0 client submit metadata to the outgoing query. This can be optionally be followed
-  by an IPv4 and/or IPv6 address. If none is specified the server's addresses are used.
+* `spray` when all backends are unhealth randomly pick one to send the traffic to (this is a failsafe).
 
 ## Policies
 
diff --git a/middleware/proxy/lookup.go b/middleware/proxy/lookup.go
index ac3da261d..0a2b809b6 100644
--- a/middleware/proxy/lookup.go
+++ b/middleware/proxy/lookup.go
@@ -1,6 +1,6 @@
 package proxy
 
-// function OTHER middleware might want to use to do lookup in the same
+// functions OTHER middleware might want to use to do lookup in the same
 // style as the proxy.
 
 import (
@@ -77,8 +77,8 @@ func (p Proxy) lookup(state middleware.State, r *dns.Msg) (*dns.Msg, error) {
 		// hosts until timeout (or until we get a nil host).
 		for time.Now().Sub(start) < tryDuration {
 			host := upstream.Select()
+
 			if host == nil {
-				// TODO(miek): if all HC fail, spray the targets.
 				return nil, errUnreachable
 			}
 
diff --git a/middleware/proxy/policy.go b/middleware/proxy/policy.go
index 077ecff33..e0c9d7e2b 100644
--- a/middleware/proxy/policy.go
+++ b/middleware/proxy/policy.go
@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"log"
 	"math/rand"
 	"sync/atomic"
 )
@@ -44,9 +45,6 @@ func (r *Random) Select(pool HostPool) *UpstreamHost {
 			}
 		}
 	}
-	if randHost == nil {
-		return new(Spray).Select(pool)
-	}
 	return randHost
 }
 
@@ -58,6 +56,7 @@ type Spray struct{}
 func (r *Spray) Select(pool HostPool) *UpstreamHost {
 	rnd := rand.Int() % len(pool)
 	randHost := pool[rnd]
+	log.Printf("[WARNING] All hosts reported as down, spraying to target: %s", randHost.Name)
 	return randHost
 }
 
@@ -93,9 +92,6 @@ func (r *LeastConn) Select(pool HostPool) *UpstreamHost {
 			}
 		}
 	}
-	if bestHost == nil {
-		return new(Spray).Select(pool)
-	}
 	return bestHost
 }
 
@@ -113,8 +109,5 @@ func (r *RoundRobin) Select(pool HostPool) *UpstreamHost {
 	for i := uint32(1); host.Down() && i < poolLen; i++ {
 		host = pool[(selection+i)%poolLen]
 	}
-	if host.Down() {
-		return new(Spray).Select(pool)
-	}
 	return host
 }
diff --git a/middleware/proxy/upstream.go b/middleware/proxy/upstream.go
index 66f07e418..556ae5869 100644
--- a/middleware/proxy/upstream.go
+++ b/middleware/proxy/upstream.go
@@ -23,6 +23,7 @@ type staticUpstream struct {
 	from   string
 	Hosts  HostPool
 	Policy Policy
+	Spray  Policy
 
 	FailTimeout time.Duration
 	MaxFails    int32
@@ -48,6 +49,7 @@ func NewStaticUpstreams(c parse.Dispenser) ([]Upstream, error) {
 			from:        "",
 			Hosts:       nil,
 			Policy:      &Random{},
+			Spray:       nil,
 			FailTimeout: 10 * time.Second,
 			MaxFails:    1,
 		}
@@ -181,11 +183,8 @@ func parseBlock(c *parse.Dispenser, u *staticUpstream) error {
 			ignoredDomains[i] = strings.ToLower(dns.Fqdn(ignoredDomains[i]))
 		}
 		u.IgnoredSubDomains = ignoredDomains
-	case "ecs":
-		ips := c.RemainingArgs()
-		if len(ips) > 0 {
-
-		}
+	case "spray":
+		u.Spray = &Spray{}
 
 	default:
 		return c.Errf("unknown property '%s'", c.Val())
@@ -228,7 +227,7 @@ func (u *staticUpstream) HealthCheckWorker(stop chan struct{}) {
 func (u *staticUpstream) Select() *UpstreamHost {
 	pool := u.Hosts
 	if len(pool) == 1 {
-		if pool[0].Down() {
+		if pool[0].Down() && u.Spray == nil {
 			return nil
 		}
 		return pool[0]
@@ -241,13 +240,29 @@ func (u *staticUpstream) Select() *UpstreamHost {
 		}
 	}
 	if allDown {
-		return nil
+		if u.Spray == nil {
+			return nil
+		}
+		return u.Spray.Select(pool)
 	}
 
 	if u.Policy == nil {
-		return (&Random{}).Select(pool)
+		h := (&Random{}).Select(pool)
+		if h == nil && u.Spray == nil {
+			return nil
+		}
+		return u.Spray.Select(pool)
+	}
+
+	h := u.Policy.Select(pool)
+	if h != nil {
+		return h
+	}
+
+	if u.Spray == nil {
+		return nil
 	}
-	return u.Policy.Select(pool)
+	return u.Spray.Select(pool)
 }
 
 func (u *staticUpstream) IsAllowedPath(name string) bool {
diff --git a/middleware/proxy/upstream_test.go b/middleware/proxy/upstream_test.go
index 3d69ca8fb..a589cf85e 100644
--- a/middleware/proxy/upstream_test.go
+++ b/middleware/proxy/upstream_test.go
@@ -10,6 +10,7 @@ func TestHealthCheck(t *testing.T) {
 		from:        "",
 		Hosts:       testPool(),
 		Policy:      &Random{},
+		Spray:       nil,
 		FailTimeout: 10 * time.Second,
 		MaxFails:    1,
 	}