plugin/forward using pkg/up (#1493)

* plugin/forward: on demand healtchecking

Only start doing health checks when we encouner an error (any error).
This uses the new pluing/pkg/up package to abstract away the actual
checking. This reduces the LOC quite a bit; does need more testing, unit
testing and tcpdumping a bit.

* fix tests

* Fix readme

* Use pkg/up for healthchecks

* remove unused channel

* more cleanups

* update readme

* * Again do go generate and go build; still referencing the wrong forward
  repo? Anyway fixed.
* Use pkg/up for doing the healtchecks to cut back on unwanted queries
  * Change up.Func to return an error instead of a boolean.
  * Drop the string target argument as it doesn't make sense.
* Add healthcheck test on failing to get an upstream answer.

TODO(miek): double check Forward and Lookup and how they interact with
HC, and if we correctly call close() on those

* actual test

* Tests here

* more tests

* try getting rid of host

* Get rid of the host indirection

* Finish removing hosts

* moar testing

* import fmt

* field is not used

* docs

* move some stuff

* bring back health_check

* maxfails=0 test

* git and merging, bah

* review

1 files changed, 10 insertions, 37 deletions

diff --git a/plugin/forward/health.go b/plugin/forward/health.go
index e277f30a6..cd4b96e27 100644
--- a/plugin/forward/health.go
+++ b/plugin/forward/health.go
@@ -1,7 +1,6 @@
 package forward
 
 import (
-	"log"
 	"sync/atomic"
 
 	"github.com/miekg/dns"
@@ -10,41 +9,25 @@ import (
 // For HC we send to . IN NS +norec message to the upstream. Dial timeouts and empty
 // replies are considered fails, basically anything else constitutes a healthy upstream.
 
-func (h *host) Check() {
-	h.Lock()
-
-	if h.checking {
-		h.Unlock()
-		return
-	}
-
-	h.checking = true
-	h.Unlock()
-
-	err := h.send()
+// Check is used as the up.Func in the up.Probe.
+func (p *Proxy) Check() error {
+	err := p.send()
 	if err != nil {
-		log.Printf("[INFO] healtheck of %s failed with %s", h.addr, err)
-
-		HealthcheckFailureCount.WithLabelValues(h.addr).Add(1)
-
-		atomic.AddUint32(&h.fails, 1)
-	} else {
-		atomic.StoreUint32(&h.fails, 0)
+		HealthcheckFailureCount.WithLabelValues(p.addr).Add(1)
+		atomic.AddUint32(&p.fails, 1)
+		return err
 	}
 
-	h.Lock()
-	h.checking = false
-	h.Unlock()
-
-	return
+	atomic.StoreUint32(&p.fails, 0)
+	return nil
 }
 
-func (h *host) send() error {
+func (p *Proxy) send() error {
 	hcping := new(dns.Msg)
 	hcping.SetQuestion(".", dns.TypeNS)
 	hcping.RecursionDesired = false
 
-	m, _, err := h.client.Exchange(hcping, h.addr)
+	m, _, err := p.client.Exchange(hcping, p.addr)
 	// If we got a header, we're alright, basically only care about I/O errors 'n stuff
 	if err != nil && m != nil {
 		// Silly check, something sane came back
@@ -55,13 +38,3 @@ func (h *host) send() error {
 
 	return err
 }
-
-// down returns true is this host has more than maxfails fails.
-func (h *host) down(maxfails uint32) bool {
-	if maxfails == 0 {
-		return false
-	}
-
-	fails := atomic.LoadUint32(&h.fails)
-	return fails > maxfails
-}