plugin/forward using pkg/up (#1493)

* plugin/forward: on demand healtchecking

Only start doing health checks when we encouner an error (any error).
This uses the new pluing/pkg/up package to abstract away the actual
checking. This reduces the LOC quite a bit; does need more testing, unit
testing and tcpdumping a bit.

* fix tests

* Fix readme

* Use pkg/up for healthchecks

* remove unused channel

* more cleanups

* update readme

* * Again do go generate and go build; still referencing the wrong forward
  repo? Anyway fixed.
* Use pkg/up for doing the healtchecks to cut back on unwanted queries
  * Change up.Func to return an error instead of a boolean.
  * Drop the string target argument as it doesn't make sense.
* Add healthcheck test on failing to get an upstream answer.

TODO(miek): double check Forward and Lookup and how they interact with
HC, and if we correctly call close() on those

* actual test

* Tests here

* more tests

* try getting rid of host

* Get rid of the host indirection

* Finish removing hosts

* moar testing

* import fmt

* field is not used

* docs

* move some stuff

* bring back health_check

* maxfails=0 test

* git and merging, bah

* review

1 files changed, 28 insertions, 8 deletions

diff --git a/plugin/forward/forward.go b/plugin/forward/forward.go
index 35885008e..8a3e7188c 100644
--- a/plugin/forward/forward.go
+++ b/plugin/forward/forward.go
@@ -20,8 +20,9 @@ import (
 // Forward represents a plugin instance that can proxy requests to another (DNS) server. It has a list
 // of proxies each representing one upstream proxy.
 type Forward struct {
-	proxies []*Proxy
-	p       Policy
+	proxies    []*Proxy
+	p          Policy
+	hcInterval time.Duration
 
 	from    string
 	ignored []string
@@ -31,22 +32,21 @@ type Forward struct {
 	maxfails      uint32
 	expire        time.Duration
 
-	forceTCP   bool          // also here for testing
-	hcInterval time.Duration // also here for testing
+	forceTCP bool // also here for testing
 
 	Next plugin.Handler
 }
 
 // New returns a new Forward.
 func New() *Forward {
-	f := &Forward{maxfails: 2, tlsConfig: new(tls.Config), expire: defaultExpire, hcInterval: hcDuration, p: new(random)}
+	f := &Forward{maxfails: 2, tlsConfig: new(tls.Config), expire: defaultExpire, p: new(random), from: ".", hcInterval: hcDuration}
 	return f
 }
 
 // SetProxy appends p to the proxy list and starts healthchecking.
 func (f *Forward) SetProxy(p *Proxy) {
 	f.proxies = append(f.proxies, p)
-	go p.healthCheck()
+	p.start(f.hcInterval)
 }
 
 // Len returns the number of configured proxies.
@@ -92,7 +92,27 @@ func (f *Forward) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg
 			child.Finish()
 		}
 
+		// If you query for instance ANY isc.org; you get a truncated query back which miekg/dns fails to unpack
+		// because the RRs are not finished. The returned message can be useful or useless. Return the original
+		// query with some header bits set that they should retry with TCP.
+		if err == dns.ErrTruncated {
+			// We may or may not have something sensible... if not reassemble something to send to the client.
+			if ret == nil {
+				ret = new(dns.Msg)
+				ret.SetReply(r)
+				ret.Truncated = true
+				ret.Authoritative = true
+				ret.Rcode = dns.RcodeSuccess
+			}
+			err = nil // and reset err to pass this back to the client.
+		}
+
 		if err != nil {
+			// Kick off health check to see if *our* upstream is broken.
+			if f.maxfails != 0 {
+				proxy.Healthcheck()
+			}
+
 			if fails < len(f.proxies) {
 				continue
 			}
@@ -140,8 +160,8 @@ func (f *Forward) isAllowedDomain(name string) bool {
 func (f *Forward) list() []*Proxy { return f.p.List(f.proxies) }
 
 var (
-	errInvalidDomain = errors.New("invalid domain for proxy")
-	errNoHealthy     = errors.New("no healthy proxies")
+	errInvalidDomain = errors.New("invalid domain for forward")
+	errNoHealthy     = errors.New("no healthy proxies or upstream error")
 	errNoForward     = errors.New("no forwarder defined")
 )