diff options
author | 2018-01-18 10:40:09 +0000 | |
---|---|---|
committer | 2018-01-18 10:40:09 +0000 | |
commit | c39e5cd01459b35fa04a4e62c798118953e15806 (patch) | |
tree | edb5c673bb0e95da412739569d9c58f0a1ca3fc0 /plugin/health | |
parent | 318bab77956a8aed04dce8d7a40823bb802282da (diff) | |
download | coredns-c39e5cd01459b35fa04a4e62c798118953e15806.tar.gz coredns-c39e5cd01459b35fa04a4e62c798118953e15806.tar.zst coredns-c39e5cd01459b35fa04a4e62c798118953e15806.zip |
plugin/health: add lameduck mode (#1379)
* plugin/health: add lameduck mode
Add a way to configure lameduck more, i.e. set health to false, stop
polling plugins. Then wait for a duration before shutting down. As the
health middleware is configured early on in the plugin list, it will
hold up all other shutdown, meaning we still answer queries.
* Add New
* More tests
* golint
* remove confusing text
Diffstat (limited to 'plugin/health')
-rw-r--r-- | plugin/health/README.md | 21 | ||||
-rw-r--r-- | plugin/health/health.go | 24 | ||||
-rw-r--r-- | plugin/health/health_test.go | 27 | ||||
-rw-r--r-- | plugin/health/setup.go | 40 | ||||
-rw-r--r-- | plugin/health/setup_test.go | 14 |
5 files changed, 112 insertions, 14 deletions
diff --git a/plugin/health/README.md b/plugin/health/README.md index 417ad167e..4389f6353 100644 --- a/plugin/health/README.md +++ b/plugin/health/README.md @@ -21,6 +21,17 @@ a 503. *health* periodically (1s) polls plugin that exports health information. plugin signals that it is unhealthy, the server will go unhealthy too. Each plugin that supports health checks has a section "Health" in their README. +More options can be set with this extended syntax: + +~~~ +health [ADDRESS] { + lameduck DURATION +} +~~~ + +* Where `lameduck` will make the process unhealthy then *wait* for **DURATION** before the process + shuts down. + ## Plugins Any plugin that implements the Healther interface will be used to report health. @@ -42,3 +53,13 @@ Run another health endpoint on http://localhost:8091. health localhost:8091 } ~~~ + +Set a lameduck duration of 1 second: + +~~~ corefile +. { + health localhost:8091 { + lameduck 1s + } +} +~~~ diff --git a/plugin/health/health.go b/plugin/health/health.go index c66f40b00..08c43d2a8 100644 --- a/plugin/health/health.go +++ b/plugin/health/health.go @@ -7,12 +7,15 @@ import ( "net" "net/http" "sync" + "time" ) var once sync.Once +// Health implements healthchecks by polling plugins. type health struct { - Addr string + Addr string + lameduck time.Duration ln net.Listener mux *http.ServeMux @@ -22,7 +25,13 @@ type health struct { sync.RWMutex ok bool // ok is the global boolean indicating an all healthy plugin stack - stop chan bool + stop chan bool + pollstop chan bool +} + +// newHealth returns a new initialized health. +func newHealth(addr string) *health { + return &health{Addr: addr, stop: make(chan bool), pollstop: make(chan bool)} } func (h *health) OnStartup() error { @@ -61,12 +70,21 @@ func (h *health) OnStartup() error { } func (h *health) OnShutdown() error { + // Stop polling plugins + h.pollstop <- true + // NACK health + h.SetOk(false) + + if h.lameduck > 0 { + log.Printf("[INFO] Going into lameduck mode for %s", h.lameduck) + time.Sleep(h.lameduck) + } + if h.ln != nil { return h.ln.Close() } h.stop <- true - return nil } diff --git a/plugin/health/health_test.go b/plugin/health/health_test.go index 0bfc50f2f..8f325c9c4 100644 --- a/plugin/health/health_test.go +++ b/plugin/health/health_test.go @@ -5,12 +5,13 @@ import ( "io/ioutil" "net/http" "testing" + "time" "github.com/coredns/coredns/plugin/erratic" ) func TestHealth(t *testing.T) { - h := health{Addr: ":0"} + h := newHealth(":0") h.h = append(h.h, &erratic.Erratic{}) if err := h.OnStartup(); err != nil { @@ -18,6 +19,11 @@ func TestHealth(t *testing.T) { } defer h.OnShutdown() + go func() { + <-h.pollstop + return + }() + // Reconstruct the http address based on the port allocated by operating system. address := fmt.Sprintf("http://%s%s", h.ln.Addr().String(), path) @@ -50,3 +56,22 @@ func TestHealth(t *testing.T) { t.Errorf("Invalid response body: expecting 'OK', got '%s'", string(content)) } } + +func TestHealthLameduck(t *testing.T) { + h := newHealth(":0") + h.lameduck = 250 * time.Millisecond + h.h = append(h.h, &erratic.Erratic{}) + + if err := h.OnStartup(); err != nil { + t.Fatalf("Unable to startup the health server: %v", err) + } + + // Both these things are behind a sync.Once, fake reading from the channels. + go func() { + <-h.pollstop + <-h.stop + return + }() + + h.OnShutdown() +} diff --git a/plugin/health/setup.go b/plugin/health/setup.go index d5285c55f..f6e92dc8a 100644 --- a/plugin/health/setup.go +++ b/plugin/health/setup.go @@ -1,6 +1,7 @@ package health import ( + "fmt" "net" "time" @@ -19,12 +20,13 @@ func init() { } func setup(c *caddy.Controller) error { - addr, err := healthParse(c) + addr, lame, err := healthParse(c) if err != nil { return plugin.Error("health", err) } - h := &health{Addr: addr, stop: make(chan bool)} + h := newHealth(addr) + h.lameduck = lame c.OnStartup(func() error { plugins := dnsserver.GetConfig(c).Handlers() @@ -41,8 +43,12 @@ func setup(c *caddy.Controller) error { h.poll() go func() { for { - <-time.After(1 * time.Second) - h.poll() + select { + case <-time.After(1 * time.Second): + h.poll() + case <-h.pollstop: + return + } } }() return nil @@ -68,8 +74,9 @@ func setup(c *caddy.Controller) error { return nil } -func healthParse(c *caddy.Controller) (string, error) { +func healthParse(c *caddy.Controller) (string, time.Duration, error) { addr := "" + dur := time.Duration(0) for c.Next() { args := c.RemainingArgs() @@ -78,11 +85,28 @@ func healthParse(c *caddy.Controller) (string, error) { case 1: addr = args[0] if _, _, e := net.SplitHostPort(addr); e != nil { - return "", e + return "", 0, e } default: - return "", c.ArgErr() + return "", 0, c.ArgErr() + } + + for c.NextBlock() { + switch c.Val() { + case "lameduck": + args := c.RemainingArgs() + if len(args) != 1 { + return "", 0, c.ArgErr() + } + l, err := time.ParseDuration(args[0]) + if err != nil { + return "", 0, fmt.Errorf("unable to parse lameduck duration value: '%v' : %v", args[0], err) + } + dur = l + default: + return "", 0, c.ArgErr() + } } } - return addr, nil + return addr, dur, nil } diff --git a/plugin/health/setup_test.go b/plugin/health/setup_test.go index 87f4fc5fd..4db6fc770 100644 --- a/plugin/health/setup_test.go +++ b/plugin/health/setup_test.go @@ -13,17 +13,27 @@ func TestSetupHealth(t *testing.T) { }{ {`health`, false}, {`health localhost:1234`, false}, + {`health localhost:1234 { + lameduck 4s +}`, false}, {`health bla:a`, false}, + {`health bla`, true}, {`health bla bla`, true}, + {`health localhost:1234 { + lameduck a +}`, true}, + {`health localhost:1234 { + lamedudk 4 +} `, true}, } for i, test := range tests { c := caddy.NewTestController("dns", test.input) - _, err := healthParse(c) + _, _, err := healthParse(c) if test.shouldErr && err == nil { - t.Errorf("Test %d: Expected error but found %s for input %s", i, err, test.input) + t.Errorf("Test %d: Expected error but found none for input %s", i, test.input) } if err != nil { |