diff options
| -rw-r--r-- | plugin/pkg/up/up.go | 24 |
1 files changed, 20 insertions, 4 deletions
diff --git a/plugin/pkg/up/up.go b/plugin/pkg/up/up.go index e2c9fe2cd..8f866311b 100644 --- a/plugin/pkg/up/up.go +++ b/plugin/pkg/up/up.go @@ -9,10 +9,12 @@ import ( // Probe is used to run a single Func until it returns true (indicating a target is healthy). If an Func // is already in progress no new one will be added, i.e. there is always a maximum of 1 checks in flight. +// When failures start to happen we will back off every second failure up to maximum of 4 intervals. type Probe struct { sync.Mutex inprogress int interval time.Duration + max time.Duration } // Func is used to determine if a target is alive. If so this function must return nil. @@ -34,17 +36,22 @@ func (p *Probe) Do(f Func) { // Passed the lock. Now run f for as long it returns false. If a true is returned // we return from the goroutine and we can accept another Func to run. go func() { + i := 1 for { if err := f(); err == nil { break } time.Sleep(interval) + if i%2 == 0 && i < 4 { // 4 is 2 doubles, so no need to increase anymore - this is *also* checked in double() + p.double() + } p.Lock() if p.inprogress == stop { p.Unlock() return } p.Unlock() + i++ } p.Lock() @@ -53,6 +60,15 @@ func (p *Probe) Do(f Func) { }() } +func (p *Probe) double() { + p.Lock() + p.interval *= 2 + if p.interval > p.max { + p.interval = p.max + } + p.Unlock() +} + // Stop stops the probing. func (p *Probe) Stop() { p.Lock() @@ -61,12 +77,10 @@ func (p *Probe) Stop() { } // Start will initialize the probe manager, after which probes can be initiated with Do. -func (p *Probe) Start(interval time.Duration) { p.SetInterval(interval) } - -// SetInterval sets the probing interval to be used by upcoming probes initiated with Do. -func (p *Probe) SetInterval(interval time.Duration) { +func (p *Probe) Start(interval time.Duration) { p.Lock() p.interval = interval + p.max = interval * multiplier p.Unlock() } @@ -74,4 +88,6 @@ const ( idle = iota active stop + + multiplier = 4 ) |