Fix forward metrics for backwards compatibility (#6178)

8 files changed, 47 insertions, 50 deletions

diff --git a/plugin/pkg/proxy/connect.go b/plugin/pkg/proxy/connect.go
index b60a1a237..d799df498 100644
--- a/plugin/pkg/proxy/connect.go
+++ b/plugin/pkg/proxy/connect.go
@@ -55,10 +55,10 @@ func (t *Transport) Dial(proto string) (*persistConn, bool, error) {
 	pc := <-t.ret
 
 	if pc != nil {
-		ConnCacheHitsCount.WithLabelValues(t.addr, proto).Add(1)
+		connCacheHitsCount.WithLabelValues(t.proxyName, t.addr, proto).Add(1)
 		return pc, true, nil
 	}
-	ConnCacheMissesCount.WithLabelValues(t.addr, proto).Add(1)
+	connCacheMissesCount.WithLabelValues(t.proxyName, t.addr, proto).Add(1)
 
 	reqTime := time.Now()
 	timeout := t.dialTimeout()
@@ -152,9 +152,7 @@ func (p *Proxy) Connect(ctx context.Context, state request.Request, opts Options
 		rc = strconv.Itoa(ret.Rcode)
 	}
 
-	RequestCount.WithLabelValues(p.addr).Add(1)
-	RcodeCount.WithLabelValues(rc, p.addr).Add(1)
-	RequestDuration.WithLabelValues(p.addr, rc).Observe(time.Since(start).Seconds())
+	requestDuration.WithLabelValues(p.proxyName, p.addr, rc).Observe(time.Since(start).Seconds())
 
 	return ret, nil
 }
diff --git a/plugin/pkg/proxy/health.go b/plugin/pkg/proxy/health.go
index a7e99560d..4b4b4cc01 100644
--- a/plugin/pkg/proxy/health.go
+++ b/plugin/pkg/proxy/health.go
@@ -32,10 +32,12 @@ type dnsHc struct {
 	c                *dns.Client
 	recursionDesired bool
 	domain           string
+
+	proxyName string
 }
 
 // NewHealthChecker returns a new HealthChecker based on transport.
-func NewHealthChecker(trans string, recursionDesired bool, domain string) HealthChecker {
+func NewHealthChecker(proxyName, trans string, recursionDesired bool, domain string) HealthChecker {
 	switch trans {
 	case transport.DNS, transport.TLS:
 		c := new(dns.Client)
@@ -47,6 +49,7 @@ func NewHealthChecker(trans string, recursionDesired bool, domain string) Health
 			c:                c,
 			recursionDesired: recursionDesired,
 			domain:           domain,
+			proxyName:        proxyName,
 		}
 	}
 
@@ -104,7 +107,7 @@ func (h *dnsHc) SetWriteTimeout(t time.Duration) {
 func (h *dnsHc) Check(p *Proxy) error {
 	err := h.send(p.addr)
 	if err != nil {
-		HealthcheckFailureCount.WithLabelValues(p.addr).Add(1)
+		healthcheckFailureCount.WithLabelValues(p.proxyName, p.addr).Add(1)
 		p.incrementFails()
 		return err
 	}
diff --git a/plugin/pkg/proxy/health_test.go b/plugin/pkg/proxy/health_test.go
index c1b5270ad..8d9acfb9c 100644
--- a/plugin/pkg/proxy/health_test.go
+++ b/plugin/pkg/proxy/health_test.go
@@ -23,11 +23,11 @@ func TestHealth(t *testing.T) {
 	})
 	defer s.Close()
 
-	hc := NewHealthChecker(transport.DNS, true, "")
+	hc := NewHealthChecker("TestHealth", transport.DNS, true, "")
 	hc.SetReadTimeout(10 * time.Millisecond)
 	hc.SetWriteTimeout(10 * time.Millisecond)
 
-	p := NewProxy(s.Addr, transport.DNS)
+	p := NewProxy("TestHealth", s.Addr, transport.DNS)
 	p.readTimeout = 10 * time.Millisecond
 	err := hc.Check(p)
 	if err != nil {
@@ -53,12 +53,12 @@ func TestHealthTCP(t *testing.T) {
 	})
 	defer s.Close()
 
-	hc := NewHealthChecker(transport.DNS, true, "")
+	hc := NewHealthChecker("TestHealthTCP", transport.DNS, true, "")
 	hc.SetTCPTransport()
 	hc.SetReadTimeout(10 * time.Millisecond)
 	hc.SetWriteTimeout(10 * time.Millisecond)
 
-	p := NewProxy(s.Addr, transport.DNS)
+	p := NewProxy("TestHealthTCP", s.Addr, transport.DNS)
 	p.readTimeout = 10 * time.Millisecond
 	err := hc.Check(p)
 	if err != nil {
@@ -84,11 +84,11 @@ func TestHealthNoRecursion(t *testing.T) {
 	})
 	defer s.Close()
 
-	hc := NewHealthChecker(transport.DNS, false, "")
+	hc := NewHealthChecker("TestHealthNoRecursion", transport.DNS, false, "")
 	hc.SetReadTimeout(10 * time.Millisecond)
 	hc.SetWriteTimeout(10 * time.Millisecond)
 
-	p := NewProxy(s.Addr, transport.DNS)
+	p := NewProxy("TestHealthNoRecursion", s.Addr, transport.DNS)
 	p.readTimeout = 10 * time.Millisecond
 	err := hc.Check(p)
 	if err != nil {
@@ -108,11 +108,11 @@ func TestHealthTimeout(t *testing.T) {
 	})
 	defer s.Close()
 
-	hc := NewHealthChecker(transport.DNS, false, "")
+	hc := NewHealthChecker("TestHealthTimeout", transport.DNS, false, "")
 	hc.SetReadTimeout(10 * time.Millisecond)
 	hc.SetWriteTimeout(10 * time.Millisecond)
 
-	p := NewProxy(s.Addr, transport.DNS)
+	p := NewProxy("TestHealthTimeout", s.Addr, transport.DNS)
 	p.readTimeout = 10 * time.Millisecond
 	err := hc.Check(p)
 	if err == nil {
@@ -134,11 +134,11 @@ func TestHealthDomain(t *testing.T) {
 	})
 	defer s.Close()
 
-	hc := NewHealthChecker(transport.DNS, true, hcDomain)
+	hc := NewHealthChecker("TestHealthDomain", transport.DNS, true, hcDomain)
 	hc.SetReadTimeout(10 * time.Millisecond)
 	hc.SetWriteTimeout(10 * time.Millisecond)
 
-	p := NewProxy(s.Addr, transport.DNS)
+	p := NewProxy("TestHealthDomain", s.Addr, transport.DNS)
 	p.readTimeout = 10 * time.Millisecond
 	err := hc.Check(p)
 	if err != nil {
diff --git a/plugin/pkg/proxy/metrics.go b/plugin/pkg/proxy/metrics.go
index 148bc6edd..e4cae97c3 100644
--- a/plugin/pkg/proxy/metrics.go
+++ b/plugin/pkg/proxy/metrics.go
@@ -9,41 +9,32 @@ import (
 
 // Variables declared for monitoring.
 var (
-	RequestCount = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: plugin.Namespace,
-		Subsystem: "proxy",
-		Name:      "requests_total",
-		Help:      "Counter of requests made per upstream.",
-	}, []string{"to"})
-	RcodeCount = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: plugin.Namespace,
-		Subsystem: "proxy",
-		Name:      "responses_total",
-		Help:      "Counter of responses received per upstream.",
-	}, []string{"rcode", "to"})
-	RequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: plugin.Namespace,
 		Subsystem: "proxy",
 		Name:      "request_duration_seconds",
 		Buckets:   plugin.TimeBuckets,
 		Help:      "Histogram of the time each request took.",
-	}, []string{"to", "rcode"})
-	HealthcheckFailureCount = promauto.NewCounterVec(prometheus.CounterOpts{
+	}, []string{"proxy_name", "to", "rcode"})
+
+	healthcheckFailureCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: plugin.Namespace,
 		Subsystem: "proxy",
 		Name:      "healthcheck_failures_total",
 		Help:      "Counter of the number of failed healthchecks.",
-	}, []string{"to"})
-	ConnCacheHitsCount = promauto.NewCounterVec(prometheus.CounterOpts{
+	}, []string{"proxy_name", "to"})
+
+	connCacheHitsCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: plugin.Namespace,
 		Subsystem: "proxy",
 		Name:      "conn_cache_hits_total",
 		Help:      "Counter of connection cache hits per upstream and protocol.",
-	}, []string{"to", "proto"})
-	ConnCacheMissesCount = promauto.NewCounterVec(prometheus.CounterOpts{
+	}, []string{"proxy_name", "to", "proto"})
+
+	connCacheMissesCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: plugin.Namespace,
 		Subsystem: "proxy",
 		Name:      "conn_cache_misses_total",
 		Help:      "Counter of connection cache misses per upstream and protocol.",
-	}, []string{"to", "proto"})
+	}, []string{"proxy_name", "to", "proto"})
 )
diff --git a/plugin/pkg/proxy/persistent.go b/plugin/pkg/proxy/persistent.go
index 2dc8bde71..49c9dd385 100644
--- a/plugin/pkg/proxy/persistent.go
+++ b/plugin/pkg/proxy/persistent.go
@@ -21,6 +21,7 @@ type Transport struct {
 	expire      time.Duration                  // After this duration a connection is expired.
 	addr        string
 	tlsConfig   *tls.Config
+	proxyName   string
 
 	dial  chan string
 	yield chan *persistConn
@@ -28,7 +29,7 @@ type Transport struct {
 	stop  chan bool
 }
 
-func newTransport(addr string) *Transport {
+func newTransport(proxyName, addr string) *Transport {
 	t := &Transport{
 		avgDialTime: int64(maxDialTimeout / 2),
 		conns:       [typeTotalCount][]*persistConn{},
@@ -38,6 +39,7 @@ func newTransport(addr string) *Transport {
 		yield:       make(chan *persistConn),
 		ret:         make(chan *persistConn),
 		stop:        make(chan bool),
+		proxyName:   proxyName,
 	}
 	return t
 }
diff --git a/plugin/pkg/proxy/persistent_test.go b/plugin/pkg/proxy/persistent_test.go
index c78bd7f1f..56d837113 100644
--- a/plugin/pkg/proxy/persistent_test.go
+++ b/plugin/pkg/proxy/persistent_test.go
@@ -17,7 +17,7 @@ func TestCached(t *testing.T) {
 	})
 	defer s.Close()
 
-	tr := newTransport(s.Addr)
+	tr := newTransport("TestCached", s.Addr)
 	tr.Start()
 	defer tr.Stop()
 
@@ -56,7 +56,7 @@ func TestCleanupByTimer(t *testing.T) {
 	})
 	defer s.Close()
 
-	tr := newTransport(s.Addr)
+	tr := newTransport("TestCleanupByTimer", s.Addr)
 	tr.SetExpire(100 * time.Millisecond)
 	tr.Start()
 	defer tr.Stop()
@@ -90,7 +90,7 @@ func TestCleanupAll(t *testing.T) {
 	})
 	defer s.Close()
 
-	tr := newTransport(s.Addr)
+	tr := newTransport("TestCleanupAll", s.Addr)
 
 	c1, _ := dns.DialTimeout("udp", tr.addr, maxDialTimeout)
 	c2, _ := dns.DialTimeout("udp", tr.addr, maxDialTimeout)
diff --git a/plugin/pkg/proxy/proxy.go b/plugin/pkg/proxy/proxy.go
index 414c34240..99fb5df78 100644
--- a/plugin/pkg/proxy/proxy.go
+++ b/plugin/pkg/proxy/proxy.go
@@ -12,8 +12,9 @@ import (
 
 // Proxy defines an upstream host.
 type Proxy struct {
-	fails uint32
-	addr  string
+	fails     uint32
+	addr      string
+	proxyName string
 
 	transport *Transport
 
@@ -25,15 +26,17 @@ type Proxy struct {
 }
 
 // NewProxy returns a new proxy.
-func NewProxy(addr, trans string) *Proxy {
+func NewProxy(proxyName, addr, trans string) *Proxy {
 	p := &Proxy{
 		addr:        addr,
 		fails:       0,
 		probe:       up.New(),
 		readTimeout: 2 * time.Second,
-		transport:   newTransport(addr),
+		transport:   newTransport(proxyName, addr),
+		health:      NewHealthChecker(proxyName, trans, true, "."),
+		proxyName:   proxyName,
 	}
-	p.health = NewHealthChecker(trans, true, ".")
+
 	runtime.SetFinalizer(p, (*Proxy).finalizer)
 	return p
 }
diff --git a/plugin/pkg/proxy/proxy_test.go b/plugin/pkg/proxy/proxy_test.go
index 17125ea68..33a7170c0 100644
--- a/plugin/pkg/proxy/proxy_test.go
+++ b/plugin/pkg/proxy/proxy_test.go
@@ -24,7 +24,7 @@ func TestProxy(t *testing.T) {
 	})
 	defer s.Close()
 
-	p := NewProxy(s.Addr, transport.DNS)
+	p := NewProxy("TestProxy", s.Addr, transport.DNS)
 	p.readTimeout = 10 * time.Millisecond
 	p.Start(5 * time.Second)
 	m := new(dns.Msg)
@@ -54,7 +54,7 @@ func TestProxyTLSFail(t *testing.T) {
 	})
 	defer s.Close()
 
-	p := NewProxy(s.Addr, transport.TLS)
+	p := NewProxy("TestProxyTLSFail", s.Addr, transport.TLS)
 	p.readTimeout = 10 * time.Millisecond
 	p.SetTLSConfig(&tls.Config{})
 	p.Start(5 * time.Second)
@@ -72,7 +72,7 @@ func TestProxyTLSFail(t *testing.T) {
 }
 
 func TestProtocolSelection(t *testing.T) {
-	p := NewProxy("bad_address", transport.DNS)
+	p := NewProxy("TestProtocolSelection", "bad_address", transport.DNS)
 	p.readTimeout = 10 * time.Millisecond
 
 	stateUDP := request.Request{W: &test.ResponseWriter{}, Req: new(dns.Msg)}
@@ -119,7 +119,7 @@ func TestProxyIncrementFails(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			p := NewProxy("bad_address", transport.DNS)
+			p := NewProxy("TestProxyIncrementFails", "bad_address", transport.DNS)
 			p.fails = tc.fails
 			p.incrementFails()
 			if p.fails != tc.expectFails {