Add cache eviction metrics to the cache plugin (#4411)

Signed-off-by: Frank Riley <fhriley@gmail.com>

plugin/cache/README.md

@@ -79,6 +79,7 @@ If monitoring is enabled (via the *prometheus* plugin) then the following metric
 * `coredns_cache_prefetch_total{server}` - Counter of times the cache has prefetched a cached item.
 * `coredns_cache_drops_total{server}` - Counter of responses excluded from the cache due to request/response question name mismatch.
 * `coredns_cache_served_stale_total{server}` - Counter of requests served from stale cache entries.
+* `coredns_cache_evictions_total{server, type}` - Counter of cache evictions.
 
 Cache types are either "denial" or "success". `Server` is the server handling the request, see the
 prometheus plugin for documentation.

plugin/cache/cache.go

@@ -190,7 +190,9 @@ func (w *ResponseWriter) set(m *dns.Msg, key uint64, mt response.Type, duration
 	switch mt {
 	case response.NoError, response.Delegation:
 		i := newItem(m, w.now(), duration)
-		w.pcache.Add(key, i)
+		if w.pcache.Add(key, i) {
+			evictions.WithLabelValues(w.server, Success).Inc()
+		}
 		// when pre-fetching, remove the negative cache entry if it exists
 		if w.prefetch {
 			w.ncache.Remove(key)
@@ -198,7 +200,9 @@ func (w *ResponseWriter) set(m *dns.Msg, key uint64, mt response.Type, duration
 
 	case response.NameError, response.NoData, response.ServerError:
 		i := newItem(m, w.now(), duration)
-		w.ncache.Add(key, i)
+		if w.ncache.Add(key, i) {
+			evictions.WithLabelValues(w.server, Denial).Inc()
+		}
 
 	case response.OtherError:
 		// don't cache these

plugin/cache/metrics.go

@@ -50,4 +50,11 @@ var (
 		Name:      "served_stale_total",
 		Help:      "The number of requests served from stale cache entries.",
 	}, []string{"server"})
+	// evictions is the counter of cache evictions.
+	evictions = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "cache",
+		Name:      "evictions_total",
+		Help:      "The count of cache evictions.",
+	}, []string{"server", "type"})
 )

plugin/pkg/cache/cache.go

@@ -45,9 +45,10 @@ func New(size int) *Cache {
 }
 
 // Add adds a new element to the cache. If the element already exists it is overwritten.
-func (c *Cache) Add(key uint64, el interface{}) {
+// Returns true if an existing element was evicted to make room for this element.
+func (c *Cache) Add(key uint64, el interface{}) bool {
 	shard := key & (shardSize - 1)
-	c.shards[shard].Add(key, el)
+	return c.shards[shard].Add(key, el)
 }
 
 // Get looks up element index under key.
@@ -75,18 +76,22 @@ func (c *Cache) Len() int {
 func newShard(size int) *shard { return &shard{items: make(map[uint64]interface{}), size: size} }
 
 // Add adds element indexed by key into the cache. Any existing element is overwritten
-func (s *shard) Add(key uint64, el interface{}) {
+// Returns true if an existing element was evicted to make room for this element.
+func (s *shard) Add(key uint64, el interface{}) bool {
+	eviction := false
 	s.Lock()
 	if len(s.items) >= s.size {
 		if _, ok := s.items[key]; !ok {
 			for k := range s.items {
 				delete(s.items, k)
+				eviction = true
 				break
 			}
 		}
 	}
 	s.items[key] = el
 	s.Unlock()
+	return eviction
 }
 
 // Remove removes the element indexed by key from the cache.