Add k8s external service CNAMEs (#677)

* Add external service cnames

* remove cruft

* update CI k8s version

* change CI k8s version

* min k8s ver for ext services

* trying k8s 1.5

* k8s 1.5 requires ports spec

* remove kruft

* update dns schema version

.travis.yml

@@ -12,7 +12,7 @@ go:
 go_import_path: github.com/coredns/coredns
 
 env:
-  - ETCD_VERSION=2.3.1 K8S_VERSION=1.3.7 KUBECTL="docker exec hyperkube /hyperkube kubectl" DNS_ARGUMENTS=""
+  - ETCD_VERSION=2.3.1 K8S_VERSION=1.5.0 KUBECTL="docker exec hyperkube /hyperkube kubectl" DNS_ARGUMENTS=""
 
 # In the Travis VM-based build environment, IPv6 networking is not
 # enabled by default. The sysctl operations below enable IPv6.

.travis/kubernetes/dns-test.yaml

@@ -183,3 +183,17 @@ spec:
   - name: c-port
     port: 1234
     protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ext-svc
+  namespace: test-1
+spec:
+  type: ExternalName
+  externalName: example.net
+  ports:
+  - name: c-port
+    port: 1234
+    protocol: UDP
+

middleware/backend_lookup.go

@@ -25,7 +25,7 @@ func A(b ServiceBackend, zone string, state request.Request, previousRecords []d
 		what, ip := serv.HostType()
 
 		switch what {
-		case dns.TypeANY:
+		case dns.TypeCNAME:
 			if Name(state.Name()).Matches(dns.Fqdn(serv.Host)) {
 				// x CNAME x is a direct loop, don't add those
 				continue
@@ -92,7 +92,7 @@ func AAAA(b ServiceBackend, zone string, state request.Request, previousRecords
 		what, ip := serv.HostType()
 
 		switch what {
-		case dns.TypeANY:
+		case dns.TypeCNAME:
 			// Try to resolve as CNAME if it's not an IP, but only if we don't create loops.
 			if Name(state.Name()).Matches(dns.Fqdn(serv.Host)) {
 				// x CNAME x is a direct loop, don't add those
@@ -182,7 +182,7 @@ func SRV(b ServiceBackend, zone string, state request.Request, opt Options) (rec
 		what, ip := serv.HostType()
 
 		switch what {
-		case dns.TypeANY:
+		case dns.TypeCNAME:
 			srv := serv.NewSRV(state.QName(), weight)
 			records = append(records, srv)
 
@@ -250,7 +250,7 @@ func MX(b ServiceBackend, zone string, state request.Request, opt Options) (reco
 		}
 		what, ip := serv.HostType()
 		switch what {
-		case dns.TypeANY:
+		case dns.TypeCNAME:
 			mx := serv.NewMX(state.QName())
 			records = append(records, mx)
 			if _, ok := lookup[mx.Mx]; ok {
@@ -364,7 +364,7 @@ func NS(b ServiceBackend, zone string, state request.Request, opt Options) (reco
 	for _, serv := range services {
 		what, ip := serv.HostType()
 		switch what {
-		case dns.TypeANY:
+		case dns.TypeCNAME:
 			return nil, nil, debug, fmt.Errorf("NS record must be an IP address: %s", serv.Host)
 
 		case dns.TypeA, dns.TypeAAAA:

middleware/etcd/msg/type.go

@@ -11,7 +11,7 @@ import (
 //
 // dns.TypeA: the service's Host field contains an A record.
 // dns.TypeAAAA: the service's Host field contains an AAAA record.
-// dns.TypeANY: the service's Host field contains a name.
+// dns.TypeCNAME: the service's Host field contains a name.
 //
 // Note that a service can double/triple as a TXT record or MX record.
 func (s *Service) HostType() (what uint16, normalized net.IP) {
@@ -20,7 +20,7 @@ func (s *Service) HostType() (what uint16, normalized net.IP) {
 
 	switch {
 	case ip == nil:
-		return dns.TypeANY, nil
+		return dns.TypeCNAME, nil
 
 	case ip.To4() != nil:
 		return dns.TypeA, ip.To4()

middleware/etcd/msg/type_test.go

@@ -11,11 +11,11 @@ func TestType(t *testing.T) {
 		serv         Service
 		expectedType uint16
 	}{
-		{Service{Host: "example.org"}, dns.TypeANY},
+		{Service{Host: "example.org"}, dns.TypeCNAME},
 		{Service{Host: "127.0.0.1"}, dns.TypeA},
 		{Service{Host: "2000::3"}, dns.TypeAAAA},
-		{Service{Host: "2000..3"}, dns.TypeANY},
-		{Service{Host: "127.0.0.257"}, dns.TypeANY},
+		{Service{Host: "2000..3"}, dns.TypeCNAME},
+		{Service{Host: "127.0.0.257"}, dns.TypeCNAME},
 		{Service{Host: "127.0.0.252", Mail: true}, dns.TypeA},
 		{Service{Host: "127.0.0.252", Mail: true, Text: "a"}, dns.TypeA},
 		{Service{Host: "127.0.0.254", Mail: false, Text: "a"}, dns.TypeA},

middleware/kubernetes/README.md

@@ -108,6 +108,13 @@ kubernetes coredns.local {
 	#
 	cidrs 10.0.0.0/24 10.0.10.0/25
 
+	# upstream <address> [<address>] ...
+	#
+	# Defines upstream resolvers used for resolving services that point to
+	# external hosts (External Services).  <address> can be an ip, and ip:port, or
+	# a path to a file structured like resolv.conf.
+	upstream 12.34.56.78:53
+	
 	# fallthrough
 	#
 	# If a query for a record in the cluster zone results in NXDOMAIN,

middleware/kubernetes/kubernetes.go

@@ -55,7 +55,7 @@ const (
 	// PodModeInsecure is where pod requests are answered without verfying they exist
 	PodModeInsecure = "insecure"
 	// DNSSchemaVersion is the schema version: https://github.com/kubernetes/dns/blob/master/docs/specification.md
-	DNSSchemaVersion = "1.0.0"
+	DNSSchemaVersion = "1.0.1"
 )
 
 type endpoint struct {
@@ -97,7 +97,7 @@ func (k *Kubernetes) Services(state request.Request, exact bool, opt middleware.
 	}
 
 	switch state.Type() {
-	case "A", "SRV":
+	case "A", "CNAME":
 		if state.Type() == "A" && isDefaultNS(state.Name(), r) {
 			// If this is an A request for "ns.dns", respond with a "fake" record for coredns.
 			// SOA records always use this hardcoded name
@@ -106,6 +106,16 @@ func (k *Kubernetes) Services(state request.Request, exact bool, opt middleware.
 		}
 		s, e := k.Records(r)
 		return s, nil, e // Haven't implemented debug queries yet.
+	case "SRV":
+		s, e := k.Records(r)
+		// SRV for external services is not yet implemented, so remove those records
+		noext := []msg.Service{}
+		for _, svc := range s {
+			if t, _ := svc.HostType(); t != dns.TypeCNAME {
+				noext = append(noext, svc)
+			}
+		}
+		return noext, nil, e
 	case "TXT":
 		err := k.recordsForTXT(r, &svcs)
 		return svcs, nil, err
@@ -373,6 +383,14 @@ func (k *Kubernetes) getRecordsForK8sItems(services []service, pods []pod, zone
 					Port: int(p.Port)}
 				records = append(records, s)
 			}
+			// If the addr is not an IP (i.e. an external service), add the record ...
+			s := msg.Service{
+				Key:  strings.Join([]string{zonePath, "svc", svc.namespace, svc.name}, "/"),
+				Host: svc.addr}
+			if t, _ := s.HostType(); t == dns.TypeCNAME {
+				records = append(records, s)
+			}
+
 		}
 	}
 
@@ -466,8 +484,16 @@ func (k *Kubernetes) findServices(r recordRequest) ([]service, error) {
 		if nsWildcard && (len(k.Namespaces) > 0) && (!dnsstrings.StringInSlice(svc.Namespace, k.Namespaces)) {
 			continue
 		}
-		s := service{name: svc.Name, namespace: svc.Namespace, addr: svc.Spec.ClusterIP}
-		if s.addr != api.ClusterIPNone {
+		s := service{name: svc.Name, namespace: svc.Namespace}
+		// External Service
+		if svc.Spec.ExternalName != "" {
+			s.addr = svc.Spec.ExternalName
+			resultItems = append(resultItems, s)
+			continue
+		}
+		// ClusterIP service
+		if svc.Spec.ClusterIP != api.ClusterIPNone {
+			s.addr = svc.Spec.ClusterIP
 			for _, p := range svc.Spec.Ports {
 				if !(symbolMatches(r.port, strings.ToLower(p.Name), portWildcard) && symbolMatches(r.protocol, strings.ToLower(string(p.Protocol)), protocolWildcard)) {
 					continue
@@ -478,6 +504,7 @@ func (k *Kubernetes) findServices(r recordRequest) ([]service, error) {
 			continue
 		}
 		// Headless service
+		s.addr = svc.Spec.ClusterIP
 		endpointsList := k.APIConn.EndpointsList()
 
 		for _, ep := range endpointsList.Items {

middleware/kubernetes/kubernetes_test.go

@@ -9,7 +9,9 @@ import (
 	"github.com/miekg/dns"
 	"k8s.io/client-go/1.5/pkg/api"
 
+	"github.com/coredns/coredns/middleware"
 	"github.com/coredns/coredns/middleware/etcd/msg"
+	"github.com/coredns/coredns/request"
 )
 
 func TestRecordForTXT(t *testing.T) {
@@ -277,3 +279,83 @@ func TestIpFromPodName(t *testing.T) {
 		}
 	}
 }
+
+type APIConnServiceTest struct{}
+
+func (APIConnServiceTest) Run() {
+	return
+}
+
+func (APIConnServiceTest) Stop() error {
+	return nil
+}
+
+func (APIConnServiceTest) ServiceList() []*api.Service {
+	svcs := []*api.Service{
+		{
+			ObjectMeta: api.ObjectMeta{
+				Name:      "external",
+				Namespace: "testns",
+			},
+			Spec: api.ServiceSpec{
+				ExternalName: "coredns.io",
+				Ports: []api.ServicePort{{
+					Name:     "http",
+					Protocol: "tcp",
+					Port:     80,
+				}},
+			},
+		},
+	}
+	return svcs
+
+}
+
+func (APIConnServiceTest) PodIndex(string) []interface{} {
+	return nil
+}
+
+func (APIConnServiceTest) EndpointsList() api.EndpointsList {
+	return api.EndpointsList{}
+}
+
+func TestServices(t *testing.T) {
+
+	k := Kubernetes{Zones: []string{"interwebs.test"}}
+	k.APIConn = &APIConnServiceTest{}
+
+	type svcAns struct {
+		host string
+		key  string
+	}
+	type svcTest struct {
+		qname  string
+		qtype  uint16
+		answer svcAns
+	}
+	tests := []svcTest{
+		// External Services
+		{qname: "external.testns.svc.interwebs.test.", qtype: dns.TypeCNAME, answer: svcAns{host: "coredns.io", key: "/coredns/test/interwebs/svc/testns/external"}},
+	}
+
+	for _, test := range tests {
+		state := request.Request{
+			Req: &dns.Msg{Question: []dns.Question{{Name: test.qname, Qtype: test.qtype}}},
+		}
+		svcs, _, e := k.Services(state, false, middleware.Options{})
+		if e != nil {
+			t.Errorf("Query '%v' got error '%v'", test.qname, e)
+		}
+		if len(svcs) != 1 {
+			t.Errorf("Query %v %v: expected expected 1 answer, got %v", test.qname, dns.TypeToString[test.qtype], len(svcs))
+		} else {
+			if test.answer.host != svcs[0].Host {
+				t.Errorf("Query %v %v: expected host '%v', got '%v'", test.qname, dns.TypeToString[test.qtype], test.answer.host, svcs[0].Host)
+			}
+			if test.answer.key != svcs[0].Key {
+				t.Errorf("Query %v %v: expected key '%v', got '%v'", test.qname, dns.TypeToString[test.qtype], test.answer.key, svcs[0].Key)
+			}
+		}
+	}
+
+}

middleware/kubernetes/setup.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/coredns/coredns/core/dnsserver"
 	"github.com/coredns/coredns/middleware"
+	"github.com/coredns/coredns/middleware/pkg/dnsutil"
+	"github.com/coredns/coredns/middleware/proxy"
 
 	"github.com/mholt/caddy"
 	unversionedapi "k8s.io/client-go/1.5/pkg/api/unversioned"
@@ -165,6 +167,16 @@ func kubernetesParse(c *caddy.Controller) (*Kubernetes, error) {
 						continue
 					}
 					return nil, c.ArgErr()
+				case "upstream":
+					args := c.RemainingArgs()
+					if len(args) == 0 {
+						return nil, c.ArgErr()
+					}
+					ups, err := dnsutil.ParseHostPortOrFile(args...)
+					if err != nil {
+						return nil, err
+					}
+					k8s.Proxy = proxy.NewLookup(ups)
 				}
 			}
 			return k8s, nil

middleware/kubernetes/setup_test.go

@@ -28,6 +28,7 @@ func TestKubernetesParse(t *testing.T) {
 		expectedPodMode       string
 		expectedCidrs         []net.IPNet
 		expectedFallthrough   bool
+		expectedUpstreams     []string
 	}{
 		// positive
 		{
@@ -42,6 +43,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"kubernetes keyword with multiple zones",
@@ -55,6 +57,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"kubernetes keyword with zone and empty braces",
@@ -69,6 +72,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"endpoint keyword with url",
@@ -84,6 +88,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"namespaces keyword with one namespace",
@@ -99,6 +104,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"namespaces keyword with multiple namespaces",
@@ -114,6 +120,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"resync period in seconds",
@@ -129,6 +136,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"resync period in minutes",
@@ -144,6 +152,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"basic label selector",
@@ -159,6 +168,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"multi-label selector",
@@ -174,6 +184,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"fully specified valid config",
@@ -193,6 +204,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			true,
+			nil,
 		},
 		// negative
 		{
@@ -207,6 +219,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"kubernetes keyword without a zone",
@@ -220,6 +233,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"endpoint keyword without an endpoint value",
@@ -235,6 +249,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"namespace keyword without a namespace value",
@@ -250,6 +265,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"resyncperiod keyword without a duration value",
@@ -265,6 +281,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"resync period no units",
@@ -280,6 +297,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"resync period invalid",
@@ -295,6 +313,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"labels with no selector value",
@@ -310,6 +329,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		{
 			"labels with invalid selector value",
@@ -325,6 +345,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		// pods disabled
 		{
@@ -341,6 +362,7 @@ func TestKubernetesParse(t *testing.T) {
 			PodModeDisabled,
 			nil,
 			false,
+			nil,
 		},
 		// pods insecure
 		{
@@ -357,6 +379,7 @@ func TestKubernetesParse(t *testing.T) {
 			PodModeInsecure,
 			nil,
 			false,
+			nil,
 		},
 		// pods verified
 		{
@@ -373,6 +396,7 @@ func TestKubernetesParse(t *testing.T) {
 			PodModeVerified,
 			nil,
 			false,
+			nil,
 		},
 		// pods invalid
 		{
@@ -389,6 +413,7 @@ func TestKubernetesParse(t *testing.T) {
 			PodModeVerified,
 			nil,
 			false,
+			nil,
 		},
 		// cidrs ok
 		{
@@ -405,6 +430,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			[]net.IPNet{parseCidr("10.0.0.0/24"), parseCidr("10.0.1.0/24")},
 			false,
+			nil,
 		},
 		// cidrs ok
 		{
@@ -421,6 +447,7 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
 		},
 		// fallthrough invalid
 		{
@@ -437,6 +464,41 @@ func TestKubernetesParse(t *testing.T) {
 			defaultPodMode,
 			nil,
 			false,
+			nil,
+		},
+		// Valid upstream
+		{
+			"valid upstream",
+			`kubernetes coredns.local {
+	upstream 13.14.15.16:53
+}`,
+			false,
+			"",
+			1,
+			0,
+			defaultResyncPeriod,
+			"",
+			defaultPodMode,
+			nil,
+			false,
+			[]string{"13.14.15.16:53"},
+		},
+		// Invalid upstream
+		{
+			"valid upstream",
+			`kubernetes coredns.local {
+	upstream 13.14.15.16orange
+}`,
+			true,
+			"not an IP address or file: \"13.14.15.16orange\"",
+			-1,
+			0,
+			defaultResyncPeriod,
+			"",
+			defaultPodMode,
+			nil,
+			false,
+			nil,
 		},
 	}
 
@@ -515,6 +577,28 @@ func TestKubernetesParse(t *testing.T) {
 		if foundFallthrough != test.expectedFallthrough {
 			t.Errorf("Test %d: Expected kubernetes controller to be initialized with fallthrough '%v'. Instead found fallthrough '%v' for input '%s'", i, test.expectedFallthrough, foundFallthrough, test.input)
 		}
+		// upstream
+		foundUpstreams := k8sController.Proxy.Upstreams
+		if test.expectedUpstreams == nil {
+			if foundUpstreams != nil {
+				t.Errorf("Test %d: Expected kubernetes controller to not be initialized with upstreams for input '%s'", i, test.input)
+			}
+		} else {
+			if foundUpstreams == nil {
+				t.Errorf("Test %d: Expected kubernetes controller to be initialized with upstreams for input '%s'", i, test.input)
+			} else {
+				if len(*foundUpstreams) != len(test.expectedUpstreams) {
+					t.Errorf("Test %d: Expected kubernetes controller to be initialized with %d upstreams. Instead found %d upstreams for input '%s'", i, len(test.expectedUpstreams), len(*foundUpstreams), test.input)
+				}
+				for j, want := range test.expectedUpstreams {
+					got := (*foundUpstreams)[j].Select().Name
+					if got != want {
+						t.Errorf("Test %d: Expected kubernetes controller to be initialized with upstream '%s'. Instead found upstream '%s' for input '%s'", i, want, got, test.input)
+					}
+				}
+
+			}
+		}
 
 	}
 }

test/kubernetes_test.go

@@ -62,6 +62,8 @@ var dnsTestCases = []test.Case{
 			test.A("svc-c.test-1.svc.cluster.local.        303    IN      A       10.0.0.115"),
 			test.A("headless-svc.test-1.svc.cluster.local.      303    IN      A       172.17.0.5"),
 			test.A("headless-svc.test-1.svc.cluster.local.      303    IN      A       172.17.0.6"),
+			test.CNAME("ext-svc.test-1.svc.cluster.local. 0 IN	CNAME	example.net."),
+			test.A("example.net.		68974	IN	A	13.14.15.16"),
 		},
 	},
 	{
@@ -73,6 +75,8 @@ var dnsTestCases = []test.Case{
 			test.A("svc-c.test-1.svc.cluster.local.        303    IN      A       10.0.0.115"),
 			test.A("headless-svc.test-1.svc.cluster.local.      303    IN      A       172.17.0.5"),
 			test.A("headless-svc.test-1.svc.cluster.local.      303    IN      A       172.17.0.6"),
+			test.CNAME("ext-svc.test-1.svc.cluster.local. 0 IN	CNAME	example.net."),
+			test.A("example.net.		68974	IN	A	13.14.15.16"),
 		},
 	},
 	{
@@ -94,6 +98,8 @@ var dnsTestCases = []test.Case{
 			test.A("svc-c.test-1.svc.cluster.local.        303    IN      A       10.0.0.115"),
 			test.A("headless-svc.test-1.svc.cluster.local.      303    IN      A       172.17.0.5"),
 			test.A("headless-svc.test-1.svc.cluster.local.      303    IN      A       172.17.0.6"),
+			test.CNAME("ext-svc.test-1.svc.cluster.local. 0 IN	CNAME	example.net."),
+			test.A("example.net.		68974	IN	A	13.14.15.16"),
 		},
 	},
 	{
@@ -104,7 +110,6 @@ var dnsTestCases = []test.Case{
 			test.A("headless-svc.test-1.svc.cluster.local.      303    IN      A       172.17.0.6"),
 		},
 	},
-	//TODO: Fix below to all use test.SRV not test.A!
 	{
 		Qname: "*._TcP.svc-1-a.test-1.svc.cluster.local.", Qtype: dns.TypeSRV,
 		Rcode: dns.RcodeSuccess,
@@ -240,6 +245,21 @@ var dnsTestCases = []test.Case{
 			test.NS("cluster.local.          0       IN      NS      kubernetes.default.svc.cluster.local."),
 		},
 	},
+	{
+		Qname: "ext-svc.test-1.svc.cluster.local.", Qtype: dns.TypeA,
+		Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.CNAME("ext-svc.test-1.svc.cluster.local. 0 IN	CNAME	example.net."),
+			test.A("example.net.		72031	IN	A	13.14.15.16"),
+		},
+	},
+	{
+		Qname: "ext-svc.test-1.svc.cluster.local.", Qtype: dns.TypeCNAME,
+		Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.CNAME("ext-svc.test-1.svc.cluster.local. 0 IN	CNAME	example.net."),
+		},
+	},
 }
 
 var dnsTestCasesPodsInsecure = []test.Case{
@@ -444,13 +464,34 @@ func doIntegrationTests(t *testing.T, corefile string, testCases []test.Case) {
 	}
 }
 
+func createUpstreamServer(t *testing.T) (func(), *caddy.Instance, string) {
+	upfile, rmfile, err := TempFile(os.TempDir(), exampleNet)
+	if err != nil {
+		t.Fatalf("Could not create file for CNAME upstream lookups: %s", err)
+	}
+	upstreamServerCorefile := `.:0 {
+    file ` + upfile + ` example.net
+	erratic . {
+		drop 0
+	}
+	`
+	server, udp := createTestServer(t, upstreamServerCorefile)
+	return rmfile, server, udp
+}
+
 func TestKubernetesIntegration(t *testing.T) {
+
+	removeUpstreamConfig, upstreamServer, udp := createUpstreamServer(t)
+	defer upstreamServer.Stop()
+	defer removeUpstreamConfig()
+
 	corefile :=
 		`.:0 {
     kubernetes cluster.local 0.0.10.in-addr.arpa {
                 endpoint http://localhost:8080
 		namespaces test-1
 		pods disabled
+		upstream ` + udp + `
     }
 	erratic . {
 		drop 0
@@ -530,6 +571,11 @@ func TestKubernetesIntegrationFallthrough(t *testing.T) {
 		t.Fatalf("Could not create TempFile for fallthrough: %s", err)
 	}
 	defer rmFunc()
+
+	removeUpstreamConfig, upstreamServer, udp := createUpstreamServer(t)
+	defer upstreamServer.Stop()
+	defer removeUpstreamConfig()
+
 	corefile :=
 		`.:0 {
     file ` + dbfile + ` cluster.local
@@ -537,6 +583,7 @@ func TestKubernetesIntegrationFallthrough(t *testing.T) {
                 endpoint http://localhost:8080
 		cidrs 10.0.0.0/24
 		namespaces test-1
+		upstream ` + udp + `
 		fallthrough
     }
     erratic {
@@ -561,3 +608,7 @@ cname.cluster.local.    IN      CNAME   www.example.net.
 
 service.namespace.svc.cluster.local.    IN      SRV     8080 10 10 cluster.local.
 `
+
+const exampleNet = `; example.net. test file for cname tests
+example.net. IN A 13.14.15.16
+`