Client-side of gRPC proxy (#511)

* WIP: Client-side of gRPC proxy

* Add tests

* gofmt

* Implement OnShutdown; add a little logging

* Update for context in Exchange change

* go fmt

* Update README

* Review comments

* Compiling is good

* More README improvements

middleware/proxy/README.md

@@ -26,7 +26,7 @@ proxy FROM TO... {
     health_check PATH:PORT [DURATION]
     except IGNORED_NAMES...
     spray
-    protocol [dns|https_google [bootstrap ADDRESS...]]
+    protocol [dns|https_google [bootstrap ADDRESS...]|grpc [insecure|CA-PEM|KEY-PEM CERT-PEM|KEY-PEM CERT-PEM CA-PEM]]
 }
 ~~~
 
@@ -40,7 +40,8 @@ proxy FROM TO... {
 * `spray` when all backends are unhealthy, randomly pick one to send the traffic to. (This is a failsafe.)
 * `protocol` specifies what protocol to use to speak to an upstream, `dns` (the default) is plain old DNS, and
   `https_google` uses `https://dns.google.com` and speaks a JSON DNS dialect. Note when using this
-  **TO** must be `dns.google.com`.
+  **TO** must be `dns.google.com`. The `grpc` option will talk to a server that has implemented the DnsService defined
+  in https://github.com/miekg/coredns/middleware/proxy/pb/dns.proto.
 
 ## Policies
 
@@ -82,6 +83,16 @@ example.org.		1799	IN	SOA	sns.dns.icann.org. noc.dns.icann.org. 2016110711 7200
 ;; ADDITIONAL SECTION:
 .			0	CH	TXT	"Response from 199.43.133.53"
 ~~~
+* `grpc`: options are used to control how the TLS connection is made to the gRPC server.
+  * None - No client authentication is used, and the system CAs are used to verify the server certificate.
+  * `insecure` - TLS is not used, the connection is made in plaintext (not good in production).
+  * CA-PEM - No client authentication is used, and the file CA-PEM is used to verify the server certificate.
+  * KEY-PEM CERT-PEM - Client authentication is used with the specified key/cert pair. The server certificate is verified
+    with the system CAs.
+  * KEY-PEM CERT-PEM CA-PEM - Client authentication is used with the specified key/cert pair. The server certificate is
+    verified using the CA-PEM file.
+
+  An out-of-tree middleware that implements the server side of this can be found at https://github.com/infobloxopen/coredns-grpc.
 
 ## Metrics
 

middleware/proxy/grpc.go

+package proxy
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+
+	"github.com/miekg/coredns/middleware/proxy/pb"
+	"github.com/miekg/coredns/request"
+
+	"github.com/miekg/dns"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+)
+
+type grpcClient struct {
+	dialOpt  grpc.DialOption
+	clients  map[string]pb.DnsServiceClient
+	conns    []*grpc.ClientConn
+	upstream *staticUpstream
+}
+
+func newGrpcClient(tls *tls.Config, u *staticUpstream) *grpcClient {
+	g := &grpcClient{upstream: u}
+
+	if tls == nil {
+		g.dialOpt = grpc.WithInsecure()
+	} else {
+		g.dialOpt = grpc.WithTransportCredentials(credentials.NewTLS(tls))
+	}
+	g.clients = map[string]pb.DnsServiceClient{}
+
+	return g
+}
+
+func (g *grpcClient) Exchange(ctx context.Context, addr string, state request.Request) (*dns.Msg, error) {
+	msg, err := state.Req.Pack()
+	if err != nil {
+		return nil, err
+	}
+
+	reply, err := g.clients[addr].Query(ctx, &pb.DnsPacket{Msg: msg})
+	if err != nil {
+		return nil, err
+	}
+	d := new(dns.Msg)
+	err = d.Unpack(reply.Msg)
+	if err != nil {
+		return nil, err
+	}
+	return d, nil
+}
+
+func (g *grpcClient) Protocol() string { return "grpc" }
+
+func (g *grpcClient) OnShutdown(p *Proxy) error {
+	for i, conn := range g.conns {
+		err := conn.Close()
+		if err != nil {
+			log.Printf("[WARNING] Error closing connection %d: %s\n", i, err)
+		}
+	}
+	return nil
+}
+
+func (g *grpcClient) OnStartup(p *Proxy) error {
+	for _, host := range g.upstream.Hosts {
+		conn, err := grpc.Dial(host.Name, g.dialOpt)
+		if err != nil {
+			log.Printf("[WARNING] Skipping gRPC host '%s' due to Dial error: %s\n", host.Name, err)
+		} else {
+			g.clients[host.Name] = pb.NewDnsServiceClient(conn)
+			g.conns = append(g.conns, conn)
+		}
+	}
+	return nil
+}

middleware/proxy/pb/dns.pb.go

+// Code generated by protoc-gen-go.
+// source: dns.proto
+// DO NOT EDIT!
+
+/*
+Package pb is a generated protocol buffer package.
+
+It is generated from these files:
+	dns.proto
+
+It has these top-level messages:
+	DnsPacket
+*/
+package pb
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+	context "golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type DnsPacket struct {
+	Msg []byte `protobuf:"bytes,1,opt,name=msg,proto3" json:"msg,omitempty"`
+}
+
+func (m *DnsPacket) Reset()                    { *m = DnsPacket{} }
+func (m *DnsPacket) String() string            { return proto.CompactTextString(m) }
+func (*DnsPacket) ProtoMessage()               {}
+func (*DnsPacket) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *DnsPacket) GetMsg() []byte {
+	if m != nil {
+		return m.Msg
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*DnsPacket)(nil), "coredns.dns.DnsPacket")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for DnsService service
+
+type DnsServiceClient interface {
+	Query(ctx context.Context, in *DnsPacket, opts ...grpc.CallOption) (*DnsPacket, error)
+}
+
+type dnsServiceClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewDnsServiceClient(cc *grpc.ClientConn) DnsServiceClient {
+	return &dnsServiceClient{cc}
+}
+
+func (c *dnsServiceClient) Query(ctx context.Context, in *DnsPacket, opts ...grpc.CallOption) (*DnsPacket, error) {
+	out := new(DnsPacket)
+	err := grpc.Invoke(ctx, "/coredns.dns.DnsService/Query", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for DnsService service
+
+type DnsServiceServer interface {
+	Query(context.Context, *DnsPacket) (*DnsPacket, error)
+}
+
+func RegisterDnsServiceServer(s *grpc.Server, srv DnsServiceServer) {
+	s.RegisterService(&_DnsService_serviceDesc, srv)
+}
+
+func _DnsService_Query_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DnsPacket)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(DnsServiceServer).Query(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/coredns.dns.DnsService/Query",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(DnsServiceServer).Query(ctx, req.(*DnsPacket))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _DnsService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "coredns.dns.DnsService",
+	HandlerType: (*DnsServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Query",
+			Handler:    _DnsService_Query_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "dns.proto",
+}
+
+func init() { proto.RegisterFile("dns.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+	// 120 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0xc9, 0x2b, 0xd6,
+	0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4e, 0xce, 0x2f, 0x4a, 0x05, 0x71, 0x53, 0xf2, 0x8a,
+	0x95, 0x64, 0xb9, 0x38, 0x5d, 0xf2, 0x8a, 0x03, 0x12, 0x93, 0xb3, 0x53, 0x4b, 0x84, 0x04, 0xb8,
+	0x98, 0x73, 0x8b, 0xd3, 0x25, 0x18, 0x15, 0x18, 0x35, 0x78, 0x82, 0x40, 0x4c, 0x23, 0x57, 0x2e,
+	0x2e, 0x97, 0xbc, 0xe2, 0xe0, 0xd4, 0xa2, 0xb2, 0xcc, 0xe4, 0x54, 0x21, 0x73, 0x2e, 0xd6, 0xc0,
+	0xd2, 0xd4, 0xa2, 0x4a, 0x21, 0x31, 0x3d, 0x24, 0x33, 0xf4, 0xe0, 0x06, 0x48, 0xe1, 0x10, 0x77,
+	0x62, 0x89, 0x62, 0x2a, 0x48, 0x4a, 0x62, 0x03, 0xdb, 0x6f, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff,
+	0xf5, 0xd1, 0x3f, 0x26, 0x8c, 0x00, 0x00, 0x00,
+}

middleware/proxy/pb/dns.proto

+syntax = "proto3";
+
+package coredns.dns;
+option go_package = "pb";
+
+message DnsPacket {
+	bytes msg = 1;
+}
+
+service DnsService {
+	rpc Query (DnsPacket) returns (DnsPacket);
+}

middleware/proxy/upstream.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/miekg/coredns/middleware"
 	"github.com/miekg/coredns/middleware/pkg/dnsutil"
+	"github.com/miekg/coredns/middleware/pkg/tls"
 
 	"github.com/mholt/caddy/caddyfile"
 	"github.com/miekg/dns"
@@ -197,6 +198,16 @@ func parseBlock(c *caddyfile.Dispenser, u *staticUpstream) error {
 			}
 
 			u.ex = newGoogle("", boot) // "" for default in google.go
+		case "grpc":
+			if len(encArgs) == 2 && encArgs[1] == "insecure" {
+				u.ex = newGrpcClient(nil, u)
+				return nil
+			}
+			tls, err := tls.NewTLSConfigFromArgs(encArgs[1:]...)
+			if err != nil {
+				return err
+			}
+			u.ex = newGrpcClient(tls, u)
 		default:
 			return fmt.Errorf("%s: %s", errInvalidProtocol, encArgs[0])
 		}

middleware/proxy/upstream_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/miekg/coredns/middleware/test"
+
 	"github.com/mholt/caddy"
 )
 
@@ -96,6 +98,14 @@ func writeTmpFile(t *testing.T, data string) (string, string) {
 }
 
 func TestProxyParse(t *testing.T) {
+	rmFunc, cert, key, ca := getPEMFiles(t)
+	defer rmFunc()
+
+	grpc1 := "proxy . 8.8.8.8:53 {\n protocol grpc " + ca + "\n}"
+	grpc2 := "proxy . 8.8.8.8:53 {\n protocol grpc " + cert + " " + key + "\n}"
+	grpc3 := "proxy . 8.8.8.8:53 {\n protocol grpc " + cert + " " + key + " " + ca + "\n}"
+	grpc4 := "proxy . 8.8.8.8:53 {\n protocol grpc " + key + "\n}"
+
 	tests := []struct {
 		inputUpstreams string
 		shouldErr      bool
@@ -174,8 +184,84 @@ proxy . 8.8.8.8:53 {
 		},
 		{
 			`
+proxy . 8.8.8.8:53 {
+	protocol grpc
+}`,
+			false,
+		},
+		{
+			`
+proxy . 8.8.8.8:53 {
+	protocol grpc insecure
+}`,
+			false,
+		},
+		{
+			`
+proxy . 8.8.8.8:53 {
+	protocol grpc a b c d
+}`,
+			true,
+		},
+		{
+			grpc1,
+			false,
+		},
+		{
+			grpc2,
+			false,
+		},
+		{
+			grpc3,
+			false,
+		},
+		{
+			grpc4,
+			true,
+		},
+		{
+			`
+proxy . 8.8.8.8:53 {
+	protocol foobar
+}`,
+			true,
+		},
+		{
+			`proxy`,
+			true,
+		},
+		{
+			`
 proxy . 8.8.8.8:53 {
 	protocol foobar
+}`,
+			true,
+		},
+		{
+			`
+proxy . 8.8.8.8:53 {
+	policy
+}`,
+			true,
+		},
+		{
+			`
+proxy . 8.8.8.8:53 {
+	fail_timeout
+}`,
+			true,
+		},
+		{
+			`
+proxy . 8.8.8.8:53 {
+	fail_timeout junky
+}`,
+			true,
+		},
+		{
+			`
+proxy . 8.8.8.8:53 {
+	health_check
 }`,
 			true,
 		},
@@ -184,7 +270,7 @@ proxy . 8.8.8.8:53 {
 		c := caddy.NewTestController("dns", test.inputUpstreams)
 		_, err := NewStaticUpstreams(&c.Dispenser)
 		if (err != nil) != test.shouldErr {
-			t.Errorf("Test %d expected no error, got %v", i+1, err)
+			t.Errorf("Test %d expected no error, got %v for %s", i+1, err, test.inputUpstreams)
 		}
 	}
 }
@@ -264,3 +350,16 @@ junky resolve.conf
 		}
 	}
 }
+
+func getPEMFiles(t *testing.T) (rmFunc func(), cert, key, ca string) {
+	tempDir, rmFunc, err := test.WritePEMFiles("")
+	if err != nil {
+		t.Fatalf("Could not write PEM files: %s", err)
+	}
+
+	cert = filepath.Join(tempDir, "cert.pem")
+	key = filepath.Join(tempDir, "key.pem")
+	ca = filepath.Join(tempDir, "ca.pem")
+
+	return
+}