create pkg/reuseport (#3455)

* create pkg/reuseport

Move the core server listening functions to a new package so plugins can
use them.

Also make *all* servers use the functions here; as only the udp/tcp
listeners where using SO_REUSEPORT (if available). This is the only
actual change in this PR; in it's core it's just a move of 2 files.

This can also be used to cleanup the dance we're doing now for
re-acquiring the sockets in e.g. the metrics plugins and the ready
plugin.
Signed-off-by: Miek Gieben <miek@miek.nl>

* Also push a small doc update
Signed-off-by: Miek Gieben <miek@miek.nl>

core/dnsserver/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/coredns/coredns/plugin/pkg/edns"
 	"github.com/coredns/coredns/plugin/pkg/log"
 	"github.com/coredns/coredns/plugin/pkg/rcode"
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/trace"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 	"github.com/coredns/coredns/request"
@@ -126,7 +127,7 @@ func (s *Server) ServePacket(p net.PacketConn) error {
 
 // Listen implements caddy.TCPServer interface.
 func (s *Server) Listen() (net.Listener, error) {
-	l, err := listen("tcp", s.Addr[len(transport.DNS+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.DNS+"://"):])
 	if err != nil {
 		return nil, err
 	}
@@ -140,7 +141,7 @@ func (s *Server) WrapListener(ln net.Listener) net.Listener {
 
 // ListenPacket implements caddy.UDPServer interface.
 func (s *Server) ListenPacket() (net.PacketConn, error) {
-	p, err := listenPacket("udp", s.Addr[len(transport.DNS+"://"):])
+	p, err := reuseport.ListenPacket("udp", s.Addr[len(transport.DNS+"://"):])
 	if err != nil {
 		return nil, err
 	}

core/dnsserver/server_grpc.go

@@ -8,6 +8,7 @@ import (
 	"net"
 
 	"github.com/coredns/coredns/pb"
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 
 	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
@@ -72,7 +73,7 @@ func (s *ServergRPC) ServePacket(p net.PacketConn) error { return nil }
 // Listen implements caddy.TCPServer interface.
 func (s *ServergRPC) Listen() (net.Listener, error) {
 
-	l, err := net.Listen("tcp", s.Addr[len(transport.GRPC+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.GRPC+"://"):])
 	if err != nil {
 		return nil, err
 	}

core/dnsserver/server_https.go

@@ -12,6 +12,7 @@ import (
 	"github.com/coredns/coredns/plugin/pkg/dnsutil"
 	"github.com/coredns/coredns/plugin/pkg/doh"
 	"github.com/coredns/coredns/plugin/pkg/response"
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 )
 
@@ -61,7 +62,7 @@ func (s *ServerHTTPS) ServePacket(p net.PacketConn) error { return nil }
 // Listen implements caddy.TCPServer interface.
 func (s *ServerHTTPS) Listen() (net.Listener, error) {
 
-	l, err := net.Listen("tcp", s.Addr[len(transport.HTTPS+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.HTTPS+"://"):])
 	if err != nil {
 		return nil, err
 	}

core/dnsserver/server_tls.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 
 	"github.com/miekg/dns"
@@ -57,7 +58,7 @@ func (s *ServerTLS) ServePacket(p net.PacketConn) error { return nil }
 
 // Listen implements caddy.TCPServer interface.
 func (s *ServerTLS) Listen() (net.Listener, error) {
-	l, err := net.Listen("tcp", s.Addr[len(transport.TLS+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.TLS+"://"):])
 	if err != nil {
 		return nil, err
 	}

plugin.md

@@ -64,6 +64,11 @@ a *Metrics* section detailing the metrics.
 If the plugin supports signalling readiness it should have a *Ready* section detailing how it
 works, and implement the `ready.Readiness` interface.
 
+## Opening Sockets
+
+See the plugin/pkg/reuseport for `Listen` and `ListenPacket` functions. Using these functions makes
+you plugin handle reload events better.
+
 ## Documentation
 
 Each plugin should have a README.md explaining what the plugin does and how it is configured. The

core/dnsserver/listen_go111.go → plugin/pkg/reuseport/listen_go111.go

 // +build go1.11
 // +build aix darwin dragonfly freebsd linux netbsd openbsd
 
-package dnsserver
+package reuseport
 
 import (
 	"context"
@@ -13,7 +13,7 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func reuseportControl(network, address string, c syscall.RawConn) error {
+func control(network, address string, c syscall.RawConn) error {
 	c.Control(func(fd uintptr) {
 		if err := unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
 			log.Warningf("Failed to set SO_REUSEPORT on socket: %s", err)
@@ -22,12 +22,16 @@ func reuseportControl(network, address string, c syscall.RawConn) error {
 	return nil
 }
 
-func listen(network, addr string) (net.Listener, error) {
-	lc := net.ListenConfig{Control: reuseportControl}
+// Listen announces on the local network address. See net.Listen for more information.
+// If SO_REUSEPORT is available it will be set on the socket.
+func Listen(network, addr string) (net.Listener, error) {
+	lc := net.ListenConfig{Control: control}
 	return lc.Listen(context.Background(), network, addr)
 }
 
-func listenPacket(network, addr string) (net.PacketConn, error) {
-	lc := net.ListenConfig{Control: reuseportControl}
+// ListenPacket announces on the local network address. See net.ListenPacket for more information.
+// If SO_REUSEPORT is available it will be set on the socket.
+func ListenPacket(network, addr string) (net.PacketConn, error) {
+	lc := net.ListenConfig{Control: control}
 	return lc.ListenPacket(context.Background(), network, addr)
 }

core/dnsserver/listen_go_not111.go → plugin/pkg/reuseport/listen_go_not111.go

 // +build !go1.11 !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd
 
-package dnsserver
+package reuseport
 
 import "net"
 
-func listen(network, addr string) (net.Listener, error) { return net.Listen(network, addr) }
+// Listen is a wrapper around net.Listen.
+func Listen(network, addr string) (net.Listener, error) { return net.Listen(network, addr) }
 
-func listenPacket(network, addr string) (net.PacketConn, error) {
+// ListenPacket is a wrapper around net.ListenPacket.
+func ListenPacket(network, addr string) (net.PacketConn, error) {
 	return net.ListenPacket(network, addr)
 }