plugin/secondary: Retry initial transfer until successful (#4663)

* retry initial transfer
Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* fix import grouping
Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* add test; use backoff timeout
Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* fix import order
Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* manual backoff
Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

plugin/secondary/setup.go

 package secondary
 
 import (
+	"time"
+
 	"github.com/coredns/caddy"
 	"github.com/coredns/coredns/core/dnsserver"
 	"github.com/coredns/coredns/plugin"
 	"github.com/coredns/coredns/plugin/file"
+	clog "github.com/coredns/coredns/plugin/pkg/log"
 	"github.com/coredns/coredns/plugin/pkg/parse"
 	"github.com/coredns/coredns/plugin/pkg/upstream"
 )
 
+var log = clog.NewWithPlugin("secondary")
+
 func init() { plugin.Register("secondary", setup) }
 
 func setup(c *caddy.Controller) error {
@@ -24,7 +29,21 @@ func setup(c *caddy.Controller) error {
 			c.OnStartup(func() error {
 				z.StartupOnce.Do(func() {
 					go func() {
-						z.TransferIn()
+						dur := time.Millisecond * 250
+						step := time.Duration(2)
+						max := time.Second * 10
+						for {
+							err := z.TransferIn()
+							if err == nil {
+								break
+							}
+							log.Warningf("All '%s' masters failed to transfer, retrying in %s: %s", n, dur.String(), err)
+							time.Sleep(dur)
+							dur = step * dur
+							if dur > max {
+								dur = max
+							}
+						}
 						z.Update()
 					}()
 				})

test/secondary_test.go

@@ -132,3 +132,87 @@ func TestIxfrResponse(t *testing.T) {
 		t.Fatalf("Serial should be %d, got %d", 2015082541, soa.Serial)
 	}
 }
+
+func TestRetryInitialTransfer(t *testing.T) {
+	// Start up a secondary that expects to transfer from a master that doesn't exist yet
+	corefile := `example.org:0 {
+		secondary {
+			transfer from 127.0.0.1:5399
+		}
+	}`
+
+	i, udp, _, err := CoreDNSServerAndPorts(corefile)
+	if err != nil {
+		t.Fatalf("Could not get CoreDNS serving instance: %s", err)
+	}
+	defer i.Stop()
+
+	m := new(dns.Msg)
+	m.SetQuestion("www.example.org.", dns.TypeA)
+	resp, err := dns.Exchange(m, udp)
+	if err != nil {
+		t.Fatal("Expected to receive reply, but didn't")
+	}
+	// Expect that the query will fail
+	if resp.Rcode != dns.RcodeServerFailure {
+		t.Fatalf("Expected reply to be a SERVFAIL, got %d", resp.Rcode)
+	}
+
+	// Now spin up the master server
+	name, rm, err := test.TempFile(".", `$ORIGIN example.org.
+@ 3600 IN SOA  sns.dns.icann.org. noc.dns.icann.org. (
+        2017042745 ; serial
+        7200       ; refresh (2 hours)
+        3600       ; retry (1 hour)
+        1209600    ; expire (2 weeks)
+        3600       ; minimum (1 hour)
+)
+
+  3600 IN NS   a.iana-servers.net.
+  3600 IN NS   b.iana-servers.net.
+
+www    IN A    127.0.0.1
+www    IN AAAA ::1
+`)
+	if err != nil {
+		t.Fatalf("Failed to create zone: %s", err)
+	}
+	defer rm()
+
+	corefileMaster := `example.org:5399 {
+		file ` + name + `
+		transfer {
+          to *
+        }
+	}`
+
+	master, _, _, err := CoreDNSServerAndPorts(corefileMaster)
+	if err != nil {
+		t.Fatalf("Could not start CoreDNS master: %s", err)
+	}
+	defer master.Stop()
+
+	retry := time.Tick(time.Millisecond * 100)
+	timeout := time.Tick(time.Second * 5)
+
+	for {
+		select {
+		case <-retry:
+			m = new(dns.Msg)
+			m.SetQuestion("www.example.org.", dns.TypeA)
+			resp, err = dns.Exchange(m, udp)
+			if err != nil {
+				continue
+			}
+			// Expect the query to succeed
+			if resp.Rcode != dns.RcodeSuccess {
+				continue
+			}
+			return
+		case <-timeout:
+			t.Fatal("Timed out trying for successful response.")
+			return
+		}
+	}
+
+}