fix record missing for zone with many of records (#4328)

* fix record missing for zone with many of records

* Update debug log

* Update debug mesg

* update test_clouddns

plugin/clouddns/clouddns.go

@@ -189,7 +189,7 @@ func (h *CloudDNS) updateZones(ctx context.Context) error {
 			for i, hostedZone := range z {
 				newZ := file.NewZone(zName, "")
 				newZ.Upstream = h.upstream
-				rrListResponse, err = h.client.listRRSets(hostedZone.projectName, hostedZone.zoneName)
+				rrListResponse, err = h.client.listRRSets(ctx, hostedZone.projectName, hostedZone.zoneName)
 				if err != nil {
 					err = fmt.Errorf("failed to list resource records for %v:%v:%v from gcp: %v", zName, hostedZone.projectName, hostedZone.zoneName, err)
 					return

plugin/clouddns/clouddns_test.go

@@ -24,7 +24,7 @@ func (c fakeGCPClient) zoneExists(projectName, hostedZoneName string) error {
 	return nil
 }
 
-func (c fakeGCPClient) listRRSets(projectName, hostedZoneName string) (*gcp.ResourceRecordSetsListResponse, error) {
+func (c fakeGCPClient) listRRSets(ctx context.Context, projectName, hostedZoneName string) (*gcp.ResourceRecordSetsListResponse, error) {
 	if projectName == "bad-project" || hostedZoneName == "bad-zone" {
 		return nil, errors.New("the 'parameters.managedZone' resource named 'bad-zone' does not exist")
 	}

plugin/clouddns/gcp.go

 package clouddns
 
-import gcp "google.golang.org/api/dns/v1"
+import (
+	"context"
+
+	gcp "google.golang.org/api/dns/v1"
+)
 
 type gcpDNS interface {
 	zoneExists(projectName, hostedZoneName string) error
-	listRRSets(projectName, hostedZoneName string) (*gcp.ResourceRecordSetsListResponse, error)
+	listRRSets(ctx context.Context, projectName, hostedZoneName string) (*gcp.ResourceRecordSetsListResponse, error)
 }
 
 type gcpClient struct {
@@ -23,10 +27,16 @@ func (c gcpClient) zoneExists(projectName, hostedZoneName string) error {
 
 // listRRSets is a wrapper method around `gcp.Service.ResourceRecordSets.List`
 // it fetches and returns the record sets for a hosted zone.
-func (c gcpClient) listRRSets(projectName, hostedZoneName string) (*gcp.ResourceRecordSetsListResponse, error) {
-	rr, err := c.ResourceRecordSets.List(projectName, hostedZoneName).Do()
-	if err != nil {
+func (c gcpClient) listRRSets(ctx context.Context, projectName, hostedZoneName string) (*gcp.ResourceRecordSetsListResponse, error) {
+	req := c.ResourceRecordSets.List(projectName, hostedZoneName)
+	var rs []*gcp.ResourceRecordSet
+	if err := req.Pages(ctx, func(page *gcp.ResourceRecordSetsListResponse) error {
+		for _, rr := range page.Rrsets {
+			rs = append(rs, rr)
+		}
+		return nil
+	}); err != nil {
 		return nil, err
 	}
-	return rr, nil
+	return &gcp.ResourceRecordSetsListResponse{Rrsets: rs}, nil
 }