fix: Add pagination to AzureDevops modified files call (#5298)

Signed-off-by: Petr Bubenik <petr.bubenik@drmax.eu>

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6
 	github.com/jpillora/backoff v1.0.0
 	github.com/kr/pretty v0.3.1
-	github.com/mcdafydd/go-azuredevops v0.12.1
+	github.com/drmaxgit/go-azuredevops v0.13.1
 	github.com/microcosm-cc/bluemonday v1.0.27
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
 	github.com/mitchellh/go-homedir v1.1.0

go.sum

@@ -114,6 +114,8 @@ github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454Wv
 github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/drmaxgit/go-azuredevops v0.13.1 h1:mqRxiwnbKraHAtqfTFa+xD75Atf84MgpjKJdA8hJeS4=
+github.com/drmaxgit/go-azuredevops v0.13.1/go.mod h1:m1pO2fW60I9FahzLHMmHYq3bM446ZMZKDpd8+AEKzxc=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

server/controllers/events/azuredevops_request_validator.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 )
 
 //go:generate pegomock generate --package mocks -o mocks/mock_azuredevops_request_validator.go AzureDevopsRequestValidator

server/controllers/events/events_controller.go

@@ -22,8 +22,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/google/go-github/v68/github"
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
 	"github.com/microcosm-cc/bluemonday"
 	"github.com/pkg/errors"
 	"github.com/runatlantis/atlantis/server/events"

server/controllers/events/events_controller_test.go

@@ -25,8 +25,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/google/go-github/v68/github"
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
 	. "github.com/petergtz/pegomock/v4"
 	events_controllers "github.com/runatlantis/atlantis/server/controllers/events"
 	"github.com/runatlantis/atlantis/server/controllers/events/mocks"

server/events/command_runner.go

@@ -17,8 +17,8 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/google/go-github/v68/github"
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
 	"github.com/pkg/errors"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 	"github.com/runatlantis/atlantis/server/events/command"

server/events/event_parser.go

@@ -23,10 +23,10 @@ import (
 
 	giteasdk "code.gitea.io/sdk/gitea"
 
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/go-playground/validator/v10"
 	"github.com/google/go-github/v68/github"
 	lru "github.com/hashicorp/golang-lru/v2"
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
 	"github.com/pkg/errors"
 	"github.com/runatlantis/atlantis/server/events/command"
 	"github.com/runatlantis/atlantis/server/events/models"

server/events/event_parser_test.go

@@ -21,8 +21,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/google/go-github/v68/github"
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
 	"github.com/mohae/deepcopy"
 	"github.com/runatlantis/atlantis/server/events"
 	"github.com/runatlantis/atlantis/server/events/command"

server/events/mocks/mock_azuredevops_pull_getter.go

@@ -4,7 +4,7 @@
 package mocks
 
 import (
-	azuredevops "github.com/mcdafydd/go-azuredevops/azuredevops"
+	azuredevops "github.com/drmaxgit/go-azuredevops/azuredevops"
 	pegomock "github.com/petergtz/pegomock/v4"
 	models "github.com/runatlantis/atlantis/server/events/models"
 	logging "github.com/runatlantis/atlantis/server/logging"

server/events/mocks/mock_event_parsing.go

@@ -5,8 +5,8 @@ package mocks
 
 import (
 	gitea "code.gitea.io/sdk/gitea"
+	azuredevops "github.com/drmaxgit/go-azuredevops/azuredevops"
 	github "github.com/google/go-github/v68/github"
-	azuredevops "github.com/mcdafydd/go-azuredevops/azuredevops"
 	pegomock "github.com/petergtz/pegomock/v4"
 	models "github.com/runatlantis/atlantis/server/events/models"
 	gitea0 "github.com/runatlantis/atlantis/server/events/vcs/gitea"

server/events/vcs/azuredevops_client.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/pkg/errors"
 	"github.com/runatlantis/atlantis/server/events/models"
 	"github.com/runatlantis/atlantis/server/events/vcs/common"
@@ -68,28 +68,40 @@ func (g *AzureDevopsClient) GetModifiedFiles(logger logging.SimpleLogging, repo
 	targetRefName := strings.Replace(pullRequest.GetTargetRefName(), "refs/heads/", "", 1)
 	sourceRefName := strings.Replace(pullRequest.GetSourceRefName(), "refs/heads/", "", 1)
 
-	r, resp, err := g.Client.Git.GetDiffs(g.ctx, owner, project, repoName, targetRefName, sourceRefName)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting pull request")
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, errors.Wrapf(err, "http response code %d getting diff %s to %s", resp.StatusCode, sourceRefName, targetRefName)
-	}
+	const pageSize = 100 // Number of files from diff call
+	var skip int
 
-	for _, change := range r.Changes {
-		item := change.GetItem()
-		// Convert the path to a relative path from the repo's root.
-		relativePath := filepath.Clean("./" + item.GetPath())
-		files = append(files, relativePath)
-
-		// If the file was renamed, we'll want to run plan in the directory
-		// it was moved from as well.
-		changeType := azuredevops.Rename.String()
-		if change.ChangeType == &changeType {
-			// Convert the path to a relative path from the repo's root.
-			relativePath = filepath.Clean("./" + change.GetSourceServerItem())
-			files = append(files, relativePath)
+	for {
+		r, resp, err := g.Client.Git.GetDiffs(g.ctx, owner, project, repoName, targetRefName, sourceRefName, &azuredevops.GitDiffListOptions{
+			Top:  pageSize,
+			Skip: skip,
+		})
+		if err != nil {
+			return nil, errors.Wrap(err, "getting pull request")
 		}
+		if resp.StatusCode != http.StatusOK {
+			return nil, errors.Wrapf(err, "http response code %d getting diff %s to %s", resp.StatusCode, sourceRefName, targetRefName)
+		}
+
+		for _, change := range r.Changes {
+			item := change.GetItem()
+			// Convert the path to a relative path from the repo's root.
+			relativePath := filepath.Clean("./" + item.GetPath())
+			files = append(files, relativePath)
+
+			// If the file was renamed, we'll want to run plan in the directory
+			// it was moved from as well.
+			changeType := azuredevops.Rename.String()
+			if change.ChangeType == &changeType {
+				relativePath = filepath.Clean("./" + change.GetSourceServerItem())
+				files = append(files, relativePath)
+			}
+		}
+
+		if len(r.Changes) < pageSize {
+			break // Break if we have reached the end
+		}
+		skip += pageSize // Move to next page
 	}
 
 	return files, nil

server/events/vcs/azuredevops_client_test.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/runatlantis/atlantis/server/events/models"
 	"github.com/runatlantis/atlantis/server/events/vcs"
 	"github.com/runatlantis/atlantis/server/events/vcs/testdata"
@@ -273,7 +273,7 @@ func TestAzureDevopsClient_GetModifiedFiles(t *testing.T) {
 			case "/owner/project/_apis/git/repositories/repo/pullrequests/1?api-version=5.1-preview.1&includeWorkItemRefs=true":
 				w.Write([]byte(testdata.ADPullJSON)) // nolint: errcheck
 			// The second should hit this URL.
-			case "/owner/project/_apis/git/repositories/repo/diffs/commits?api-version=5.1&baseVersion=new_feature&targetVersion=npaulk/my_work":
+			case "/owner/project/_apis/git/repositories/repo/diffs/commits?%24top=100&api-version=5.1&baseVersion=new_feature&targetVersion=npaulk%2Fmy_work":
 				// We write a header that means there's an additional page.
 				w.Write([]byte(resp)) // nolint: errcheck
 				return

server/events/vcs/testdata/fixtures.go

@@ -21,9 +21,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/drmaxgit/go-azuredevops/azuredevops"
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/google/go-github/v68/github"
-	"github.com/mcdafydd/go-azuredevops/azuredevops"
 )
 
 var PullEvent = github.PullRequestEvent{