Indexing from CLI now falls back to full reindex if necessary (#9517)

Notably, this PR adds a check for codesearch-enablement, and injection
of the repo access token, to the main app server, before forwarding the
indexing request to the codesearch server.

I was able to mostly test this locally, but I wasn't able to launch the
server with the github integration enabled, so I'm not confident in
codesearch.go `getGitRepoAccessToken` code. I pulled most of this from
[service.go](https://github.com/buildbuddy-io/buildbuddy/blob/master/enterprise/server/workflow/service/service.go#L629).

Author: jdelfino

cli/index/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//proto:index_go_proto",
         "//server/util/git",
         "//server/util/grpc_client",
+        "//server/util/status",
         "@org_golang_google_grpc//metadata",
     ],
 )

cli/index/index.go

@@ -13,6 +13,7 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/codesearch/github"
 	"github.com/buildbuddy-io/buildbuddy/server/util/git"
 	"github.com/buildbuddy-io/buildbuddy/server/util/grpc_client"
+	"github.com/buildbuddy-io/buildbuddy/server/util/status"
 	"google.golang.org/grpc/metadata"
 
 	bbspb "github.com/buildbuddy-io/buildbuddy/proto/buildbuddy_service"
@@ -71,6 +72,40 @@ func getRepoInfo(gc github.GitClient) (*git.RepoURL, string, error) {
 	return parseRepoURL, headSHA, nil
 }
 
+func buildIndexRequest(gc github.GitClient, repoURL *git.RepoURL, headSHA, lastIndexSHA string) (*inpb.IndexRequest, error) {
+	req := &inpb.IndexRequest{
+		GitRepo: &gitpb.GitRepo{
+			RepoUrl:  repoURL.String(),
+			Username: repoURL.Owner,
+		},
+	}
+
+	if lastIndexSHA != "" {
+		update, err := github.ComputeIncrementalUpdate(gc, lastIndexSHA, headSHA)
+		if err != nil {
+			if status.IsFailedPreconditionError(err) {
+				log.Printf("Failed to compute incremental update, falling back to full re-index: %s", err)
+			} else {
+				return nil, fmt.Errorf("failed to compute incremental update: %w", err)
+			}
+		} else {
+			req.ReplacementStrategy = inpb.ReplacementStrategy_INCREMENTAL
+			req.Update = update
+			return req, nil
+		}
+	} else {
+		log.Printf("No previous index found for repo %s, performing full re-index.", repoURL)
+	}
+
+	req.ReplacementStrategy = inpb.ReplacementStrategy_REPLACE_REPO
+	req.RepoState = &gitpb.RepoState{
+		CommitSha: headSHA,
+	}
+	req.Async = true // Don't wait for full re-indexes to complete.
+	// Access token will be added by the server based on user auth.
+	return req, nil
+}
+
 func indexRepo() error {
 	ctx := context.Background()
 
@@ -101,27 +136,24 @@ func indexRepo() error {
 		return fmt.Errorf("failed to get repo status: %w", err)
 	}
 
-	update, err := github.ComputeIncrementalUpdate(gc, rsp.GetLastIndexedCommitSha(), headSHA)
+	update, err := buildIndexRequest(gc, parsedRepoURL, headSHA, rsp.GetLastIndexedCommitSha())
 	if err != nil {
-		return fmt.Errorf("incremental update aborted: %w", err)
-	}
-
-	req := &inpb.IndexRequest{
-		GitRepo: &gitpb.GitRepo{
-			RepoUrl: parsedRepoURL.String(),
-		},
-		ReplacementStrategy: inpb.ReplacementStrategy_INCREMENTAL,
-		Update:              update,
+		return fmt.Errorf("failed to create index request: %w", err)
 	}
 
-	_, err = client.Index(ctx, req)
+	_, err = client.Index(ctx, update)
 	if err != nil {
 		return err
 	}
 
-	firstSha := update.Commits[0].GetSha()
-	lastSha := update.Commits[len(update.Commits)-1].GetSha()
-	log.Printf("Completed incremental update for %s, %s..%s", parsedRepoURL.String(), firstSha, lastSha)
+	if update.ReplacementStrategy == inpb.ReplacementStrategy_REPLACE_REPO {
+		log.Printf("Re-indexing entire repo %s at commit %s", parsedRepoURL.String(), headSHA)
+	} else {
+		commits := update.GetUpdate().GetCommits()
+		firstSha := commits[0].GetSha()
+		lastSha := commits[len(commits)-1].GetSha()
+		log.Printf("Completed incremental update for %s, %s..%s", parsedRepoURL.String(), firstSha, lastSha)
+	}
 	return nil
 }
 

codesearch/github/github.go

@@ -284,10 +284,10 @@ func ComputeIncrementalUpdate(gc GitClient, firstSha, lastSha string) (*inpb.Inc
 	changes := strings.Split(strings.TrimSpace(changesStr), "\n")
 
 	if len(changes) > maxAllowedChanges {
-		return nil, fmt.Errorf("too many changes in commit range %s..%s: %d", firstSha, lastSha, len(changes))
+		return nil, status.FailedPreconditionErrorf("too many changes in commit range %s..%s: %d", firstSha, lastSha, len(changes))
 	}
 	if len(changes) == 0 || (len(changes) == 1 && len(changes[0]) == 0) {
-		return nil, fmt.Errorf("no commits found between %s and %s", firstSha, lastSha)
+		return nil, status.FailedPreconditionErrorf("no commits found between %s and %s", firstSha, lastSha)
 	}
 
 	result := &inpb.IncrementalUpdate{

codesearch/server/server.go

@@ -369,7 +369,10 @@ func (css *codesearchServer) RepoStatus(ctx context.Context, req *inpb.RepoStatu
 
 	rev, err := github.GetLastIndexedCommitSha(r, repoURL)
 	if err != nil {
-		return nil, err
+		// If there's no status, return an empty commit SHA, but don't error.
+		if !status.IsNotFoundError(err) {
+			return nil, err
+		}
 	}
 
 	return &inpb.RepoStatusResponse{

codesearch/server/server_test.go

@@ -109,8 +109,10 @@ func TestRepoStatus_NoStatus(t *testing.T) {
 	response, err := server.RepoStatus(context.Background(), &inpb.RepoStatusRequest{
 		RepoUrl: "github.com/buildbuddy-io/buildbuddy",
 	})
-	assert.True(t, status.IsNotFoundError(err), "expected status.NotFoundError, got: %v", err)
-	assert.Nil(t, response)
+	require.NoError(t, err)
+	assert.Equal(t, &inpb.RepoStatusResponse{
+		LastIndexedCommitSha: "",
+	}, response)
 }
 
 func TestRepoStatus_InvalidRepo(t *testing.T) {

enterprise/server/backends/codesearch/BUILD

@@ -11,6 +11,9 @@ go_library(
         "//proto:search_go_proto",
         "//server/environment",
         "//server/real_environment",
+        "//server/tables",
+        "//server/util/claims",
+        "//server/util/db",
         "//server/util/flag",
         "//server/util/grpc_client",
         "//server/util/status",

enterprise/server/backends/codesearch/codesearch.go

@@ -5,6 +5,9 @@ import (
 
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
 	"github.com/buildbuddy-io/buildbuddy/server/real_environment"
+	"github.com/buildbuddy-io/buildbuddy/server/tables"
+	"github.com/buildbuddy-io/buildbuddy/server/util/claims"
+	"github.com/buildbuddy-io/buildbuddy/server/util/db"
 	"github.com/buildbuddy-io/buildbuddy/server/util/flag"
 	"github.com/buildbuddy-io/buildbuddy/server/util/grpc_client"
 	"github.com/buildbuddy-io/buildbuddy/server/util/status"
@@ -20,6 +23,7 @@ var (
 
 type CodesearchService struct {
 	client csspb.CodesearchServiceClient
+	env    environment.Env
 }
 
 func Register(realEnv *real_environment.RealEnv) error {
@@ -41,6 +45,7 @@ func New(env environment.Env) (*CodesearchService, error) {
 	}
 	return &CodesearchService{
 		client: csspb.NewCodesearchServiceClient(conn),
+		env:    env,
 	}, nil
 }
 
@@ -49,7 +54,61 @@ func (css *CodesearchService) Search(ctx context.Context, req *srpb.SearchReques
 	return css.client.Search(ctx, req)
 }
 
+func (css *CodesearchService) getGitRepoAccessToken(ctx context.Context, groupId, repoURL string) (string, error) {
+	ghas := css.env.GetGitHubAppService()
+	if ghas == nil {
+		return "", status.InternalError("GitHub App service is not configured")
+	}
+
+	gha, err := ghas.GetGitHubAppForOwner(ctx, repoURL)
+	if err != nil {
+		return "", status.UnavailableErrorf("could not get GitHub app for repo %q: %s", repoURL, err)
+	}
+
+	gitRepository := &tables.GitRepository{}
+	err = css.env.GetDBHandle().NewQuery(ctx, "hosted_runner_get_for_repo").Raw(`
+		SELECT *
+		FROM "GitRepositories"
+		WHERE group_id = ?
+		AND repo_url = ?
+	`, groupId, repoURL).Take(gitRepository)
+	if err != nil {
+		if db.IsRecordNotFound(err) {
+			return "", status.NotFoundErrorf("repo %q not found", repoURL)
+		}
+		return "", status.InternalErrorf("failed to look up repo %s: %s", repoURL, err)
+	}
+
+	token, err := gha.GetRepositoryInstallationToken(ctx, gitRepository)
+	if err != nil {
+		return "", status.UnavailableErrorf("could not get access token for repo %q: %s", repoURL, err)
+	}
+
+	return token, nil
+}
+
 func (css *CodesearchService) Index(ctx context.Context, req *inpb.IndexRequest) (*inpb.IndexResponse, error) {
+	claims, err := claims.ClaimsFromContext(ctx)
+	if err != nil {
+		return nil, status.UnauthenticatedErrorf("failed to get claims from context: %s", err)
+	}
+
+	g, err := css.env.GetUserDB().GetGroupByID(ctx, claims.GroupID)
+	if err != nil {
+		return nil, err
+	}
+	if !g.CodeSearchEnabled {
+		return nil, status.FailedPreconditionError("codesearch is not enabled")
+	}
+
+	if req.GetReplacementStrategy() == inpb.ReplacementStrategy_REPLACE_REPO {
+		token, err := css.getGitRepoAccessToken(ctx, claims.GroupID, req.GetGitRepo().GetRepoUrl())
+		if err != nil {
+			return nil, err
+		}
+		req.GetGitRepo().AccessToken = token
+	}
+
 	return css.client.Index(ctx, req)
 }