From fcc685b163edc6061786046a4697055bec6d131f Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 4 Apr 2024 12:56:29 +0100 Subject: [PATCH 1/6] introduce AAM webhook --- backend/controllers/github.go | 46 ++++-- backend/controllers/github_after_merge.go | 181 ++++++++++++++++++++++ backend/main.go | 1 + libs/digger_config/config.go | 1 + libs/digger_config/converters.go | 6 + libs/digger_config/yaml.go | 1 + libs/orchestrator/github/github.go | 116 +++++++++----- libs/orchestrator/utils.go | 8 +- 8 files changed, 310 insertions(+), 50 deletions(-) create mode 100644 backend/controllers/github_after_merge.go diff --git a/backend/controllers/github.go b/backend/controllers/github.go index a5e55ed14..1e8b29c8c 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -425,9 +425,9 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR cloneURL := *payload.Repo.CloneURL prNumber := *payload.PullRequest.Number - diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber) + diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber) if err != nil { - log.Printf("getDiggerConfig error: %v", err) + log.Printf("getDiggerConfigForPR error: %v", err) return fmt.Errorf("error getting digger config") } @@ -502,23 +502,17 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return nil } -func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) { +func getDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) { ghService, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) if err != nil { log.Printf("Error getting github service: %v", err) - return "", nil, nil, nil, nil, fmt.Errorf("error getting github service") - } - var prBranch string - prBranch, err = ghService.GetBranchName(prNumber) - if err != nil { - log.Printf("Error getting branch name: %v", err) - return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name") + return "", nil, nil, nil, fmt.Errorf("error getting github service") } var config *dg_configuration.DiggerConfig var diggerYmlStr string var dependencyGraph graph.Graph[string, dg_configuration.Project] - err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) error { + err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir) @@ -530,7 +524,31 @@ func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFu }) if err != nil { log.Printf("Error generating projects: %v", err) - return "", nil, nil, nil, nil, fmt.Errorf("error generating projects") + return "", nil, nil, nil, fmt.Errorf("error generating projects") + } + + log.Printf("Digger config loadded successfully\n") + return diggerYmlStr, ghService, config, dependencyGraph, nil +} + +func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) { + ghService, _, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) + if err != nil { + log.Printf("Error getting github service: %v", err) + return "", nil, nil, nil, nil, fmt.Errorf("error getting github service") + } + + var prBranch string + prBranch, err = ghService.GetBranchName(prNumber) + if err != nil { + log.Printf("Error getting branch name: %v", err) + return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name") + } + + diggerYmlStr, ghService, config, dependencyGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch) + if err != nil { + log.Printf("Error loading digger.yml: %v", err) + return "", nil, nil, nil, nil, fmt.Errorf("error loading digger.yml") } log.Printf("Digger config loadded successfully\n") @@ -583,9 +601,9 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu return nil } - diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber) + diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber) if err != nil { - log.Printf("getDiggerConfig error: %v", err) + log.Printf("getDiggerConfigForPR error: %v", err) return fmt.Errorf("error getting digger config") } diff --git a/backend/controllers/github_after_merge.go b/backend/controllers/github_after_merge.go new file mode 100644 index 000000000..31a6ac1d7 --- /dev/null +++ b/backend/controllers/github_after_merge.go @@ -0,0 +1,181 @@ +package controllers + +import ( + "fmt" + "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/backend/utils" + dg_github "github.com/diggerhq/digger/libs/orchestrator/github" + "github.com/gin-gonic/gin" + "github.com/google/go-github/v58/github" + "log" + "net/http" + "os" + "path" + "reflect" + "strings" +) + +func GithubAppWebHookAfterMerge(c *gin.Context) { + c.Header("Content-Type", "application/json") + gh := &utils.DiggerGithubRealClientProvider{} + log.Printf("GithubAppWebHook") + + payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET"))) + if err != nil { + log.Printf("Error validating github app webhook's payload: %v", err) + c.String(http.StatusBadRequest, "Error validating github app webhook's payload") + return + } + + webhookType := github.WebHookType(c.Request) + event, err := github.ParseWebHook(webhookType, payload) + if err != nil { + log.Printf("Failed to parse Github Event. :%v\n", err) + c.String(http.StatusInternalServerError, "Failed to parse Github Event") + return + } + + log.Printf("github event type: %v\n", reflect.TypeOf(event)) + + switch event := event.(type) { + case *github.InstallationEvent: + log.Printf("InstallationEvent, action: %v\n", *event.Action) + if *event.Action == "created" { + err := handleInstallationCreatedEvent(event) + if err != nil { + c.String(http.StatusInternalServerError, "Failed to handle webhook event.") + return + } + } + + if *event.Action == "deleted" { + err := handleInstallationDeletedEvent(event) + if err != nil { + c.String(http.StatusInternalServerError, "Failed to handle webhook event.") + return + } + } + case *github.InstallationRepositoriesEvent: + log.Printf("InstallationRepositoriesEvent, action: %v\n", *event.Action) + if *event.Action == "added" { + err := handleInstallationRepositoriesAddedEvent(gh, event) + if err != nil { + c.String(http.StatusInternalServerError, "Failed to handle installation repo added event.") + } + } + if *event.Action == "removed" { + err := handleInstallationRepositoriesDeletedEvent(event) + if err != nil { + c.String(http.StatusInternalServerError, "Failed to handle installation repo deleted event.") + } + } + + case *github.IssueCommentEvent: + log.Printf("IssueCommentEvent, action: %v IN APPLY AFTER MERGE\n", *event.Action) + if event.Sender.Type != nil && *event.Sender.Type == "Bot" { + c.String(http.StatusOK, "OK") + return + } + err := handleIssueCommentEvent(gh, event) + if err != nil { + log.Printf("handleIssueCommentEvent error: %v", err) + c.String(http.StatusInternalServerError, err.Error()) + return + } + case *github.PullRequestEvent: + log.Printf("Got pull request event for %d IN APPLY AFTER MERGE", *event.PullRequest.ID) + err := handlePullRequestEvent(gh, event) + if err != nil { + log.Printf("handlePullRequestEvent error: %v", err) + c.String(http.StatusInternalServerError, err.Error()) + return + } + case *github.PushEvent: + log.Printf("Got push event for %d", event.Repo.URL) + err := handlePushEventApplyAfterMerge(gh, event) + if err != nil { + log.Printf("handlePushEvent error: %v", err) + c.String(http.StatusInternalServerError, err.Error()) + return + } + default: + log.Printf("Unhandled event, event type %v", reflect.TypeOf(event)) + } + + c.JSON(200, "ok") +} + +func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *github.PushEvent) error { + installationId := *payload.Installation.ID + repoName := *payload.Repo.Name + repoFullName := *payload.Repo.FullName + repoOwner := *payload.Repo.Owner.Login + cloneURL := *payload.Repo.CloneURL + commitId := *payload.After + ref := *payload.Ref + defaultBranch := *payload.Repo.DefaultBranch + + if strings.HasSuffix(ref, defaultBranch) { + link, err := models.DB.GetGithubAppInstallationLink(installationId) + if err != nil { + log.Printf("Error getting GetGithubAppInstallationLink: %v", err) + return fmt.Errorf("error getting github app link") + } + + orgId := link.OrganisationId + diggerRepoName := strings.ReplaceAll(repoFullName, "/", "-") + repo, err := models.DB.GetRepo(orgId, diggerRepoName) + if err != nil { + log.Printf("Error getting Repo: %v", err) + return fmt.Errorf("error getting github app link") + } + if repo == nil { + log.Printf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) + return fmt.Errorf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) + } + + service, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) + if err != nil { + log.Printf("Error getting github service: %v", err) + return fmt.Errorf("error getting github service") + } + utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error { + dat, err := os.ReadFile(path.Join(dir, "digger.yml")) + //TODO: fail here and return failure to main fn (need to refactor CloneGitRepoAndDoAction for that + if err != nil { + log.Printf("ERROR fetching digger.yml file: %v", err) + } + models.DB.UpdateRepoDiggerConfig(link.OrganisationId, string(dat), repo) + return nil + }) + + // ==== starting apply after merge part ======= + diggerYmlStr, ghService, config, projectsGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, commitId) + if err != nil { + log.Printf("getDiggerConfigForPR error: %v", err) + return fmt.Errorf("error getting digger config") + } + + impactedProjects, requestedProject, _, err := dg_github.ProcessGitHubPushEvent(payload, config, projectsGraph, ghService) + if err != nil { + log.Printf("Error processing event: %v", err) + return fmt.Errorf("error processing event") + } + log.Printf("GitHub IssueComment event processed successfully\n") + + // create 2 jobspecs (digger plan, digger apply) using commitID + + // create job for the jobSpecs + + // create batch for each job + + // create Run + + // create RunStage for each batch attached to run + + // queue a RunQueueItem for the created Run + + } + + return nil +} diff --git a/backend/main.go b/backend/main.go index 7dc8a010a..6201fd955 100644 --- a/backend/main.go +++ b/backend/main.go @@ -74,6 +74,7 @@ func main() { r.GET("/", web.RedirectToLoginOrProjects) r.POST("/github-app-webhook", controllers.GithubAppWebHook) + r.POST("/github-app-webhook/aam", controllers.GithubAppWebHookAfterMerge) tenantActionsGroup := r.Group("/tenants") tenantActionsGroup.Use(middleware.CORSMiddleware()) diff --git a/libs/digger_config/config.go b/libs/digger_config/config.go index 11f60a13c..7d616bc74 100644 --- a/libs/digger_config/config.go +++ b/libs/digger_config/config.go @@ -1,6 +1,7 @@ package digger_config type DiggerConfig struct { + ApplyAfterMerge bool DependencyConfiguration DependencyConfiguration Projects []Project AutoMerge bool diff --git a/libs/digger_config/converters.go b/libs/digger_config/converters.go index 251cb394d..a699f212a 100644 --- a/libs/digger_config/converters.go +++ b/libs/digger_config/converters.go @@ -157,6 +157,12 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml) (*DiggerConfig, gra diggerConfig.AutoMerge = false } + if diggerYaml.ApplyAfterMerge != nil { + diggerConfig.ApplyAfterMerge = *diggerYaml.ApplyAfterMerge + } else { + diggerConfig.ApplyAfterMerge = false + } + if diggerYaml.MentionDriftedProjectsInPR != nil { diggerConfig.MentionDriftedProjectsInPR = *diggerYaml.MentionDriftedProjectsInPR } else { diff --git a/libs/digger_config/yaml.go b/libs/digger_config/yaml.go index b168882af..675390bf4 100644 --- a/libs/digger_config/yaml.go +++ b/libs/digger_config/yaml.go @@ -7,6 +7,7 @@ import ( ) type DiggerConfigYaml struct { + ApplyAfterMerge *bool `yaml:"apply_after_merge"` DependencyConfiguration *DependencyConfigurationYaml `yaml:"dependency_configuration"` Projects []*ProjectYaml `yaml:"projects"` AutoMerge *bool `yaml:"auto_merge"` diff --git a/libs/orchestrator/github/github.go b/libs/orchestrator/github/github.go index 0b0c796f7..4596065e3 100644 --- a/libs/orchestrator/github/github.go +++ b/libs/orchestrator/github/github.go @@ -75,6 +75,30 @@ func (svc GithubService) GetChangedFiles(prNumber int) ([]string, error) { return fileNames, nil } +func (svc GithubService) GetChangedFilesForCommit(owner string, repo string, commitID string) ([]string, error) { + var fileNames []string + opts := github.ListOptions{PerPage: 100} + + for { + commit, resp, err := svc.Client.Repositories.GetCommit(context.Background(), owner, repo, commitID, &opts) + if err != nil { + log.Fatalf("error getting commitfiles: %v", err) + } + for _, file := range commit.Files { + fileNames = append(fileNames, *file.Filename) + if file.PreviousFilename != nil { + fileNames = append(fileNames, *file.PreviousFilename) + } + } + + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + return fileNames, nil +} + func (svc GithubService) ListIssues() ([]*orchestrator.Issue, error) { allIssues := make([]*orchestrator.Issue, 0) opts := &github.IssueListByRepoOptions{ @@ -353,6 +377,8 @@ func GetRunEnvVars(defaultBranch string, prBranch string, projectName string, pr func ConvertGithubIssueCommentEventToJobs(payload *github.IssueCommentEvent, impactedProjects []digger_config.Project, requestedProject *digger_config.Project, workflows map[string]digger_config.Workflow, prBranchName string) ([]orchestrator.Job, bool, error) { jobs := make([]orchestrator.Job, 0) + repoFullName := *payload.Repo.FullName + requestedBy := *payload.Sender.Login defaultBranch := *payload.Repo.DefaultBranch prBranch := prBranchName @@ -371,55 +397,56 @@ func ConvertGithubIssueCommentEventToJobs(payload *github.IssueCommentEvent, imp return jobs, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) } } - diggerCommand := strings.ToLower(*payload.Comment.Body) diggerCommand = strings.TrimSpace(diggerCommand) - for _, command := range supportedCommands { if strings.HasPrefix(diggerCommand, command) { for _, project := range runForProjects { - workflow, ok := workflows[project.Workflow] - if !ok { - return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) - } issueNumber := payload.Issue.Number - runEnvVars := GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir) - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) - StateEnvProvider, CommandEnvProvider := orchestrator.GetStateAndCommandProviders(project) - workspace := project.Workspace - workspaceOverride, err := orchestrator.ParseWorkspace(*payload.Comment.Body) + job, err := CreateJobForProject(&project, command, "issue_comment", repoFullName, requestedBy, workflows, issueNumber, nil, defaultBranch, prBranch) if err != nil { - return []orchestrator.Job{}, false, err + return nil, false, err } - if workspaceOverride != "" { - workspace = workspaceOverride - } - jobs = append(jobs, orchestrator.Job{ - ProjectName: project.Name, - ProjectDir: project.Dir, - ProjectWorkspace: workspace, - ProjectWorkflow: project.Workflow, - Terragrunt: project.Terragrunt, - OpenTofu: project.OpenTofu, - Commands: []string{command}, - ApplyStage: orchestrator.ToConfigStage(workflow.Apply), - PlanStage: orchestrator.ToConfigStage(workflow.Plan), - RunEnvVars: runEnvVars, - CommandEnvVars: commandEnvVars, - StateEnvVars: stateEnvVars, - PullRequestNumber: issueNumber, - EventName: "issue_comment", - Namespace: *payload.Repo.FullName, - RequestedBy: *payload.Sender.Login, - StateEnvProvider: StateEnvProvider, - CommandEnvProvider: CommandEnvProvider, - }) + jobs = append(jobs, *job) + } } } return jobs, coversAllImpactedProjects, nil } +func CreateJobForProject(project *digger_config.Project, command string, event string, repoFullName string, requestedBy string, workflows map[string]digger_config.Workflow, issueNumber *int, commitSha *string, defaultBranch string, prBranch string) (*orchestrator.Job, error) { + workflow, ok := workflows[project.Workflow] + if !ok { + return nil, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) + } + runEnvVars := GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + StateEnvProvider, CommandEnvProvider := orchestrator.GetStateAndCommandProviders(project) + workspace := project.Workspace + + return &orchestrator.Job{ + ProjectName: project.Name, + ProjectDir: project.Dir, + ProjectWorkspace: workspace, + ProjectWorkflow: project.Workflow, + Terragrunt: project.Terragrunt, + OpenTofu: project.OpenTofu, + Commands: []string{command}, + ApplyStage: orchestrator.ToConfigStage(workflow.Apply), + PlanStage: orchestrator.ToConfigStage(workflow.Plan), + RunEnvVars: runEnvVars, + CommandEnvVars: commandEnvVars, + StateEnvVars: stateEnvVars, + PullRequestNumber: issueNumber, + EventName: event, //"issue_comment", + Namespace: repoFullName, + RequestedBy: requestedBy, + StateEnvProvider: StateEnvProvider, + CommandEnvProvider: CommandEnvProvider, + }, nil +} + func ProcessGitHubEvent(ghEvent interface{}, diggerConfig *digger_config.DiggerConfig, ciService orchestrator.PullRequestService) ([]digger_config.Project, *digger_config.Project, int, error) { var impactedProjects []digger_config.Project var prNumber int @@ -531,6 +558,25 @@ func FindAllProjectsDependantOnImpactedProjects(impactedProjects []digger_config return impactedProjectsWithDependantProjects, nil } +func ProcessGitHubPushEvent(payload *github.PushEvent, diggerConfig *digger_config.DiggerConfig, dependencyGraph graph.Graph[string, digger_config.Project], ciService orchestrator.PullRequestService) ([]digger_config.Project, *digger_config.Project, int, error) { + var impactedProjects []digger_config.Project + var prNumber int + + commitId := *payload.After + owner := *payload.Repo.Owner.Login + repo := *payload.Repo.Name + + // TODO: Refactor to make generic interface + changedFiles, err := ciService.(GithubService).GetChangedFilesForCommit(owner, repo, commitId) + if err != nil { + return nil, nil, 0, fmt.Errorf("could not get changed files") + } + + impactedProjects = diggerConfig.GetModifiedProjects(changedFiles) + return impactedProjects, nil, prNumber, nil + +} + func ProcessGitHubIssueCommentEvent(payload *github.IssueCommentEvent, diggerConfig *digger_config.DiggerConfig, dependencyGraph graph.Graph[string, digger_config.Project], ciService orchestrator.PullRequestService) ([]digger_config.Project, *digger_config.Project, int, error) { var impactedProjects []digger_config.Project var prNumber int diff --git a/libs/orchestrator/utils.go b/libs/orchestrator/utils.go index 63150ec42..9d7ae7d38 100644 --- a/libs/orchestrator/utils.go +++ b/libs/orchestrator/utils.go @@ -16,7 +16,13 @@ func ParseWorkspace(comment string) (string, error) { if len(matches) > 1 { return "", errors.New("more than one -w flag found") } - + workspaceOverride, err := orchestrator.ParseWorkspace(*payload.Comment.Body) + if err != nil { + return nil, err + } + if workspaceOverride != "" { + workspace = workspaceOverride + } if len(matches[0]) < 2 || matches[0][1] == "" { return "", errors.New("no value found after -w flag") } From 8db0d1e3fee21ff26134e284c1b34bb70263a70e Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 4 Apr 2024 14:56:36 +0100 Subject: [PATCH 2/6] fix build and make func create jobs for multiple projects --- backend/controllers/github_after_merge.go | 3 + libs/orchestrator/github/github.go | 86 +++++++++++++---------- libs/orchestrator/utils.go | 26 ------- 3 files changed, 50 insertions(+), 65 deletions(-) diff --git a/backend/controllers/github_after_merge.go b/backend/controllers/github_after_merge.go index 31a6ac1d7..8c056989f 100644 --- a/backend/controllers/github_after_merge.go +++ b/backend/controllers/github_after_merge.go @@ -163,6 +163,9 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith } log.Printf("GitHub IssueComment event processed successfully\n") + // TODO: delete this line + fmt.Sprintf(diggerYmlStr, impactedProjects, requestedProject, service) + // create 2 jobspecs (digger plan, digger apply) using commitID // create job for the jobSpecs diff --git a/libs/orchestrator/github/github.go b/libs/orchestrator/github/github.go index 4596065e3..97729c091 100644 --- a/libs/orchestrator/github/github.go +++ b/libs/orchestrator/github/github.go @@ -379,6 +379,7 @@ func ConvertGithubIssueCommentEventToJobs(payload *github.IssueCommentEvent, imp jobs := make([]orchestrator.Job, 0) repoFullName := *payload.Repo.FullName requestedBy := *payload.Sender.Login + issueNumber := *payload.Issue.Number defaultBranch := *payload.Repo.DefaultBranch prBranch := prBranchName @@ -399,52 +400,59 @@ func ConvertGithubIssueCommentEventToJobs(payload *github.IssueCommentEvent, imp } diggerCommand := strings.ToLower(*payload.Comment.Body) diggerCommand = strings.TrimSpace(diggerCommand) + isSupportedCommand := false for _, command := range supportedCommands { if strings.HasPrefix(diggerCommand, command) { - for _, project := range runForProjects { - issueNumber := payload.Issue.Number - job, err := CreateJobForProject(&project, command, "issue_comment", repoFullName, requestedBy, workflows, issueNumber, nil, defaultBranch, prBranch) - if err != nil { - return nil, false, err - } - jobs = append(jobs, *job) - - } + isSupportedCommand = true } } + if !isSupportedCommand { + return nil, false, fmt.Errorf("command is not supported: %v", diggerCommand) + } + + jobs, err := CreateJobsForProjects(runForProjects, diggerCommand, "issue_comment", repoFullName, requestedBy, workflows, &issueNumber, nil, defaultBranch, prBranch) + if err != nil { + return nil, false, err + } + return jobs, coversAllImpactedProjects, nil } -func CreateJobForProject(project *digger_config.Project, command string, event string, repoFullName string, requestedBy string, workflows map[string]digger_config.Workflow, issueNumber *int, commitSha *string, defaultBranch string, prBranch string) (*orchestrator.Job, error) { - workflow, ok := workflows[project.Workflow] - if !ok { - return nil, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) - } - runEnvVars := GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir) - stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) - StateEnvProvider, CommandEnvProvider := orchestrator.GetStateAndCommandProviders(project) - workspace := project.Workspace - - return &orchestrator.Job{ - ProjectName: project.Name, - ProjectDir: project.Dir, - ProjectWorkspace: workspace, - ProjectWorkflow: project.Workflow, - Terragrunt: project.Terragrunt, - OpenTofu: project.OpenTofu, - Commands: []string{command}, - ApplyStage: orchestrator.ToConfigStage(workflow.Apply), - PlanStage: orchestrator.ToConfigStage(workflow.Plan), - RunEnvVars: runEnvVars, - CommandEnvVars: commandEnvVars, - StateEnvVars: stateEnvVars, - PullRequestNumber: issueNumber, - EventName: event, //"issue_comment", - Namespace: repoFullName, - RequestedBy: requestedBy, - StateEnvProvider: StateEnvProvider, - CommandEnvProvider: CommandEnvProvider, - }, nil +func CreateJobsForProjects(projects []digger_config.Project, command string, event string, repoFullName string, requestedBy string, workflows map[string]digger_config.Workflow, issueNumber *int, commitSha *string, defaultBranch string, prBranch string) ([]orchestrator.Job, error) { + jobs := make([]orchestrator.Job, 0) + + for _, project := range projects { + workflow, ok := workflows[project.Workflow] + if !ok { + return nil, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) + } + + runEnvVars := GetRunEnvVars(defaultBranch, prBranch, project.Name, project.Dir) + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) + StateEnvProvider, CommandEnvProvider := orchestrator.GetStateAndCommandProviders(project) + workspace := project.Workspace + jobs = append(jobs, orchestrator.Job{ + ProjectName: project.Name, + ProjectDir: project.Dir, + ProjectWorkspace: workspace, + ProjectWorkflow: project.Workflow, + Terragrunt: project.Terragrunt, + OpenTofu: project.OpenTofu, + Commands: []string{command}, + ApplyStage: orchestrator.ToConfigStage(workflow.Apply), + PlanStage: orchestrator.ToConfigStage(workflow.Plan), + RunEnvVars: runEnvVars, + CommandEnvVars: commandEnvVars, + StateEnvVars: stateEnvVars, + PullRequestNumber: issueNumber, + EventName: event, //"issue_comment", + Namespace: repoFullName, + RequestedBy: requestedBy, + StateEnvProvider: StateEnvProvider, + CommandEnvProvider: CommandEnvProvider, + }) + } + return jobs, nil } func ProcessGitHubEvent(ghEvent interface{}, diggerConfig *digger_config.DiggerConfig, ciService orchestrator.PullRequestService) ([]digger_config.Project, *digger_config.Project, int, error) { diff --git a/libs/orchestrator/utils.go b/libs/orchestrator/utils.go index 9d7ae7d38..4a76f377c 100644 --- a/libs/orchestrator/utils.go +++ b/libs/orchestrator/utils.go @@ -1,35 +1,9 @@ package orchestrator import ( - "errors" "regexp" ) -func ParseWorkspace(comment string) (string, error) { - re := regexp.MustCompile(`-w(?:\s+(\S+)|$)`) - matches := re.FindAllStringSubmatch(comment, -1) - - if len(matches) == 0 { - return "", nil - } - - if len(matches) > 1 { - return "", errors.New("more than one -w flag found") - } - workspaceOverride, err := orchestrator.ParseWorkspace(*payload.Comment.Body) - if err != nil { - return nil, err - } - if workspaceOverride != "" { - workspace = workspaceOverride - } - if len(matches[0]) < 2 || matches[0][1] == "" { - return "", errors.New("no value found after -w flag") - } - - return matches[0][1], nil -} - func ParseProjectName(comment string) string { re := regexp.MustCompile(`-p ([0-9a-zA-Z\-_]+)`) match := re.FindStringSubmatch(comment) From 740f2b78e9088fee1e492acfd2cb266687016116 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 4 Apr 2024 16:57:31 +0100 Subject: [PATCH 3/6] creation of models --- backend/controllers/github_after_merge.go | 95 +++++++++++++++++++++-- backend/models/storage.go | 14 ++++ 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/backend/controllers/github_after_merge.go b/backend/controllers/github_after_merge.go index 8c056989f..537f3d87c 100644 --- a/backend/controllers/github_after_merge.go +++ b/backend/controllers/github_after_merge.go @@ -1,10 +1,14 @@ package controllers import ( + "encoding/json" "fmt" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" + dg_configuration "github.com/diggerhq/digger/libs/digger_config" + "github.com/diggerhq/digger/libs/orchestrator" dg_github "github.com/diggerhq/digger/libs/orchestrator/github" + "github.com/diggerhq/digger/libs/orchestrator/scheduler" "github.com/gin-gonic/gin" "github.com/google/go-github/v58/github" "log" @@ -112,6 +116,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith repoOwner := *payload.Repo.Owner.Login cloneURL := *payload.Repo.CloneURL commitId := *payload.After + requestedBy := *payload.Sender.Login ref := *payload.Ref defaultBranch := *payload.Repo.DefaultBranch @@ -167,16 +172,96 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith fmt.Sprintf(diggerYmlStr, impactedProjects, requestedProject, service) // create 2 jobspecs (digger plan, digger apply) using commitID + planJobs, err := dg_github.CreateJobsForProjects(impactedProjects, "digger plan", "push", repoFullName, requestedBy, config.Workflows, nil, &commitId, defaultBranch, "") + if err != nil { + log.Printf("Error creating jobs: %v", err) + return fmt.Errorf("error creating jobs") + } + + applyJobs, err := dg_github.CreateJobsForProjects(impactedProjects, "digger apply", "push", repoFullName, requestedBy, config.Workflows, nil, &commitId, defaultBranch, "") + if err != nil { + log.Printf("Error creating jobs: %v", err) + return fmt.Errorf("error creating jobs") + } - // create job for the jobSpecs + if len(planJobs) == 0 || len(applyJobs) == 0 { + log.Printf("no projects impacated, succeeding") + return nil + } - // create batch for each job + impactedProjectsMap := make(map[string]dg_configuration.Project) + for _, p := range impactedProjects { + impactedProjectsMap[p.Name] = p + } - // create Run + impactedProjectsJobMap := make(map[string]orchestrator.Job) + for _, j := range planJobs { + impactedProjectsJobMap[j.ProjectName] = j + } - // create RunStage for each batch attached to run + for i, _ := range planJobs { + planJob := planJobs[i] + applyJob := applyJobs[i] + planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, impactedProjects[i])) + if err != nil { + log.Printf("Error creating jobspec: %v %v", planJob.ProjectName, err) + return fmt.Errorf("error creating jobspec") - // queue a RunQueueItem for the created Run + } + + applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, impactedProjects[i])) + if err != nil { + log.Printf("Error creating jobs: %v %v", applyJob.ProjectName, err) + return fmt.Errorf("error creating jobs") + } + // create batches + planBatch, err := models.DB.CreateDiggerBatch(installationId, repoOwner, repoName, repoFullName, 0, diggerYmlStr, defaultBranch, scheduler.BatchTypePlan, nil) + if err != nil { + log.Printf("Error creating batch: %v", err) + return fmt.Errorf("error creating batch") + } + + applyBatch, err := models.DB.CreateDiggerBatch(installationId, repoOwner, repoName, repoFullName, 0, diggerYmlStr, defaultBranch, scheduler.BatchTypeApply, nil) + if err != nil { + log.Printf("Error creating batch: %v", err) + return fmt.Errorf("error creating batch") + } + + // create jobs + _, err = models.DB.CreateDiggerJob(planBatch.ID, planJobSpec, impactedProjects[i].WorkflowFile) + if err != nil { + log.Printf("Error creating digger job: %v", err) + return fmt.Errorf("error creating digger job") + } + + _, err = models.DB.CreateDiggerJob(planBatch.ID, applyJobSpec, impactedProjects[i].WorkflowFile) + if err != nil { + log.Printf("Error creating digger job: %v", err) + return fmt.Errorf("error creating digger job") + } + + diggerRun, err := models.DB.CreateDiggerRun("push", 0, models.RunQueued, commitId, diggerYmlStr, installationId, 0, 0, models.PlanAndApply) + if err != nil { + log.Printf("Error creating digger run: %v", err) + return fmt.Errorf("error creating digger run") + } + + // creating run stages + _, err = models.DB.CreateDiggerRunStage(diggerRun.ID, planBatch.ID.String()) + if err != nil { + log.Printf("Error creating digger run stage: %v", err) + return fmt.Errorf("error creating digger run stage") + } + + _, err = models.DB.CreateDiggerRunStage(diggerRun.ID, applyBatch.ID.String()) + if err != nil { + log.Printf("Error creating digger run stage: %v", err) + return fmt.Errorf("error creating digger run stage") + } + + models.DB.CreateDiggerRunQueueItem(0, diggerRun.ID) + + } } diff --git a/backend/models/storage.go b/backend/models/storage.go index a103fd816..bce4a8522 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -676,6 +676,20 @@ func (db *Database) CreateDiggerRun(Triggertype string, PrNumber int, Status Dig return dr, nil } +func (db *Database) CreateDiggerRunStage(runId uint, batchId string) (*DiggerRunStage, error) { + drs := &DiggerRunStage{ + RunID: runId, + BatchID: &batchId, + } + result := db.GormDB.Save(drs) + if result.Error != nil { + log.Printf("Failed to create DiggerRunStage: %v, error: %v\n", drs.ID, result.Error) + return nil, result.Error + } + log.Printf("DiggerRunStage %v, has been created successfully\n", drs.ID) + return drs, nil +} + func (db *Database) GetLastDiggerRunForProject(projectId uint) (*DiggerRun, error) { diggerRun := &DiggerRun{} result := db.GormDB.Where("project_id = ? AND status <> ?", projectId, RunQueued).Order("created_at Desc").First(diggerRun) From d3c045ac35ea72d586a1ece039904e41609a8dfd Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 4 Apr 2024 17:30:08 +0100 Subject: [PATCH 4/6] migrate fields --- backend/controllers/github_after_merge.go | 23 ++++++++++++----------- backend/migrations/20240404160724.sql | 2 ++ backend/migrations/20240404161121.sql | 2 ++ backend/migrations/20240404161723.sql | 4 ++++ backend/migrations/atlas.sum | 5 ++++- backend/models/runs.go | 17 ++++++++--------- backend/models/storage.go | 18 +++++++----------- backend/tasks/runs.go | 22 +++++++++++----------- 8 files changed, 50 insertions(+), 43 deletions(-) create mode 100644 backend/migrations/20240404160724.sql create mode 100644 backend/migrations/20240404161121.sql create mode 100644 backend/migrations/20240404161723.sql diff --git a/backend/controllers/github_after_merge.go b/backend/controllers/github_after_merge.go index 537f3d87c..963872a8b 100644 --- a/backend/controllers/github_after_merge.go +++ b/backend/controllers/github_after_merge.go @@ -202,16 +202,17 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith for i, _ := range planJobs { planJob := planJobs[i] applyJob := applyJobs[i] + projectName := planJob.ProjectName planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, impactedProjects[i])) if err != nil { - log.Printf("Error creating jobspec: %v %v", planJob.ProjectName, err) + log.Printf("Error creating jobspec: %v %v", projectName, err) return fmt.Errorf("error creating jobspec") } applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, impactedProjects[i])) if err != nil { - log.Printf("Error creating jobs: %v %v", applyJob.ProjectName, err) + log.Printf("Error creating jobs: %v %v", projectName, err) return fmt.Errorf("error creating jobs") } // create batches @@ -240,26 +241,26 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith return fmt.Errorf("error creating digger job") } - diggerRun, err := models.DB.CreateDiggerRun("push", 0, models.RunQueued, commitId, diggerYmlStr, installationId, 0, 0, models.PlanAndApply) - if err != nil { - log.Printf("Error creating digger run: %v", err) - return fmt.Errorf("error creating digger run") - } - // creating run stages - _, err = models.DB.CreateDiggerRunStage(diggerRun.ID, planBatch.ID.String()) + planStage, err := models.DB.CreateDiggerRunStage(planBatch.ID.String()) if err != nil { log.Printf("Error creating digger run stage: %v", err) return fmt.Errorf("error creating digger run stage") } - _, err = models.DB.CreateDiggerRunStage(diggerRun.ID, applyBatch.ID.String()) + applyStage, err := models.DB.CreateDiggerRunStage(applyBatch.ID.String()) if err != nil { log.Printf("Error creating digger run stage: %v", err) return fmt.Errorf("error creating digger run stage") } - models.DB.CreateDiggerRunQueueItem(0, diggerRun.ID) + diggerRun, err := models.DB.CreateDiggerRun("push", 0, models.RunQueued, commitId, diggerYmlStr, installationId, repo.ID, projectName, models.PlanAndApply, &planStage.ID, &applyStage.ID) + if err != nil { + log.Printf("Error creating digger run: %v", err) + return fmt.Errorf("error creating digger run") + } + + models.DB.CreateDiggerRunQueueItem(diggerRun.ID) } diff --git a/backend/migrations/20240404160724.sql b/backend/migrations/20240404160724.sql new file mode 100644 index 000000000..42b03f35b --- /dev/null +++ b/backend/migrations/20240404160724.sql @@ -0,0 +1,2 @@ +-- Modify "digger_run_queue_items" table +ALTER TABLE "public"."digger_run_queue_items" DROP COLUMN "project_id"; diff --git a/backend/migrations/20240404161121.sql b/backend/migrations/20240404161121.sql new file mode 100644 index 000000000..2e6df41af --- /dev/null +++ b/backend/migrations/20240404161121.sql @@ -0,0 +1,2 @@ +-- Modify "digger_run_stages" table +ALTER TABLE "public"."digger_run_stages" DROP COLUMN "run_id"; diff --git a/backend/migrations/20240404161723.sql b/backend/migrations/20240404161723.sql new file mode 100644 index 000000000..1cf8f9ccc --- /dev/null +++ b/backend/migrations/20240404161723.sql @@ -0,0 +1,4 @@ +-- Modify "digger_runs" table +ALTER TABLE "public"."digger_runs" ADD COLUMN "plan_stage_id" bigint NULL, ADD COLUMN "apply_stage_id" bigint NULL, ADD + CONSTRAINT "fk_digger_runs_apply_stage" FOREIGN KEY ("apply_stage_id") REFERENCES "public"."digger_run_stages" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION, ADD + CONSTRAINT "fk_digger_runs_plan_stage" FOREIGN KEY ("plan_stage_id") REFERENCES "public"."digger_run_stages" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index 9a6bb5a50..a97a5466f 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:AwUIRwmx9E+aRp469t6izImEEgfQt6IHwtAKXumn4jo= +h1:wCerMVvOi8Co+XSFeE+18Vcex7+cmseQ9ZJp4sHZwaY= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -11,3 +11,6 @@ h1:AwUIRwmx9E+aRp469t6izImEEgfQt6IHwtAKXumn4jo= 20240402110915.sql h1:bG2Dvbzm3ZvFa29Feb0Bwj6KtAtZy1Vyuje6yV31msQ= 20240403155357_drop_dup_idx.sql h1:6LyRtGfutHQompownriYYrq8us+Cdj4FTgWa7VPsXFA= 20240403155456.sql h1:XJgyne416JMAV4xHA3IweHZos0ULrjFEJBqhWFjGNho= +20240404160724.sql h1:cjEZiC7JC0dCJIRSIh1OBBhi2IBuRZEBj/ifoYqNzAw= +20240404161121.sql h1:ZbMLfHRom6Tws+2M3BcnMu1lcjz/YFwAI8kGmC+I+H4= +20240404161723.sql h1:z3bJcKs0ZJSyTJewqgE0GSHpn33sX7zgc2rmCMF99Qo= diff --git a/backend/models/runs.go b/backend/models/runs.go index b7f1ad285..0d8a020ed 100644 --- a/backend/models/runs.go +++ b/backend/models/runs.go @@ -30,8 +30,6 @@ const ( type DiggerRunQueueItem struct { gorm.Model - ProjectId uint `gorm:"index:idx_digger_run_queue_project_id"` - Project *Project DiggerRunId uint `gorm:"index:idx_digger_run_queue_run_id"` DiggerRun DiggerRun time time.Time @@ -47,15 +45,16 @@ type DiggerRun struct { GithubInstallationId int64 RepoId uint Repo *Repo - Project *Project - ProjectID uint + ProjectName string RunType RunType + PlanStage *DiggerRunStage + PlanStageId *uint + ApplyStage *DiggerRunStage + ApplyStageId *uint } type DiggerRunStage struct { gorm.Model - Run *DiggerRun - RunID uint `gorm:"index:idx_digger_run_stage_id"` Batch *DiggerBatch BatchID *string `gorm:"index:idx_digger_run_batch_id"` } @@ -78,9 +77,9 @@ func (r *DiggerRunStage) MapToJsonStruct() (interface{}, error) { } return SerializedRunStage{ - DiggerJobId: job.DiggerJobID, - Status: job.Status, - ProjectName: r.Run.Project.Name, + DiggerJobId: job.DiggerJobID, + Status: job.Status, + //ProjectName: r.Run.Project.Name, WorkflowRunUrl: job.WorkflowRunUrl, ResourcesCreated: job.DiggerJobSummary.ResourcesCreated, ResourcesUpdated: job.DiggerJobSummary.ResourcesUpdated, diff --git a/backend/models/storage.go b/backend/models/storage.go index bce4a8522..3cbf02ef8 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -655,7 +655,7 @@ func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, wor return job, nil } -func (db *Database) CreateDiggerRun(Triggertype string, PrNumber int, Status DiggerRunStatus, CommitId string, DiggerConfig string, GithubInstallationId int64, RepoId uint, ProjectID uint, RunType RunType) (*DiggerRun, error) { +func (db *Database) CreateDiggerRun(Triggertype string, PrNumber int, Status DiggerRunStatus, CommitId string, DiggerConfig string, GithubInstallationId int64, RepoId uint, ProjectName string, RunType RunType, planStageId *uint, applyStageId *uint) (*DiggerRun, error) { dr := &DiggerRun{ Triggertype: Triggertype, PrNumber: &PrNumber, @@ -664,8 +664,10 @@ func (db *Database) CreateDiggerRun(Triggertype string, PrNumber int, Status Dig DiggerConfig: DiggerConfig, GithubInstallationId: GithubInstallationId, RepoId: RepoId, - ProjectID: ProjectID, + ProjectName: ProjectName, RunType: RunType, + PlanStageId: planStageId, + ApplyStageId: applyStageId, } result := db.GormDB.Save(dr) if result.Error != nil { @@ -676,9 +678,8 @@ func (db *Database) CreateDiggerRun(Triggertype string, PrNumber int, Status Dig return dr, nil } -func (db *Database) CreateDiggerRunStage(runId uint, batchId string) (*DiggerRunStage, error) { +func (db *Database) CreateDiggerRunStage(batchId string) (*DiggerRunStage, error) { drs := &DiggerRunStage{ - RunID: runId, BatchID: &batchId, } result := db.GormDB.Save(drs) @@ -709,9 +710,8 @@ func (db *Database) GetDiggerRun(id uint) (*DiggerRun, error) { return dr, nil } -func (db *Database) CreateDiggerRunQueueItem(projectId uint, diggeRrunId uint) (*DiggerRunQueueItem, error) { +func (db *Database) CreateDiggerRunQueueItem(diggeRrunId uint) (*DiggerRunQueueItem, error) { drq := &DiggerRunQueueItem{ - ProjectId: projectId, DiggerRunId: diggeRrunId, } result := db.GormDB.Save(drq) @@ -796,15 +796,11 @@ WHERE // 2. Preload Project and DiggerRun for every DiggerrunQueue item (front of queue) var runqueuesWithData []DiggerRunQueueItem - projectIds := lo.Map(runqueues, func(run DiggerRunQueueItem, index int) uint { - return run.ProjectId - }) diggerRunIds := lo.Map(runqueues, func(run DiggerRunQueueItem, index int) uint { return run.DiggerRunId }) - tx = db.GormDB.Preload("Project").Preload("DiggerRun"). - Where("digger_run_queues.project_id in ?", projectIds). + tx = db.GormDB.Preload("DiggerRun"). Where("digger_run_queues.digger_run_id in ?", diggerRunIds).Find(&runqueuesWithData) if tx.Error != nil { diff --git a/backend/tasks/runs.go b/backend/tasks/runs.go index caf7ab59e..0dbe3bd1a 100644 --- a/backend/tasks/runs.go +++ b/backend/tasks/runs.go @@ -18,7 +18,7 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, CIBackend ci_ba dr.Status = models.RunPlanning err := models.DB.UpdateDiggerRun(&dr) if err != nil { - log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } case models.RunPlanning: // Check the status of the batch @@ -30,11 +30,11 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, CIBackend ci_ba dr.Status = models.RunFailed err := models.DB.UpdateDiggerRun(&dr) if err != nil { - log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } err = models.DB.DequeueRunItem(queueItem) if err != nil { - log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } } @@ -43,13 +43,13 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, CIBackend ci_ba dr.Status = models.RunPendingApproval err := models.DB.UpdateDiggerRun(&dr) if err != nil { - log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } } else { dr.Status = models.RunApproved err := models.DB.UpdateDiggerRun(&dr) if err != nil { - log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } } @@ -61,7 +61,7 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, CIBackend ci_ba dr.Status = models.RunApplying err := models.DB.UpdateDiggerRun(&dr) if err != nil { - log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } case models.RunApplying: @@ -73,11 +73,11 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, CIBackend ci_ba dr.Status = models.RunFailed err := models.DB.UpdateDiggerRun(&dr) if err != nil { - log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } err = models.DB.DequeueRunItem(queueItem) if err != nil { - log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } } @@ -86,7 +86,7 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, CIBackend ci_ba dr.Status = models.RunSucceeded err := models.DB.UpdateDiggerRun(&dr) if err != nil { - log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to update Digger Run for queueID: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } } @@ -94,13 +94,13 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, CIBackend ci_ba // dequeue err := models.DB.DequeueRunItem(queueItem) if err != nil { - log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } case models.RunFailed: // dequeue err := models.DB.DequeueRunItem(queueItem) if err != nil { - log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.ProjectId) + log.Printf("ERROR: Failed to delete queueItem item: %v [%v %v]", queueItem.ID, queueItem.DiggerRunId, queueItem.DiggerRun.ProjectName) } default: log.Printf("WARN: Recieived unknown DiggerRunStatus: %v", queueItem.DiggerRun.Status) From 19a3cb47366b5e8cb9e0a5c6a75723f25033f692 Mon Sep 17 00:00:00 2001 From: motatoes Date: Fri, 5 Apr 2024 14:15:38 +0100 Subject: [PATCH 5/6] add endpoint for project runs --- backend/controllers/github_after_merge.go | 46 ++++++++------- backend/controllers/runs.go | 71 ++++++++++++++++++++++ backend/main.go | 1 + backend/migrations/20240404165910.sql | 2 + backend/migrations/atlas.sum | 3 +- backend/models/runs.go | 72 +++++++++++++++++------ backend/models/storage.go | 31 ++++++++++ libs/orchestrator/github/github.go | 2 +- 8 files changed, 187 insertions(+), 41 deletions(-) create mode 100644 backend/controllers/runs.go create mode 100644 backend/migrations/20240404165910.sql diff --git a/backend/controllers/github_after_merge.go b/backend/controllers/github_after_merge.go index 963872a8b..2a1c998cd 100644 --- a/backend/controllers/github_after_merge.go +++ b/backend/controllers/github_after_merge.go @@ -74,26 +74,26 @@ func GithubAppWebHookAfterMerge(c *gin.Context) { } } - case *github.IssueCommentEvent: - log.Printf("IssueCommentEvent, action: %v IN APPLY AFTER MERGE\n", *event.Action) - if event.Sender.Type != nil && *event.Sender.Type == "Bot" { - c.String(http.StatusOK, "OK") - return - } - err := handleIssueCommentEvent(gh, event) - if err != nil { - log.Printf("handleIssueCommentEvent error: %v", err) - c.String(http.StatusInternalServerError, err.Error()) - return - } - case *github.PullRequestEvent: - log.Printf("Got pull request event for %d IN APPLY AFTER MERGE", *event.PullRequest.ID) - err := handlePullRequestEvent(gh, event) - if err != nil { - log.Printf("handlePullRequestEvent error: %v", err) - c.String(http.StatusInternalServerError, err.Error()) - return - } + //case *github.IssueCommentEvent: + // log.Printf("IssueCommentEvent, action: %v IN APPLY AFTER MERGE\n", *event.Action) + // if event.Sender.Type != nil && *event.Sender.Type == "Bot" { + // c.String(http.StatusOK, "OK") + // return + // } + // err := handleIssueCommentEvent(gh, event) + // if err != nil { + // log.Printf("handleIssueCommentEvent error: %v", err) + // c.String(http.StatusInternalServerError, err.Error()) + // return + // } + //case *github.PullRequestEvent: + // log.Printf("Got pull request event for %d IN APPLY AFTER MERGE", *event.PullRequest.ID) + // err := handlePullRequestEvent(gh, event) + // if err != nil { + // log.Printf("handlePullRequestEvent error: %v", err) + // c.String(http.StatusInternalServerError, err.Error()) + // return + // } case *github.PushEvent: log.Printf("Got push event for %d", event.Repo.URL) err := handlePushEventApplyAfterMerge(gh, event) @@ -110,6 +110,7 @@ func GithubAppWebHookAfterMerge(c *gin.Context) { } func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *github.PushEvent) error { + print("*** HANDLING PUSH EVENT *****") installationId := *payload.Installation.ID repoName := *payload.Repo.Name repoFullName := *payload.Repo.FullName @@ -155,7 +156,8 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith }) // ==== starting apply after merge part ======= - diggerYmlStr, ghService, config, projectsGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, commitId) + // TODO: Replace branch with actual commitID + diggerYmlStr, ghService, config, projectsGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, defaultBranch) if err != nil { log.Printf("getDiggerConfigForPR error: %v", err) return fmt.Errorf("error getting digger config") @@ -235,7 +237,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith return fmt.Errorf("error creating digger job") } - _, err = models.DB.CreateDiggerJob(planBatch.ID, applyJobSpec, impactedProjects[i].WorkflowFile) + _, err = models.DB.CreateDiggerJob(applyBatch.ID, applyJobSpec, impactedProjects[i].WorkflowFile) if err != nil { log.Printf("Error creating digger job: %v", err) return fmt.Errorf("error creating digger job") diff --git a/backend/controllers/runs.go b/backend/controllers/runs.go new file mode 100644 index 000000000..3c5fcfcbc --- /dev/null +++ b/backend/controllers/runs.go @@ -0,0 +1,71 @@ +package controllers + +import ( + "errors" + "fmt" + "github.com/diggerhq/digger/backend/middleware" + "github.com/diggerhq/digger/backend/models" + "github.com/gin-gonic/gin" + "gorm.io/gorm" + "log" + "net/http" + "strconv" +) + +func RunsForProject(c *gin.Context) { + currentOrg, exists := c.Get(middleware.ORGANISATION_ID_KEY) + projectIdStr := c.Param("project_id") + + if projectIdStr == "" { + c.String(http.StatusBadRequest, "ProjectId not specified") + return + } + + projectId, err := strconv.Atoi(projectIdStr) + if err != nil { + c.String(http.StatusBadRequest, "Invalid ProjectId") + return + } + + if !exists { + c.String(http.StatusForbidden, "Not allowed to access this resource") + return + } + + var org models.Organisation + err = models.DB.GormDB.Where("id = ?", currentOrg).First(&org).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + c.String(http.StatusNotFound, fmt.Sprintf("Could not find organisation: %v", currentOrg)) + } else { + c.String(http.StatusInternalServerError, "Unknown error occurred while fetching database") + } + return + } + + project, err := models.DB.GetProject(uint(projectId)) + if err != nil { + log.Printf("could not fetch project: %v", err) + c.String(http.StatusInternalServerError, "Could not fetch project") + return + } + + runs, err := models.DB.ListDiggerRunsForProject(project.Name, project.Repo.ID) + if err != nil { + log.Printf("could not fetch runs: %v", err) + c.String(http.StatusInternalServerError, "Could not fetch runs") + return + } + + serializedRuns := make([]interface{}, 0) + for _, run := range runs { + serializedRun, err := run.MapToJsonStruct() + if err != nil { + log.Printf("could not unmarshal run: %v", err) + c.String(http.StatusInternalServerError, "Could not unmarshal runs") + return + } + serializedRuns = append(serializedRuns, serializedRun) + } + c.JSON(http.StatusOK, serializedRuns) +} diff --git a/backend/main.go b/backend/main.go index 6201fd955..263c8b88d 100644 --- a/backend/main.go +++ b/backend/main.go @@ -165,6 +165,7 @@ func main() { projectsApiGroup := apiGroup.Group("/projects") projectsApiGroup.Use(middleware.GetWebMiddleware()) projectsApiGroup.GET("/", controllers.FindProjectsForOrg) + projectsApiGroup.GET("/:project_id/runs", controllers.RunsForProject) fronteggWebhookProcessor.POST("/create-org-from-frontegg", controllers.CreateFronteggOrgFromWebhook) diff --git a/backend/migrations/20240404165910.sql b/backend/migrations/20240404165910.sql new file mode 100644 index 000000000..c75b501b0 --- /dev/null +++ b/backend/migrations/20240404165910.sql @@ -0,0 +1,2 @@ +-- Modify "digger_runs" table +ALTER TABLE "public"."digger_runs" DROP COLUMN "project_id", ADD COLUMN "project_name" text NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index a97a5466f..e4e35e3e6 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:wCerMVvOi8Co+XSFeE+18Vcex7+cmseQ9ZJp4sHZwaY= +h1:xkkYzycO+Sb/rIvE5VwTvzTb6dA6kyuQQ1/ZT/Oy9no= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -14,3 +14,4 @@ h1:wCerMVvOi8Co+XSFeE+18Vcex7+cmseQ9ZJp4sHZwaY= 20240404160724.sql h1:cjEZiC7JC0dCJIRSIh1OBBhi2IBuRZEBj/ifoYqNzAw= 20240404161121.sql h1:ZbMLfHRom6Tws+2M3BcnMu1lcjz/YFwAI8kGmC+I+H4= 20240404161723.sql h1:z3bJcKs0ZJSyTJewqgE0GSHpn33sX7zgc2rmCMF99Qo= +20240404165910.sql h1:ofwrBzkvnxFz7sOrtaF3vb2xHsenPmUTSSBHvO1NEdI= diff --git a/backend/models/runs.go b/backend/models/runs.go index 0d8a020ed..ece504487 100644 --- a/backend/models/runs.go +++ b/backend/models/runs.go @@ -47,9 +47,9 @@ type DiggerRun struct { Repo *Repo ProjectName string RunType RunType - PlanStage *DiggerRunStage + PlanStage DiggerRunStage PlanStageId *uint - ApplyStage *DiggerRunStage + ApplyStage DiggerRunStage ApplyStageId *uint } @@ -60,29 +60,67 @@ type DiggerRunStage struct { } type SerializedRunStage struct { - DiggerJobId string `json:"digger_job_id"` - Status orchestrator_scheduler.DiggerJobStatus `json:"status"` - ProjectName string `json:"project_name"` - WorkflowRunUrl *string `json:"workflow_run_url"` - ResourcesCreated uint `json:"resources_created"` - ResourcesDeleted uint `json:"resources_deleted"` - ResourcesUpdated uint `json:"resources_updated"` + //DiggerRunId uint `json:"digger_run_id"` + DiggerJobId string `json:"digger_job_id"` + Status orchestrator_scheduler.DiggerJobStatus `json:"status"` + ProjectName string `json:"project_name"` + WorkflowRunUrl *string `json:"workflow_run_url"` + ResourcesCreated uint `json:"resources_created"` + ResourcesDeleted uint `json:"resources_deleted"` + ResourcesUpdated uint `json:"resources_updated"` + LastActivityTimeStamp string `json:"last_activity_timestamp"` } -func (r *DiggerRunStage) MapToJsonStruct() (interface{}, error) { - job, err := DB.GetDiggerJobFromRunStage(*r) +func (r *DiggerRun) MapToJsonStruct() (interface{}, error) { + planStage, err := r.PlanStage.MapToJsonStruct() + if err != nil { + log.Printf("error serializing run: %v", err) + return nil, err + } + + applyStage, err := r.ApplyStage.MapToJsonStruct() + if err != nil { + log.Printf("error serializing run: %v", err) + return nil, err + } + + x := struct { + Id uint `json:"id"` + Status string `json:"status"` + Type string `json:"type"` + ApprovalAuthor string `json:"approval_author"` + ApprovalStatus string `json:"approval_status"` + ApprovalDate string `json:"approval_date"` + LastActivityTimeStamp string `json:"last_activity_time_stamp"` + PlanStage SerializedRunStage `json:"plan_stage"` + ApplyStage SerializedRunStage `json:"apply_stage"` + }{ + Id: r.ID, + Status: string(r.Status), + Type: string(r.RunType), + LastActivityTimeStamp: r.UpdatedAt.String(), + PlanStage: *planStage, + ApplyStage: *applyStage, + } + + return x, nil +} + +func (r DiggerRunStage) MapToJsonStruct() (*SerializedRunStage, error) { + job, err := DB.GetDiggerJobFromRunStage(r) if err != nil { log.Printf("Could not retrive job from run") return nil, err } - return SerializedRunStage{ + return &SerializedRunStage{ DiggerJobId: job.DiggerJobID, Status: job.Status, - //ProjectName: r.Run.Project.Name, - WorkflowRunUrl: job.WorkflowRunUrl, - ResourcesCreated: job.DiggerJobSummary.ResourcesCreated, - ResourcesUpdated: job.DiggerJobSummary.ResourcesUpdated, - ResourcesDeleted: job.DiggerJobSummary.ResourcesDeleted, + //ProjectName: r.Run.ProjectName, + WorkflowRunUrl: job.WorkflowRunUrl, + ResourcesCreated: job.DiggerJobSummary.ResourcesCreated, + ResourcesUpdated: job.DiggerJobSummary.ResourcesUpdated, + ResourcesDeleted: job.DiggerJobSummary.ResourcesDeleted, + LastActivityTimeStamp: r.UpdatedAt.String(), }, nil } diff --git a/backend/models/storage.go b/backend/models/storage.go index 3cbf02ef8..db7cc8d77 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -169,6 +169,22 @@ func (db *Database) GetProjectByProjectId(c *gin.Context, projectId uint, orgIdK return &project, true } +func (db *Database) GetProject(projectId uint) (*Project, error) { + log.Printf("GetProject, project id: %v\n", projectId) + var project Project + + err := db.GormDB.Preload("Organisation").Preload("Repo"). + Where("id = ?", projectId). + First(&project).Error + + if err != nil { + log.Printf("Unknown error occurred while fetching database, %v\n", err) + return nil, err + } + + return &project, nil +} + // GetProjectByName return project for specified org and repo // if record doesn't exist return nil func (db *Database) GetProjectByName(orgId any, repo *Repo, name string) (*Project, error) { @@ -655,6 +671,21 @@ func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, wor return job, nil } +func (db *Database) ListDiggerRunsForProject(projectName string, repoId uint) ([]DiggerRun, error) { + var runs []DiggerRun + + err := db.GormDB.Preload("PlanStage").Preload("ApplyStage"). + Where("project_name = ? AND repo_id= ?", projectName, repoId).Order("created_at desc").Find(&runs).Error + + if err != nil { + log.Printf("Unknown error occurred while fetching database, %v\n", err) + return nil, err + } + + log.Printf("ListDiggerRunsForProject, number of runs:%d\n", len(runs)) + return runs, nil +} + func (db *Database) CreateDiggerRun(Triggertype string, PrNumber int, Status DiggerRunStatus, CommitId string, DiggerConfig string, GithubInstallationId int64, RepoId uint, ProjectName string, RunType RunType, planStageId *uint, applyStageId *uint) (*DiggerRun, error) { dr := &DiggerRun{ Triggertype: Triggertype, diff --git a/libs/orchestrator/github/github.go b/libs/orchestrator/github/github.go index 97729c091..369b9c680 100644 --- a/libs/orchestrator/github/github.go +++ b/libs/orchestrator/github/github.go @@ -575,7 +575,7 @@ func ProcessGitHubPushEvent(payload *github.PushEvent, diggerConfig *digger_conf repo := *payload.Repo.Name // TODO: Refactor to make generic interface - changedFiles, err := ciService.(GithubService).GetChangedFilesForCommit(owner, repo, commitId) + changedFiles, err := ciService.(*GithubService).GetChangedFilesForCommit(owner, repo, commitId) if err != nil { return nil, nil, 0, fmt.Errorf("could not get changed files") } From 3da0ff4d331283d3f4bda8103590315bf11e3a17 Mon Sep 17 00:00:00 2001 From: motatoes Date: Fri, 5 Apr 2024 14:35:25 +0100 Subject: [PATCH 6/6] fix tests --- backend/models/storage.go | 2 +- backend/tasks/runs_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/models/storage.go b/backend/models/storage.go index db7cc8d77..a33717e4b 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -756,7 +756,7 @@ func (db *Database) CreateDiggerRunQueueItem(diggeRrunId uint) (*DiggerRunQueueI func (db *Database) GetDiggerRunQueueItem(id uint) (*DiggerRunQueueItem, error) { dr := &DiggerRunQueueItem{} - result := db.GormDB.Preload("Project").Preload("DiggerRun").Where("id=? ", id).Find(dr) + result := db.GormDB.Preload("DiggerRun").Where("id=? ", id).Find(dr) if result.Error != nil { return nil, result.Error } diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index 997f1ca4f..8fe18ec11 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -139,8 +139,8 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) { ciBackend := MockCiBackend{} batch, _ := models.DB.CreateDiggerBatch(123, "", "", "", 22, "", "", "", nil) project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), nil, nil) - diggerRun, _ := models.DB.CreateDiggerRun("", 1, testParam.InitialStatus, "sha", "", 123, 1, project.ID, "apply") - queueItem, _ := models.DB.CreateDiggerRunQueueItem(project.ID, diggerRun.ID) + diggerRun, _ := models.DB.CreateDiggerRun("", 1, testParam.InitialStatus, "sha", "", 123, 1, project.Name, "apply", nil, nil) + queueItem, _ := models.DB.CreateDiggerRunQueueItem(diggerRun.ID) batch.Status = testParam.BatchStatus models.DB.UpdateDiggerBatch(batch) queueItem, _ = models.DB.GetDiggerRunQueueItem(queueItem.ID)