Skip to content

Commit 63f509f

Browse files
motatoesmotatoes
authored
Creation of run on workflows (#1325)
* introduce AAM webhook * refactor func create jobs for multiple projects * add endpoint for project runs
1 parent 7d05530 commit 63f509f

18 files changed

+618
-121
lines changed

‎backend/controllers/github.go

+32-14
Original file line numberDiff line numberDiff line change
@@ -425,9 +425,9 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
425425
cloneURL := *payload.Repo.CloneURL
426426
prNumber := *payload.PullRequest.Number
427427

428-
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
428+
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
429429
if err != nil {
430-
log.Printf("getDiggerConfig error: %v", err)
430+
log.Printf("getDiggerConfigForPR error: %v", err)
431431
return fmt.Errorf("error getting digger config")
432432
}
433433

@@ -502,23 +502,17 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
502502
return nil
503503
}
504504

505-
func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) {
505+
func getDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) {
506506
ghService, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
507507
if err != nil {
508508
log.Printf("Error getting github service: %v", err)
509-
return "", nil, nil, nil, nil, fmt.Errorf("error getting github service")
510-
}
511-
var prBranch string
512-
prBranch, err = ghService.GetBranchName(prNumber)
513-
if err != nil {
514-
log.Printf("Error getting branch name: %v", err)
515-
return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name")
509+
return "", nil, nil, nil, fmt.Errorf("error getting github service")
516510
}
517511

518512
var config *dg_configuration.DiggerConfig
519513
var diggerYmlStr string
520514
var dependencyGraph graph.Graph[string, dg_configuration.Project]
521-
err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) error {
515+
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
522516
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
523517
diggerYmlStr = string(diggerYmlBytes)
524518
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir)
@@ -530,7 +524,31 @@ func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFu
530524
})
531525
if err != nil {
532526
log.Printf("Error generating projects: %v", err)
533-
return "", nil, nil, nil, nil, fmt.Errorf("error generating projects")
527+
return "", nil, nil, nil, fmt.Errorf("error generating projects")
528+
}
529+
530+
log.Printf("Digger config loadded successfully\n")
531+
return diggerYmlStr, ghService, config, dependencyGraph, nil
532+
}
533+
534+
func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) {
535+
ghService, _, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
536+
if err != nil {
537+
log.Printf("Error getting github service: %v", err)
538+
return "", nil, nil, nil, nil, fmt.Errorf("error getting github service")
539+
}
540+
541+
var prBranch string
542+
prBranch, err = ghService.GetBranchName(prNumber)
543+
if err != nil {
544+
log.Printf("Error getting branch name: %v", err)
545+
return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name")
546+
}
547+
548+
diggerYmlStr, ghService, config, dependencyGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch)
549+
if err != nil {
550+
log.Printf("Error loading digger.yml: %v", err)
551+
return "", nil, nil, nil, nil, fmt.Errorf("error loading digger.yml")
534552
}
535553

536554
log.Printf("Digger config loadded successfully\n")
@@ -583,9 +601,9 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
583601
return nil
584602
}
585603

586-
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
604+
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
587605
if err != nil {
588-
log.Printf("getDiggerConfig error: %v", err)
606+
log.Printf("getDiggerConfigForPR error: %v", err)
589607
return fmt.Errorf("error getting digger config")
590608
}
591609

‎backend/controllers/github_after_merge.go

+272
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
package controllers
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"github.com/diggerhq/digger/backend/models"
7+
"github.com/diggerhq/digger/backend/utils"
8+
dg_configuration "github.com/diggerhq/digger/libs/digger_config"
9+
"github.com/diggerhq/digger/libs/orchestrator"
10+
dg_github "github.com/diggerhq/digger/libs/orchestrator/github"
11+
"github.com/diggerhq/digger/libs/orchestrator/scheduler"
12+
"github.com/gin-gonic/gin"
13+
"github.com/google/go-github/v58/github"
14+
"log"
15+
"net/http"
16+
"os"
17+
"path"
18+
"reflect"
19+
"strings"
20+
)
21+
22+
func GithubAppWebHookAfterMerge(c *gin.Context) {
23+
c.Header("Content-Type", "application/json")
24+
gh := &utils.DiggerGithubRealClientProvider{}
25+
log.Printf("GithubAppWebHook")
26+
27+
payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET")))
28+
if err != nil {
29+
log.Printf("Error validating github app webhook's payload: %v", err)
30+
c.String(http.StatusBadRequest, "Error validating github app webhook's payload")
31+
return
32+
}
33+
34+
webhookType := github.WebHookType(c.Request)
35+
event, err := github.ParseWebHook(webhookType, payload)
36+
if err != nil {
37+
log.Printf("Failed to parse Github Event. :%v\n", err)
38+
c.String(http.StatusInternalServerError, "Failed to parse Github Event")
39+
return
40+
}
41+
42+
log.Printf("github event type: %v\n", reflect.TypeOf(event))
43+
44+
switch event := event.(type) {
45+
case *github.InstallationEvent:
46+
log.Printf("InstallationEvent, action: %v\n", *event.Action)
47+
if *event.Action == "created" {
48+
err := handleInstallationCreatedEvent(event)
49+
if err != nil {
50+
c.String(http.StatusInternalServerError, "Failed to handle webhook event.")
51+
return
52+
}
53+
}
54+
55+
if *event.Action == "deleted" {
56+
err := handleInstallationDeletedEvent(event)
57+
if err != nil {
58+
c.String(http.StatusInternalServerError, "Failed to handle webhook event.")
59+
return
60+
}
61+
}
62+
case *github.InstallationRepositoriesEvent:
63+
log.Printf("InstallationRepositoriesEvent, action: %v\n", *event.Action)
64+
if *event.Action == "added" {
65+
err := handleInstallationRepositoriesAddedEvent(gh, event)
66+
if err != nil {
67+
c.String(http.StatusInternalServerError, "Failed to handle installation repo added event.")
68+
}
69+
}
70+
if *event.Action == "removed" {
71+
err := handleInstallationRepositoriesDeletedEvent(event)
72+
if err != nil {
73+
c.String(http.StatusInternalServerError, "Failed to handle installation repo deleted event.")
74+
}
75+
}
76+
77+
//case *github.IssueCommentEvent:
78+
// log.Printf("IssueCommentEvent, action: %v IN APPLY AFTER MERGE\n", *event.Action)
79+
// if event.Sender.Type != nil && *event.Sender.Type == "Bot" {
80+
// c.String(http.StatusOK, "OK")
81+
// return
82+
// }
83+
// err := handleIssueCommentEvent(gh, event)
84+
// if err != nil {
85+
// log.Printf("handleIssueCommentEvent error: %v", err)
86+
// c.String(http.StatusInternalServerError, err.Error())
87+
// return
88+
// }
89+
//case *github.PullRequestEvent:
90+
// log.Printf("Got pull request event for %d IN APPLY AFTER MERGE", *event.PullRequest.ID)
91+
// err := handlePullRequestEvent(gh, event)
92+
// if err != nil {
93+
// log.Printf("handlePullRequestEvent error: %v", err)
94+
// c.String(http.StatusInternalServerError, err.Error())
95+
// return
96+
// }
97+
case *github.PushEvent:
98+
log.Printf("Got push event for %d", event.Repo.URL)
99+
err := handlePushEventApplyAfterMerge(gh, event)
100+
if err != nil {
101+
log.Printf("handlePushEvent error: %v", err)
102+
c.String(http.StatusInternalServerError, err.Error())
103+
return
104+
}
105+
default:
106+
log.Printf("Unhandled event, event type %v", reflect.TypeOf(event))
107+
}
108+
109+
c.JSON(200, "ok")
110+
}
111+
112+
func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *github.PushEvent) error {
113+
print("*** HANDLING PUSH EVENT *****")
114+
installationId := *payload.Installation.ID
115+
repoName := *payload.Repo.Name
116+
repoFullName := *payload.Repo.FullName
117+
repoOwner := *payload.Repo.Owner.Login
118+
cloneURL := *payload.Repo.CloneURL
119+
commitId := *payload.After
120+
requestedBy := *payload.Sender.Login
121+
ref := *payload.Ref
122+
defaultBranch := *payload.Repo.DefaultBranch
123+
124+
if strings.HasSuffix(ref, defaultBranch) {
125+
link, err := models.DB.GetGithubAppInstallationLink(installationId)
126+
if err != nil {
127+
log.Printf("Error getting GetGithubAppInstallationLink: %v", err)
128+
return fmt.Errorf("error getting github app link")
129+
}
130+
131+
orgId := link.OrganisationId
132+
diggerRepoName := strings.ReplaceAll(repoFullName, "/", "-")
133+
repo, err := models.DB.GetRepo(orgId, diggerRepoName)
134+
if err != nil {
135+
log.Printf("Error getting Repo: %v", err)
136+
return fmt.Errorf("error getting github app link")
137+
}
138+
if repo == nil {
139+
log.Printf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName)
140+
return fmt.Errorf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName)
141+
}
142+
143+
service, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
144+
if err != nil {
145+
log.Printf("Error getting github service: %v", err)
146+
return fmt.Errorf("error getting github service")
147+
}
148+
utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error {
149+
dat, err := os.ReadFile(path.Join(dir, "digger.yml"))
150+
//TODO: fail here and return failure to main fn (need to refactor CloneGitRepoAndDoAction for that
151+
if err != nil {
152+
log.Printf("ERROR fetching digger.yml file: %v", err)
153+
}
154+
models.DB.UpdateRepoDiggerConfig(link.OrganisationId, string(dat), repo)
155+
return nil
156+
})
157+
158+
// ==== starting apply after merge part =======
159+
// TODO: Replace branch with actual commitID
160+
diggerYmlStr, ghService, config, projectsGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, defaultBranch)
161+
if err != nil {
162+
log.Printf("getDiggerConfigForPR error: %v", err)
163+
return fmt.Errorf("error getting digger config")
164+
}
165+
166+
impactedProjects, requestedProject, _, err := dg_github.ProcessGitHubPushEvent(payload, config, projectsGraph, ghService)
167+
if err != nil {
168+
log.Printf("Error processing event: %v", err)
169+
return fmt.Errorf("error processing event")
170+
}
171+
log.Printf("GitHub IssueComment event processed successfully\n")
172+
173+
// TODO: delete this line
174+
fmt.Sprintf(diggerYmlStr, impactedProjects, requestedProject, service)
175+
176+
// create 2 jobspecs (digger plan, digger apply) using commitID
177+
planJobs, err := dg_github.CreateJobsForProjects(impactedProjects, "digger plan", "push", repoFullName, requestedBy, config.Workflows, nil, &commitId, defaultBranch, "")
178+
if err != nil {
179+
log.Printf("Error creating jobs: %v", err)
180+
return fmt.Errorf("error creating jobs")
181+
}
182+
183+
applyJobs, err := dg_github.CreateJobsForProjects(impactedProjects, "digger apply", "push", repoFullName, requestedBy, config.Workflows, nil, &commitId, defaultBranch, "")
184+
if err != nil {
185+
log.Printf("Error creating jobs: %v", err)
186+
return fmt.Errorf("error creating jobs")
187+
}
188+
189+
if len(planJobs) == 0 || len(applyJobs) == 0 {
190+
log.Printf("no projects impacated, succeeding")
191+
return nil
192+
}
193+
194+
impactedProjectsMap := make(map[string]dg_configuration.Project)
195+
for _, p := range impactedProjects {
196+
impactedProjectsMap[p.Name] = p
197+
}
198+
199+
impactedProjectsJobMap := make(map[string]orchestrator.Job)
200+
for _, j := range planJobs {
201+
impactedProjectsJobMap[j.ProjectName] = j
202+
}
203+
204+
for i, _ := range planJobs {
205+
planJob := planJobs[i]
206+
applyJob := applyJobs[i]
207+
projectName := planJob.ProjectName
208+
planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, impactedProjects[i]))
209+
if err != nil {
210+
log.Printf("Error creating jobspec: %v %v", projectName, err)
211+
return fmt.Errorf("error creating jobspec")
212+
213+
}
214+
215+
applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, impactedProjects[i]))
216+
if err != nil {
217+
log.Printf("Error creating jobs: %v %v", projectName, err)
218+
return fmt.Errorf("error creating jobs")
219+
}
220+
// create batches
221+
planBatch, err := models.DB.CreateDiggerBatch(installationId, repoOwner, repoName, repoFullName, 0, diggerYmlStr, defaultBranch, scheduler.BatchTypePlan, nil)
222+
if err != nil {
223+
log.Printf("Error creating batch: %v", err)
224+
return fmt.Errorf("error creating batch")
225+
}
226+
227+
applyBatch, err := models.DB.CreateDiggerBatch(installationId, repoOwner, repoName, repoFullName, 0, diggerYmlStr, defaultBranch, scheduler.BatchTypeApply, nil)
228+
if err != nil {
229+
log.Printf("Error creating batch: %v", err)
230+
return fmt.Errorf("error creating batch")
231+
}
232+
233+
// create jobs
234+
_, err = models.DB.CreateDiggerJob(planBatch.ID, planJobSpec, impactedProjects[i].WorkflowFile)
235+
if err != nil {
236+
log.Printf("Error creating digger job: %v", err)
237+
return fmt.Errorf("error creating digger job")
238+
}
239+
240+
_, err = models.DB.CreateDiggerJob(applyBatch.ID, applyJobSpec, impactedProjects[i].WorkflowFile)
241+
if err != nil {
242+
log.Printf("Error creating digger job: %v", err)
243+
return fmt.Errorf("error creating digger job")
244+
}
245+
246+
// creating run stages
247+
planStage, err := models.DB.CreateDiggerRunStage(planBatch.ID.String())
248+
if err != nil {
249+
log.Printf("Error creating digger run stage: %v", err)
250+
return fmt.Errorf("error creating digger run stage")
251+
}
252+
253+
applyStage, err := models.DB.CreateDiggerRunStage(applyBatch.ID.String())
254+
if err != nil {
255+
log.Printf("Error creating digger run stage: %v", err)
256+
return fmt.Errorf("error creating digger run stage")
257+
}
258+
259+
diggerRun, err := models.DB.CreateDiggerRun("push", 0, models.RunQueued, commitId, diggerYmlStr, installationId, repo.ID, projectName, models.PlanAndApply, &planStage.ID, &applyStage.ID)
260+
if err != nil {
261+
log.Printf("Error creating digger run: %v", err)
262+
return fmt.Errorf("error creating digger run")
263+
}
264+
265+
models.DB.CreateDiggerRunQueueItem(diggerRun.ID)
266+
267+
}
268+
269+
}
270+
271+
return nil
272+
}

0 commit comments

Comments
 (0)