remove the need for a digger token injected to job

.github/workflows/tasks_run_test.yml

@@ -0,0 +1,31 @@
+name: Libs tests
+on:
+  push:
+  pull_request:
+    types: [opened, reopened]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.1
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Deps
+        run: |
+          pwd
+          go get -v ./...
+        working-directory: backend/tasks
+
+
+      - name: Test
+        run: go test -v ./...
+        working-directory: backend/tasks

backend/controllers/github.go

@@ -424,6 +424,12 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	repoFullName := *payload.Repo.FullName
 	cloneURL := *payload.Repo.CloneURL
 	prNumber := *payload.PullRequest.Number
+	link, err := models.DB.GetGithubAppInstallationLink(installationId)
+	if err != nil {
+		log.Printf("Error getting GetGithubAppInstallationLink: %v", err)
+		return fmt.Errorf("error getting github app link")
+	}
+	organisationId := link.OrganisationId
 
 	diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
 	if err != nil {
@@ -485,7 +491,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	}
 
 	batchType := getBatchType(jobsForImpactedProjects)
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -591,6 +597,13 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	cloneURL := *payload.Repo.CloneURL
 	issueNumber := *payload.Issue.Number
 
+	link, err := models.DB.GetGithubAppInstallationLink(installationId)
+	if err != nil {
+		log.Printf("Error getting GetGithubAppInstallationLink: %v", err)
+		return fmt.Errorf("error getting github app link")
+	}
+	orgId := link.OrganisationId
+
 	if *payload.Action != "created" {
 		log.Printf("comment is not of type 'created', ignoring")
 		return nil
@@ -673,7 +686,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	}
 
 	batchType := getBatchType(jobs)
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))

backend/controllers/github_after_merge.go

@@ -86,14 +86,14 @@ func GithubAppWebHookAfterMerge(c *gin.Context) {
 	//		c.String(http.StatusInternalServerError, err.Error())
 	//		return
 	//	}
-	//case *github.PullRequestEvent:
-	//	log.Printf("Got pull request event for %d  IN APPLY AFTER MERGE", *event.PullRequest.ID)
-	//	err := handlePullRequestEvent(gh, event)
-	//	if err != nil {
-	//		log.Printf("handlePullRequestEvent error: %v", err)
-	//		c.String(http.StatusInternalServerError, err.Error())
-	//		return
-	//	}
+	case *github.PullRequestEvent:
+		log.Printf("Got pull request event for %d  IN APPLY AFTER MERGE", *event.PullRequest.ID)
+		err := handlePullRequestEvent(gh, event)
+		if err != nil {
+			log.Printf("handlePullRequestEvent error: %v", err)
+			c.String(http.StatusInternalServerError, err.Error())
+			return
+		}
 	case *github.PushEvent:
 		log.Printf("Got push event for %d", event.Repo.URL)
 		err := handlePushEventApplyAfterMerge(gh, event)
@@ -120,6 +120,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 	requestedBy := *payload.Sender.Login
 	ref := *payload.Ref
 	defaultBranch := *payload.Repo.DefaultBranch
+	backendHostName := os.Getenv("HOSTNAME")
 
 	if strings.HasSuffix(ref, defaultBranch) {
 		link, err := models.DB.GetGithubAppInstallationLink(installationId)
@@ -129,6 +130,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 		}
 
 		orgId := link.OrganisationId
+		orgName := link.Organisation.Name
 		diggerRepoName := strings.ReplaceAll(repoFullName, "/", "-")
 		repo, err := models.DB.GetRepo(orgId, diggerRepoName)
 		if err != nil {
@@ -208,18 +210,32 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 			planJob := planJobs[i]
 			applyJob := applyJobs[i]
 			projectName := planJob.ProjectName
-			planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, impactedProjects[i]))
+
+			planJobToken, err := models.DB.CreateDiggerJobToken(orgId)
+			if err != nil {
+				log.Printf("Error creating job token: %v %v", projectName, err)
+				return fmt.Errorf("error creating job token")
+			}
+
+			planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, orgName, planJobToken.Value, backendHostName, impactedProjects[i]))
 			if err != nil {
 				log.Printf("Error creating jobspec: %v %v", projectName, err)
 				return fmt.Errorf("error creating jobspec")
 
 			}
 
-			applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, impactedProjects[i]))
+			applyJobToken, err := models.DB.CreateDiggerJobToken(orgId)
+			if err != nil {
+				log.Printf("Error creating job token: %v %v", projectName, err)
+				return fmt.Errorf("error creating job token")
+			}
+
+			applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, orgName, applyJobToken.Value, backendHostName, impactedProjects[i]))
 			if err != nil {
 				log.Printf("Error creating jobs: %v %v", projectName, err)
 				return fmt.Errorf("error creating jobs")
 			}
+
 			// create batches
 			planBatch, err := models.DB.CreateDiggerBatch(installationId, repoOwner, repoName, repoFullName, issueNumber, diggerYmlStr, defaultBranch, scheduler.BatchTypePlan, nil)
 			if err != nil {

backend/controllers/github_test.go

@@ -591,7 +591,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {
 	// migrate tables
 	err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{},
 		&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubApp{}, &models.GithubAppInstallationLink{},
-		&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{})
+		&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.JobToken{})
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -732,7 +732,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) {
 	graph, err := configuration.CreateProjectDependencyGraph(projects)
 	assert.NoError(t, err)
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -761,7 +761,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 
@@ -794,7 +794,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -839,7 +839,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) {
 	projectMap["555"] = project5
 	projectMap["666"] = project6
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 6, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID)

backend/main.go

@@ -121,7 +121,7 @@ func main() {
 	checkoutGroup.GET("/checkout", web.Checkout)
 
 	authorized := r.Group("/")
-	authorized.Use(middleware.GetApiMiddleware(), middleware.AccessLevel(models.AccessPolicyType, models.AdminPolicyType))
+	authorized.Use(middleware.GetApiMiddleware(), middleware.AccessLevel(models.CliJobAccessType, models.AccessPolicyType, models.AdminPolicyType))
 
 	admin := r.Group("/")
 	admin.Use(middleware.GetApiMiddleware(), middleware.AccessLevel(models.AdminPolicyType))

backend/middleware/jwt.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 )
 
 func SetContextParameters(c *gin.Context, auth services.Auth, token *jwt.Token) error {
@@ -193,7 +194,32 @@ func JWTBearerTokenAuth(auth services.Auth) gin.HandlerFunc {
 			return
 		}
 
-		if strings.HasPrefix(token, "t:") {
+		if strings.HasPrefix(token, "cli:") {
+			var dbToken models.Token
+
+			jobToken, err := models.DB.GetJobToken(token)
+			if jobToken == nil {
+				c.String(http.StatusForbidden, "Invalid bearer token")
+				c.Abort()
+				return
+			}
+
+			if time.Now().After(jobToken.Expiry) {
+				log.Printf("Token has already expired: %v", err)
+				c.String(http.StatusForbidden, "Token has expired")
+				c.Abort()
+				return
+			}
+
+			if err != nil {
+				log.Printf("Error while fetching token from database: %v", err)
+				c.String(http.StatusInternalServerError, "Error occurred while fetching database")
+				c.Abort()
+				return
+			}
+			c.Set(ORGANISATION_ID_KEY, dbToken.OrganisationID)
+			c.Set(ACCESS_LEVEL_KEY, dbToken.Type)
+		} else if strings.HasPrefix(token, "t:") {
 			var dbToken models.Token
 
 			token, err := models.DB.GetToken(token)

backend/migrations/20240409161739.sql

@@ -0,0 +1,15 @@
+-- Create "job_tokens" table
+CREATE TABLE "public"."job_tokens" (
+  "id" bigserial NOT NULL,
+  "created_at" timestamptz NULL,
+  "updated_at" timestamptz NULL,
+  "deleted_at" timestamptz NULL,
+  "value" text NULL,
+  "expiry" timestamptz NULL,
+  "organisation_id" bigint NULL,
+  "type" text NULL,
+  PRIMARY KEY ("id"),
+  CONSTRAINT "fk_job_tokens_organisation" FOREIGN KEY ("organisation_id") REFERENCES "public"."organisations" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION
+);
+-- Create index "idx_job_tokens_deleted_at" to table: "job_tokens"
+CREATE INDEX "idx_job_tokens_deleted_at" ON "public"."job_tokens" ("deleted_at");

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:IoMAe7wNz/XdmLMzpgmtCvO0Pu1Kw0lZsufwdmjf0dE=
+h1:yIWTCl8ClNyBna56KDV5hF7yq68qD8QAv8SgM/GiA7E=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -17,3 +17,4 @@ h1:IoMAe7wNz/XdmLMzpgmtCvO0Pu1Kw0lZsufwdmjf0dE=
 20240404165910.sql h1:ofwrBzkvnxFz7sOrtaF3vb2xHsenPmUTSSBHvO1NEdI=
 20240405150942.sql h1:0JIQlXqQmfgfBcill47gAef3LnnfdwK6ry98eHraUbo=
 20240405160110.sql h1:8bXZtrh8ZFFuCEXWIZ4fSjca0SSk1gsa2BqK7dIZ0To=
+20240409161739.sql h1:x0dZOsILJhmeQ6w8JKkllXZb2oz+QqV/PGLo+8R2pWI=

backend/models/orgs.go

@@ -123,4 +123,5 @@ type Token struct {
 const (
 	AccessPolicyType = "access"
 	AdminPolicyType  = "admin"
+	CliJobAccessType = "cli_access"
 )

backend/models/scheduler.go

@@ -52,6 +52,16 @@ type DiggerJobSummary struct {
 	ResourcesUpdated uint
 }
 
+// These tokens will be pre
+type JobToken struct {
+	gorm.Model
+	Value          string `gorm:"uniqueJobTokenIndex:idx_token"`
+	Expiry         time.Time
+	OrganisationID uint
+	Organisation   Organisation
+	Type           string // AccessTokenType starts with j:
+}
+
 type DiggerJobLinkStatus int8
 
 const (

backend/models/storage.go

@@ -1037,6 +1037,38 @@ func (db *Database) GetToken(tenantId any) (*Token, error) {
 	return token, nil
 }
 
+func (db *Database) CreateDiggerJobToken(organisationId uint) (*JobToken, error) {
+
+	// create a digger job token
+	// prefixing token to make easier to retire this type of tokens later
+	token := "cli:" + uuid.New().String()
+	jobToken := &JobToken{
+		Value:          token,
+		OrganisationID: organisationId,
+		Type:           CliJobAccessType,
+		Expiry:         time.Now().Add(time.Hour * 2), // some jobs can take >30 mins (k8s cluster)
+	}
+	err := db.GormDB.Create(jobToken).Error
+	if err != nil {
+		log.Printf("failed to create token: %v", err)
+		return nil, err
+	}
+	return jobToken, nil
+}
+
+func (db *Database) GetJobToken(tenantId any) (*JobToken, error) {
+	token := &JobToken{}
+	result := db.GormDB.Take(token, "value = ?", tenantId)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return nil, nil
+		} else {
+			return nil, result.Error
+		}
+	}
+	return token, nil
+}
+
 func (db *Database) CreateGithubAppInstallation(installationId int64, githubAppId int64, login string, accountId int, repoFullName string) (*GithubAppInstallation, error) {
 	installation := &GithubAppInstallation{
 		GithubInstallationId: installationId,

backend/utils/graphs.go

@@ -11,16 +11,31 @@ import (
 	"github.com/dominikbraun/graph"
 	"github.com/google/uuid"
 	"log"
+	"os"
 )
 
 // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value
-func ConvertJobsToDiggerJobs(jobsMap map[string]orchestrator.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commentId int64, diggerConfigStr string, batchType orchestrator_scheduler.DiggerBatchType) (*uuid.UUID, map[string]*models.DiggerJob, error) {
+func ConvertJobsToDiggerJobs(organisationId uint, jobsMap map[string]orchestrator.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commentId int64, diggerConfigStr string, batchType orchestrator_scheduler.DiggerBatchType) (*uuid.UUID, map[string]*models.DiggerJob, error) {
 	result := make(map[string]*models.DiggerJob)
+	organisation, err := models.DB.GetOrganisationById(organisationId)
+	if err != nil {
+		log.Printf("Error getting organisation: %v %v", organisationId, err)
+		return nil, nil, fmt.Errorf("error retriving organisation")
+	}
+	organisationName := organisation.Name
+
+	backendHostName := os.Getenv("HOSTNAME")
 
 	log.Printf("Number of Jobs: %v\n", len(jobsMap))
 	marshalledJobsMap := map[string][]byte{}
 	for projectName, job := range jobsMap {
-		marshalled, err := json.Marshal(orchestrator.JobToJson(job, projectMap[projectName]))
+		jobToken, err := models.DB.CreateDiggerJobToken(organisationId)
+		if err != nil {
+			log.Printf("Error creating job token: %v %v", projectName, err)
+			return nil, nil, fmt.Errorf("error creating job token")
+		}
+
+		marshalled, err := json.Marshal(orchestrator.JobToJson(job, organisationName, jobToken.Value, backendHostName, projectMap[projectName]))
 		if err != nil {
 			return nil, nil, err
 		}

cli/cmd/digger/main.go

@@ -96,9 +96,6 @@ func gitHubCI(lock core_locking.Lock, policyChecker core_policy.Checker, backend
 
 	// this is used when called from api by the backend and exits in the end of if statement
 	if wdEvent, ok := ghEvent.(github.WorkflowDispatchEvent); ok && runningMode != "manual" && runningMode != "drift-detection" {
-		if !strings.Contains(os.Getenv("DIGGER_HOSTNAME"), "cloud.digger.dev") {
-			usage.SendUsageRecord(githubActor, "log", "selfhosted")
-		}
 
 		type Inputs struct {
 			JobString string `json:"job"`
@@ -125,8 +122,19 @@ func gitHubCI(lock core_locking.Lock, policyChecker core_policy.Checker, backend
 		var job orchestrator.JobJson
 
 		err = json.Unmarshal([]byte(inputs.JobString), &job)
+		if err != nil {
+			reportErrorAndExit(githubActor, fmt.Sprintf("Failed unmarshall job string: %v", err), 4)
+		}
 		commentId64, err := strconv.ParseInt(inputs.CommentId, 10, 64)
 
+		if job.BackendHostname != "" && job.BackendOrganisationName != "" && job.BackendJobToken != "" {
+			log.Printf("Found settings sent by backend in job string, overriding backendApi and policyCheckecd r. setting: (orgName: %v BackedHost: %v token: %v)", job.BackendOrganisationName, job.BackendHostname, "****")
+			backendApi = NewBackendApi(job.BackendHostname, job.BackendJobToken)
+			policyChecker = NewPolicyChecker(job.BackendHostname, job.BackendOrganisationName, job.BackendJobToken)
+		} else {
+			reportErrorAndExit(githubActor, fmt.Sprintf("Missing values from job spec: hostname, orgName, token: %v %v", job.BackendHostname, job.BackendOrganisationName), 4)
+		}
+
 		err = githubPrService.SetOutput(*job.PullRequestNumber, "DIGGER_PR_NUMBER", fmt.Sprintf("%v", *job.PullRequestNumber))
 		if err != nil {
 			reportErrorAndExit(githubActor, fmt.Sprintf("Failed to set job output. Exiting. %s", err), 4)