load projects on push

backend/controllers/github.go

@@ -100,6 +100,13 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 				return
 			}
 		}
+	case *github.PushEvent:
+		slog.Info("Processing PushEvent",
+			"action", *event.Action,
+			"repo", *event.Repo.FullName,
+		)
+
+		go handlePushEvent(gh, event, appId64)
 
 	case *github.IssueCommentEvent:
 		slog.Info("Processing IssueCommentEvent",
@@ -383,6 +390,33 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent, appI
 	return nil
 }
 
+func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent, appId int64) error {
+	slog.Debug("Handling push event", "appId", appId, "payload", payload)
+	defer func() {
+		if r := recover(); r != nil {
+			stack := string(debug.Stack())
+			slog.Error("Recovered from panic in handlePushEvent", "error", r, slog.Group("stack", slog.String("trace", stack)))
+		}
+	}()
+
+	installationId := *payload.Installation.ID
+	repoName := *payload.Repo.Name
+	repoOwner := *payload.Repo.Owner.Login
+	repoFullName := *payload.Repo.FullName
+	cloneURL := *payload.Repo.CloneURL
+	ref := *payload.Ref
+	defaultBranch := *payload.Repo.DefaultBranch
+
+	if strings.HasSuffix(ref, defaultBranch) {
+		err := services.LoadProjectsFromGithubRepo(gh, strconv.FormatInt(installationId, 10), repoFullName, repoOwner, repoName, cloneURL, defaultBranch)
+		if err != nil {
+			slog.Error("Failed to load projects from GitHub repo", "error", err)
+		}
+	}
+
+	return nil
+}
+
 func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error {
 	defer func() {
 		if r := recover(); r != nil {

backend/controllers/policies.go

@@ -249,7 +249,7 @@ func upsertPolicyForRepoAndProject(c *gin.Context, policyType string) {
 	if projectResult.RowsAffected == 0 {
 		projectModel = models.Project{
 			OrganisationID: orgID.(uint),
-			RepoID:         repoModel.ID,
+			RepoFullName:   repoModel.RepoFullName,
 			Name:           projectName,
 		}
 		err := models.DB.GormDB.Create(&projectModel).Error

backend/controllers/projects.go

@@ -251,7 +251,7 @@ func ProjectDetails(c *gin.Context) {
 	slog.Info("Successfully retrieved project details",
 		"projectId", projectId,
 		"projectName", project.Name,
-		"repoName", project.Repo.Name,
+		"repoFullName", project.RepoFullName,
 	)
 
 	c.JSON(http.StatusOK, project.MapToJsonStruct())
@@ -355,9 +355,8 @@ func ReportProjectsForRepo(c *gin.Context) {
 			project := models.Project{
 				Name:              request.Name,
 				ConfigurationYaml: request.ConfigurationYaml,
-				RepoID:            repo.ID,
 				OrganisationID:    org.ID,
-				Repo:              &repo,
+				RepoFullName:      repo.RepoFullName,
 				Organisation:      org,
 			}
 

backend/controllers/runs.go

@@ -57,6 +57,13 @@ func RunsForProject(c *gin.Context) {
 		return
 	}
 
+	repo, err := models.DB.GetRepoByFullName(org.ID, project.RepoFullName)
+	if err != nil {
+		slog.Error("Could not fetch repo", "projectId", projectId, "repoFullName", project.RepoFullName, "error", err)
+		c.String(http.StatusInternalServerError, "Could not fetch repo")
+		return
+	}
+
 	if project.OrganisationID != org.ID {
 		slog.Warn("Forbidden access: not allowed to access project",
 			"projectOrgId", project.OrganisationID,
@@ -66,9 +73,9 @@ func RunsForProject(c *gin.Context) {
 		return
 	}
 
-	runs, err := models.DB.ListDiggerRunsForProject(project.Name, project.Repo.ID)
+	runs, err := models.DB.ListDiggerRunsForProject(project.Name, repo.ID)
 	if err != nil {
-		slog.Error("Could not fetch runs", "projectId", projectId, "repoId", project.Repo.ID, "error", err)
+		slog.Error("Could not fetch runs", "projectId", projectId, "repoId", repo.ID, "error", err)
 		c.String(http.StatusInternalServerError, "Could not fetch runs")
 		return
 	}
@@ -87,7 +94,7 @@ func RunsForProject(c *gin.Context) {
 	slog.Info("Successfully fetched runs for project",
 		"projectId", projectId,
 		"projectName", project.Name,
-		"repoId", project.Repo.ID,
+		"repoId", repo.ID,
 		"runCount", len(runs))
 
 	response := make(map[string]interface{})

backend/models/orgs.go

@@ -55,8 +55,6 @@ func (p *ProjectRun) MapToJsonStruct() interface{} {
 		Id:            p.ID,
 		ProjectID:     p.ProjectID,
 		ProjectName:   p.Project.Name,
-		RepoUrl:       p.Project.Repo.RepoUrl,
-		RepoFullName:  p.Project.Repo.RepoFullName,
 		StartedAt:     time.UnixMilli(p.StartedAt),
 		EndedAt:       time.UnixMilli(p.EndedAt),
 		Status:        p.Status,
@@ -78,8 +76,7 @@ type Project struct {
 	Name              string `gorm:"uniqueIndex:idx_project"`
 	OrganisationID    uint   `gorm:"uniqueIndex:idx_project"`
 	Organisation      *Organisation
-	RepoID            uint `gorm:"uniqueIndex:idx_project"`
-	Repo              *Repo
+	RepoFullName      string
 	ConfigurationYaml string // TODO: probably needs to be deleted
 	Status            ProjectStatus
 	IsGenerated       bool
@@ -112,12 +109,8 @@ func (p *Project) MapToJsonStruct() interface{} {
 		Id:                    p.ID,
 		Name:                  p.Name,
 		OrganisationID:        p.OrganisationID,
-		RepoID:                p.RepoID,
 		OrganisationName:      p.Organisation.Name,
-		RepoFullName:          p.Repo.RepoFullName,
-		RepoName:              p.Repo.RepoName,
-		RepoOrg:               p.Repo.RepoOrganisation,
-		RepoUrl:               p.Repo.RepoUrl,
+		RepoFullName:          p.RepoFullName,
 		LastActivityTimestamp: p.UpdatedAt.String(),
 		LastActivityAuthor:    "unknown",
 		LastActivityStatus:    string(status),

backend/models/storage.go

@@ -215,34 +215,33 @@ func (db *Database) GetProject(projectId uint) (*Project, error) {
 
 // GetProjectByName return project for specified org and repo
 // if record doesn't exist return nil
-func (db *Database) GetProjectByName(orgId any, repo *Repo, name string) (*Project, error) {
+func (db *Database) GetProjectByName(orgId any, repoFullName string, name string) (*Project, error) {
 	slog.Info("getting project by name",
 		slog.Group("project",
 			"orgId", orgId,
-			"repoId", repo.ID,
+			"repoFullName", repoFullName,
 			"name", name))
 
 	var project Project
 
-	err := db.GormDB.Preload("Organisation").Preload("Repo").
-		Joins("INNER JOIN repos ON projects.repo_id = repos.id").
+	err := db.GormDB.Preload("Organisation").
 		Joins("INNER JOIN organisations ON projects.organisation_id = organisations.id").
 		Where("projects.organisation_id = ?", orgId).
-		Where("repos.id = ?", repo.ID).
+		Where("repo_full_name = ?", repoFullName).
 		Where("projects.name = ?", name).First(&project).Error
 
 	if err != nil {
 		if errors.Is(err, gorm.ErrRecordNotFound) {
 			slog.Debug("project not found",
 				"orgId", orgId,
-				"repoId", repo.ID,
+				"repoFullName", repoFullName,
 				"name", name)
 			return nil, nil
 		}
 		slog.Error("error fetching project from database",
 			"error", err,
 			"orgId", orgId,
-			"repoId", repo.ID,
+			"repoFullname", repoFullName,
 			"name", name)
 		return nil, err
 	}
@@ -378,6 +377,38 @@ func (db *Database) GetRepo(orgIdKey any, repoName string) (*Repo, error) {
 	return &repo, nil
 }
 
+func (db *Database) GetRepoByFullName(orgIdKey any, repoFullName string) (*Repo, error) {
+	slog.Info("getting repo by name",
+		"orgId", orgIdKey,
+		"repoName", repoFullName)
+
+	var repo Repo
+
+	err := db.GormDB.Preload("Organisation").
+		Joins("INNER JOIN organisations ON repos.organisation_id = organisations.id").
+		Where("organisations.id = ? AND repos.repo_full_name=?", orgIdKey, repoFullName).First(&repo).Error
+
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Debug("repo not found",
+				"orgId", orgIdKey,
+				"repoFullName", repoFullName)
+			return nil, fmt.Errorf("repo not found %v", repoFullName)
+		}
+		slog.Error("failed to find digger repo",
+			"error", err,
+			"orgId", orgIdKey,
+			"repoFullName", repoFullName)
+		return nil, err
+	}
+
+	slog.Debug("found repo",
+		"repoId", repo.ID,
+		"orgId", orgIdKey,
+		"repoName", repoFullName)
+	return &repo, nil
+}
+
 // GetRepoById returns digger repo by organisationId and repo name (diggerhq-digger)
 func (db *Database) GetRepoById(orgIdKey any, repoId any) (*Repo, error) {
 	var repo Repo
@@ -834,7 +865,7 @@ func (db *Database) ListDiggerRunsForProject(projectName string, repoId uint) ([
 	var runs []DiggerRun
 
 	err := db.GormDB.Preload("PlanStage").Preload("ApplyStage").
-		Where("project_name = ? AND repo_id=  ?", projectName, repoId).Order("created_at desc").Find(&runs).Error
+		Where("project_name = ? AND repo_id = ?", projectName, repoId).Order("created_at desc").Find(&runs).Error
 
 	if err != nil {
 		slog.Error("error fetching digger runs for project",
@@ -1371,11 +1402,11 @@ func (db *Database) CreateOrganisation(name string, externalSource string, tenan
 	return org, nil
 }
 
-func (db *Database) CreateProject(name string, org *Organisation, repo *Repo, isGenerated bool, isInMainBranch bool) (*Project, error) {
+func (db *Database) CreateProject(name string, org *Organisation, repoFullName string, isGenerated bool, isInMainBranch bool) (*Project, error) {
 	project := &Project{
 		Name:           name,
 		Organisation:   org,
-		Repo:           repo,
+		RepoFullName:   repoFullName,
 		Status:         ProjectActive,
 		IsGenerated:    isGenerated,
 		IsInMainBranch: isInMainBranch,
@@ -1385,7 +1416,7 @@ func (db *Database) CreateProject(name string, org *Organisation, repo *Repo, is
 		slog.Error("failed to create project",
 			"name", name,
 			"orgId", org.ID,
-			"repoId", repo.ID,
+			"repoFullName", repoFullName,
 			"error", result.Error)
 		return nil, result.Error
 	}
@@ -1394,7 +1425,7 @@ func (db *Database) CreateProject(name string, org *Organisation, repo *Repo, is
 			"id", project.ID,
 			"name", name,
 			"orgId", org.ID,
-			"repoId", repo.ID,
+			"repoFullName", repoFullName,
 			"isGenerated", isGenerated,
 			"isInMainBranch", isInMainBranch))
 	return project, nil
@@ -1600,105 +1631,34 @@ func validateDiggerConfigYaml(configYaml string) (*configuration.DiggerConfig, e
 	return diggerConfig, nil
 }
 
-func (db *Database) UpdateRepoDiggerConfig(orgId any, config configuration.DiggerConfigYaml, repo *Repo, isMainBranch bool) error {
-	slog.Info("updating repo digger config",
-		"repoId", repo.ID,
-		"repoName", repo.Name,
-		"orgId", orgId,
-		"isMainBranch", isMainBranch)
+func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.DiggerConfigYaml, repoFullName string) error {
+	slog.Debug("UpdateRepoDiggerConfig, repo", "repoFullName", repoFullName)
 
 	org, err := db.GetOrganisationById(orgId)
 	if err != nil {
-		slog.Error("failed to get organisation",
-			"orgId", orgId,
-			"error", err)
-		return err
+		return fmt.Errorf("error retrieving org by name: %v", err)
 	}
 
 	err = db.GormDB.Transaction(func(tx *gorm.DB) error {
-		if isMainBranch {
-			// we reset all projects already in main branch to create new projects
-			repoProjects, err := db.GetProjectByRepo(orgId, repo)
-			if err != nil {
-				slog.Error("could not get repo projects",
-					"repoId", repo.ID,
-					"orgId", orgId,
-					"error", err)
-				return fmt.Errorf("could not get repo projects: %v", err)
-			}
-
-			slog.Info("resetting main branch flag for existing projects",
-				"projectCount", len(repoProjects),
-				"repoId", repo.ID)
-
-			for _, rp := range repoProjects {
-				rp.IsInMainBranch = false
-				err = db.UpdateProject(&rp)
-				if err != nil {
-					slog.Error("could not update existing main branch project",
-						"projectId", rp.ID,
-						"error", err)
-					return fmt.Errorf("could not update existing main branch projects: %v", err)
-				}
-			}
-		}
-
-		slog.Info("processing projects from config",
-			"projectCount", len(config.Projects),
-			"repoId", repo.ID)
-
 		for _, dc := range config.Projects {
 			projectName := dc.Name
-			p, err := db.GetProjectByName(orgId, repo, projectName)
+			p, err := db.GetProjectByName(orgId, repoFullName, projectName)
 			if err != nil {
-				slog.Error("error retrieving project by name",
-					"projectName", projectName,
-					"repoId", repo.ID,
-					"error", err)
 				return fmt.Errorf("error retrieving project by name: %v", err)
 			}
-
 			if p == nil {
-				slog.Info("creating new project",
-					"projectName", projectName,
-					"repoId", repo.ID,
-					"isGenerated", dc.Generated,
-					"isMainBranch", isMainBranch)
-
-				_, err := db.CreateProject(projectName, org, repo, dc.Generated, isMainBranch)
+				_, err := db.CreateProject(projectName, org, repoFullName, false, true)
 				if err != nil {
-					slog.Error("could not create project",
-						"projectName", projectName,
-						"error", err)
 					return fmt.Errorf("could not create project: %v", err)
 				}
-			} else {
-				slog.Info("updating existing project",
-					"projectId", p.ID,
-					"projectName", projectName,
-					"isGenerated", dc.Generated,
-					"isMainBranch", isMainBranch)
-
-				if isMainBranch == true {
-					p.IsInMainBranch = isMainBranch
-				}
-				p.IsGenerated = dc.Generated
-				db.UpdateProject(p)
 			}
 		}
 		return nil
 	})
 
 	if err != nil {
-		slog.Error("error while updating projects from config",
-			"repoId", repo.ID,
-			"error", err)
 		return fmt.Errorf("error while updating projects from config: %v", err)
 	}
-
-	slog.Info("successfully updated repo digger config",
-		"repoId", repo.ID,
-		"projectCount", len(config.Projects))
 	return nil
 }