diff --git a/backend/bootstrap/main.go b/backend/bootstrap/main.go index 3b7ed656a..8bbd078dd 100644 --- a/backend/bootstrap/main.go +++ b/backend/bootstrap/main.go @@ -229,6 +229,9 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController reposApiGroup.GET("/", controllers.ListReposApi) reposApiGroup.GET("/:repo_id/jobs", controllers.GetJobsForRepoApi) + projectsApiGroup := apiGroup.Group("/projects") + projectsApiGroup.GET("/", controllers.ListProjectsApi) + githubApiGroup := apiGroup.Group("/github") githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi) diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index 7aa4dc1b9..577a39798 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -614,7 +614,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) { // create test project projectName := "test project" - _, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false) + _, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false) if err != nil { panic(err) } diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go index d9dd833ba..bbc1f9d4f 100644 --- a/backend/controllers/projects.go +++ b/backend/controllers/projects.go @@ -25,8 +25,50 @@ import ( "gorm.io/gorm" ) -func ListProjects(c *gin.Context) { +func ListProjectsApi(c *gin.Context) { + // assume all exists as validated in middleware + organisationId := c.GetString(middleware.ORGANISATION_ID_KEY) + organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY) + var org models.Organisation + err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + slog.Info("Organisation not found", "organisationId", organisationId, "source", organisationSource) + c.String(http.StatusNotFound, "Could not find organisation: "+organisationId) + } else { + slog.Error("Error fetching organisation", "organisationId", organisationId, "source", organisationSource, "error", err) + c.String(http.StatusInternalServerError, "Error fetching organisation") + } + return + } + + var projects []models.Project + + err = models.DB.GormDB.Preload("Organisation"). + Where("projects.organisation_id = ?", org.ID). + Order("name"). + Find(&projects).Error + + if err != nil { + slog.Error("Error fetching projects", "organisationId", organisationId, "orgId", org.ID, "error", err) + c.String(http.StatusInternalServerError, "Unknown error occurred while fetching database") + return + } + + marshalledRepos := make([]interface{}, 0) + + for _, p := range projects { + marshalled := p.MapToJsonStruct() + marshalledRepos = append(marshalledRepos, marshalled) + } + + slog.Info("Successfully fetched projects", "organisationId", organisationId, "orgId", org.ID, "projectCount", len(projects)) + + response := make(map[string]interface{}) + response["result"] = marshalledRepos + + c.JSON(http.StatusOK, response) } func FindProjectsForRepo(c *gin.Context) { @@ -353,11 +395,10 @@ func ReportProjectsForRepo(c *gin.Context) { ) project := models.Project{ - Name: request.Name, - ConfigurationYaml: request.ConfigurationYaml, - OrganisationID: org.ID, - RepoFullName: repo.RepoFullName, - Organisation: org, + Name: request.Name, + OrganisationID: org.ID, + RepoFullName: repo.RepoFullName, + Organisation: org, } err = models.DB.GormDB.Create(&project).Error diff --git a/backend/go.mod b/backend/go.mod index 89cff3326..80b5b83b3 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -20,9 +20,9 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/go-github/v61 v61.0.0 github.com/google/uuid v1.6.0 + github.com/imdatngo/slog-gorm/v2 v2.0.0 github.com/ktrysmt/go-bitbucket v0.9.81 github.com/migueleliasweb/go-github-mock v0.0.23 - github.com/orandin/slog-gorm v1.4.0 github.com/robfig/cron v1.2.0 github.com/samber/lo v1.39.0 github.com/samber/slog-gin v1.15.0 diff --git a/backend/go.sum b/backend/go.sum index 1f0472ddd..1f2a41443 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1421,6 +1421,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdatngo/slog-gorm/v2 v2.0.0 h1:SrMVqpExd3USDZjiFlJoFg3FJvZOreJX/d8N8G6PQ4w= +github.com/imdatngo/slog-gorm/v2 v2.0.0/go.mod h1:hp2V0UOoVXwMT9YUbB/EbaXCN7g/7txxekKaalO/6fU= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -1683,8 +1685,6 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM= github.com/opencontainers/runc v1.1.9/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= -github.com/orandin/slog-gorm v1.4.0 h1:FgA8hJufF9/jeNSYoEXmHPPBwET2gwlF3B85JdpsTUU= -github.com/orandin/slog-gorm v1.4.0/go.mod h1:MoZ51+b7xE9lwGNPYEhxcUtRNrYzjdcKvA8QXQQGEPA= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/owenrumney/go-sarif v1.1.1 h1:QNObu6YX1igyFKhdzd7vgzmw7XsWN3/6NMGuDzBgXmE= diff --git a/backend/migrations/20250704024948.sql b/backend/migrations/20250704024948.sql new file mode 100644 index 000000000..94c08954c --- /dev/null +++ b/backend/migrations/20250704024948.sql @@ -0,0 +1,2 @@ +-- Modify "projects" table +ALTER TABLE "public"."projects" ADD COLUMN "directory" text NULL; diff --git a/backend/migrations/20250704025208.sql b/backend/migrations/20250704025208.sql new file mode 100644 index 000000000..8f680348c --- /dev/null +++ b/backend/migrations/20250704025208.sql @@ -0,0 +1,2 @@ +-- Modify "projects" table +ALTER TABLE "public"."projects" DROP COLUMN "configuration_yaml"; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index da97b48cd..1937eb05d 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:IeQ6uHSz71Ox7tBqG6sy05KmaRMpYqhr0SeipkVSEbM= +h1:E1MYgdCTYDdHVINjSBVtGufzvYspZC8Wur7nvPK7fN0= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -50,3 +50,5 @@ h1:IeQ6uHSz71Ox7tBqG6sy05KmaRMpYqhr0SeipkVSEbM= 20250512172515.sql h1:iIZIhFq3TTyjTzsxNWv3k4FcIkXfOCdHtmHNmYqjBMM= 20250512213729.sql h1:n8bYIXWko9xOUs+FPcG7lS3GXMVQBSygXX7Hpj20eVs= 20250530015921.sql h1:FidRW+0ur3mCDBPgP7V/sqr2jjfmi/CFJPRpNxTmqGs= +20250704024948.sql h1:ki8bOrfd8y7Mfq40Dz5lH6MUuQqPVah+oimeRDVHOOQ= +20250704025208.sql h1:XRY66dphINSj53mbxl5ALmyWqebuB1swPvUuUPvNR88= diff --git a/backend/models/orgs.go b/backend/models/orgs.go index d64e3d526..de1e9e1b0 100644 --- a/backend/models/orgs.go +++ b/backend/models/orgs.go @@ -73,14 +73,14 @@ const ( type Project struct { gorm.Model - Name string `gorm:"uniqueIndex:idx_project_org"` - OrganisationID uint `gorm:"uniqueIndex:idx_project_org"` - Organisation *Organisation - RepoFullName string `gorm:"uniqueIndex:idx_project_org"` - ConfigurationYaml string // TODO: probably needs to be deleted - Status ProjectStatus - IsGenerated bool - IsInMainBranch bool + Name string `gorm:"uniqueIndex:idx_project_org"` + Directory string + OrganisationID uint `gorm:"uniqueIndex:idx_project_org"` + Organisation *Organisation + RepoFullName string `gorm:"uniqueIndex:idx_project_org"` + Status ProjectStatus + IsGenerated bool + IsInMainBranch bool } func (p *Project) MapToJsonStruct() interface{} { @@ -97,9 +97,6 @@ func (p *Project) MapToJsonStruct() interface{} { OrganisationName string `json:"organisation_name"` RepoID uint `json:"repo_id"` RepoFullName string `json:"repo_full_name"` - RepoName string `json:"repo_name"` - RepoOrg string `json:"repo_org"` - RepoUrl string `json:"repo_url"` IsInMainBranch bool `json:"is_in_main_branch"` IsGenerated bool `json:"is_generated"` LastActivityTimestamp string `json:"last_activity_timestamp"` @@ -108,6 +105,7 @@ func (p *Project) MapToJsonStruct() interface{} { }{ Id: p.ID, Name: p.Name, + Directory: p.Directory, OrganisationID: p.OrganisationID, OrganisationName: p.Organisation.Name, RepoFullName: p.RepoFullName, diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go index 6d08010e0..526b5549e 100644 --- a/backend/models/scheduler_test.go +++ b/backend/models/scheduler_test.go @@ -54,7 +54,7 @@ func setupSuiteScheduler(tb testing.TB) (func(tb testing.TB), *Database) { } projectName := "test project" - _, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false) + _, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false) if err != nil { panic(err) } diff --git a/backend/models/setup.go b/backend/models/setup.go index 6b5306944..8a3103d85 100644 --- a/backend/models/setup.go +++ b/backend/models/setup.go @@ -1,13 +1,13 @@ package models import ( - "log/slog" - "os" - - slogGorm "github.com/orandin/slog-gorm" + sloggorm "github.com/imdatngo/slog-gorm/v2" "gorm.io/driver/postgres" _ "gorm.io/driver/postgres" "gorm.io/gorm" + "log/slog" + "os" + "time" ) type Database struct { @@ -20,8 +20,19 @@ var DEFAULT_ORG_NAME = "digger" var DB *Database func ConnectDatabase() { + slogger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + + cfg := sloggorm.NewConfig(slogger.Handler()). + WithGroupKey("db"). + WithSlowThreshold(time.Second). + WithIgnoreRecordNotFoundError(true) + + if os.Getenv("DIGGER_LOG_LEVEL") == "DEBUG" { + cfg.WithTraceAll(true) + } + glogger := sloggorm.NewWithConfig(cfg) database, err := gorm.Open(postgres.Open(os.Getenv("DATABASE_URL")), &gorm.Config{ - Logger: slogGorm.New(), + Logger: glogger, }) if err != nil { slog.Error("Failed to connect to database", "error", err) diff --git a/backend/models/storage.go b/backend/models/storage.go index d7357ff18..4acfb28a5 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -378,7 +378,7 @@ func (db *Database) GetRepo(orgIdKey any, repoName string) (*Repo, error) { } func (db *Database) GetRepoByFullName(orgIdKey any, repoFullName string) (*Repo, error) { - slog.Info("getting repo by name", + slog.Info("getting repo by full name", "orgId", orgIdKey, "repoName", repoFullName) @@ -1402,9 +1402,10 @@ func (db *Database) CreateOrganisation(name string, externalSource string, tenan return org, nil } -func (db *Database) CreateProject(name string, org *Organisation, repoFullName string, isGenerated bool, isInMainBranch bool) (*Project, error) { +func (db *Database) CreateProject(name string, directory string, org *Organisation, repoFullName string, isGenerated bool, isInMainBranch bool) (*Project, error) { project := &Project{ Name: name, + Directory: directory, Organisation: org, RepoFullName: repoFullName, Status: ProjectActive, @@ -1642,15 +1643,19 @@ func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.D err = db.GormDB.Transaction(func(tx *gorm.DB) error { for _, dc := range config.Projects { projectName := dc.Name + projectDirectory := dc.Dir p, err := db.GetProjectByName(orgId, repoFullName, projectName) if err != nil { return fmt.Errorf("error retrieving project by name: %v", err) } if p == nil { - _, err := db.CreateProject(projectName, org, repoFullName, false, true) + _, err := db.CreateProject(projectName, projectDirectory, org, repoFullName, false, true) if err != nil { return fmt.Errorf("could not create project: %v", err) } + } else { + p.Directory = projectDirectory + db.GormDB.Save(p) } } return nil diff --git a/backend/services/repos.go b/backend/services/repos.go index ce16108af..1efb9a916 100644 --- a/backend/services/repos.go +++ b/backend/services/repos.go @@ -4,15 +4,13 @@ import ( "fmt" "github.com/diggerhq/digger/backend/models" utils3 "github.com/diggerhq/digger/backend/utils" - "github.com/diggerhq/digger/ee/drift/utils" dg_configuration "github.com/diggerhq/digger/libs/digger_config" - utils2 "github.com/diggerhq/digger/next/utils" "log/slog" "strconv" "strings" ) -func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId string, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string) error { +func LoadProjectsFromGithubRepo(gh utils3.GithubClientProvider, installationId string, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string) error { installationId64, err := strconv.ParseInt(installationId, 10, 64) if err != nil { slog.Error("failed to convert installation id %v to int64", "insallationId", installationId) @@ -36,7 +34,8 @@ func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId s return fmt.Errorf("repo not found: Org: %v | repo: %v", orgId, diggerRepoName) } - _, token, err := utils.GetGithubService(gh, installationId64, repoFullName, repoOwner, repoName) + slog.Debug("getting github service", "installationId", installationId, "repoFullName", repoFullName, "repoOwner", repoOwner, "repoName", repoName) + _, token, err := utils3.GetGithubService(gh, installationId64, repoFullName, repoOwner, repoName) if err != nil { slog.Error("getting github service", "error", err) return fmt.Errorf("error getting github service") diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index 9f74d3135..323c8496c 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -74,7 +74,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) { // create test project projectName := "test project" - _, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false) + _, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false) if err != nil { panic(err) } @@ -140,7 +140,7 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) { for i, testParam := range testParameters { ciService := github2.MockCiService{} batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, true, nil) - project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), nil, "", false, false) + project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), "", nil, "", false, false) planStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) applyStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) diggerRun, _ := models.DB.CreateDiggerRun("", 1, testParam.InitialStatus, "sha", "", 123, 1, project.Name, "apply", &planStage.ID, &applyStage.ID) diff --git a/ee/drift/utils/github.go b/ee/drift/utils/github.go index a843a8db7..14441ad19 100644 --- a/ee/drift/utils/github.go +++ b/ee/drift/utils/github.go @@ -13,6 +13,7 @@ import ( "github.com/dominikbraun/graph" "github.com/google/go-github/v61/github" "log" + "log/slog" net "net/http" "os" "path" @@ -20,6 +21,9 @@ import ( ) func GetGithubClient(gh utils.GithubClientProvider, installationId int64, repoFullName string) (*github.Client, *string, error) { + slog.Debug("Getting GitHub client", + "installationId", installationId, + "repoFullName", repoFullName) repo, err := dbmodels.DB.GetRepoByInstllationIdAndRepoFullName(strconv.FormatInt(installationId, 10), repoFullName) if err != nil { log.Printf("Error getting repo: %v", err) @@ -31,6 +35,7 @@ func GetGithubClient(gh utils.GithubClientProvider, installationId int64, repoFu } func GetGithubService(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string) (*github2.GithubService, *string, error) { + slog.Debug("getting github client", "installationId", installationId, "repoFullName", repoFullName) ghClient, token, err := GetGithubClient(gh, installationId, repoFullName) if err != nil { log.Printf("Error creating github app client: %v", err)