From ee391d50de744579032d2697643268184f8f02e9 Mon Sep 17 00:00:00 2001 From: motatoes Date: Fri, 4 Jul 2025 20:38:35 -0700 Subject: [PATCH 1/6] projects refresh service --- Dockerfile_bg_projects_refresh | 28 +++++++++ backend/controllers/cache.go | 3 +- backend/controllers/github.go | 3 +- backend/services/repos.go | 3 +- backend/utils/bitbucket.go | 3 +- backend/utils/github.go | 60 ------------------ backend/utils/github_test.go | 9 +-- backend/utils/gitlab.go | 3 +- .../projects-refresh-service/.gitignore | 1 + background/projects-refresh-service/go.mod | 3 + background/projects-refresh-service/main.go | 40 ++++++++++++ ee/cli/pkg/policy/policy.go | 4 +- ee/cli/pkg/utils/github.go | 35 ----------- ee/drift/tasks/github.go | 2 +- ee/drift/utils/github.go | 2 +- go.work | 1 + .../git_utils/clone_utils.go | 63 ++++++++++++++++++- next/controllers/github.go | 3 +- next/services/config.go | 2 +- 19 files changed, 157 insertions(+), 111 deletions(-) create mode 100644 Dockerfile_bg_projects_refresh create mode 100644 background/projects-refresh-service/.gitignore create mode 100644 background/projects-refresh-service/go.mod create mode 100644 background/projects-refresh-service/main.go delete mode 100644 ee/cli/pkg/utils/github.go rename backend/utils/gitshell.go => libs/git_utils/clone_utils.go (70%) diff --git a/Dockerfile_bg_projects_refresh b/Dockerfile_bg_projects_refresh new file mode 100644 index 000000000..2179d4f20 --- /dev/null +++ b/Dockerfile_bg_projects_refresh @@ -0,0 +1,28 @@ +FROM golang:1.24.0 as builder +ARG COMMIT_SHA +RUN echo "commit sha: ${COMMIT_SHA}" + +# Set the working directory +WORKDIR $GOPATH/src/github.com/diggerhq/digger + +# Copy all required source, blacklist files that are not required through `.dockerignore` +COPY . . + +RUN go build -ldflags="-X 'main.Version=${COMMIT_SHA}'" -o projects_refresh_exe ./background/projects-refresh-service + +# Multi-stage build will just copy the binary to an alpine image. +FROM ubuntu:24.04 as runner +ARG COMMIT_SHA +WORKDIR /app + +RUN apt-get update && apt-get install -y ca-certificates curl && apt-get install -y git && apt-get clean all +RUN update-ca-certificates + +RUN echo "commit sha: ${COMMIT_SHA}" + +# Copy the binary to the corresponding folder +COPY --from=builder /go/src/github.com/diggerhq/digger/projects_refresh_exe /app/projects_refresh_exe +RUN chmod +x projects_refresh_exe + +# Run the binary +ENTRYPOINT ["/bin/bash", "/app/entrypoint.sh"] diff --git a/backend/controllers/cache.go b/backend/controllers/cache.go index 86f04dce7..8f519fd03 100644 --- a/backend/controllers/cache.go +++ b/backend/controllers/cache.go @@ -2,6 +2,7 @@ package controllers import ( "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "net/http" "os" @@ -67,7 +68,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) { // update the cache here, do it async for immediate response go func() { - err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 539eeb136..95b9d34b9 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "math/rand" "net/http" @@ -915,7 +916,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 var diggerYmlStr string var dependencyGraph graph.Graph[string, dg_configuration.Project] - err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { slog.Debug("Reading Digger config from cloned repository", "directory", dir) diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir) diff --git a/backend/services/repos.go b/backend/services/repos.go index 1efb9a916..3bd51136e 100644 --- a/backend/services/repos.go +++ b/backend/services/repos.go @@ -5,6 +5,7 @@ import ( "github.com/diggerhq/digger/backend/models" utils3 "github.com/diggerhq/digger/backend/utils" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "strconv" "strings" @@ -41,7 +42,7 @@ func LoadProjectsFromGithubRepo(gh utils3.GithubClientProvider, installationId s return fmt.Errorf("error getting github service") } - err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) if err != nil { slog.Error("failed to load digger.yml: %v", "error", err) diff --git a/backend/utils/bitbucket.go b/backend/utils/bitbucket.go index c47468ae0..7a14ab5ce 100644 --- a/backend/utils/bitbucket.go +++ b/backend/utils/bitbucket.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "net/http" "os" @@ -94,7 +95,7 @@ func GetDiggerConfigForBitbucketBranch(bb BitbucketProvider, token string, repoF "changedFilesCount", len(changedFiles), ) - err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "x-token-auth", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "x-token-auth", func(dir string) error { diggerYmlPath := path.Join(dir, "digger.yml") diggerYmlBytes, err := os.ReadFile(diggerYmlPath) if err != nil { diff --git a/backend/utils/github.go b/backend/utils/github.go index d35e4821f..c45f7a568 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -18,66 +18,6 @@ import ( "github.com/google/go-github/v61/github" ) -func createTempDir() string { - tempDir, err := os.MkdirTemp("", "repo") - if err != nil { - slog.Error("Failed to create temporary directory", "error", err) - panic(err) - } - return tempDir -} - -type action func(string) error - -func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, tokenUsername string, action action) error { - dir := createTempDir() - - slog.Debug("Cloning git repository", - "repoUrl", repoUrl, - "branch", branch, - "commitHash", commitHash, - "directory", dir, - ) - - git := NewGitShellWithTokenAuth(dir, token, tokenUsername) - err := git.Clone(repoUrl, branch) - if err != nil { - slog.Error("Failed to clone repository", - "repoUrl", repoUrl, - "branch", branch, - "error", err, - ) - return err - } - - if commitHash != "" { - err := git.Checkout(commitHash) - if err != nil { - slog.Error("Failed to checkout commit", - "commitHash", commitHash, - "error", err, - ) - return err - } - } - - defer func() { - slog.Debug("Removing cloned directory", "directory", dir) - ferr := os.RemoveAll(dir) - if ferr != nil { - slog.Warn("Failed to remove directory", "directory", dir, "error", ferr) - } - }() - - err = action(dir) - if err != nil { - slog.Error("Error performing action on repository", "directory", dir, "error", err) - return err - } - - return nil -} - // just a wrapper around github client to be able to use mocks type DiggerGithubRealClientProvider struct { } diff --git a/backend/utils/github_test.go b/backend/utils/github_test.go index def6d613e..51c0510c8 100644 --- a/backend/utils/github_test.go +++ b/backend/utils/github_test.go @@ -1,6 +1,7 @@ package utils import ( + "github.com/diggerhq/digger/libs/git_utils" "os" "testing" @@ -9,14 +10,14 @@ import ( func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) { f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", "", f) assert.NotNil(t, err) } func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, "", f) assert.Nil(t, err) } @@ -27,13 +28,13 @@ func TestGithubCloneWithPrivateRepoAndValidTokenThrowsNoError(t *testing.T) { return } f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, "", f) assert.Nil(t, err) } func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, "", f) + err := git_utils.CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, "", f) assert.NotNil(t, err) } diff --git a/backend/utils/gitlab.go b/backend/utils/gitlab.go index 187ff8e93..78c9b8aff 100644 --- a/backend/utils/gitlab.go +++ b/backend/utils/gitlab.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/diggerhq/digger/libs/git_utils" "log/slog" "os" "path" @@ -113,7 +114,7 @@ func GetDiggerConfigForBranchGitlab(gh GitlabProvider, projectId int, repoFullNa "changedFilesCount", len(changedFiles), ) - err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { diggerYmlPath := path.Join(dir, "digger.yml") diggerYmlBytes, err := os.ReadFile(diggerYmlPath) if err != nil { diff --git a/background/projects-refresh-service/.gitignore b/background/projects-refresh-service/.gitignore new file mode 100644 index 000000000..88d050b19 --- /dev/null +++ b/background/projects-refresh-service/.gitignore @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/background/projects-refresh-service/go.mod b/background/projects-refresh-service/go.mod new file mode 100644 index 000000000..164852bf7 --- /dev/null +++ b/background/projects-refresh-service/go.mod @@ -0,0 +1,3 @@ +module github.com/diggerhq/digger/backgorund/projects-refresh-service + +go 1.24.0 diff --git a/background/projects-refresh-service/main.go b/background/projects-refresh-service/main.go new file mode 100644 index 000000000..5243644b3 --- /dev/null +++ b/background/projects-refresh-service/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + "github.com/diggerhq/digger/backend/models" + dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils3 "github.com/diggerhq/digger/libs/git_utils" + "log/slog" + "os" +) + +func main() { + cloneUrl := os.Getenv("DIGGER_GITHUB_REPO_CLONE_URL") + branch := os.Getenv("DIGGER_GITHUB_REPO_CLONE_BRANCH") + token := os.Getenv("DIGGER_GITHUB_TOKEN") + orgId := os.Getenv("DIGGER_ORG_ID") + repoFullName := os.Getenv("DIGGER_REPO_FULL_NAME") + + if cloneUrl == "" && branch == "" && token == "" && orgId == "" && repoFullName == "" { + slog.Info("smoketests mode, skipping this run") + os.Exit(0) + } + + models.ConnectDatabase() + + err := utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { + config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) + if err != nil { + slog.Error("failed to load digger.yml: %v", "error", err) + return fmt.Errorf("error loading digger.yml %v", err) + } + models.DB.RefreshProjectsFromRepo(orgId, *config, repoFullName) + return nil + }) + if err != nil { + slog.Error("error while cloning repo: %v", err) + os.Exit(1) + } + +} diff --git a/ee/cli/pkg/policy/policy.go b/ee/cli/pkg/policy/policy.go index c53983a5d..3919fcdef 100644 --- a/ee/cli/pkg/policy/policy.go +++ b/ee/cli/pkg/policy/policy.go @@ -1,7 +1,7 @@ package policy import ( - "github.com/diggerhq/digger/ee/cli/pkg/utils" + "github.com/diggerhq/digger/libs/git_utils" "github.com/samber/lo" "os" "path" @@ -62,7 +62,7 @@ func GetPrefixesForPath(path string, fileName string) []string { func (p DiggerRepoPolicyProvider) getPolicyFileContents(repo string, projectName string, projectDir string, fileName string) (string, error) { var contents string - err := utils.CloneGitRepoAndDoAction(p.ManagementRepoUrl, "main", p.GitToken, "", func(basePath string) error { + err := git_utils.CloneGitRepoAndDoAction(p.ManagementRepoUrl, "main", "", p.GitToken, "", func(basePath string) error { // we start with the project directory path prefixes as the highest priority prefixes := GetPrefixesForPath(path.Join(basePath, projectDir), fileName) diff --git a/ee/cli/pkg/utils/github.go b/ee/cli/pkg/utils/github.go deleted file mode 100644 index 3655ac6f3..000000000 --- a/ee/cli/pkg/utils/github.go +++ /dev/null @@ -1,35 +0,0 @@ -package utils - -import ( - "github.com/diggerhq/digger/backend/utils" - "log" - "os" -) - -func createTempDir() string { - tempDir, err := os.MkdirTemp("", "repo") - if err != nil { - log.Fatal(err) - } - return tempDir -} - -type action func(string) error - -func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, tokenUsername string, action action) error { - dir := createTempDir() - git := utils.NewGitShellWithTokenAuth(dir, token, tokenUsername) - err := git.Clone(repoUrl, branch) - if err != nil { - return err - } - defer os.RemoveAll(dir) - err = action(dir) - if err != nil { - log.Printf("error performing action: %v", err) - return err - } - - return nil - -} diff --git a/ee/drift/tasks/github.go b/ee/drift/tasks/github.go index f7e6af4b9..ba9995555 100644 --- a/ee/drift/tasks/github.go +++ b/ee/drift/tasks/github.go @@ -2,10 +2,10 @@ package tasks import ( "fmt" - utils3 "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/ee/drift/dbmodels" "github.com/diggerhq/digger/ee/drift/utils" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils3 "github.com/diggerhq/digger/libs/git_utils" utils2 "github.com/diggerhq/digger/next/utils" "log" "strconv" diff --git a/ee/drift/utils/github.go b/ee/drift/utils/github.go index 14441ad19..0dee042e5 100644 --- a/ee/drift/utils/github.go +++ b/ee/drift/utils/github.go @@ -5,10 +5,10 @@ import ( "encoding/base64" "fmt" "github.com/bradleyfalzon/ghinstallation/v2" - utils2 "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/ee/drift/dbmodels" github2 "github.com/diggerhq/digger/libs/ci/github" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils2 "github.com/diggerhq/digger/libs/git_utils" "github.com/diggerhq/digger/next/utils" "github.com/dominikbraun/graph" "github.com/google/go-github/v61/github" diff --git a/go.work b/go.work index c4da94096..4122be1a9 100644 --- a/go.work +++ b/go.work @@ -2,6 +2,7 @@ go 1.24.0 use ( ./backend + ./background/projects-refresh-service ./cli ./cli_e2e ./dgctl diff --git a/backend/utils/gitshell.go b/libs/git_utils/clone_utils.go similarity index 70% rename from backend/utils/gitshell.go rename to libs/git_utils/clone_utils.go index 9abcca0b6..6158506f6 100644 --- a/backend/utils/gitshell.go +++ b/libs/git_utils/clone_utils.go @@ -1,9 +1,10 @@ -package utils +package git_utils import ( "bytes" "context" "fmt" + "log/slog" "net/url" "os" "os/exec" @@ -11,6 +12,66 @@ import ( "time" ) +func createTempDir() string { + tempDir, err := os.MkdirTemp("", "repo") + if err != nil { + slog.Error("Failed to create temporary directory", "error", err) + panic(err) + } + return tempDir +} + +type action func(string) error + +func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, tokenUsername string, action action) error { + dir := createTempDir() + + slog.Debug("Cloning git repository", + "repoUrl", repoUrl, + "branch", branch, + "commitHash", commitHash, + "directory", dir, + ) + + git := NewGitShellWithTokenAuth(dir, token, tokenUsername) + err := git.Clone(repoUrl, branch) + if err != nil { + slog.Error("Failed to clone repository", + "repoUrl", repoUrl, + "branch", branch, + "error", err, + ) + return err + } + + if commitHash != "" { + err := git.Checkout(commitHash) + if err != nil { + slog.Error("Failed to checkout commit", + "commitHash", commitHash, + "error", err, + ) + return err + } + } + + defer func() { + slog.Debug("Removing cloned directory", "directory", dir) + ferr := os.RemoveAll(dir) + if ferr != nil { + slog.Warn("Failed to remove directory", "directory", dir, "error", ferr) + } + }() + + err = action(dir) + if err != nil { + slog.Error("Error performing action on repository", "directory", dir, "error", err) + return err + } + + return nil +} + type GitAuth struct { Username string Password string // Can be either password or access token diff --git a/next/controllers/github.go b/next/controllers/github.go index b14e84bb2..ecbd39e92 100644 --- a/next/controllers/github.go +++ b/next/controllers/github.go @@ -12,6 +12,7 @@ import ( "github.com/diggerhq/digger/libs/ci" dg_github "github.com/diggerhq/digger/libs/ci/github" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + "github.com/diggerhq/digger/libs/git_utils" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/diggerhq/digger/next/ci_backends" "github.com/diggerhq/digger/next/dbmodels" @@ -538,7 +539,7 @@ func getDiggerConfigForBranch(gh next_utils.GithubClientProvider, installationId log.Printf("Error getting changed files: %v", err) return "", nil, nil, nil, fmt.Errorf("error getting changed files") } - err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles) diff --git a/next/services/config.go b/next/services/config.go index f00eee483..29d79af04 100644 --- a/next/services/config.go +++ b/next/services/config.go @@ -2,8 +2,8 @@ package services import ( "fmt" - utils3 "github.com/diggerhq/digger/backend/utils" dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils3 "github.com/diggerhq/digger/libs/git_utils" "github.com/diggerhq/digger/next/dbmodels" "github.com/diggerhq/digger/next/utils" "log" From dbc686eb8ee6cdfc8021ef81f76ba247a2d96ede Mon Sep 17 00:00:00 2001 From: motatoes Date: Sat, 5 Jul 2025 20:45:15 -0700 Subject: [PATCH 2/6] tmp commit --- backend/service_clients/projects_service.go | 99 +++++++++++++++++++++ backend/services/repos.go | 18 ++-- fly-projects-refresh-service.toml | 23 +++++ 3 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 backend/service_clients/projects_service.go create mode 100644 fly-projects-refresh-service.toml diff --git a/backend/service_clients/projects_service.go b/backend/service_clients/projects_service.go new file mode 100644 index 000000000..9dedb231f --- /dev/null +++ b/backend/service_clients/projects_service.go @@ -0,0 +1,99 @@ +package service_clients + +import ( + "bytes" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "os" + "time" +) + +type MachineConfig struct { + Name string `json:"name"` + Config struct { + Image string `json:"image"` + Env struct { + CloneUrl string `json:"DIGGER_GITHUB_REPO_CLONE_URL"` + Branch string `json:"DIGGER_GITHUB_REPO_CLONE_BRANCH"` + GithubToken string `json:"DIGGER_GITHUB_TOKEN"` + RepoFullName string `json:"DIGGER_REPO_FULL_NAME"` + OrgId string `json:"DIGGER_ORG_ID"` + } `json:"env"` + Guest struct { + CPUs int `json:"cpus"` + CPUKind string `json:"cpu_kind"` + MemoryMB int `json:"memory_mb"` + } `json:"guest"` + AutoDestroy bool `json:"auto_destroy"` + } `json:"config"` +} + +type MachineResponse struct { + ID string `json:"id"` +} + +type QueuedResponse struct { + Queued string `json:"queued"` +} + +func TriggerProjectsRefreshService(cloneUrl string, branch string, githubToken string, repoFullName string, orgId string) (*MachineResponse, error) { + + slog.Debug("awaiting machine fetch") + + // Prepare machine configuration + machineConfig := MachineConfig{ + Name: fmt.Sprintf("hello-%d", time.Now().UnixMilli()), + } + + machineConfig.Config.Image = "registry.fly.io/projects-refresh-service:latest" + machineConfig.Config.Env.CloneUrl = cloneUrl + machineConfig.Config.Env.Branch = branch + machineConfig.Config.Env.GithubToken = githubToken + machineConfig.Config.Env.RepoFullName = repoFullName + machineConfig.Config.Env.OrgId = orgId + + machineConfig.Config.Guest.CPUs = 1 + machineConfig.Config.Guest.CPUKind = "shared" + machineConfig.Config.Guest.MemoryMB = 256 + machineConfig.Config.AutoDestroy = true + + // Marshal JSON payload + payload, err := json.Marshal(machineConfig) + if err != nil { + slog.Error("Error creating machine config", "error", err) + return nil, err + } + + // Create HTTP request + apiURL := fmt.Sprintf("https://api.machines.dev/v1/apps/%s/machines", os.Getenv("JOB_APP")) + req2, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(payload)) + if err != nil { + slog.Error("Error creating request", "error", err) + return nil, err + } + + // Set headers + req2.Header.Set("Authorization", "Bearer "+os.Getenv("FLY_PROJECTS_SVC_API_TOKEN")) + req2.Header.Set("Content-Type", "application/json") + + // Make HTTP request + client := &http.Client{} + resp, err := client.Do(req2) + if err != nil { + slog.Error("Error making request", "error", err) + return nil, err + } + defer resp.Body.Close() + + // Parse response + var machineResp MachineResponse + if err := json.NewDecoder(resp.Body).Decode(&machineResp); err != nil { + slog.Error("Error parsing response", "error", err) + return nil, err + } + + slog.Debug("triggered projects refresh service", "machineId", machineResp.ID) + return &MachineResponse{ID: machineResp.ID}, nil +} diff --git a/backend/services/repos.go b/backend/services/repos.go index 3bd51136e..dce68940f 100644 --- a/backend/services/repos.go +++ b/backend/services/repos.go @@ -3,9 +3,8 @@ package services import ( "fmt" "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/backend/service_clients" utils3 "github.com/diggerhq/digger/backend/utils" - dg_configuration "github.com/diggerhq/digger/libs/digger_config" - "github.com/diggerhq/digger/libs/git_utils" "log/slog" "strconv" "strings" @@ -42,17 +41,12 @@ func LoadProjectsFromGithubRepo(gh utils3.GithubClientProvider, installationId s return fmt.Errorf("error getting github service") } - err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { - config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) - if err != nil { - slog.Error("failed to load digger.yml: %v", "error", err) - return fmt.Errorf("error loading digger.yml %v", err) - } - models.DB.RefreshProjectsFromRepo(strconv.Itoa(int(link.OrganisationId)), *config, repoFullName) - return nil - }) + resp, err := service_clients.TriggerProjectsRefreshService(cloneUrl, branch, *token, repoFullName, strconv.Itoa(int(link.OrganisationId))) if err != nil { - return fmt.Errorf("error while cloning repo: %v", err) + return fmt.Errorf("error triggering projects refresh service: %v", err) + } + if resp == nil { + return fmt.Errorf("error triggering projects refresh service: response nil") } return nil diff --git a/fly-projects-refresh-service.toml b/fly-projects-refresh-service.toml new file mode 100644 index 000000000..c101319c4 --- /dev/null +++ b/fly-projects-refresh-service.toml @@ -0,0 +1,23 @@ +# fly.toml app configuration file generated for projects-refresh-service on 2025-07-05T13:54:02-07:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + +app = 'projects-refresh-service' +primary_region = 'lhr' + +[build] + dockerfile = 'Dockerfile_bg_projects_refresh' + +[http_service] + internal_port = 8080 + force_https = true + auto_stop_machines = 'stop' + auto_start_machines = true + min_machines_running = 0 + processes = ['app'] + +[[vm]] + memory = '1gb' + cpu_kind = 'shared' + cpus = 1 From 681e3caf1766d6d10cd4f2dd5797ec600b430392 Mon Sep 17 00:00:00 2001 From: motatoes Date: Sun, 6 Jul 2025 11:38:10 -0700 Subject: [PATCH 3/6] initial version of projects refresh svc --- Dockerfile_bg_projects_refresh | 2 +- backend/models/storage.go | 1 + backend/service_clients/projects_service.go | 12 +++++-- .../{main.go => projects_refesh_main.go} | 32 +++++++++++++++++-- 4 files changed, 42 insertions(+), 5 deletions(-) rename background/projects-refresh-service/{main.go => projects_refesh_main.go} (54%) diff --git a/Dockerfile_bg_projects_refresh b/Dockerfile_bg_projects_refresh index 2179d4f20..02356e654 100644 --- a/Dockerfile_bg_projects_refresh +++ b/Dockerfile_bg_projects_refresh @@ -25,4 +25,4 @@ COPY --from=builder /go/src/github.com/diggerhq/digger/projects_refresh_exe /app RUN chmod +x projects_refresh_exe # Run the binary -ENTRYPOINT ["/bin/bash", "/app/entrypoint.sh"] +ENTRYPOINT ["./projects_refresh_exe"] diff --git a/backend/models/storage.go b/backend/models/storage.go index 4acfb28a5..3cc1e0c4a 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -1642,6 +1642,7 @@ func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.D err = db.GormDB.Transaction(func(tx *gorm.DB) error { for _, dc := range config.Projects { + slog.Debug("refreshing for project", "name", dc.Name, "dir", dc.Dir) projectName := dc.Name projectDirectory := dc.Dir p, err := db.GetProjectByName(orgId, repoFullName, projectName) diff --git a/backend/service_clients/projects_service.go b/backend/service_clients/projects_service.go index 9dedb231f..cec8c7719 100644 --- a/backend/service_clients/projects_service.go +++ b/backend/service_clients/projects_service.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "log/slog" "net/http" "os" @@ -67,7 +68,7 @@ func TriggerProjectsRefreshService(cloneUrl string, branch string, githubToken s } // Create HTTP request - apiURL := fmt.Sprintf("https://api.machines.dev/v1/apps/%s/machines", os.Getenv("JOB_APP")) + apiURL := fmt.Sprintf("https://api.machines.dev/v1/apps/%s/machines", os.Getenv("DIGGER_PROJECTS_SVC_APP_NAME")) req2, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(payload)) if err != nil { slog.Error("Error creating request", "error", err) @@ -87,6 +88,12 @@ func TriggerProjectsRefreshService(cloneUrl string, branch string, githubToken s } defer resp.Body.Close() + if resp.StatusCode != 200 { + body, err2 := io.ReadAll(resp.Body) + slog.Error("Error triggering projects refresh service", "statusCode", resp.StatusCode, "body", body, "readyErr", err2) + return nil, fmt.Errorf("error triggering projects refresh service") + } + // Parse response var machineResp MachineResponse if err := json.NewDecoder(resp.Body).Decode(&machineResp); err != nil { @@ -94,6 +101,7 @@ func TriggerProjectsRefreshService(cloneUrl string, branch string, githubToken s return nil, err } - slog.Debug("triggered projects refresh service", "machineId", machineResp.ID) + slog.Debug("triggered projects refresh service", "machineId", machineResp.ID, "statusCode", resp.StatusCode) + return &MachineResponse{ID: machineResp.ID}, nil } diff --git a/background/projects-refresh-service/main.go b/background/projects-refresh-service/projects_refesh_main.go similarity index 54% rename from background/projects-refresh-service/main.go rename to background/projects-refresh-service/projects_refesh_main.go index 5243644b3..0623237c5 100644 --- a/background/projects-refresh-service/main.go +++ b/background/projects-refresh-service/projects_refesh_main.go @@ -9,12 +9,33 @@ import ( "os" ) +func init() { + logLevel := os.Getenv("DIGGER_LOG_LEVEL") + var level slog.Leveler + + if logLevel == "DEBUG" { + level = slog.LevelDebug + } else if logLevel == "WARN" { + level = slog.LevelWarn + } else if logLevel == "ERROR" { + level = slog.LevelError + } else { + level = slog.LevelInfo + } + + handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: level, + }) + logger := slog.New(handler) + slog.SetDefault(logger) +} + func main() { cloneUrl := os.Getenv("DIGGER_GITHUB_REPO_CLONE_URL") branch := os.Getenv("DIGGER_GITHUB_REPO_CLONE_BRANCH") token := os.Getenv("DIGGER_GITHUB_TOKEN") - orgId := os.Getenv("DIGGER_ORG_ID") repoFullName := os.Getenv("DIGGER_REPO_FULL_NAME") + orgId := os.Getenv("DIGGER_ORG_ID") if cloneUrl == "" && branch == "" && token == "" && orgId == "" && repoFullName == "" { slog.Info("smoketests mode, skipping this run") @@ -23,13 +44,20 @@ func main() { models.ConnectDatabase() + slog.Info("refreshing projects from repo", "repoFullName", repoFullName) err := utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) if err != nil { slog.Error("failed to load digger.yml: %v", "error", err) return fmt.Errorf("error loading digger.yml %v", err) } - models.DB.RefreshProjectsFromRepo(orgId, *config, repoFullName) + slog.Debug("RefreshProjectsFromRepo", "repoFullName", repoFullName) + err = models.DB.RefreshProjectsFromRepo(orgId, *config, repoFullName) + slog.Debug("finished RefreshProjectsFromRepo", "repoFullName", repoFullName, "error", err) + if err != nil { + slog.Error("failed to refresh projects from repo: %v", "error", err) + return fmt.Errorf("error refreshing projects from repo %v", err) + } return nil }) if err != nil { From c4051219b6ab9ebf8f6c197e5e163605f21df79e Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 7 Jul 2025 11:56:54 -0700 Subject: [PATCH 4/6] fix env variables --- background/projects-refresh-service/go.mod | 2 +- fly-pro.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/background/projects-refresh-service/go.mod b/background/projects-refresh-service/go.mod index 164852bf7..f743f6e4d 100644 --- a/background/projects-refresh-service/go.mod +++ b/background/projects-refresh-service/go.mod @@ -1,3 +1,3 @@ -module github.com/diggerhq/digger/backgorund/projects-refresh-service +module github.com/diggerhq/digger/background/projects-refresh-service go 1.24.0 diff --git a/fly-pro.toml b/fly-pro.toml index 132608001..df2f375ca 100644 --- a/fly-pro.toml +++ b/fly-pro.toml @@ -10,6 +10,9 @@ primary_region = 'lhr' HOSTNAME = 'https://ui-backend.digger.dev' DIGGER_HOSTNAME = 'https://ui-backend.digger.dev' JWT_AUTH = 'true' + DIGGER_PROJECTS_SVC_APP_NAME = 'projects-refresh-service' + DIGGER_LOAD_PROJECTS_ON_PUSH = 'true' + [build] dockerfile = 'Dockerfile_backend_ee' From fc510a7bcba54315cb742f437ffe0fd948b5083e Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 7 Jul 2025 12:08:08 -0700 Subject: [PATCH 5/6] projects refresh workflow --- .github/workflows/drift_deploy.yml | 17 ----------------- .github/workflows/pro-deploy.yml | 12 ++++++------ 2 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 .github/workflows/drift_deploy.yml diff --git a/.github/workflows/drift_deploy.yml b/.github/workflows/drift_deploy.yml deleted file mode 100644 index b42591950..000000000 --- a/.github/workflows/drift_deploy.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: driftapp Deploy -on: - push: - branches: - - develop # change to main if needed - - feat-drift-app -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time - steps: - - uses: actions/checkout@v4 - - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy --remote-only --config fly-drift.toml - env: - FLY_API_TOKEN: ${{ secrets.FLYIO_DRIFT_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/pro-deploy.yml b/.github/workflows/pro-deploy.yml index 6f37d4212..4545f1457 100644 --- a/.github/workflows/pro-deploy.yml +++ b/.github/workflows/pro-deploy.yml @@ -1,19 +1,19 @@ # See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ -name: Deploy pro-backend +name: Deploy projects-refresh-service on: push: branches: - develop - - pro jobs: deploy: - name: Deploy app + name: Deploy projects-refresh-service app runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time + concurrency: projects-refresh # optional: ensure only one action runs at a time steps: - uses: actions/checkout@v4 - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy --remote-only --config fly-pro.toml + - run: flyctl deploy --remote-only --config fly-projects-refresh-service.toml --image-label latest + env: - FLY_API_TOKEN: ${{ secrets.FLYIO_PRO_TOKEN }} + FLY_API_TOKEN: ${{ secrets.FLYIO_PROJECTS_REFRESH_SVC_TOKEN }} From 3a911ec11eee291450d73e9b5b19a94914dd42b4 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 7 Jul 2025 12:11:52 -0700 Subject: [PATCH 6/6] do not panic in createtempdir --- libs/git_utils/clone_utils.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libs/git_utils/clone_utils.go b/libs/git_utils/clone_utils.go index 6158506f6..5bd8b3cc8 100644 --- a/libs/git_utils/clone_utils.go +++ b/libs/git_utils/clone_utils.go @@ -12,19 +12,23 @@ import ( "time" ) -func createTempDir() string { +func createTempDir() (string, error) { tempDir, err := os.MkdirTemp("", "repo") if err != nil { slog.Error("Failed to create temporary directory", "error", err) - panic(err) + return "", err } - return tempDir + return tempDir, nil } type action func(string) error func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, tokenUsername string, action action) error { - dir := createTempDir() + dir, err := createTempDir() + if err != nil { + slog.Error("Failed to create temporary directory", "error", err) + return err + } slog.Debug("Cloning git repository", "repoUrl", repoUrl, @@ -34,7 +38,7 @@ func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, t ) git := NewGitShellWithTokenAuth(dir, token, tokenUsername) - err := git.Clone(repoUrl, branch) + err = git.Clone(repoUrl, branch) if err != nil { slog.Error("Failed to clone repository", "repoUrl", repoUrl,