feat: redesign architecture and move to graphql (#109)

* chore: remove old codebase

* feat: graphql initiated

* feat: types updated

* chore: database models updated

* feat: enums updated

* chore: code restructure

* chore: update driver files

* chore: cleanup code

* feat: graphql routes for git credentials added

* chore: remove unnecessary code

* feat: graphql endpoints for image repository credentials added

* feat: added functions for test git credential againsta a repo

* feat: script added to generate graphql codes for gqlgen

* chore: rename service name of application to just name

* chore: update docs

* feat: updated db design to remove json field

* feat: git credential operations added

* feat: image registry credential operations added

* feat: new graphql schemas added

* feat: functions added for persistent volume

* feat: resolvers added for persistent volume

* chore: format code

* fix: bug fix persistent volume is exist check

* feat: add resolvers and operations for applications

* chore: update resolvers

* chore: soft deletion for application added

* chore: application create function added

* chore: application create validation added

* feat: enums added for types

* chore: application create and fix bug fixed and resolvers added

* chore: move to id from int to uint by scalar type to eliminate type casting

* chore: if id is zero for a object return blank object

* feat: sub resolvers added

* chore: resolvers and domain related gql schema added

* feat: for all deletion operation return boolean

* feat: added functions to update applications

* feat: resolvers added for domain, ingress rule and redirect rule

* feat: checks added for duplicate mounting paths

* feat: packages renamed

* feat: init pubusb module

* feat: added local pubsub functions

* fix: stuck on channel full has been resolve

* chore: make bufferlength configurable

* fix: concurrent read write from same map fixed by mutex

* fix: race conditions

* feat: remote pubsub added

* feat: updated code to ensure no duplicate channel closure in remote

* feat: updated code for example

* chore: cleanup

* chore: delete dump.rdb

* feat: function for register function against a queue added for local task queue

* feat: function for task enqeue has been added

* feat: added start consumers function to interface

* feat: local task queue using go channels has been added

* feat: added code for consumer in remote queue

* feat: added code for publish tasks in rabbitmq

* chore: cleanup

* feat: added amqp connection establishment checks in consumers as well'

* chore: load local task queue and pubsub client

* feat: boilerplates added for worker

* feat: application build worker completed

* chore: code cleanup

* chore: deploy application worker has been added

* chore: ssl generation worker has been added

* chore: ingress rule apply worker has been added

* chore: redirect rule apply worker has been added

* feat: ingress and redirect rule deletion workers added

* feat: cronjobs added

* feat: push deployment logs to redis

* feat: subscription endpoint added for deployment log

* feat: subscription endpoint added for runtime log

* feat: added code upload route

* feat: publish tasks to queue has been completed

* feat: ssl generate push to queue added

* feat: application deletion added

* chore: run workers with server

* fix: application creation bugs fixed

* feat: bug fixed application creation and log

* feat: bug fixed application deletion

* chore: remove debug statement

* fix: ingress rule issue fixed

* feat: cronjobs added

* fix: source code based application deployment issues fixed

* fix: docker config generator added and tested

* fix: image build from git issues fixed

* chore: todo added

* chore: todo added

* chore: remove junk codes

Author: tanmoysrt

.gitignore

@@ -12,4 +12,6 @@ dev.log
 .images
 .idea
 node_modules
-swiftwave
+swiftwave
+dump.rdb
+ignore_*

container_manager/image.go

@@ -10,7 +10,7 @@ import (
 )
 
 /*
-`CreateImage` builds a Docker image from a Dockerfile and returns a scanner to read the build logs.
+CreateImage builds a Docker image from a Dockerfile and returns a scanner to read the build logs.
 It takes the Dockerfile content as a string, a map of build arguments, the path to the code directory, and the name of the image to be built.
 It returns a scanner to read the build logs and an error if any.
 */
@@ -48,3 +48,23 @@ func (m Manager) CreateImage(dockerfile string, buildargs map[string]string, cod
 	scanner := bufio.NewScanner(response.Body)
 	return scanner, nil
 }
+
+// PullImage pulls a Docker image from a remote registry and returns a scanner to read the pull logs.
+func (m Manager) PullImage(image string) (*bufio.Scanner, error) {
+	// Pull the image
+	scanner, err := m.client.ImagePull(m.ctx, image, types.ImagePullOptions{})
+	if err != nil {
+		return nil, errors.New("failed to pull the image")
+	}
+	return bufio.NewScanner(scanner), nil
+}
+
+// ExistsImage checks if a Docker image exists locally.
+func (m Manager) ExistsImage(image string) bool {
+	// Check if the image exists locally
+	_, _, err := m.client.ImageInspectWithRaw(m.ctx, image)
+	if err != nil {
+		return false
+	}
+	return true
+}

container_manager/volume.go

@@ -2,12 +2,10 @@ package containermanger
 
 import (
 	"errors"
-	"fmt"
-
 	"github.com/docker/docker/api/types/volume"
 )
 
-// Create a new volume, return id of the volume
+// CreateVolume : Create a new volume, return id of the volume
 func (m Manager) CreateVolume(name string) error {
 	_, err := m.client.VolumeCreate(m.ctx, volume.CreateOptions{
 		Name: name,
@@ -18,7 +16,7 @@ func (m Manager) CreateVolume(name string) error {
 	return nil
 }
 
-// Remove a volume by id
+// RemoveVolume : Remove a volume by id
 func (m Manager) RemoveVolume(id string) error {
 	err := m.client.VolumeRemove(m.ctx, id, true)
 	if err != nil {
@@ -27,14 +25,13 @@ func (m Manager) RemoveVolume(id string) error {
 	return nil
 }
 
-// Check if volume exists
+// ExistsVolume : Check if volume exists
 func (m Manager) ExistsVolume(id string) bool {
-	d, err := m.client.VolumeInspect(m.ctx, id)
-	fmt.Println(d)
+	_, err := m.client.VolumeInspect(m.ctx, id)
 	return err == nil
 }
 
-// Fetch all volumes
+// FetchVolumes Fetch all volumes
 func (m Manager) FetchVolumes() ([]string, error) {
 	volumes, err := m.client.VolumeList(m.ctx, volume.ListOptions{})
 	if err != nil {

dev.linux.sh

@@ -35,8 +35,10 @@ fi
 echo -e "${GREEN}Fetching required images...${NC}"
 sudo docker pull ghcr.io/swiftwave-org/dashboard:develop
 sudo docker pull haproxytech/haproxy-debian:2.9
+sudo docker pull postgres:14
 sudo docker save -o ./.images/swiftwave-dashboard.tar ghcr.io/swiftwave-org/dashboard:develop
 sudo docker save -o ./.images/haproxy-debian.tar haproxytech/haproxy-debian:2.9
+sudo docker save -o ./.images/postgres.tar postgres:14
 
 # build docker image
 echo "${GREEN}Building docker image...${NC}"

dev/Dockerfile

@@ -4,7 +4,7 @@ FROM docker:dind
 USER root
 
 # Install packages
-RUN apk update --no-cache && apk add --no-cache gcc g++ make sqlite iproute2 net-tools bind-tools bash apache2-utils sudo shadow redis
+RUN apk update --no-cache && apk add --no-cache gcc g++ make git sqlite iproute2 net-tools bind-tools bash apache2-utils sudo shadow redis htop
 
 ARG NEWUSER=user
 # Create non-root user
@@ -20,9 +20,9 @@ RUN addgroup docker
 RUN addgroup ${NEWUSER} docker
 
 # Download and install Go
-RUN wget https://go.dev/dl/go1.21.0.linux-amd64.tar.gz
-RUN tar -C /usr/local -xzf go1.21.0.linux-amd64.tar.gz
-RUN rm go1.21.0.linux-amd64.tar.gz
+RUN wget https://go.dev/dl/go1.21.3.linux-amd64.tar.gz
+RUN tar -C /usr/local -xzf go1.21.3.linux-amd64.tar.gz
+RUN rm go1.21.3.linux-amd64.tar.gz
 RUN echo "export PATH=\$PATH:/usr/local/go/bin" >> /home/${NEWUSER}/.bashrc
 
 

dev/docker-compose.dev.yml

@@ -1,5 +1,18 @@
 version: '3.5'
 services:
+  postgres:
+    image: postgres:14
+    environment:
+      POSTGRES_DB: swiftwave_development
+      POSTGRES_PASSWORD: postgres
+      PGDATA: /var/lib/postgresql/data/pgdata
+    volumes:
+      - postgres_data:/var/lib/postgresql@14/data/pgdata
+    ports:
+      - target: 5432
+        published: 5432
+        protocol: tcp
+        mode: host
   haproxy:
     image: "haproxytech/haproxy-debian:2.9"
     volumes:
@@ -28,6 +41,8 @@ services:
         published: 1212
         protocol: tcp
         mode: host
+volumes:
+  postgres_data:
 networks:
   swarm_network:
     external: true

dev/start.sh

@@ -124,6 +124,7 @@ echo "$haproxy_cfg" >"$SWIFTWAVE_HAPROXY_FOLDER/haproxy.cfg"
 # Pull docker images
 sudo docker load --input /app/.images/swiftwave-dashboard.tar
 sudo docker load --input /app/.images/haproxy-debian.tar
+sudo docker load --input /app/.images/postgres.tar
 # Start the services
 sudo docker stack deploy -c /app/dev/docker-compose.dev.yml $STACK_NAME
 
@@ -143,9 +144,7 @@ export ADMIN_PASSWORD="$SWIFTWAVE_ADMIN_PASSWORD_HASH"
 export CODE_TARBALL_DIR="$SWIFTWAVE_APP_TARBALL_FOLDER"
 export SWARM_NETWORK="$SWARM_NETWORK"
 export HAPROXY_SERVICE_NAME="swiftwave_haproxy"
-export DATABASE_TYPE="sqlite"
-export SQLITE_DATABASE="/app/.data/gorm.db"
-export POSTGRESQL_URI=""
+export POSTGRESQL_DSN="host=localhost user=postgres password=postgres dbname=swiftwave_development port=5432 sslmode=disable TimeZone=Asia/Kolkata"
 export REDIS_ADDRESS="127.0.0.1:6379"
 export REDIS_PASSWORD=""
 export ACCOUNT_EMAIL_ID="$SWIFTWAVE_ADMIN_EMAIL"

docker_config_generator/utils.go

@@ -132,15 +132,15 @@ func (m Manager) generateConfigFromSourceCodeDirectory(directory string) (Docker
 }
 
 // Generate DockerConfig from custom dockerfile. If GenerateConfigFromGitRepository fails to detect service, this function will be used.
-func (m Manager) GenerateConfigFromCustomDocker(dockerfile string) DockerFileConfig {
+func (m Manager) GenerateConfigFromCustomDocker(dockerfile string) (DockerFileConfig, error) {
 	dockerConfig := DockerFileConfig{}
 	dockerConfig.DetectedService = "Custom Dockerfile"
 	dockerConfig.DockerFile = dockerfile
 	dockerConfig.Variables = ParseBuildArgsFromDockerfile(dockerfile)
 	if dockerConfig.Variables == nil {
 		dockerConfig.Variables = map[string]Variable{}
 	}
-	return dockerConfig
+	return dockerConfig, nil
 }
 
 // DefaultArgs returns default arguments for a service.

generate_graphql.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+cd ./swiftwave_service && go run github.com/99designs/gqlgen generate

go.mod

@@ -1,49 +1,56 @@
 module github.com/swiftwave-org/swiftwave
 
-go 1.20
+go 1.21
+
+toolchain go1.21.3
 
 require (
+	github.com/99designs/gqlgen v0.17.39
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/go-git/go-git/v5 v5.10.0
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/gorilla/websocket v1.5.0
+	github.com/hashicorp/go-set v0.1.14
 	github.com/labstack/echo/v4 v4.11.2
 	github.com/mholt/acmez v1.2.0
-	github.com/vmihailenco/taskq/v3 v3.2.9
+	github.com/rabbitmq/amqp091-go v1.9.0
+	github.com/vektah/gqlparser/v2 v2.5.10
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/postgres v1.5.4
-	gorm.io/driver/sqlite v1.5.4
 	gorm.io/gorm v1.25.5
 )
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/agnivade/levenshtein v1.1.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.3 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.4.3 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/skeema/knownhosts v1.2.0 // indirect
+	github.com/sosodev/duration v1.1.0 // indirect
+	github.com/urfave/cli/v2 v2.25.5 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/time v0.3.0 // indirect
 )
 
 require (
-	github.com/bsm/redislock v0.7.2 // indirect
-	github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/containerd/containerd v1.7.2 // indirect
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/go-redis/redis_rate/v9 v9.1.2 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/joho/godotenv v1.5.1
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
@@ -53,8 +60,6 @@ require (
 	github.com/opencontainers/runc v1.1.8 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
-	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )
@@ -71,7 +76,6 @@ require (
 	github.com/labstack/gommon v0.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -82,7 +86,7 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/crypto v0.14.0
+	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect