feat: move to yml based config (#131)

* feat: move to yml based config

* feat: replaced env vars with system config

* chore: rename serviceconfig to systemconfig

* chore: remopve http support from haproxy image

* chore: haproxy no need to expose 5555 anymore

* chore: update config variable names

* chore: configuration example added

* chore: configuration example added

* chore: tidy dependecies

Author: tanmoysrt

config.example.yml

@@ -0,0 +1,49 @@
+version: 1.0
+mode: standalone # standalone or cluster
+service:
+  auto_tls: true # true or false
+  bind_address: 0.0.0.0
+  bind_port: 3333 # choose any other ports except 80 and 443
+  network_name: swiftwave_network # docker swarm overflow network name
+  data_dir: ~/swiftwave/data
+  docker_unix_socket_path: /var/run/docker.sock
+  restricted_ports: # ports that can't be exposed and bound to haproxy
+    - 2377 # docker swarm port
+    - 7946 # docker swarm port
+    - 4789 # docker swarm port
+    - 3333 # swiftwave port
+haproxy:
+  service_name: haproxy
+  image: ghcr.io/swiftwave-org/haproxy:2.9
+  unix_socket_path: ~/swiftwave/dataplaneapi.sock
+  user: admin
+  password: admin
+postgresql:
+  host: localhost
+  port: 5432
+  user: postgres
+  password: postgres
+  database: swiftwave
+  time_zone: Asia/Kolkata
+lets_encrypt:
+  staging_environment: false # true or false
+  email_id: [email protected]
+  account_private_key_path: ~/swiftwave/account.key
+pubsub:
+  mode: local # local or remote
+  buffer_length: 1000
+  redis: # all the info should be filled if mode is remote
+    host: localhost
+    port: 6379
+    password: ""
+    database_id: 0
+task_queue:
+  mode: local # local or remote
+  max_outstanding_messages_per_queue: 1000
+  amqp: # all the info should be filled if mode is remote
+    protocol: amqp # amqp or amqps
+    host: localhost
+    user: guest
+    password: guest
+    vhost:
+    client_name: swiftwave

go.mod

@@ -52,7 +52,6 @@ require (
 	github.com/containerd/containerd v1.7.2 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/joho/godotenv v1.5.1
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/moby/patternmatcher v0.5.0 // indirect

go.sum

@@ -99,8 +99,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
-github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

haproxy_manager/constructor.go

@@ -2,17 +2,9 @@ package haproxymanager
 
 // Init HaProxy Manager with a unix socket
 func (s *Manager) InitUnixSocket(unixSocketPath string) {
-	s.isUnix = true
 	s.unixSocketPath = unixSocketPath
 }
 
-// Init HaProxy Manager with tcp socket info (host, port)
-func (s *Manager) InitTcpSocket(host string, port int) {
-	s.isUnix = false
-	s.Host = host
-	s.Port = port
-}
-
 // Update auth credentials for HaProxy Manager
 func (s *Manager) Auth(username string, password string) {
 	s.username = username

haproxy_manager/request.go

@@ -2,21 +2,19 @@ package haproxymanager
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"io"
 	"mime/multipart"
+	"net"
 	"net/http"
 	"path/filepath"
-	"strconv"
 	"strings"
 )
 
 // Generate Base URI for HAProxy Server
 func (s Manager) URI() string {
-	if s.isUnix {
-		return "unix://" + s.unixSocketPath
-	}
-	return "http://" + s.Host + ":" + strconv.Itoa(s.Port) + "/v2"
+	return "http://unix/v2"
 }
 
 // Wrapper to send request to HAProxy Server
@@ -30,7 +28,13 @@ func (s Manager) getRequest(route string, queryParams QueryParameters) (*http.Re
 		return nil, err
 	}
 	req.SetBasicAuth(s.username, s.password)
-	client := &http.Client{}
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", s.unixSocketPath)
+			},
+		},
+	}
 	return client.Do(req)
 }
 
@@ -45,7 +49,13 @@ func (s Manager) deleteRequest(route string, queryParams QueryParameters) (*http
 		return nil, err
 	}
 	req.SetBasicAuth(s.username, s.password)
-	client := &http.Client{}
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", s.unixSocketPath)
+			},
+		},
+	}
 	return client.Do(req)
 }
 
@@ -61,7 +71,13 @@ func (s Manager) postRequest(route string, queryParams QueryParameters, body io.
 	}
 	req.SetBasicAuth(s.username, s.password)
 	req.Header.Add("Content-Type", "application/json")
-	client := &http.Client{}
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", s.unixSocketPath)
+			},
+		},
+	}
 	return client.Do(req)
 }
 
@@ -77,7 +93,13 @@ func (s Manager) putRequest(route string, queryParams QueryParameters, body io.R
 	}
 	req.SetBasicAuth(s.username, s.password)
 	req.Header.Add("Content-Type", "application/json")
-	client := &http.Client{}
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", s.unixSocketPath)
+			},
+		},
+	}
 	return client.Do(req)
 }
 
@@ -113,7 +135,13 @@ func (s Manager) uploadSSL(route string, domain string, file io.Reader) (*http.R
 	}
 	req.SetBasicAuth(s.username, s.password)
 	req.Header.Add("Content-Type", writer.FormDataContentType())
-	client := &http.Client{}
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", s.unixSocketPath)
+			},
+		},
+	}
 	return client.Do(req)
 }
 
@@ -138,6 +166,12 @@ func (s Manager) replaceSSL(route string, domain string, file io.Reader) (*http.
 	}
 	req.SetBasicAuth(s.username, s.password)
 	req.Header.Add("Content-Type", "text/plain")
-	client := &http.Client{}
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", s.unixSocketPath)
+			},
+		},
+	}
 	return client.Do(req)
 }

haproxy_manager/types.go

@@ -1,9 +1,6 @@
 package haproxymanager
 
 type Manager struct {
-	Host           string
-	Port           int
-	isUnix         bool // If true, Host is a path to a unix socket
 	unixSocketPath string
 	username       string
 	password       string

main.go

@@ -1,36 +1,43 @@
 package main
 
 import (
-	"github.com/joho/godotenv"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	swiftwave "github.com/swiftwave-org/swiftwave/swiftwave_service"
 	"github.com/swiftwave-org/swiftwave/swiftwave_service/core"
 	"github.com/swiftwave-org/swiftwave/swiftwave_service/cronjob"
 	"github.com/swiftwave-org/swiftwave/swiftwave_service/worker"
+	"github.com/swiftwave-org/swiftwave/system_config"
 	"log"
+	"os"
 )
 
 func main() {
-	err := godotenv.Load()
+	// Load config path from environment variable
+	systemConfigPath := os.Getenv("SWIFTWAVE_CONFIG_PATH")
+	if systemConfigPath == "" {
+		systemConfigPath = "~/swiftwave/config.yaml"
+		log.Println("SWIFTWAVE_CONFIG_PATH environment variable not set, using default path > ", systemConfigPath)
+	}
+	// Load the config
+	config, err := system_config.ReadFromFile(systemConfigPath)
 	if err != nil {
-		log.Println("WARN: error loading .env file. Ignoring")
+		panic(err)
 	}
+
 	// Load the manager
-	config := core.ServiceConfig{}
 	manager := core.ServiceManager{}
-	config.Load()
-	manager.Load()
+	manager.Load(*config)
 
 	// Create the worker manager
-	workerManager := worker.NewManager(&config, &manager)
+	workerManager := worker.NewManager(config, &manager)
 	err = workerManager.StartConsumers(true)
 	if err != nil {
 		panic(err)
 	}
 
 	// Create the cronjob manager
-	cronjobManager := cronjob.NewManager(&config, &manager)
+	cronjobManager := cronjob.NewManager(config, &manager)
 	cronjobManager.Start(true)
 
 	// create a channel to block the main thread
@@ -43,7 +50,7 @@ func main() {
 	echoServer.Use(middleware.CORS())
 
 	// Start the swift wave server
-	go swiftwave.StartServer(&config, &manager, echoServer, workerManager, true)
+	go swiftwave.StartServer(config, &manager, echoServer, workerManager, true)
 	// Wait for consumers
 	go workerManager.WaitForConsumers()
 	// Wait for cronjobs

swiftwave_service/core/create_db_client.go

@@ -3,12 +3,11 @@ package core
 import (
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
-	"os"
 )
 
-func createDbClient() (*gorm.DB, error) {
+func createDbClient(dsn string) (*gorm.DB, error) {
 	// Initiating database client
-	dbDialect := postgres.Open(os.Getenv("POSTGRESQL_DSN"))
+	dbDialect := postgres.Open(dsn)
 	client, err := gorm.Open(dbDialect, &gorm.Config{})
 	if err != nil {
 		return nil, err