feat: share models by url (#1522)

* feat: allow to pass by models via args

* expose it also as an env/arg

* docs: enhancements to build/requirements

* do not display status always

* print download status

* not all mesages are debug

Author: mudler

api/api.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	config "github.com/go-skynet/LocalAI/api/config"
@@ -16,6 +17,7 @@ import (
 	"github.com/go-skynet/LocalAI/metrics"
 	"github.com/go-skynet/LocalAI/pkg/assets"
 	"github.com/go-skynet/LocalAI/pkg/model"
+	"github.com/go-skynet/LocalAI/pkg/utils"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/cors"
@@ -36,6 +38,26 @@ func Startup(opts ...options.AppOption) (*options.Option, *config.ConfigLoader,
 	log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.Loader.ModelPath)
 	log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
 
+	modelPath := options.Loader.ModelPath
+	if len(options.ModelsURL) > 0 {
+		for _, url := range options.ModelsURL {
+			if utils.LooksLikeURL(url) {
+				// md5 of model name
+				md5Name := utils.MD5(url)
+
+				// check if file exists
+				if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) {
+					err := utils.DownloadFile(url, filepath.Join(modelPath, md5Name)+".yaml", "", func(fileName, current, total string, percent float64) {
+						utils.DisplayDownloadFunction(fileName, current, total, percent)
+					})
+					if err != nil {
+						log.Error().Msgf("error loading model: %s", err.Error())
+					}
+				}
+			}
+		}
+	}
+
 	cl := config.NewConfigLoader()
 	if err := cl.LoadConfigs(options.Loader.ModelPath); err != nil {
 		log.Error().Msgf("error loading config files: %s", err.Error())

api/config/config.go

@@ -286,7 +286,7 @@ func (cm *ConfigLoader) Preload(modelPath string) error {
 			// check if file exists
 			if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) {
 				err := utils.DownloadFile(modelURL, filepath.Join(modelPath, md5Name), "", func(fileName, current, total string, percent float64) {
-					log.Info().Msgf("Downloading %s: %s/%s (%.2f%%)", fileName, current, total, percent)
+					utils.DisplayDownloadFunction(fileName, current, total, percent)
 				})
 				if err != nil {
 					return err

api/options/options.go

@@ -40,9 +40,12 @@ type Option struct {
 	SingleBackend           bool
 	ParallelBackendRequests bool
 
-	WatchDogIdle                             bool
-	WatchDogBusy                             bool
-	WatchDog                                 bool
+	WatchDogIdle bool
+	WatchDogBusy bool
+	WatchDog     bool
+
+	ModelsURL []string
+
 	WatchDogBusyTimeout, WatchDogIdleTimeout time.Duration
 }
 
@@ -63,6 +66,12 @@ func NewOptions(o ...AppOption) *Option {
 	return opt
 }
 
+func WithModelsURL(urls ...string) AppOption {
+	return func(o *Option) {
+		o.ModelsURL = urls
+	}
+}
+
 func WithCors(b bool) AppOption {
 	return func(o *Option) {
 		o.CORS = b

docs/content/advanced/_index.en.md

@@ -359,15 +359,7 @@ docker run --env REBUILD=true localai
 docker run --env-file .env localai
 ```
 
-### Build only a single backend
 
-You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only:
-
-```bash
-make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build
-```
-
-By default, all the backends are built.
 
 ### Extra backends
 

docs/content/build/_index.en.md

@@ -7,32 +7,69 @@ url = '/basics/build/'
 
 +++
 
-### Build locally
+### Build
+
+#### Container image
 
 Requirements:
 
-Either Docker/podman, or
-- Golang >= 1.21
-- Cmake/make
-- GCC
+- Docker or podman, or a container engine
 
-In order to build the `LocalAI` container image locally you can use `docker`:
+In order to build the `LocalAI` container image locally you can use `docker`, for example:
 
 ```
 # build the image
 docker build -t localai .
 docker run localai
 ```
 
-Or you can build the manually binary with `make`:
+#### Locally
+
+In order to build LocalAI locally, you need the following requirements:
+
+- Golang >= 1.21
+- Cmake/make
+- GCC
+- GRPC
+
+To install the dependencies follow the instructions below:
+
+{{< tabs >}}
+{{% tab name="Apple" %}}
+
+```bash
+brew install abseil cmake go grpc protobuf wget
+```
+
+{{% /tab %}}
+{{% tab name="Debian" %}}
+
+```bash
+apt install protobuf-compiler-grpc libgrpc-dev make cmake
+```
+
+{{% /tab %}}
+{{% tab name="From source" %}}
+
+Specify `BUILD_GRPC_FOR_BACKEND_LLAMA=true` to build automatically the gRPC dependencies
+
+```bash
+make ... BUILD_GRPC_FOR_BACKEND_LLAMA=true build
+```
+
+{{% /tab %}}
+{{< /tabs >}}
+
+
+To build LocalAI with `make`:
 
 ```
 git clone https://github.com/go-skynet/LocalAI
 cd LocalAI
 make build
 ```
 
-To run: `./local-ai`
+This should produce the binary `local-ai`
 
 {{% notice note %}}
 
@@ -54,7 +91,7 @@ docker run --rm -ti -p 8080:8080 -e DEBUG=true -e MODELS_PATH=/models -e THREADS
 
 {{% /notice %}}
 
-### Build on mac
+### Example: Build on mac
 
 Building on Mac (M1 or M2) works, but you may need to install some prerequisites using `brew`. 
 
@@ -188,6 +225,16 @@ make BUILD_TYPE=metal build
 # Note: only models quantized with q4_0 are supported!
 ```
 
+### Build only a single backend
+
+You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only:
+
+```bash
+make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build
+```
+
+By default, all the backends are built.
+
 ### Windows compatibility
 
 Make sure to give enough resources to the running container. See https://github.com/go-skynet/LocalAI/issues/2

main.go

@@ -99,6 +99,11 @@ func main() {
 				Usage:   "A List of models to apply in JSON at start",
 				EnvVars: []string{"PRELOAD_MODELS"},
 			},
+			&cli.StringFlag{
+				Name:    "models",
+				Usage:   "A List of models URLs configurations.",
+				EnvVars: []string{"MODELS"},
+			},
 			&cli.StringFlag{
 				Name:    "preload-models-config",
 				Usage:   "A List of models to apply at startup. Path to a YAML config file",
@@ -222,6 +227,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
 				options.WithBackendAssetsOutput(ctx.String("backend-assets-path")),
 				options.WithUploadLimitMB(ctx.Int("upload-limit")),
 				options.WithApiKeys(ctx.StringSlice("api-keys")),
+				options.WithModelsURL(append(ctx.StringSlice("models"), ctx.Args().Slice()...)...),
 			}
 
 			idleWatchDog := ctx.Bool("enable-watchdog-idle")

pkg/model/initializers.go

@@ -239,10 +239,10 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) {
 	for _, b := range o.externalBackends {
 		allBackendsToAutoLoad = append(allBackendsToAutoLoad, b)
 	}
-	log.Debug().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", "))
+	log.Info().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", "))
 
 	for _, b := range allBackendsToAutoLoad {
-		log.Debug().Msgf("[%s] Attempting to load", b)
+		log.Info().Msgf("[%s] Attempting to load", b)
 		options := []Option{
 			WithBackendString(b),
 			WithModel(o.model),
@@ -257,14 +257,14 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) {
 
 		model, modelerr := ml.BackendLoader(options...)
 		if modelerr == nil && model != nil {
-			log.Debug().Msgf("[%s] Loads OK", b)
+			log.Info().Msgf("[%s] Loads OK", b)
 			return model, nil
 		} else if modelerr != nil {
 			err = multierror.Append(err, modelerr)
-			log.Debug().Msgf("[%s] Fails: %s", b, modelerr.Error())
+			log.Info().Msgf("[%s] Fails: %s", b, modelerr.Error())
 		} else if model == nil {
 			err = multierror.Append(err, fmt.Errorf("backend returned no usable model"))
-			log.Debug().Msgf("[%s] Fails: %s", b, "backend returned no usable model")
+			log.Info().Msgf("[%s] Fails: %s", b, "backend returned no usable model")
 		}
 	}
 

pkg/utils/logging.go

@@ -29,9 +29,9 @@ func DisplayDownloadFunction(fileName string, current string, total string, perc
 		}
 
 		if total != "" {
-			log.Debug().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta)
+			log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta)
 		} else {
-			log.Debug().Msgf("Downloading: %s", current)
+			log.Info().Msgf("Downloading: %s", current)
 		}
 	}
 }

pkg/utils/uri.go

@@ -15,27 +15,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	githubURI = "github:"
-)
-
 func GetURI(url string, f func(url string, i []byte) error) error {
-	if strings.HasPrefix(url, githubURI) {
-		parts := strings.Split(url, ":")
-		repoParts := strings.Split(parts[1], "@")
-		branch := "main"
-
-		if len(repoParts) > 1 {
-			branch = repoParts[1]
-		}
-
-		repoPath := strings.Split(repoParts[0], "/")
-		org := repoPath[0]
-		project := repoPath[1]
-		projectPath := strings.Join(repoPath[2:], "/")
-
-		url = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
-	}
+	url = ConvertURL(url)
 
 	if strings.HasPrefix(url, "file://") {
 		rawURL := strings.TrimPrefix(url, "file://")
@@ -73,14 +54,53 @@ func GetURI(url string, f func(url string, i []byte) error) error {
 
 const (
 	HuggingFacePrefix = "huggingface://"
+	HTTPPrefix        = "http://"
+	HTTPSPrefix       = "https://"
+	GithubURI         = "github:"
+	GithubURI2        = "github://"
 )
 
 func LooksLikeURL(s string) bool {
-	return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, HuggingFacePrefix)
+	return strings.HasPrefix(s, HTTPPrefix) ||
+		strings.HasPrefix(s, HTTPSPrefix) ||
+		strings.HasPrefix(s, HuggingFacePrefix) ||
+		strings.HasPrefix(s, GithubURI) ||
+		strings.HasPrefix(s, GithubURI2)
 }
 
 func ConvertURL(s string) string {
 	switch {
+	case strings.HasPrefix(s, GithubURI2):
+		repository := strings.Replace(s, GithubURI2, "", 1)
+
+		repoParts := strings.Split(repository, "@")
+		branch := "main"
+
+		if len(repoParts) > 1 {
+			branch = repoParts[1]
+		}
+
+		repoPath := strings.Split(repoParts[0], "/")
+		org := repoPath[0]
+		project := repoPath[1]
+		projectPath := strings.Join(repoPath[2:], "/")
+
+		return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
+	case strings.HasPrefix(s, GithubURI):
+		parts := strings.Split(s, ":")
+		repoParts := strings.Split(parts[1], "@")
+		branch := "main"
+
+		if len(repoParts) > 1 {
+			branch = repoParts[1]
+		}
+
+		repoPath := strings.Split(repoParts[0], "/")
+		org := repoPath[0]
+		project := repoPath[1]
+		projectPath := strings.Join(repoPath[2:], "/")
+
+		return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
 	case strings.HasPrefix(s, HuggingFacePrefix):
 		repository := strings.Replace(s, HuggingFacePrefix, "", 1)
 		// convert repository to a full URL.