Update CI scripts

Author: ibuildthecloud

Makefile

@@ -4,5 +4,9 @@ build:
 test:
 	go test -v ./...
 
+validate:
+	go vet ./...
+	golangci-lint run
+
 ci: build
 	./bin/gptscript ./scripts/ci.gpt

go.mod

@@ -5,6 +5,7 @@ go 1.21.5
 require (
 	github.com/acorn-io/cmd v0.0.0-20240203032901-e9e631185ddb
 	github.com/adrg/xdg v0.4.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/hexops/autogold/v2 v2.1.0
 	github.com/sashabaranov/go-openai v1.18.3
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -31,6 +31,8 @@ github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYd
 github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY=
 github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=
 github.com/hexops/autogold v1.3.1/go.mod h1:sQO+mQUCVfxOKPht+ipDSkJ2SCJ7BNJVHZexsXqWMx4=

pkg/builtin/builtin.go

@@ -85,6 +85,21 @@ var Tools = map[string]types.Tool{
 			return string(data), nil
 		},
 	},
+	"sys.abort": {
+		Description: "Aborts execution",
+		Arguments: types.ObjectSchema(
+			"message", "The description of the error or unexpected result that caused abort to be called",
+		),
+		BuiltinFunc: func(ctx context.Context, env []string, input string) (string, error) {
+			var params struct {
+				Message string `json:"message,omitempty"`
+			}
+			if err := json.Unmarshal([]byte(input), &params); err != nil {
+				return "", err
+			}
+			return "", fmt.Errorf("ABORT: %s", params.Message)
+		},
+	},
 	"sys.http.post": {
 		Description: "Write contents to a http or https URL using the POST method",
 		Arguments: types.ObjectSchema(

pkg/cli/gptscript.go

@@ -136,10 +136,10 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
 	} else {
 		if !r.Quiet {
 			if toolInput != "" {
-				_, _ = fmt.Fprintln(os.Stderr, "\nINPUT:\n")
+				_, _ = fmt.Fprint(os.Stderr, "\nINPUT:\n\n")
 				_, _ = fmt.Fprintln(os.Stderr, toolInput)
 			}
-			_, _ = fmt.Fprintln(os.Stderr, "\nOUTPUT:\n")
+			_, _ = fmt.Fprint(os.Stderr, "\nOUTPUT:\n\n")
 		}
 		fmt.Print(s)
 		if !strings.HasSuffix(s, "\n") {

pkg/engine/engine.go

@@ -14,6 +14,7 @@ import (
 	"github.com/acorn-io/gptscript/pkg/openai"
 	"github.com/acorn-io/gptscript/pkg/types"
 	"github.com/acorn-io/gptscript/pkg/version"
+	"github.com/google/shlex"
 )
 
 // InternalSystemPrompt is added to all threads. Changing this is very dangerous as it has a
@@ -152,19 +153,24 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
 	dec := json.NewDecoder(bytes.NewReader([]byte(input)))
 	dec.UseNumber()
 
+	envMap := map[string]string{}
 	if err := json.Unmarshal([]byte(input), &data); err == nil {
 		for k, v := range data {
-			envName := fmt.Sprintf("%s", strings.ToUpper(strings.ReplaceAll(k, "-", "_")))
+			envName := strings.ToUpper(strings.ReplaceAll(k, "-", "_"))
 			switch val := v.(type) {
 			case string:
+				envMap[envName] = val
 				env = append(env, envName+"="+val)
 			case json.Number:
+				envMap[envName] = string(val)
 				env = append(env, envName+"="+string(val))
 			case bool:
+				envMap[envName] = fmt.Sprint(val)
 				env = append(env, envName+"="+fmt.Sprint(val))
 			default:
 				data, err := json.Marshal(val)
 				if err == nil {
+					envMap[envName] = string(data)
 					env = append(env, envName+"="+string(data))
 				}
 			}
@@ -184,9 +190,19 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
 		return "", err
 	}
 	interpreter = strings.TrimSpace(interpreter)[2:]
+
+	interpreter = os.Expand(interpreter, func(s string) string {
+		return envMap[s]
+	})
+
+	args, err := shlex.Split(interpreter)
+	if err != nil {
+		return "", err
+	}
+
 	output := &bytes.Buffer{}
 
-	cmd := exec.Command(interpreter, f.Name())
+	cmd := exec.Command(args[0], append(args[1:], f.Name())...)
 	cmd.Env = env
 	cmd.Stdin = strings.NewReader(input)
 	cmd.Stderr = os.Stderr
@@ -198,7 +214,7 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
 		return "", err
 	}
 
-	return string(output.Bytes()), nil
+	return output.String(), nil
 }
 
 func (e *Engine) Start(ctx Context, input string) (*Return, error) {

pkg/mvl/log.go

@@ -1,7 +1,6 @@
 package mvl
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"runtime"
@@ -34,24 +33,6 @@ func (f formatter) Format(entry *logrus.Entry) ([]byte, error) {
 		msg)), nil
 }
 
-type verbose struct {
-}
-
-func (f verbose) Format(entry *logrus.Entry) ([]byte, error) {
-	buf, err := json.MarshalIndent(struct {
-		Time    time.Time `json:"time,omitempty"`
-		Level   string    `json:"level,omitempty"`
-		Message string    `json:"message,omitempty"`
-		Data    any       `json:"data,omitempty"`
-	}{
-		Time:    entry.Time,
-		Level:   entry.Level.String(),
-		Message: entry.Message,
-		Data:    entry.Data,
-	}, "", "  ")
-	return append(buf, []byte("\n")...), err
-}
-
 func SetDebug() {
 	logrus.SetFormatter(&logrus.JSONFormatter{
 		PrettyPrint: os.Getenv("GPTSCRIPT_JSON_LOG_SINGLE_LINE") != "true",

pkg/test/examples_test.go

@@ -17,19 +17,17 @@ const (
 )
 
 func TestExamples(t *testing.T) {
-	l, err := os.ReadDir(examplePath)
-	require.NoError(t, err)
-	if err != nil {
-		t.Fatal(err)
+	tests := []string{
+		"fib.gpt",
+		"echo.gpt",
+		"helloworld.gpt",
 	}
-
-	for _, entry := range l {
-		entry.Name()
-		t.Run(entry.Name(), func(t *testing.T) {
+	for _, entry := range tests {
+		t.Run(entry, func(t *testing.T) {
 			r, err := runner.New()
 			require.NoError(t, err)
 
-			prg, err := loader.Program(context.Background(), filepath.Join(examplePath, entry.Name()), "")
+			prg, err := loader.Program(context.Background(), filepath.Join(examplePath, entry), "")
 			require.NoError(t, err)
 
 			output, err := r.Run(context.Background(), prg, os.Environ(), "")

scripts/ci.gpt

@@ -1,20 +1,15 @@
-tools: make
+tools: make, sys.abort
 
-Run each step sequentially
+Run each step sequentially, if either step fails abort
 
 1. Run "make" to compile
 2. Run "make test" to test
+3. Run "make validate" to test
 
 Then print SUCCESS
 ---
 tool: make
 description: Runs the "make"
 args: arg: the args to pass to make
 
-#!/bin/sh
-set -e
-
-echo "START MAKE OUTPUT:"
-echo "Running make ${ARG}"
-make ${ARG}
-echo "END MAKE OUTPUT"
+#!make ${ARG}