Convert from packr2 to go:embed for bundling files, fixes #2343 (#2872)

* Convert from packr2 to go:embed, fixes #2343
* Move some examples to global commands
* Move settings path creation to app.Start() instead of post-start action

Author: rfay

.ci-scripts/linux_arm64_setup.sh

@@ -5,6 +5,7 @@ set -o errexit
 # Basic tools
 
 set -x
+export GO_VERSION=1.16.2
 
 if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then
   set +x
@@ -22,6 +23,8 @@ echo "capath=/etc/ssl/certs/" >>~/.curlrc
 
 . ~/.bashrc
 
+curl -sSL https://golang.org/dl/go${GO_VERSION}.linux-arm64.tar.gz -o /tmp/go.tgz && sudo rm -rf /usr/local/go && sudo tar -zxf /tmp/go.tgz -C /usr/local
+
 git clone --branch v1.2.1 https://github.com/bats-core/bats-core.git /tmp/bats-core && pushd /tmp/bats-core >/dev/null && sudo ./install.sh /usr/local
 
 # Install mkcert

Makefile

@@ -158,20 +158,6 @@ setup:
 	@mkdir -p $(TESTTMP)
 	@mkdir -p $(DOWNLOADTMP)
 
-packr2:
-	docker run -t --rm -u $(shell id -u):$(shell id -g)    \
-          	    -v "/$(PWD):/workdir$(DOCKERMOUNTFLAG)"  \
-          	    -v "/$(PWD)/$(GOTMP)/bin:$(S)/go/bin" \
-          	    -e GOCACHE="//workdir/$(GOTMP)/.cache" \
-          	    -w //workdir/cmd/ddev/cmd       \
-          	    $(BUILD_IMAGE) packr2
-	docker run -t --rm -u $(shell id -u):$(shell id -g)    \
-          	    -v "/$(PWD):/workdir$(DOCKERMOUNTFLAG)"  \
-          	    -v "/$(PWD)/$(GOTMP)/bin:$(S)/go/bin" \
-          	    -e GOCACHE="//workdir/$(GOTMP)/.cache" \
-          	    -w //workdir/pkg/ddevapp       \
-          	    $(BUILD_IMAGE) packr2
-
 # Required static analysis targets used in circleci - these cause fail if they don't work
 staticrequired: setup golangci-lint markdownlint mkdocs pyspelling
 

cmd/ddev/cmd/cmd-packr.go

@@ -2,9 +2,9 @@ package cmd
 
 import (
 	"bufio"
+	"embed"
 	"fmt"
 	"github.com/drud/ddev/pkg/nodeps"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -16,7 +16,6 @@ import (
 	"github.com/drud/ddev/pkg/fileutil"
 	"github.com/drud/ddev/pkg/globalconfig"
 	"github.com/drud/ddev/pkg/util"
-	"github.com/gobuffalo/packr/v2"
 	"github.com/mattn/go-isatty"
 	"github.com/spf13/cobra"
 )
@@ -309,60 +308,26 @@ func findDirectivesInScriptCommand(script string) map[string]string {
 	return directives
 }
 
-// populateExamplesCommandsHomeadditions grabs packr2 assets
-// When the items in the assets directory are changed, the packr2 command
-// must be run again in this directory (cmd/ddev/cmd) to update the saved
-// embedded files.
-// "make packr2" can be used to update the packr2 cache.
+//The bundled assets for the project .ddev directory are in directory dotddev_assets
+//The bundled assets for the global .ddev directory are in directory global_dotddev_assets
+//go:embed dotddev_assets global_dotddev_assets
+var bundledAssets embed.FS
+
+// populateExamplesCommandsHomeadditions grabs embedded assets
 func populateExamplesCommandsHomeadditions() error {
 	app, err := ddevapp.GetActiveApp("")
 	if err != nil {
 		return nil
 	}
-	box := packr.New("customcommands", "./dotddev_assets")
-
-	list := box.List()
-	for _, file := range list {
-		localPath := app.GetConfigPath(file)
-		sigFound, err := fileutil.FgrepStringInFile(localPath, ddevapp.DdevFileSignature)
-		if sigFound || err != nil {
-			content, err := box.Find(file)
-			if err != nil {
-				return err
-			}
-			err = os.MkdirAll(filepath.Dir(localPath), 0755)
-			if err != nil {
-				return err
-			}
-			err = ioutil.WriteFile(localPath, content, 0755)
-			if err != nil {
-				return err
-			}
-		}
-	}
 
-	// This brings in both the commands and the homeadditions files
-	box = packr.New("global_dotddev", "./global_dotddev_assets")
-
-	list = box.List()
-	globalDdevDir := globalconfig.GetGlobalDdevDir()
-	for _, file := range list {
-		localPath := filepath.Join(globalDdevDir, file)
-		sigFound, err := fileutil.FgrepStringInFile(localPath, ddevapp.DdevFileSignature)
-		if sigFound || err != nil {
-			content, err := box.Find(file)
-			if err != nil {
-				return err
-			}
-			err = os.MkdirAll(filepath.Dir(localPath), 0755)
-			if err != nil {
-				return err
-			}
-			err = ioutil.WriteFile(localPath, content, 0755)
-			if err != nil {
-				return err
-			}
-		}
+	err = ddevapp.CopyEmbedAssets(bundledAssets, "dotddev_assets", app.GetConfigPath(""))
+	if err != nil {
+		return err
 	}
+	err = ddevapp.CopyEmbedAssets(bundledAssets, "global_dotddev_assets", globalconfig.GetGlobalDdevDir())
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

cmd/ddev/cmd/commands.go

@@ -22,7 +22,7 @@ import (
 // TestCustomCommands does basic checks to make sure custom commands work OK.
 func TestCustomCommands(t *testing.T) {
 	assert := asrt.New(t)
-	runTime := util.TimeTrack(time.Now(), "ddev list")
+	runTime := util.TimeTrack(time.Now(), t.Name())
 
 	tmpHome := testcommon.CreateTmpDir(t.Name() + "tempHome")
 	origHome := os.Getenv("HOME")
@@ -68,7 +68,6 @@ func TestCustomCommands(t *testing.T) {
 	require.NoError(t, err)
 
 	assert.FileExists(filepath.Join(projectCommandsDir, "db", "mysql"))
-	assert.FileExists(filepath.Join(projectCommandsDir, "host", "mysqlworkbench.example"))
 	out, err := exec.RunCommand(DdevBin, []string{})
 	assert.NoError(err)
 	assert.Contains(out, "mysql client in db container")
@@ -185,11 +184,14 @@ func TestCustomCommands(t *testing.T) {
 		assert.NoError(err, "expected to find command %s for app.Type=%s", c, app.Type)
 	}
 
-	// Make sure that the non-command stuff we installed is there
-	for _, f := range []string{"db/mysqldump.example", "db/README.txt", "web/README.txt", "host/README.txt", "host/phpstorm.example"} {
-		assert.FileExists(filepath.Join(projectCommandsDir, f))
+	// Make sure that the non-command stuff we installed is in globalCommandsDir
+	for _, f := range []string{"db/mysqldump.example", "db/README.txt", "host/heidisql", "host/mysqlworkbench.example", "host/phpstorm.example", "host/README.txt", "host/sequelace", "host/sequelpro", "host/tableplus", "web/README.txt"} {
 		assert.FileExists(filepath.Join(globalCommandsDir, f))
 	}
+	// Make sure that the non-command stuff we installed is in project commands dir
+	for _, f := range []string{"db/mysql", "db/README.txt", "host/launch", "host/README.txt", "host/solrtail.example", "solr/README.txt", "solr/solrtail.example", "web/live", "web/README.txt", "web/xdebug"} {
+		assert.FileExists(filepath.Join(projectCommandsDir, f))
+	}
 
 	// Make sure we haven't accidentally created anything inappropriate in ~/.ddev
 	assert.False(fileutil.FileExists(filepath.Join(tmpHome, ".ddev", ".globalcommands")))