From 225620ab4ffb4496f5918bdeab1da2b9bedbbaa5 Mon Sep 17 00:00:00 2001 From: vishal Date: Thu, 12 Sep 2019 18:06:51 +0000 Subject: [PATCH 1/5] Limit file size --- cli/cmd/deploy.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index 700cf018ec..9e37110fa1 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -32,6 +32,7 @@ import ( "github.com/cortexlabs/cortex/pkg/operator/api/userconfig" ) +var MaxProjectSize = 1024 * 1024 * 50 var flagDeployForce bool func init() { @@ -73,7 +74,7 @@ func deploy(force bool, ignoreCache bool) { files.IgnorePythonGeneratedFiles, ) if err != nil { - errors.Exit(err) + errors.Exit(errors.New("zipped project folder exceeds 50 MB")) } projectZipBytes, err := zip.ToMem(&zip.Input{ @@ -89,6 +90,10 @@ func deploy(force bool, ignoreCache bool) { errors.Exit(errors.Wrap(err, "failed to zip project folder")) } + if len(projectZipBytes) != MaxProjectSize { + errors.Exit(errors.New("")) + } + uploadInput := &HTTPUploadInput{ Bytes: map[string][]byte{ "cortex.yaml": configBytes, From 6b15aec6a6c96cca666fdbb655353abe11a28104 Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 13 Sep 2019 07:42:56 -0400 Subject: [PATCH 2/5] Read config from CLI to determine zip --- cli/cmd/delete.go | 3 +- cli/cmd/deploy.go | 64 ++++++++++++++++----------- cli/cmd/lib_config_reader.go | 12 +++-- pkg/operator/api/userconfig/config.go | 8 ++-- pkg/operator/endpoints/deploy.go | 27 ++++++----- 5 files changed, 68 insertions(+), 46 deletions(-) diff --git a/cli/cmd/delete.go b/cli/cmd/delete.go index 89ee3669d1..d583b6615c 100644 --- a/cli/cmd/delete.go +++ b/cli/cmd/delete.go @@ -46,10 +46,11 @@ var deleteCmd = &cobra.Command{ if len(args) == 1 { appName = args[0] } else { - appName, err = appNameFromConfig() + config, err := readConfig() if err != nil { errors.Exit(err) } + appName = config.App.Name } params := map[string]string{ diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index 9e37110fa1..a9ecc3085b 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -52,7 +52,7 @@ var deployCmd = &cobra.Command{ func deploy(force bool, ignoreCache bool) { root := mustAppRoot() - _, err := appNameFromConfig() // Check proper cortex.yaml + config, err := readConfig() // Check proper cortex.yaml if err != nil { errors.Exit(err) } @@ -67,38 +67,52 @@ func deploy(force bool, ignoreCache bool) { errors.Exit(errors.Wrap(err, "cortex.yaml", userconfig.ErrorReadConfig().Error())) } - projectPaths, err := files.ListDirRecursive(root, false, - files.IgnoreCortexYAML, - files.IgnoreHiddenFiles, - files.IgnoreHiddenFolders, - files.IgnorePythonGeneratedFiles, - ) - if err != nil { - errors.Exit(errors.New("zipped project folder exceeds 50 MB")) + uploadBytes := map[string][]byte{ + "cortex.yaml": configBytes, + } + + isProjectZipRequired := false + for _, api := range config.APIs { + if api.RequestHandler != nil { + isProjectZipRequired = true + break + } } - projectZipBytes, err := zip.ToMem(&zip.Input{ - FileLists: []zip.FileListInput{ - { - Sources: projectPaths, - RemovePrefix: root, + if isProjectZipRequired { + fmt.Println("isProjectZipRequired") + projectPaths, err := files.ListDirRecursive(root, false, + files.IgnoreCortexYAML, + files.IgnoreHiddenFiles, + files.IgnoreHiddenFolders, + files.IgnorePythonGeneratedFiles, + ) + if err != nil { + errors.Exit(errors.New("zipped project folder exceeds 50 MB")) + } + + projectZipBytes, err := zip.ToMem(&zip.Input{ + FileLists: []zip.FileListInput{ + { + Sources: projectPaths, + RemovePrefix: root, + }, }, - }, - }) + }) - if err != nil { - errors.Exit(errors.Wrap(err, "failed to zip project folder")) - } + if err != nil { + errors.Exit(errors.Wrap(err, "failed to zip project folder")) + } + + if len(projectZipBytes) != MaxProjectSize { + errors.Exit(errors.New("")) + } - if len(projectZipBytes) != MaxProjectSize { - errors.Exit(errors.New("")) + uploadBytes["project.zip"] = projectZipBytes } uploadInput := &HTTPUploadInput{ - Bytes: map[string][]byte{ - "cortex.yaml": configBytes, - "project.zip": projectZipBytes, - }, + Bytes: uploadBytes, } response, err := HTTPUpload("/deploy", uploadInput, params) diff --git a/cli/cmd/lib_config_reader.go b/cli/cmd/lib_config_reader.go index e610a96276..2430d50496 100644 --- a/cli/cmd/lib_config_reader.go +++ b/cli/cmd/lib_config_reader.go @@ -66,9 +66,13 @@ func pythonPaths(dir string) []string { return pyPaths } -func appNameFromConfig() (string, error) { +func readConfig() (*userconfig.Config, error) { appRoot := mustAppRoot() - return userconfig.ReadAppName(filepath.Join(appRoot, "cortex.yaml"), "cortex.yaml") + config, err := userconfig.ReadConfigFile(filepath.Join(appRoot, "cortex.yaml"), "cortex.yaml") + if err != nil { + return nil, err + } + return config, nil } func AppNameFromFlagOrConfig() (string, error) { @@ -76,12 +80,12 @@ func AppNameFromFlagOrConfig() (string, error) { return flagAppName, nil } - appName, err := appNameFromConfig() + config, err := readConfig() if err != nil { return "", err } - return appName, nil + return config.App.Name, nil } func IsAppNameSpecified() bool { diff --git a/pkg/operator/api/userconfig/config.go b/pkg/operator/api/userconfig/config.go index 2be02c1a0c..74415b7927 100644 --- a/pkg/operator/api/userconfig/config.go +++ b/pkg/operator/api/userconfig/config.go @@ -118,16 +118,16 @@ func New(filePath string, configBytes []byte) (*Config, error) { return config, nil } -func ReadAppName(filePath string, relativePath string) (string, error) { +func ReadConfigFile(filePath string, relativePath string) (*Config, error) { configBytes, err := files.ReadFileBytes(filePath) if err != nil { - return "", errors.Wrap(err, relativePath, ErrorReadConfig().Error()) + return nil, errors.Wrap(err, relativePath, ErrorReadConfig().Error()) } config, err := New(relativePath, configBytes) if err != nil { - return "", err + return nil, err } - return config.App.Name, nil + return config, nil } diff --git a/pkg/operator/endpoints/deploy.go b/pkg/operator/endpoints/deploy.go index 9dda4cbe5a..3e00027093 100644 --- a/pkg/operator/endpoints/deploy.go +++ b/pkg/operator/endpoints/deploy.go @@ -17,6 +17,7 @@ limitations under the License. package endpoints import ( + "fmt" "net/http" "github.com/cortexlabs/cortex/pkg/lib/errors" @@ -47,27 +48,29 @@ func Deploy(w http.ResponseWriter, r *http.Request) { return } - projectBytes, err := files.ReadReqFile(r, "project.zip") - if err != nil { - RespondError(w, errors.WithStack(err)) - return - } + projectBytes := []byte{} - userconf, err := userconfig.New("cortex.yaml", configBytes) - if err != nil { - RespondError(w, err) - return + if _, ok := r.Form["project.zip"]; ok { + fmt.Println("print.zip") + projectBytes, err = files.ReadReqFile(r, "project.zip") + if err != nil { + RespondError(w, errors.WithStack(err)) + return + } } - err = userconf.Validate(projectBytes) + userconf, err := userconfig.New("cortex.yaml", configBytes) if err != nil { RespondError(w, err) return } if len(projectBytes) == 0 { - RespondError(w, ErrorFormFileMustBeProvided("project.zip")) - return + err = userconf.Validate(projectBytes) + if err != nil { + RespondError(w, err) + return + } } ctx, err := ocontext.New(userconf, projectBytes, ignoreCache) From 058d02465a41d02d3eba7a463f4b2ae9a76a691a Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 13 Sep 2019 08:23:00 -0400 Subject: [PATCH 3/5] Refactor and cleanup --- cli/cmd/deploy.go | 16 ++++------------ pkg/operator/api/userconfig/apis.go | 13 +++++++++++++ pkg/operator/api/userconfig/config.go | 14 ++++++++++---- pkg/operator/context/context.go | 13 +++++-------- pkg/operator/endpoints/deploy.go | 15 +++------------ 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index a9ecc3085b..e2219d6188 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -71,15 +71,7 @@ func deploy(force bool, ignoreCache bool) { "cortex.yaml": configBytes, } - isProjectZipRequired := false - for _, api := range config.APIs { - if api.RequestHandler != nil { - isProjectZipRequired = true - break - } - } - - if isProjectZipRequired { + if config.APIs.AreProjectFilesRequired() { fmt.Println("isProjectZipRequired") projectPaths, err := files.ListDirRecursive(root, false, files.IgnoreCortexYAML, @@ -88,7 +80,7 @@ func deploy(force bool, ignoreCache bool) { files.IgnorePythonGeneratedFiles, ) if err != nil { - errors.Exit(errors.New("zipped project folder exceeds 50 MB")) + errors.Exit(err) } projectZipBytes, err := zip.ToMem(&zip.Input{ @@ -104,8 +96,8 @@ func deploy(force bool, ignoreCache bool) { errors.Exit(errors.Wrap(err, "failed to zip project folder")) } - if len(projectZipBytes) != MaxProjectSize { - errors.Exit(errors.New("")) + if len(projectZipBytes) > MaxProjectSize { + errors.Exit(errors.New("zipped project folder exceeds " + s.Int(MaxProjectSize) + " bytes")) } uploadBytes["project.zip"] = projectZipBytes diff --git a/pkg/operator/api/userconfig/apis.go b/pkg/operator/api/userconfig/apis.go index a7b5a79490..2857647f98 100644 --- a/pkg/operator/api/userconfig/apis.go +++ b/pkg/operator/api/userconfig/apis.go @@ -282,6 +282,10 @@ func (api *API) Validate(projectFileMap map[string][]byte) error { return nil } +func (api *API) AreProjectFilesRequired() bool { + return api.RequestHandler != nil +} + func (api *API) GetResourceType() resource.Type { return resource.APIType } @@ -293,3 +297,12 @@ func (apis APIs) Names() []string { } return names } + +func (apis APIs) AreProjectFilesRequired() bool { + for _, api := range apis { + if api.AreProjectFilesRequired() { + return true + } + } + return false +} diff --git a/pkg/operator/api/userconfig/config.go b/pkg/operator/api/userconfig/config.go index 74415b7927..b035f11b7f 100644 --- a/pkg/operator/api/userconfig/config.go +++ b/pkg/operator/api/userconfig/config.go @@ -37,15 +37,21 @@ var typeFieldValidation = &cr.StructFieldValidation{ } func (config *Config) Validate(projectBytes []byte) error { - if err := config.App.Validate(); err != nil { - return err - } + err := config.App.Validate() - projectFileMap, err := zip.UnzipMemToMem(projectBytes) if err != nil { return err } + projectFileMap := map[string][]byte{} + + if config.APIs.AreProjectFilesRequired() { + projectFileMap, err = zip.UnzipMemToMem(projectBytes) + if err != nil { + return err + } + } + if config.APIs != nil { if err := config.APIs.Validate(projectFileMap); err != nil { return err diff --git a/pkg/operator/context/context.go b/pkg/operator/context/context.go index 79a432e6c1..5459c103c7 100644 --- a/pkg/operator/context/context.go +++ b/pkg/operator/context/context.go @@ -69,14 +69,11 @@ func New( } ctx.APIs = apis - for _, api := range ctx.APIs { - if api.RequestHandler != nil { - ctx.ProjectID = projectID - ctx.ProjectKey = filepath.Join(consts.ProjectsDir, ctx.ProjectID+".zip") - if err = config.AWS.UploadBytesToS3(projectBytes, ctx.ProjectKey); err != nil { - return nil, err - } - break + if userconf.APIs.AreProjectFilesRequired() { + ctx.ProjectID = projectID + ctx.ProjectKey = filepath.Join(consts.ProjectsDir, ctx.ProjectID+".zip") + if err = config.AWS.UploadBytesToS3(projectBytes, ctx.ProjectKey); err != nil { + return nil, err } } diff --git a/pkg/operator/endpoints/deploy.go b/pkg/operator/endpoints/deploy.go index 3e00027093..5fd86e5b24 100644 --- a/pkg/operator/endpoints/deploy.go +++ b/pkg/operator/endpoints/deploy.go @@ -17,7 +17,6 @@ limitations under the License. package endpoints import ( - "fmt" "net/http" "github.com/cortexlabs/cortex/pkg/lib/errors" @@ -50,8 +49,7 @@ func Deploy(w http.ResponseWriter, r *http.Request) { projectBytes := []byte{} - if _, ok := r.Form["project.zip"]; ok { - fmt.Println("print.zip") + if _, ok := r.MultipartForm.File["project.zip"]; ok { projectBytes, err = files.ReadReqFile(r, "project.zip") if err != nil { RespondError(w, errors.WithStack(err)) @@ -65,20 +63,13 @@ func Deploy(w http.ResponseWriter, r *http.Request) { return } - if len(projectBytes) == 0 { - err = userconf.Validate(projectBytes) - if err != nil { - RespondError(w, err) - return - } - } - - ctx, err := ocontext.New(userconf, projectBytes, ignoreCache) + err = userconf.Validate(projectBytes) if err != nil { RespondError(w, err) return } + ctx, err := ocontext.New(userconf, projectBytes, ignoreCache) if err != nil { RespondError(w, err) return From 21e6d3733d12ecfddf7b666cebf38d076cdeaaa7 Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 13 Sep 2019 08:24:39 -0400 Subject: [PATCH 4/5] Remove println --- cli/cmd/deploy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index e2219d6188..e99d2d5e81 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -72,7 +72,6 @@ func deploy(force bool, ignoreCache bool) { } if config.APIs.AreProjectFilesRequired() { - fmt.Println("isProjectZipRequired") projectPaths, err := files.ListDirRecursive(root, false, files.IgnoreCortexYAML, files.IgnoreHiddenFiles, From 48b4b2f184a62d5c847cb89e1f9f6418486a4fdb Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 13 Sep 2019 14:42:52 -0400 Subject: [PATCH 5/5] Address PR comments --- cli/cmd/deploy.go | 2 +- pkg/operator/api/userconfig/config.go | 8 ++++++-- pkg/operator/context/context.go | 2 +- pkg/operator/endpoints/deploy.go | 10 +--------- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index e99d2d5e81..528af80123 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -71,7 +71,7 @@ func deploy(force bool, ignoreCache bool) { "cortex.yaml": configBytes, } - if config.APIs.AreProjectFilesRequired() { + if config.AreProjectFilesRequired() { projectPaths, err := files.ListDirRecursive(root, false, files.IgnoreCortexYAML, files.IgnoreHiddenFiles, diff --git a/pkg/operator/api/userconfig/config.go b/pkg/operator/api/userconfig/config.go index b035f11b7f..aca07a0a02 100644 --- a/pkg/operator/api/userconfig/config.go +++ b/pkg/operator/api/userconfig/config.go @@ -43,9 +43,9 @@ func (config *Config) Validate(projectBytes []byte) error { return err } - projectFileMap := map[string][]byte{} + projectFileMap := make(map[string][]byte) - if config.APIs.AreProjectFilesRequired() { + if config.AreProjectFilesRequired() { projectFileMap, err = zip.UnzipMemToMem(projectBytes) if err != nil { return err @@ -61,6 +61,10 @@ func (config *Config) Validate(projectBytes []byte) error { return nil } +func (config *Config) AreProjectFilesRequired() bool { + return config.APIs.AreProjectFilesRequired() +} + func New(filePath string, configBytes []byte) (*Config, error) { var err error diff --git a/pkg/operator/context/context.go b/pkg/operator/context/context.go index 5459c103c7..453516c9e2 100644 --- a/pkg/operator/context/context.go +++ b/pkg/operator/context/context.go @@ -69,7 +69,7 @@ func New( } ctx.APIs = apis - if userconf.APIs.AreProjectFilesRequired() { + if userconf.AreProjectFilesRequired() { ctx.ProjectID = projectID ctx.ProjectKey = filepath.Join(consts.ProjectsDir, ctx.ProjectID+".zip") if err = config.AWS.UploadBytesToS3(projectBytes, ctx.ProjectKey); err != nil { diff --git a/pkg/operator/endpoints/deploy.go b/pkg/operator/endpoints/deploy.go index 5fd86e5b24..968563c7ec 100644 --- a/pkg/operator/endpoints/deploy.go +++ b/pkg/operator/endpoints/deploy.go @@ -47,15 +47,7 @@ func Deploy(w http.ResponseWriter, r *http.Request) { return } - projectBytes := []byte{} - - if _, ok := r.MultipartForm.File["project.zip"]; ok { - projectBytes, err = files.ReadReqFile(r, "project.zip") - if err != nil { - RespondError(w, errors.WithStack(err)) - return - } - } + projectBytes, err := files.ReadReqFile(r, "project.zip") userconf, err := userconfig.New("cortex.yaml", configBytes) if err != nil {