diff --git a/cli/cmd/lib_config_reader.go b/cli/cmd/lib_config_reader.go index dcf83d4cc5..0a68e4a450 100644 --- a/cli/cmd/lib_config_reader.go +++ b/cli/cmd/lib_config_reader.go @@ -92,7 +92,7 @@ func allConfigPaths(root string) []string { func appNameFromConfig() (string, error) { appRoot := mustAppRoot() - return userconfig.ReadAppName(filepath.Join(appRoot, "app.yaml")) + return userconfig.ReadAppName(filepath.Join(appRoot, "app.yaml"), "app.yaml") } func AppNameFromFlagOrConfig() (string, error) { diff --git a/pkg/api/context/models.go b/pkg/api/context/models.go index a5f16dc176..0490c8fb3a 100644 --- a/pkg/api/context/models.go +++ b/pkg/api/context/models.go @@ -36,26 +36,18 @@ type Model struct { } type TrainingDataset struct { + userconfig.ResourceConfigFields *ComputedResourceFields - Name string `json:"name"` ModelName string `json:"model_name"` TrainKey string `json:"train_key"` EvalKey string `json:"eval_key"` MetadataKey string `json:"metadata_key"` } -func (trainingDataset *TrainingDataset) GetName() string { - return trainingDataset.Name -} - func (trainingDataset *TrainingDataset) GetResourceType() resource.Type { return resource.TrainingDatasetType } -func (trainingDataset *TrainingDataset) GetFilePath() string { - return "" -} - func (models Models) OneByID(id string) *Model { for _, model := range models { if model.ID == id { diff --git a/pkg/api/context/python_packages.go b/pkg/api/context/python_packages.go index 10d3a0cf14..3d3866ed32 100644 --- a/pkg/api/context/python_packages.go +++ b/pkg/api/context/python_packages.go @@ -18,25 +18,18 @@ package context import ( "github.com/cortexlabs/cortex/pkg/api/resource" + "github.com/cortexlabs/cortex/pkg/api/userconfig" ) type PythonPackages map[string]*PythonPackage type PythonPackage struct { - Name string `json:"name"` + userconfig.ResourceConfigFields + *ComputedResourceFields SrcKey string `json:"src_key"` PackageKey string `json:"package_key"` - *ComputedResourceFields -} - -func (pythonPackage *PythonPackage) GetName() string { - return pythonPackage.Name } func (pythonPackage *PythonPackage) GetResourceType() resource.Type { return resource.PythonPackageType } - -func (pythonPackage *PythonPackage) GetFilePath() string { - return "" -} diff --git a/pkg/api/resource/errors.go b/pkg/api/resource/errors.go index 27367e15cb..82c89b5398 100644 --- a/pkg/api/resource/errors.go +++ b/pkg/api/resource/errors.go @@ -31,6 +31,8 @@ const ( ErrNameNotFound ErrNameOrTypeNotFound ErrInvalidType + ErrTemplateInTemplate + ErrEmbedInTemplate ) var ( @@ -41,6 +43,8 @@ var ( "err_name_not_found", "err_name_or_type_not_found", "err_invalid_type", + "err_template_in_template", + "err_embed_in_template", } ) @@ -121,3 +125,17 @@ func ErrorUnknownKind(name string) error { message: fmt.Sprintf("unknown kind %s", s.UserStr(name)), } } + +func ErrorTemplateInTemplate() error { + return ResourceError{ + Kind: ErrTemplateInTemplate, + message: "templates cannot be defined inside of templates", + } +} + +func ErrorEmbedInTemplate() error { + return ResourceError{ + Kind: ErrEmbedInTemplate, + message: "embeds cannot be defined inside of templates", + } +} diff --git a/pkg/api/userconfig/aggregates.go b/pkg/api/userconfig/aggregates.go index aaf4c8e77e..9a8be8e7a9 100644 --- a/pkg/api/userconfig/aggregates.go +++ b/pkg/api/userconfig/aggregates.go @@ -25,12 +25,11 @@ import ( type Aggregates []*Aggregate type Aggregate struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Aggregator string `json:"aggregator" yaml:"aggregator"` Inputs *Inputs `json:"inputs" yaml:"inputs"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path" yaml:"-"` } var aggregateValidation = &cr.StructValidation{ @@ -69,18 +68,10 @@ func (aggregates Aggregates) Validate() error { return nil } -func (aggregate *Aggregate) GetName() string { - return aggregate.Name -} - func (aggregate *Aggregate) GetResourceType() resource.Type { return resource.AggregateType } -func (aggregate *Aggregate) GetFilePath() string { - return aggregate.FilePath -} - func (aggregates Aggregates) Names() []string { names := make([]string, len(aggregates)) for i, aggregate := range aggregates { diff --git a/pkg/api/userconfig/aggregators.go b/pkg/api/userconfig/aggregators.go index 3890318c14..0e894a1f31 100644 --- a/pkg/api/userconfig/aggregators.go +++ b/pkg/api/userconfig/aggregators.go @@ -24,11 +24,10 @@ import ( type Aggregators []*Aggregator type Aggregator struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Inputs *Inputs `json:"inputs" yaml:"inputs"` OutputType interface{} `json:"output_type" yaml:"output_type"` Path string `json:"path" yaml:"path"` - FilePath string `json:"file_path" yaml:"-"` } var aggregatorValidation = &cr.StructValidation{ @@ -84,18 +83,10 @@ func (aggregators Aggregators) Get(name string) *Aggregator { return nil } -func (aggregator *Aggregator) GetName() string { - return aggregator.Name -} - func (aggregator *Aggregator) GetResourceType() resource.Type { return resource.AggregatorType } -func (aggregator *Aggregator) GetFilePath() string { - return aggregator.FilePath -} - func (aggregators Aggregators) Names() []string { names := make([]string, len(aggregators)) for i, aggregator := range aggregators { diff --git a/pkg/api/userconfig/apis.go b/pkg/api/userconfig/apis.go index 3c6f6e5df8..b2d5eea66f 100644 --- a/pkg/api/userconfig/apis.go +++ b/pkg/api/userconfig/apis.go @@ -24,11 +24,10 @@ import ( type APIs []*API type API struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields ModelName string `json:"model_name" yaml:"model_name"` Compute *APICompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path" yaml:"-"` } var apiValidation = &cr.StructValidation{ @@ -67,18 +66,10 @@ func (apis APIs) Validate() error { return nil } -func (api *API) GetName() string { - return api.Name -} - func (api *API) GetResourceType() resource.Type { return resource.APIType } -func (api *API) GetFilePath() string { - return api.FilePath -} - func (apis APIs) Names() []string { names := make([]string, len(apis)) for i, api := range apis { diff --git a/pkg/api/userconfig/app.go b/pkg/api/userconfig/app.go index 99eff2c7b2..3be178e602 100644 --- a/pkg/api/userconfig/app.go +++ b/pkg/api/userconfig/app.go @@ -17,13 +17,11 @@ limitations under the License. package userconfig import ( - "github.com/cortexlabs/cortex/pkg/api/resource" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" ) type App struct { - Name string `json:"name" yaml:"name"` - FilePath string `json:"file_path" yaml:"-"` + Name string `json:"name" yaml:"name"` } var appValidation = &cr.StructValidation{ @@ -40,14 +38,6 @@ var appValidation = &cr.StructValidation{ }, } -func (app *App) GetName() string { - return app.Name -} - -func (app *App) GetResourceType() resource.Type { - return resource.AppType -} - func (app *App) Validate() error { return nil } diff --git a/pkg/api/userconfig/columns.go b/pkg/api/userconfig/columns.go index 20d08068eb..a6c9a97e13 100644 --- a/pkg/api/userconfig/columns.go +++ b/pkg/api/userconfig/columns.go @@ -25,7 +25,7 @@ import ( ) type Column interface { - GetName() string + Resource IsRaw() bool } diff --git a/pkg/api/userconfig/config.go b/pkg/api/userconfig/config.go index 85e5460be8..bb305db369 100644 --- a/pkg/api/userconfig/config.go +++ b/pkg/api/userconfig/config.go @@ -226,13 +226,16 @@ func (config *Config) Validate(envName string) error { return nil } -func (config *Config) MergeBytes(configBytes []byte, filePath string) (*Config, error) { +func (config *Config) MergeBytes(configBytes []byte, filePath string, emb *Embed, template *Template) (*Config, error) { sliceData, err := cr.ReadYAMLBytes(configBytes) if err != nil { - return nil, err + if emb == nil { + return nil, errors.Wrap(err, filePath) + } + return nil, errors.Wrap(err, Identify(template), YAMLKey) } - subConfig, err := newPartial(sliceData, filePath) + subConfig, err := newPartial(sliceData, filePath, emb, template) if err != nil { return nil, err } @@ -244,94 +247,122 @@ func (config *Config) MergeBytes(configBytes []byte, filePath string) (*Config, return config, nil } -func newPartial(configData interface{}, filePath string) (*Config, error) { +func newPartial(configData interface{}, filePath string, emb *Embed, template *Template) (*Config, error) { configDataSlice, ok := cast.InterfaceToStrInterfaceMapSlice(configData) if !ok { - return nil, ErrorMalformedConfig() + if emb == nil { + return nil, errors.Wrap(ErrorMalformedConfig(), filePath) + } + return nil, errors.Wrap(ErrorMalformedConfig(), Identify(template), YAMLKey) } config := &Config{} for i, data := range configDataSlice { - kindStr, ok := data[KindKey].(string) + kindInterface, ok := data[KindKey] + if !ok { + return nil, errors.New(identify(filePath, resource.UnknownType, "", i, emb), KindKey, s.ErrMustBeDefined) + } + kindStr, ok := kindInterface.(string) if !ok { - return nil, errors.New("resource at "+s.Index(i), KindKey, s.ErrMustBeDefined) + return nil, errors.New(identify(filePath, resource.UnknownType, "", i, emb), KindKey, s.ErrInvalidPrimitiveType(kindInterface, s.PrimTypeString)) } var errs []error - switch resource.TypeFromKindString(kindStr) { + resourceType := resource.TypeFromKindString(kindStr) + var newResource Resource + switch resourceType { case resource.AppType: app := &App{} errs = cr.Struct(app, data, appValidation) - app.FilePath = filePath config.App = app case resource.RawColumnType: var rawColumnInter interface{} rawColumnInter, errs = cr.InterfaceStruct(data, rawColumnValidation) - if rawColumnInter != nil { - rawColumn := rawColumnInter.(RawColumn) - rawColumn.SetFilePath(filePath) - config.RawColumns = append(config.RawColumns, rawColumn) + if !errors.HasErrors(errs) && rawColumnInter != nil { + newResource = rawColumnInter.(RawColumn) + config.RawColumns = append(config.RawColumns, newResource.(RawColumn)) } case resource.TransformedColumnType: - transformedColumn := &TransformedColumn{} - errs = cr.Struct(transformedColumn, data, transformedColumnValidation) - transformedColumn.FilePath = filePath - config.TransformedColumns = append(config.TransformedColumns, transformedColumn) + newResource = &TransformedColumn{} + errs = cr.Struct(newResource, data, transformedColumnValidation) + if !errors.HasErrors(errs) { + config.TransformedColumns = append(config.TransformedColumns, newResource.(*TransformedColumn)) + } case resource.AggregateType: - aggregate := &Aggregate{} - errs = cr.Struct(aggregate, data, aggregateValidation) - aggregate.FilePath = filePath - config.Aggregates = append(config.Aggregates, aggregate) + newResource = &Aggregate{} + errs = cr.Struct(newResource, data, aggregateValidation) + if !errors.HasErrors(errs) { + config.Aggregates = append(config.Aggregates, newResource.(*Aggregate)) + } case resource.ConstantType: - constant := &Constant{} - errs = cr.Struct(constant, data, constantValidation) - constant.FilePath = filePath - config.Constants = append(config.Constants, constant) + newResource = &Constant{} + errs = cr.Struct(newResource, data, constantValidation) + if !errors.HasErrors(errs) { + config.Constants = append(config.Constants, newResource.(*Constant)) + } case resource.APIType: - api := &API{} - errs = cr.Struct(api, data, apiValidation) - api.FilePath = filePath - config.APIs = append(config.APIs, api) + newResource = &API{} + errs = cr.Struct(newResource, data, apiValidation) + if !errors.HasErrors(errs) { + config.APIs = append(config.APIs, newResource.(*API)) + } case resource.ModelType: - model := &Model{} - errs = cr.Struct(model, data, modelValidation) - model.FilePath = filePath - config.Models = append(config.Models, model) + newResource = &Model{} + errs = cr.Struct(newResource, data, modelValidation) + if !errors.HasErrors(errs) { + config.Models = append(config.Models, newResource.(*Model)) + } case resource.EnvironmentType: - environment := &Environment{} - errs = cr.Struct(environment, data, environmentValidation) - environment.FilePath = filePath - config.Environments = append(config.Environments, environment) + newResource = &Environment{} + errs = cr.Struct(newResource, data, environmentValidation) + if !errors.HasErrors(errs) { + config.Environments = append(config.Environments, newResource.(*Environment)) + } case resource.AggregatorType: - aggregator := &Aggregator{} - errs = cr.Struct(aggregator, data, aggregatorValidation) - aggregator.FilePath = filePath - config.Aggregators = append(config.Aggregators, aggregator) + newResource = &Aggregator{} + errs = cr.Struct(newResource, data, aggregatorValidation) + if !errors.HasErrors(errs) { + config.Aggregators = append(config.Aggregators, newResource.(*Aggregator)) + } case resource.TransformerType: - transformer := &Transformer{} - errs = cr.Struct(transformer, data, transformerValidation) - transformer.FilePath = filePath - config.Transformers = append(config.Transformers, transformer) + newResource = &Transformer{} + errs = cr.Struct(newResource, data, transformerValidation) + if !errors.HasErrors(errs) { + config.Transformers = append(config.Transformers, newResource.(*Transformer)) + } case resource.TemplateType: - template := &Template{} - errs = cr.Struct(template, data, templateValidation) - template.FilePath = filePath - config.Templates = append(config.Templates, template) + if emb != nil { + errs = []error{resource.ErrorTemplateInTemplate()} + } else { + newResource = &Template{} + errs = cr.Struct(newResource, data, templateValidation) + if !errors.HasErrors(errs) { + config.Templates = append(config.Templates, newResource.(*Template)) + } + } case resource.EmbedType: - embed := &Embed{} - errs = cr.Struct(embed, data, embedValidation) - embed.FilePath = filePath - config.Embeds = append(config.Embeds, embed) + if emb != nil { + errs = []error{resource.ErrorEmbedInTemplate()} + } else { + newResource = &Embed{} + errs = cr.Struct(newResource, data, embedValidation) + if !errors.HasErrors(errs) { + config.Embeds = append(config.Embeds, newResource.(*Embed)) + } + } default: - return nil, errors.Wrap(resource.ErrorUnknownKind(kindStr), "resource at "+s.Index(i)) + return nil, errors.Wrap(resource.ErrorUnknownKind(kindStr), identify(filePath, resource.UnknownType, "", i, emb)) } if errors.HasErrors(errs) { - wrapStr := fmt.Sprintf("%s at %s", kindStr, s.Index(i)) - if resourceName, ok := data[NameKey].(string); ok { - wrapStr = fmt.Sprintf("%s: %s", kindStr, resourceName) - } - return nil, errors.Wrap(errors.FirstError(errs...), wrapStr) + name, _ := data[NameKey].(string) + return nil, errors.Wrap(errors.FirstError(errs...), identify(filePath, resourceType, name, i, emb)) + } + + if newResource != nil { + newResource.SetIndex(i) + newResource.SetFilePath(filePath) + newResource.SetEmbed(emb) } } @@ -353,7 +384,7 @@ func NewPartialPath(filePath string) (*Config, error) { if err != nil { return nil, errors.Wrap(err, filePath, ErrorParseConfig().Error()) } - return newPartial(configData, filePath) + return newPartial(configData, filePath, nil, nil) } func New(configs map[string][]byte, envName string) (*Config, error) { @@ -363,7 +394,7 @@ func New(configs map[string][]byte, envName string) (*Config, error) { if !util.IsFilePathYAML(filePath) { continue } - config, err = config.MergeBytes(configBytes, filePath) + config, err = config.MergeBytes(configBytes, filePath, nil, nil) if err != nil { return nil, err } @@ -373,17 +404,17 @@ func New(configs map[string][]byte, envName string) (*Config, error) { for _, emb := range config.Embeds { template, ok := templates[emb.Template] if !ok { - return nil, errors.Wrap(ErrorUndefinedResource(emb.Template, resource.TemplateType), resource.EmbedType.String()) + return nil, errors.Wrap(ErrorUndefinedResource(emb.Template, resource.TemplateType), Identify(emb)) } populatedTemplate, err := template.Populate(emb) if err != nil { - return nil, errors.Wrap(err, emb.FilePath, resource.EmbedType.String()) + return nil, errors.Wrap(err, Identify(emb)) } - config, err = config.MergeBytes([]byte(populatedTemplate), emb.FilePath) + config, err = config.MergeBytes([]byte(populatedTemplate), emb.FilePath, emb, template) if err != nil { - return nil, errors.Wrap(err, resource.EmbedType.String(), fmt.Sprintf("%s %s", resource.TemplateType.String(), s.UserStr(template.Name))) + return nil, err } } @@ -393,22 +424,22 @@ func New(configs map[string][]byte, envName string) (*Config, error) { return config, nil } -func ReadAppName(configPath string) (string, error) { - configBytes, err := ioutil.ReadFile(configPath) +func ReadAppName(filePath string, relativePath string) (string, error) { + configBytes, err := ioutil.ReadFile(filePath) if err != nil { - return "", errors.Wrap(err, ErrorReadConfig().Error(), configPath) + return "", errors.Wrap(err, ErrorReadConfig().Error(), relativePath) } configData, err := cr.ReadYAMLBytes(configBytes) if err != nil { - return "", errors.Wrap(err, ErrorParseConfig().Error(), configPath) + return "", errors.Wrap(err, ErrorParseConfig().Error(), relativePath) } configDataSlice, ok := cast.InterfaceToStrInterfaceMapSlice(configData) if !ok { - return "", errors.Wrap(ErrorMalformedConfig(), configPath) + return "", errors.Wrap(ErrorMalformedConfig(), relativePath) } if len(configDataSlice) == 0 { - return "", errors.Wrap(ErrorMissingAppDefinition(), configPath) + return "", errors.Wrap(ErrorMissingAppDefinition(), relativePath) } var appName string @@ -416,28 +447,28 @@ func ReadAppName(configPath string) (string, error) { kindStr, _ := configItem[KindKey].(string) if resource.TypeFromKindString(kindStr) == resource.AppType { if appName != "" { - return "", errors.Wrap(ErrorDuplicateConfig(resource.AppType), configPath) + return "", errors.Wrap(ErrorDuplicateConfig(resource.AppType), relativePath) } wrapStr := fmt.Sprintf("%s at %s", resource.AppType.String(), s.Index(i)) appNameInter, ok := configItem[NameKey] if !ok { - return "", errors.New(configPath, wrapStr, NameKey, s.ErrMustBeDefined) + return "", errors.New(relativePath, wrapStr, NameKey, s.ErrMustBeDefined) } appName, ok = appNameInter.(string) if !ok { - return "", errors.New(configPath, wrapStr, s.ErrInvalidPrimitiveType(appNameInter, s.PrimTypeString)) + return "", errors.New(relativePath, wrapStr, s.ErrInvalidPrimitiveType(appNameInter, s.PrimTypeString)) } if appName == "" { - return "", errors.New(configPath, wrapStr, s.ErrCannotBeEmpty) + return "", errors.New(relativePath, wrapStr, s.ErrCannotBeEmpty) } } } if appName == "" { - return "", errors.Wrap(ErrorMissingAppDefinition(), configPath) + return "", errors.Wrap(ErrorMissingAppDefinition(), relativePath) } return appName, nil diff --git a/pkg/api/userconfig/config_key.go b/pkg/api/userconfig/config_key.go index 14524fa794..0bdfeb7a5d 100644 --- a/pkg/api/userconfig/config_key.go +++ b/pkg/api/userconfig/config_key.go @@ -35,6 +35,7 @@ const ( TransformerKey = "transformer" PathKey = "path" ValueKey = "value" + YAMLKey = "yaml" // environment LimitKey = "limit" diff --git a/pkg/api/userconfig/constants.go b/pkg/api/userconfig/constants.go index 0cc9d9472d..18d8246a90 100644 --- a/pkg/api/userconfig/constants.go +++ b/pkg/api/userconfig/constants.go @@ -25,11 +25,10 @@ import ( type Constants []*Constant type Constant struct { - Name string `json:"name" yaml:"name"` - Type interface{} `json:"type" yaml:"type"` - Value interface{} `json:"value" yaml:"value"` - Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path" yaml:"-"` + ResourceConfigFields + Type interface{} `json:"type" yaml:"type"` + Value interface{} `json:"value" yaml:"value"` + Tags Tags `json:"tags" yaml:"tags"` } var constantValidation = &cr.StructValidation{ @@ -98,18 +97,10 @@ func (constant *Constant) GetType() interface{} { return constant.Type } -func (constant *Constant) GetName() string { - return constant.Name -} - func (constant *Constant) GetResourceType() resource.Type { return resource.ConstantType } -func (constant *Constant) GetFilePath() string { - return constant.FilePath -} - func (constants Constants) Names() []string { names := make([]string, len(constants)) for i, constant := range constants { diff --git a/pkg/api/userconfig/embed.go b/pkg/api/userconfig/embed.go index 5d7a5a2191..b22b32a555 100644 --- a/pkg/api/userconfig/embed.go +++ b/pkg/api/userconfig/embed.go @@ -17,15 +17,16 @@ limitations under the License. package userconfig import ( + "github.com/cortexlabs/cortex/pkg/api/resource" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" ) type Embeds []*Embed type Embed struct { + ResourceConfigFields Template string `json:"template" yaml:"template"` Args map[string]interface{} `json:"args" yaml:"args"` - FilePath string `json:"file_path" yaml:"-"` } var embedValidation = &cr.StructValidation{ @@ -47,3 +48,7 @@ var embedValidation = &cr.StructValidation{ typeFieldValidation, }, } + +func (embed *Embed) GetResourceType() resource.Type { + return resource.EmbedType +} diff --git a/pkg/api/userconfig/environments.go b/pkg/api/userconfig/environments.go index a5f7040814..750907e83c 100644 --- a/pkg/api/userconfig/environments.go +++ b/pkg/api/userconfig/environments.go @@ -27,11 +27,10 @@ import ( type Environments []*Environment type Environment struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields LogLevel *LogLevel `json:"log_level" yaml:"log_level"` Limit *Limit `json:"limit" yaml:"limit"` Data Data `json:"-" yaml:"-"` - FilePath string `json:"file_path" yaml:"-"` } var environmentValidation = &cr.StructValidation{ @@ -390,18 +389,10 @@ func (parqData *ParquetData) GetIngestedColumns() []string { return column_names } -func (env *Environment) GetName() string { - return env.Name -} - func (env *Environment) GetResourceType() resource.Type { return resource.EnvironmentType } -func (env *Environment) GetFilePath() string { - return env.FilePath -} - func (environments Environments) Names() []string { names := make([]string, len(environments)) for i, env := range environments { diff --git a/pkg/api/userconfig/errors.go b/pkg/api/userconfig/errors.go index 3b1e382601..31c60e8a95 100644 --- a/pkg/api/userconfig/errors.go +++ b/pkg/api/userconfig/errors.go @@ -23,7 +23,7 @@ import ( "github.com/cortexlabs/cortex/pkg/api/resource" s "github.com/cortexlabs/cortex/pkg/api/strings" "github.com/cortexlabs/cortex/pkg/utils/cast" - "github.com/cortexlabs/cortex/pkg/utils/util" + "github.com/cortexlabs/cortex/pkg/utils/sets/strset" ) type ErrorKind int @@ -134,16 +134,43 @@ func (e ConfigError) Error() string { } func ErrorDuplicateResourceName(resources ...Resource) error { - filePaths := make([]string, len(resources)) - resourceTypes := make(resource.Types, len(resources)) - for i, res := range resources { - filePaths[i] = res.GetFilePath() - resourceTypes[i] = res.GetResourceType() + filePaths := strset.New() + embededFilePaths := strset.New() + templates := strset.New() + resourceTypes := strset.New() + + for _, res := range resources { + resourceTypes.Add(res.GetResourceType().Plural()) + if emb := res.GetEmbed(); emb != nil { + embededFilePaths.Add(res.GetFilePath()) + templates.Add(emb.Template) + } else { + filePaths.Add(res.GetFilePath()) + } + } + + var pathStrs []string + + if len(filePaths) > 0 { + pathStrs = append(pathStrs, "defined in "+s.StrsAnd(filePaths.List())) } + if len(embededFilePaths) > 0 { + embStr := "embedded in " + s.StrsAnd(embededFilePaths.List()) + if len(templates) > 1 { + embStr += " via templates " + } else { + embStr += " via template " + } + embStr += s.UserStrsAnd(templates.List()) + pathStrs = append(pathStrs, embStr) + } + + pathStr := strings.Join(pathStrs, ", ") + return ConfigError{ Kind: ErrDuplicateConfigName, - message: fmt.Sprintf("name %s must be unique across %s (defined in %s)", s.UserStr(resources[0].GetName()), s.StrsAnd(util.UniqueStrs(resourceTypes.PluralList())), s.StrsAnd(util.UniqueStrs(filePaths))), + message: fmt.Sprintf("name %s must be unique across %s (%s)", s.UserStr(resources[0].GetName()), s.StrsAnd(resourceTypes.List()), pathStr), } } diff --git a/pkg/api/userconfig/models.go b/pkg/api/userconfig/models.go index fed46dd7ab..86b51c7b06 100644 --- a/pkg/api/userconfig/models.go +++ b/pkg/api/userconfig/models.go @@ -28,7 +28,7 @@ import ( type Models []*Model type Model struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Type string `json:"type" yaml:"type"` Path string `json:"path" yaml:"path"` TargetColumn string `json:"target_column" yaml:"target_column"` @@ -42,7 +42,6 @@ type Model struct { Evaluation *ModelEvaluation `json:"evaluation" yaml:"evaluation"` Compute *TFCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path"` } var modelValidation = &cr.StructValidation{ @@ -366,18 +365,10 @@ func (model *Model) AllColumnNames() []string { return util.MergeStrSlices(model.FeatureColumns, model.TrainingColumns, []string{model.TargetColumn}) } -func (model *Model) GetName() string { - return model.Name -} - func (model *Model) GetResourceType() resource.Type { return resource.ModelType } -func (model *Model) GetFilePath() string { - return model.FilePath -} - func (models Models) Names() []string { names := make([]string, len(models)) for i, model := range models { diff --git a/pkg/api/userconfig/raw_columns.go b/pkg/api/userconfig/raw_columns.go index d3d5285e78..32860d95f9 100644 --- a/pkg/api/userconfig/raw_columns.go +++ b/pkg/api/userconfig/raw_columns.go @@ -26,9 +26,6 @@ type RawColumn interface { GetType() string GetCompute() *SparkCompute GetUserConfig() Resource - GetResourceType() resource.Type - GetFilePath() string - SetFilePath(string) } type RawColumns []RawColumn @@ -53,7 +50,7 @@ var rawColumnValidation = &cr.InterfaceStructValidation{ } type RawIntColumn struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Type string `json:"type" yaml:"type"` Required bool `json:"required" yaml:"required"` Min *int64 `json:"min" yaml:"min"` @@ -61,7 +58,6 @@ type RawIntColumn struct { Values []int64 `json:"values" yaml:"values"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path" yaml:"-"` } var rawIntColumnFieldValidations = []*cr.StructFieldValidation{ @@ -103,7 +99,7 @@ var rawIntColumnFieldValidations = []*cr.StructFieldValidation{ } type RawFloatColumn struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Type string `json:"type" yaml:"type"` Required bool `json:"required" yaml:"required"` Min *float32 `json:"min" yaml:"min"` @@ -111,7 +107,6 @@ type RawFloatColumn struct { Values []float32 `json:"values" yaml:"values"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path" yaml:"-"` } var rawFloatColumnFieldValidations = []*cr.StructFieldValidation{ @@ -153,13 +148,12 @@ var rawFloatColumnFieldValidations = []*cr.StructFieldValidation{ } type RawStringColumn struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Type string `json:"type" yaml:"type"` Required bool `json:"required" yaml:"required"` Values []string `json:"values" yaml:"values"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path" yaml:"-"` } var rawStringColumnFieldValidations = []*cr.StructFieldValidation{ @@ -221,18 +215,6 @@ func (rawColumns RawColumns) Get(name string) RawColumn { return nil } -func (column *RawIntColumn) GetName() string { - return column.Name -} - -func (column *RawFloatColumn) GetName() string { - return column.Name -} - -func (column *RawStringColumn) GetName() string { - return column.Name -} - func (column *RawIntColumn) GetType() string { return column.Type } @@ -292,27 +274,3 @@ func (column *RawFloatColumn) GetUserConfig() Resource { func (column *RawStringColumn) GetUserConfig() Resource { return column } - -func (column *RawIntColumn) SetFilePath(path string) { - column.FilePath = path -} - -func (column *RawFloatColumn) SetFilePath(path string) { - column.FilePath = path -} - -func (column *RawStringColumn) SetFilePath(path string) { - column.FilePath = path -} - -func (column *RawIntColumn) GetFilePath() string { - return column.FilePath -} - -func (column *RawFloatColumn) GetFilePath() string { - return column.FilePath -} - -func (column *RawStringColumn) GetFilePath() string { - return column.FilePath -} diff --git a/pkg/api/userconfig/resource.go b/pkg/api/userconfig/resource.go index 5ff379f3b5..672d413db9 100644 --- a/pkg/api/userconfig/resource.go +++ b/pkg/api/userconfig/resource.go @@ -19,16 +19,85 @@ import ( "fmt" "github.com/cortexlabs/cortex/pkg/api/resource" + s "github.com/cortexlabs/cortex/pkg/api/strings" ) type Resource interface { GetName() string GetResourceType() resource.Type + GetIndex() int + SetIndex(int) GetFilePath() string + SetFilePath(string) + GetEmbed() *Embed + SetEmbed(*Embed) +} + +type ResourceConfigFields struct { + Name string `json:"name" yaml:"name"` + Index int `json:"index" yaml:"-"` + FilePath string `json:"file_path" yaml:"-"` + Embed *Embed `json:"embed" yaml:"-"` +} + +func (resourceConfigFields *ResourceConfigFields) GetName() string { + return resourceConfigFields.Name +} + +func (resourceConfigFields *ResourceConfigFields) GetIndex() int { + return resourceConfigFields.Index +} + +func (resourceConfigFields *ResourceConfigFields) SetIndex(index int) { + resourceConfigFields.Index = index +} + +func (resourceConfigFields *ResourceConfigFields) GetFilePath() string { + return resourceConfigFields.FilePath +} + +func (resourceConfigFields *ResourceConfigFields) SetFilePath(filePath string) { + resourceConfigFields.FilePath = filePath +} + +func (resourceConfigFields *ResourceConfigFields) GetEmbed() *Embed { + return resourceConfigFields.Embed +} + +func (resourceConfigFields *ResourceConfigFields) SetEmbed(embed *Embed) { + resourceConfigFields.Embed = embed } func Identify(r Resource) string { - return fmt.Sprintf("%s: %s: %s", r.GetFilePath(), r.GetResourceType().String(), r.GetName()) + return identify(r.GetFilePath(), r.GetResourceType(), r.GetName(), r.GetIndex(), r.GetEmbed()) +} + +func identify(filePath string, resourceType resource.Type, name string, index int, embed *Embed) string { + resourceTypeStr := resourceType.String() + if resourceType == resource.UnknownType { + resourceTypeStr = "resource" + } + + str := "" + + if filePath != "" { + str += filePath + ": " + } + + if embed != nil { + if embed.Index >= 0 { + str += fmt.Sprintf("%s at %s (%s \"%s\"): ", resource.EmbedType.String(), s.Index(embed.Index), resource.TemplateType.String(), embed.Template) + } else { + str += fmt.Sprintf("%s (%s \"%s\"): ", resource.EmbedType.String(), resource.TemplateType.String(), embed.Template) + } + } + + if name != "" { + return str + resourceTypeStr + ": " + name + } else if index >= 0 { + return str + resourceTypeStr + " at " + s.Index(index) + } + return str + resourceTypeStr } func FindDuplicateResourceName(resources ...Resource) []Resource { diff --git a/pkg/api/userconfig/templates.go b/pkg/api/userconfig/templates.go index a41d09a4fc..b8f981e9be 100644 --- a/pkg/api/userconfig/templates.go +++ b/pkg/api/userconfig/templates.go @@ -31,9 +31,8 @@ var templateVarRegex = regexp.MustCompile("\\{\\s*([a-zA-Z0-9_-]+)\\s*\\}") type Templates []*Template type Template struct { - Name string `json:"name" yaml:"name"` - YAML string `json:"yaml" yaml:"yaml"` - FilePath string `json:"file_path" yaml:"-"` + ResourceConfigFields + YAML string `json:"yaml" yaml:"yaml"` } var templateValidation = &cr.StructValidation{ @@ -136,14 +135,6 @@ func (template *Template) Populate(emb *Embed) (string, error) { return populatedTemplate, nil } -func (template *Template) GetName() string { - return template.Name -} - func (template *Template) GetResourceType() resource.Type { return resource.TemplateType } - -func (template *Template) GetFilePath() string { - return template.FilePath -} diff --git a/pkg/api/userconfig/transformed_columns.go b/pkg/api/userconfig/transformed_columns.go index d2686bd5c6..9e3a44f1f4 100644 --- a/pkg/api/userconfig/transformed_columns.go +++ b/pkg/api/userconfig/transformed_columns.go @@ -25,12 +25,11 @@ import ( type TransformedColumns []*TransformedColumn type TransformedColumn struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Transformer string `json:"transformer" yaml:"transformer"` Inputs *Inputs `json:"inputs" yaml:"inputs"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` - FilePath string `json:"file_path" yaml:"-"` } var transformedColumnValidation = &cr.StructValidation{ @@ -74,18 +73,10 @@ func (column *TransformedColumn) IsRaw() bool { return false } -func (transformedColumn *TransformedColumn) GetName() string { - return transformedColumn.Name -} - func (transformedColumn *TransformedColumn) GetResourceType() resource.Type { return resource.TransformedColumnType } -func (transformedColumn *TransformedColumn) GetFilePath() string { - return transformedColumn.FilePath -} - func (transformedColumns TransformedColumns) Names() []string { names := make([]string, len(transformedColumns)) for i, transformedColumn := range transformedColumns { diff --git a/pkg/api/userconfig/transformers.go b/pkg/api/userconfig/transformers.go index b90c845078..9cc45753dc 100644 --- a/pkg/api/userconfig/transformers.go +++ b/pkg/api/userconfig/transformers.go @@ -24,11 +24,10 @@ import ( type Transformers []*Transformer type Transformer struct { - Name string `json:"name" yaml:"name"` + ResourceConfigFields Inputs *Inputs `json:"inputs" yaml:"inputs"` OutputType string `json:"output_type" yaml:"output_type"` Path string `json:"path" yaml:"path"` - FilePath string `json:"file_path" yaml:"-"` } var transformerValidation = &cr.StructValidation{ @@ -83,18 +82,10 @@ func (transformers Transformers) Get(name string) *Transformer { return nil } -func (transformer *Transformer) GetName() string { - return transformer.Name -} - func (transformer *Transformer) GetResourceType() resource.Type { return resource.TransformerType } -func (transformer *Transformer) GetFilePath() string { - return transformer.FilePath -} - func (transformers Transformers) Names() []string { names := make([]string, len(transformers)) for i, transformer := range transformers { diff --git a/pkg/operator/context/autogenerator.go b/pkg/operator/context/autogenerator.go index 5e79bec11b..1e7cbde9c2 100644 --- a/pkg/operator/context/autogenerator.go +++ b/pkg/operator/context/autogenerator.go @@ -60,7 +60,9 @@ func autoGenerateConfig( }, "/") constant := &userconfig.Constant{ - Name: constantName, + ResourceConfigFields: userconfig.ResourceConfigFields{ + Name: constantName, + }, Type: argType, Value: argVal, Tags: make(map[string]interface{}), @@ -99,7 +101,9 @@ func autoGenerateConfig( }, "/") constant := &userconfig.Constant{ - Name: constantName, + ResourceConfigFields: userconfig.ResourceConfigFields{ + Name: constantName, + }, Type: argType, Value: argVal, Tags: make(map[string]interface{}), diff --git a/pkg/operator/context/models.go b/pkg/operator/context/models.go index 48309462f3..c9e3619bf4 100644 --- a/pkg/operator/context/models.go +++ b/pkg/operator/context/models.go @@ -103,6 +103,11 @@ func getModels( ImplID: modelImplID, ImplKey: modelImplKey, Dataset: &context.TrainingDataset{ + ResourceConfigFields: userconfig.ResourceConfigFields{ + Name: trainingDatasetName, + FilePath: modelConfig.FilePath, + Embed: modelConfig.Embed, + }, ComputedResourceFields: &context.ComputedResourceFields{ ResourceFields: &context.ResourceFields{ ID: datasetID, @@ -110,7 +115,6 @@ func getModels( ResourceType: resource.TrainingDatasetType, }, }, - Name: trainingDatasetName, ModelName: modelConfig.Name, TrainKey: filepath.Join(datasetRoot, "train.tfrecord"), EvalKey: filepath.Join(datasetRoot, "eval.tfrecord"), diff --git a/pkg/operator/context/python_packages.go b/pkg/operator/context/python_packages.go index 0388605a8a..616790b7b6 100644 --- a/pkg/operator/context/python_packages.go +++ b/pkg/operator/context/python_packages.go @@ -23,6 +23,7 @@ import ( "github.com/cortexlabs/cortex/pkg/api/context" "github.com/cortexlabs/cortex/pkg/api/resource" + "github.com/cortexlabs/cortex/pkg/api/userconfig" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/operator/aws" "github.com/cortexlabs/cortex/pkg/utils/errors" @@ -53,13 +54,15 @@ func loadPythonPackages(files map[string][]byte) (context.PythonPackages, error) buf.Write(reqFileBytes) id := util.HashBytes(buf.Bytes()) pythonPackage := context.PythonPackage{ + ResourceConfigFields: userconfig.ResourceConfigFields{ + Name: consts.RequirementsTxt, + }, ComputedResourceFields: &context.ComputedResourceFields{ ResourceFields: &context.ResourceFields{ ID: id, ResourceType: resource.PythonPackageType, }, }, - Name: consts.RequirementsTxt, SrcKey: filepath.Join(consts.PythonPackagesDir, id, "src.txt"), PackageKey: filepath.Join(consts.PythonPackagesDir, id, "package.zip"), } @@ -87,13 +90,15 @@ func loadPythonPackages(files map[string][]byte) (context.PythonPackages, error) } id := util.HashBytes(buf.Bytes()) pythonPackage := context.PythonPackage{ + ResourceConfigFields: userconfig.ResourceConfigFields{ + Name: consts.RequirementsTxt, + }, ComputedResourceFields: &context.ComputedResourceFields{ ResourceFields: &context.ResourceFields{ ID: id, ResourceType: resource.PythonPackageType, }, }, - Name: packageName, SrcKey: filepath.Join(consts.PythonPackagesDir, id, "src.zip"), PackageKey: filepath.Join(consts.PythonPackagesDir, id, "package.zip"), }