diff --git a/pkg/api/context/models.go b/pkg/api/context/models.go index 9b80739f12..a5f16dc176 100644 --- a/pkg/api/context/models.go +++ b/pkg/api/context/models.go @@ -52,6 +52,10 @@ func (trainingDataset *TrainingDataset) GetResourceType() resource.Type { return resource.TrainingDatasetType } +func (trainingDataset *TrainingDataset) GetFilePath() string { + return "" +} + func (models Models) OneByID(id string) *Model { for _, model := range models { if model.ID == id { diff --git a/pkg/api/userconfig/aggregates.go b/pkg/api/userconfig/aggregates.go index cb96937dc6..aaf4c8e77e 100644 --- a/pkg/api/userconfig/aggregates.go +++ b/pkg/api/userconfig/aggregates.go @@ -30,6 +30,7 @@ type Aggregate struct { Inputs *Inputs `json:"inputs" yaml:"inputs"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path" yaml:"-"` } var aggregateValidation = &cr.StructValidation{ @@ -56,9 +57,14 @@ var aggregateValidation = &cr.StructValidation{ } func (aggregates Aggregates) Validate() error { - dups := util.FindDuplicateStrs(aggregates.Names()) + resources := make([]Resource, len(aggregates)) + for i, res := range aggregates { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.AggregateType) + return ErrorDuplicateResourceName(dups...) } return nil } @@ -71,6 +77,10 @@ func (aggregate *Aggregate) GetResourceType() resource.Type { return resource.AggregateType } +func (aggregate *Aggregate) GetFilePath() string { + return aggregate.FilePath +} + func (aggregates Aggregates) Names() []string { names := make([]string, len(aggregates)) for i, aggregate := range aggregates { diff --git a/pkg/api/userconfig/aggregators.go b/pkg/api/userconfig/aggregators.go index 714270e3ce..3890318c14 100644 --- a/pkg/api/userconfig/aggregators.go +++ b/pkg/api/userconfig/aggregators.go @@ -19,7 +19,6 @@ package userconfig import ( "github.com/cortexlabs/cortex/pkg/api/resource" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" - "github.com/cortexlabs/cortex/pkg/utils/util" ) type Aggregators []*Aggregator @@ -29,6 +28,7 @@ type Aggregator struct { Inputs *Inputs `json:"inputs" yaml:"inputs"` OutputType interface{} `json:"output_type" yaml:"output_type"` Path string `json:"path" yaml:"path"` + FilePath string `json:"file_path" yaml:"-"` } var aggregatorValidation = &cr.StructValidation{ @@ -63,9 +63,14 @@ var aggregatorValidation = &cr.StructValidation{ } func (aggregators Aggregators) Validate() error { - dups := util.FindDuplicateStrs(aggregators.Names()) + resources := make([]Resource, len(aggregators)) + for i, res := range aggregators { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.AggregatorType) + return ErrorDuplicateResourceName(dups...) } return nil } @@ -87,6 +92,10 @@ func (aggregator *Aggregator) GetResourceType() resource.Type { return resource.AggregatorType } +func (aggregator *Aggregator) GetFilePath() string { + return aggregator.FilePath +} + func (aggregators Aggregators) Names() []string { names := make([]string, len(aggregators)) for i, aggregator := range aggregators { diff --git a/pkg/api/userconfig/apis.go b/pkg/api/userconfig/apis.go index 7e4b64e8f5..3c6f6e5df8 100644 --- a/pkg/api/userconfig/apis.go +++ b/pkg/api/userconfig/apis.go @@ -19,7 +19,6 @@ package userconfig import ( "github.com/cortexlabs/cortex/pkg/api/resource" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" - "github.com/cortexlabs/cortex/pkg/utils/util" ) type APIs []*API @@ -29,6 +28,7 @@ type API struct { ModelName string `json:"model_name" yaml:"model_name"` Compute *APICompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path" yaml:"-"` } var apiValidation = &cr.StructValidation{ @@ -55,9 +55,14 @@ var apiValidation = &cr.StructValidation{ } func (apis APIs) Validate() error { - dups := util.FindDuplicateStrs(apis.Names()) + resources := make([]Resource, len(apis)) + for i, res := range apis { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.APIType) + return ErrorDuplicateResourceName(dups...) } return nil } @@ -70,6 +75,10 @@ func (api *API) GetResourceType() resource.Type { return resource.APIType } +func (api *API) GetFilePath() string { + return api.FilePath +} + func (apis APIs) Names() []string { names := make([]string, len(apis)) for i, api := range apis { diff --git a/pkg/api/userconfig/app.go b/pkg/api/userconfig/app.go index 0633a1daef..99eff2c7b2 100644 --- a/pkg/api/userconfig/app.go +++ b/pkg/api/userconfig/app.go @@ -22,7 +22,8 @@ import ( ) type App struct { - Name string `json:"name" yaml:"name"` + Name string `json:"name" yaml:"name"` + FilePath string `json:"file_path" yaml:"-"` } var appValidation = &cr.StructValidation{ diff --git a/pkg/api/userconfig/columns.go b/pkg/api/userconfig/columns.go index 1ca373ef90..20d08068eb 100644 --- a/pkg/api/userconfig/columns.go +++ b/pkg/api/userconfig/columns.go @@ -30,9 +30,18 @@ type Column interface { } func (config *Config) ValidateColumns() error { - dups := util.FindDuplicateStrs(config.ColumnNames()) + columnResources := make([]Resource, len(config.RawColumns)+len(config.TransformedColumns)) + for i, res := range config.RawColumns { + columnResources[i] = res + } + + for i, res := range config.TransformedColumns { + columnResources[i+len(config.RawColumns)] = res + } + + dups := FindDuplicateResourceName(columnResources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.RawColumnType, resource.TransformedColumnType) + return ErrorDuplicateResourceName(dups...) } for _, aggregate := range config.Aggregates { diff --git a/pkg/api/userconfig/config.go b/pkg/api/userconfig/config.go index 1c04137631..9cb1d832d5 100644 --- a/pkg/api/userconfig/config.go +++ b/pkg/api/userconfig/config.go @@ -45,15 +45,6 @@ type Config struct { Embeds Embeds `json:"embeds" yaml:"embeds"` } -type Resource interface { - GetName() string - GetResourceType() resource.Type -} - -func Identify(r Resource) string { - return fmt.Sprintf("%s: %s", r.GetResourceType().String(), r.GetName()) -} - var typeFieldValidation = &cr.StructFieldValidation{ Key: "kind", Nil: true, @@ -235,13 +226,13 @@ func (config *Config) Validate(envName string) error { return nil } -func (config *Config) MergeBytes(configBytes []byte) (*Config, error) { +func (config *Config) MergeBytes(configBytes []byte, filePath string) (*Config, error) { sliceData, err := cr.ReadYAMLBytes(configBytes) if err != nil { return nil, err } - subConfig, err := newPartial(sliceData) + subConfig, err := newPartial(sliceData, filePath) if err != nil { return nil, err } @@ -253,7 +244,7 @@ func (config *Config) MergeBytes(configBytes []byte) (*Config, error) { return config, nil } -func newPartial(configData interface{}) (*Config, error) { +func newPartial(configData interface{}, filePath string) (*Config, error) { configDataSlice, ok := cast.InterfaceToStrInterfaceMapSlice(configData) if !ok { return nil, ErrorMalformedConfig() @@ -271,52 +262,65 @@ func newPartial(configData interface{}) (*Config, error) { case resource.AppType: app := &App{} errs = cr.Struct(app, data, appValidation) + app.FilePath = filePath config.App = app case resource.RawColumnType: var rawColumnInter interface{} rawColumnInter, errs = cr.InterfaceStruct(data, rawColumnValidation) if rawColumnInter != nil { - config.RawColumns = append(config.RawColumns, rawColumnInter.(RawColumn)) + rawColumn := rawColumnInter.(RawColumn) + rawColumn.SetFilePath(filePath) + config.RawColumns = append(config.RawColumns, rawColumn) } case resource.TransformedColumnType: transformedColumn := &TransformedColumn{} errs = cr.Struct(transformedColumn, data, transformedColumnValidation) + transformedColumn.FilePath = filePath config.TransformedColumns = append(config.TransformedColumns, transformedColumn) case resource.AggregateType: aggregate := &Aggregate{} errs = cr.Struct(aggregate, data, aggregateValidation) + aggregate.FilePath = filePath config.Aggregates = append(config.Aggregates, aggregate) case resource.ConstantType: constant := &Constant{} errs = cr.Struct(constant, data, constantValidation) + constant.FilePath = filePath config.Constants = append(config.Constants, constant) case resource.APIType: api := &API{} errs = cr.Struct(api, data, apiValidation) + api.FilePath = filePath config.APIs = append(config.APIs, api) case resource.ModelType: model := &Model{} errs = cr.Struct(model, data, modelValidation) + model.FilePath = filePath config.Models = append(config.Models, model) case resource.EnvironmentType: environment := &Environment{} errs = cr.Struct(environment, data, environmentValidation) + environment.FilePath = filePath config.Environments = append(config.Environments, environment) case resource.AggregatorType: aggregator := &Aggregator{} errs = cr.Struct(aggregator, data, aggregatorValidation) + aggregator.FilePath = filePath config.Aggregators = append(config.Aggregators, aggregator) case resource.TransformerType: transformer := &Transformer{} errs = cr.Struct(transformer, data, transformerValidation) + transformer.FilePath = filePath config.Transformers = append(config.Transformers, transformer) case resource.TemplateType: template := &Template{} errs = cr.Struct(template, data, templateValidation) + template.FilePath = filePath config.Templates = append(config.Templates, template) case resource.EmbedType: embed := &Embed{} errs = cr.Struct(embed, data, embedValidation) + embed.FilePath = filePath config.Embeds = append(config.Embeds, embed) default: return nil, errors.Wrap(resource.ErrorUnknownKind(kindStr), "resource at "+s.Index(i)) @@ -339,29 +343,29 @@ func newPartial(configData interface{}) (*Config, error) { return config, nil } -func NewPartialPath(configPath string) (*Config, error) { - configBytes, err := ioutil.ReadFile(configPath) +func NewPartialPath(filePath string) (*Config, error) { + configBytes, err := ioutil.ReadFile(filePath) if err != nil { - return nil, errors.Wrap(err, configPath, ErrorReadConfig().Error()) + return nil, errors.Wrap(err, filePath, ErrorReadConfig().Error()) } configData, err := cr.ReadYAMLBytes(configBytes) if err != nil { - return nil, errors.Wrap(err, configPath, ErrorParseConfig().Error()) + return nil, errors.Wrap(err, filePath, ErrorParseConfig().Error()) } - return newPartial(configData) + return newPartial(configData, filePath) } func New(configs map[string][]byte, envName string) (*Config, error) { var err error config := &Config{} - for configPath, configBytes := range configs { - if !util.IsFilePathYAML(configPath) { + for filePath, configBytes := range configs { + if !util.IsFilePathYAML(filePath) { continue } - config, err = config.MergeBytes(configBytes) + config, err = config.MergeBytes(configBytes, filePath) if err != nil { - return nil, errors.Wrap(err, configPath) + return nil, err } } @@ -374,10 +378,10 @@ func New(configs map[string][]byte, envName string) (*Config, error) { populatedTemplate, err := template.Populate(emb) if err != nil { - return nil, errors.Wrap(err, resource.EmbedType.String()) + return nil, errors.Wrap(err, emb.FilePath, resource.EmbedType.String()) } - config, err = config.MergeBytes([]byte(populatedTemplate)) + config, err = config.MergeBytes([]byte(populatedTemplate), emb.FilePath) if err != nil { return nil, errors.Wrap(err, resource.EmbedType.String(), fmt.Sprintf("%s %s", resource.TemplateType.String(), s.UserStr(template.Name))) } diff --git a/pkg/api/userconfig/constants.go b/pkg/api/userconfig/constants.go index d1b695e21d..0cc9d9472d 100644 --- a/pkg/api/userconfig/constants.go +++ b/pkg/api/userconfig/constants.go @@ -20,16 +20,16 @@ import ( "github.com/cortexlabs/cortex/pkg/api/resource" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" "github.com/cortexlabs/cortex/pkg/utils/errors" - "github.com/cortexlabs/cortex/pkg/utils/util" ) type Constants []*Constant type Constant struct { - Name string `json:"name" yaml:"name"` - Type interface{} `json:"type" yaml:"type"` - Value interface{} `json:"value" yaml:"value"` - Tags Tags `json:"tags" yaml:"tags"` + Name string `json:"name" yaml:"name"` + Type interface{} `json:"type" yaml:"type"` + Value interface{} `json:"value" yaml:"value"` + Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path" yaml:"-"` } var constantValidation = &cr.StructValidation{ @@ -71,9 +71,14 @@ func (constants Constants) Validate() error { } } - dups := util.FindDuplicateStrs(constants.Names()) + resources := make([]Resource, len(constants)) + for i, res := range constants { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.ConstantType) + return ErrorDuplicateResourceName(dups...) } return nil @@ -101,6 +106,10 @@ func (constant *Constant) GetResourceType() resource.Type { return resource.ConstantType } +func (constant *Constant) GetFilePath() string { + return constant.FilePath +} + func (constants Constants) Names() []string { names := make([]string, len(constants)) for i, constant := range constants { diff --git a/pkg/api/userconfig/embed.go b/pkg/api/userconfig/embed.go index ab7e1e0d11..5d7a5a2191 100644 --- a/pkg/api/userconfig/embed.go +++ b/pkg/api/userconfig/embed.go @@ -25,6 +25,7 @@ type Embeds []*Embed type Embed struct { Template string `json:"template" yaml:"template"` Args map[string]interface{} `json:"args" yaml:"args"` + FilePath string `json:"file_path" yaml:"-"` } var embedValidation = &cr.StructValidation{ diff --git a/pkg/api/userconfig/environments.go b/pkg/api/userconfig/environments.go index 6bff10169b..9da647a3ec 100644 --- a/pkg/api/userconfig/environments.go +++ b/pkg/api/userconfig/environments.go @@ -30,6 +30,7 @@ type Environment struct { Name string `json:"name" yaml:"name"` LogLevel *LogLevel `json:"log_level" yaml:"log_level"` Data Data `json:"-" yaml:"-"` + FilePath string `json:"file_path" yaml:"-"` } var environmentValidation = &cr.StructValidation{ @@ -191,9 +192,14 @@ func (environments Environments) Validate() error { } } - dups := util.FindDuplicateStrs(environments.Names()) + resources := make([]Resource, len(environments)) + for i, res := range environments { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.EnvironmentType) + return ErrorDuplicateResourceName(dups...) } return nil @@ -248,6 +254,10 @@ func (env *Environment) GetResourceType() resource.Type { return resource.EnvironmentType } +func (env *Environment) GetFilePath() string { + return env.FilePath +} + func (environments Environments) Names() []string { names := make([]string, len(environments)) for i, env := range environments { diff --git a/pkg/api/userconfig/errors.go b/pkg/api/userconfig/errors.go index abab331f55..c6f359409a 100644 --- a/pkg/api/userconfig/errors.go +++ b/pkg/api/userconfig/errors.go @@ -23,6 +23,7 @@ import ( "github.com/cortexlabs/cortex/pkg/api/resource" s "github.com/cortexlabs/cortex/pkg/api/strings" "github.com/cortexlabs/cortex/pkg/utils/cast" + "github.com/cortexlabs/cortex/pkg/utils/util" ) type ErrorKind int @@ -130,11 +131,17 @@ func (e ConfigError) Error() string { return e.message } -func ErrorDuplicateConfigName(name string, resourcesTypes ...resource.Type) error { - resourceTypes := resource.Types(resourcesTypes) +func ErrorDuplicateResourceName(resources ...Resource) error { + filePaths := make([]string, len(resources)) + resourceTypes := make(resource.Types, len(resources)) + for i, res := range resources { + filePaths[i] = res.GetFilePath() + resourceTypes[i] = res.GetResourceType() + } + return ConfigError{ Kind: ErrDuplicateConfigName, - message: fmt.Sprintf("name %s must be unique across %s", s.UserStr(name), s.StrsAnd(resourceTypes.PluralList())), + message: fmt.Sprintf("name %s must be unique across %s (defined in %s)", s.UserStr(resources[0].GetName()), s.StrsAnd(util.UniqueStrs(resourceTypes.PluralList())), s.StrsAnd(util.UniqueStrs(filePaths))), } } diff --git a/pkg/api/userconfig/models.go b/pkg/api/userconfig/models.go index 61d56a8e71..fed46dd7ab 100644 --- a/pkg/api/userconfig/models.go +++ b/pkg/api/userconfig/models.go @@ -42,6 +42,7 @@ type Model struct { Evaluation *ModelEvaluation `json:"evaluation" yaml:"evaluation"` Compute *TFCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path"` } var modelValidation = &cr.StructValidation{ @@ -313,9 +314,14 @@ func (models Models) Validate() error { } } - dups := util.FindDuplicateStrs(models.Names()) + resources := make([]Resource, len(models)) + for i, res := range models { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.ModelType) + return ErrorDuplicateResourceName(dups...) } return nil @@ -368,6 +374,10 @@ func (model *Model) GetResourceType() resource.Type { return resource.ModelType } +func (model *Model) GetFilePath() string { + return model.FilePath +} + func (models Models) Names() []string { names := make([]string, len(models)) for i, model := range models { diff --git a/pkg/api/userconfig/raw_columns.go b/pkg/api/userconfig/raw_columns.go index 3db9cc3856..d3d5285e78 100644 --- a/pkg/api/userconfig/raw_columns.go +++ b/pkg/api/userconfig/raw_columns.go @@ -19,7 +19,6 @@ package userconfig import ( "github.com/cortexlabs/cortex/pkg/api/resource" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" - "github.com/cortexlabs/cortex/pkg/utils/util" ) type RawColumn interface { @@ -28,6 +27,8 @@ type RawColumn interface { GetCompute() *SparkCompute GetUserConfig() Resource GetResourceType() resource.Type + GetFilePath() string + SetFilePath(string) } type RawColumns []RawColumn @@ -60,6 +61,7 @@ type RawIntColumn struct { Values []int64 `json:"values" yaml:"values"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path" yaml:"-"` } var rawIntColumnFieldValidations = []*cr.StructFieldValidation{ @@ -109,6 +111,7 @@ type RawFloatColumn struct { Values []float32 `json:"values" yaml:"values"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path" yaml:"-"` } var rawFloatColumnFieldValidations = []*cr.StructFieldValidation{ @@ -156,6 +159,7 @@ type RawStringColumn struct { Values []string `json:"values" yaml:"values"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path" yaml:"-"` } var rawStringColumnFieldValidations = []*cr.StructFieldValidation{ @@ -186,11 +190,17 @@ var rawStringColumnFieldValidations = []*cr.StructFieldValidation{ typeFieldValidation, } -func (rawColumns *RawColumns) Validate() error { - dups := util.FindDuplicateStrs(rawColumns.Names()) +func (rawColumns RawColumns) Validate() error { + resources := make([]Resource, len(rawColumns)) + for i, res := range rawColumns { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.RawColumnType) + return ErrorDuplicateResourceName(dups...) } + return nil } @@ -282,3 +292,27 @@ func (column *RawFloatColumn) GetUserConfig() Resource { func (column *RawStringColumn) GetUserConfig() Resource { return column } + +func (column *RawIntColumn) SetFilePath(path string) { + column.FilePath = path +} + +func (column *RawFloatColumn) SetFilePath(path string) { + column.FilePath = path +} + +func (column *RawStringColumn) SetFilePath(path string) { + column.FilePath = path +} + +func (column *RawIntColumn) GetFilePath() string { + return column.FilePath +} + +func (column *RawFloatColumn) GetFilePath() string { + return column.FilePath +} + +func (column *RawStringColumn) GetFilePath() string { + return column.FilePath +} diff --git a/pkg/api/userconfig/resource.go b/pkg/api/userconfig/resource.go new file mode 100644 index 0000000000..5ff379f3b5 --- /dev/null +++ b/pkg/api/userconfig/resource.go @@ -0,0 +1,47 @@ +/* +Copyright 2019 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package userconfig + +import ( + "fmt" + + "github.com/cortexlabs/cortex/pkg/api/resource" +) + +type Resource interface { + GetName() string + GetResourceType() resource.Type + GetFilePath() string +} + +func Identify(r Resource) string { + return fmt.Sprintf("%s: %s: %s", r.GetFilePath(), r.GetResourceType().String(), r.GetName()) +} + +func FindDuplicateResourceName(resources ...Resource) []Resource { + names := make(map[string][]Resource) + for _, r := range resources { + names[r.GetName()] = append(names[r.GetName()], r) + } + + for name := range names { + if len(names[name]) > 1 { + return names[name] + } + } + + return nil +} diff --git a/pkg/api/userconfig/templates.go b/pkg/api/userconfig/templates.go index 47f32f0581..a41d09a4fc 100644 --- a/pkg/api/userconfig/templates.go +++ b/pkg/api/userconfig/templates.go @@ -24,7 +24,6 @@ import ( "github.com/cortexlabs/cortex/pkg/api/resource" s "github.com/cortexlabs/cortex/pkg/api/strings" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" - "github.com/cortexlabs/cortex/pkg/utils/util" ) var templateVarRegex = regexp.MustCompile("\\{\\s*([a-zA-Z0-9_-]+)\\s*\\}") @@ -32,8 +31,9 @@ var templateVarRegex = regexp.MustCompile("\\{\\s*([a-zA-Z0-9_-]+)\\s*\\}") type Templates []*Template type Template struct { - Name string `json:"name" yaml:"name"` - YAML string `json:"yaml" yaml:"yaml"` + Name string `json:"name" yaml:"name"` + YAML string `json:"yaml" yaml:"yaml"` + FilePath string `json:"file_path" yaml:"-"` } var templateValidation = &cr.StructValidation{ @@ -56,9 +56,14 @@ var templateValidation = &cr.StructValidation{ } func (templates Templates) Validate() error { - dups := util.FindDuplicateStrs(templates.Names()) + resources := make([]Resource, len(templates)) + for i, res := range templates { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.TemplateType) + return ErrorDuplicateResourceName(dups...) } return nil @@ -138,3 +143,7 @@ func (template *Template) GetName() string { func (template *Template) GetResourceType() resource.Type { return resource.TemplateType } + +func (template *Template) GetFilePath() string { + return template.FilePath +} diff --git a/pkg/api/userconfig/transformed_columns.go b/pkg/api/userconfig/transformed_columns.go index e25ce82fe5..d2686bd5c6 100644 --- a/pkg/api/userconfig/transformed_columns.go +++ b/pkg/api/userconfig/transformed_columns.go @@ -30,6 +30,7 @@ type TransformedColumn struct { Inputs *Inputs `json:"inputs" yaml:"inputs"` Compute *SparkCompute `json:"compute" yaml:"compute"` Tags Tags `json:"tags" yaml:"tags"` + FilePath string `json:"file_path" yaml:"-"` } var transformedColumnValidation = &cr.StructValidation{ @@ -56,10 +57,16 @@ var transformedColumnValidation = &cr.StructValidation{ } func (transformedColumns TransformedColumns) Validate() error { - dups := util.FindDuplicateStrs(transformedColumns.Names()) + resources := make([]Resource, len(transformedColumns)) + for i, res := range transformedColumns { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.TransformedColumnType) + return ErrorDuplicateResourceName(dups...) } + return nil } @@ -75,6 +82,10 @@ func (transformedColumn *TransformedColumn) GetResourceType() resource.Type { return resource.TransformedColumnType } +func (transformedColumn *TransformedColumn) GetFilePath() string { + return transformedColumn.FilePath +} + func (transformedColumns TransformedColumns) Names() []string { names := make([]string, len(transformedColumns)) for i, transformedColumn := range transformedColumns { diff --git a/pkg/api/userconfig/transformers.go b/pkg/api/userconfig/transformers.go index b7f1f0e23f..b90c845078 100644 --- a/pkg/api/userconfig/transformers.go +++ b/pkg/api/userconfig/transformers.go @@ -19,7 +19,6 @@ package userconfig import ( "github.com/cortexlabs/cortex/pkg/api/resource" cr "github.com/cortexlabs/cortex/pkg/utils/configreader" - "github.com/cortexlabs/cortex/pkg/utils/util" ) type Transformers []*Transformer @@ -29,6 +28,7 @@ type Transformer struct { Inputs *Inputs `json:"inputs" yaml:"inputs"` OutputType string `json:"output_type" yaml:"output_type"` Path string `json:"path" yaml:"path"` + FilePath string `json:"file_path" yaml:"-"` } var transformerValidation = &cr.StructValidation{ @@ -61,10 +61,16 @@ var transformerValidation = &cr.StructValidation{ } func (transformers Transformers) Validate() error { - dups := util.FindDuplicateStrs(transformers.Names()) + resources := make([]Resource, len(transformers)) + for i, res := range transformers { + resources[i] = res + } + + dups := FindDuplicateResourceName(resources...) if len(dups) > 0 { - return ErrorDuplicateConfigName(dups[0], resource.TransformerType) + return ErrorDuplicateResourceName(dups...) } + return nil } @@ -85,6 +91,10 @@ func (transformer *Transformer) GetResourceType() resource.Type { return resource.TransformerType } +func (transformer *Transformer) GetFilePath() string { + return transformer.FilePath +} + func (transformers Transformers) Names() []string { names := make([]string, len(transformers)) for i, transformer := range transformers { diff --git a/pkg/operator/context/aggregates.go b/pkg/operator/context/aggregates.go index b001a2dda5..5d148d1b2a 100644 --- a/pkg/operator/context/aggregates.go +++ b/pkg/operator/context/aggregates.go @@ -41,7 +41,7 @@ func getAggregates( for _, aggregateConfig := range config.Aggregates { if _, ok := constants[aggregateConfig.Name]; ok { - return nil, userconfig.ErrorDuplicateConfigName(aggregateConfig.Name, resource.AggregateType, resource.ConstantType) + return nil, userconfig.ErrorDuplicateResourceName(aggregateConfig, constants[aggregateConfig.Name]) } aggregator, err := getAggregator(aggregateConfig.Aggregator, userAggregators)