Fix cx panic when object and prefix present on bkt

Author: RobertLucian

pkg/lib/gcp/gcs.go

@@ -38,7 +38,7 @@ func SplitGCSPath(gcsPath string) (string, string, error) {
 	if !IsValidGCSPath(gcsPath) {
 		return "", "", ErrorInvalidGCSPath(gcsPath)
 	}
-	fullPath := gcsPath[len("s3://"):]
+	fullPath := gcsPath[len("gs://"):]
 	slashIndex := strings.Index(fullPath, "/")
 	if slashIndex == -1 {
 		return fullPath, "", nil

pkg/lib/slices/string.go

@@ -98,6 +98,16 @@ func RemoveEmptiesAndUnique(strs []string) []string {
 	return out
 }
 
+func RemoveIfHasString(strs []string, str string) []string {
+	var filteredStrs []string
+	for _, elem := range strs {
+		if elem != str {
+			filteredStrs = append(filteredStrs, elem)
+		}
+	}
+	return filteredStrs
+}
+
 func HasDuplicateStr(in []string) bool {
 	keys := strset.New()
 	for _, elem := range in {

pkg/types/spec/errors.go

@@ -55,10 +55,11 @@ const (
 
 	ErrShmSizeCannotExceedMem = "spec.shm_size_cannot_exceed_mem"
 
-	ErrFileNotFound              = "spec.file_not_found"
-	ErrDirIsEmpty                = "spec.dir_is_empty"
-	ErrMustBeRelativeProjectPath = "spec.must_be_relative_project_path"
-	ErrPythonPathNotFound        = "spec.python_path_not_found"
+	ErrFileNotFound                = "spec.file_not_found"
+	ErrDirIsEmpty                  = "spec.dir_is_empty"
+	ErrMustBeRelativeProjectPath   = "spec.must_be_relative_project_path"
+	ErrPythonPathNotFound          = "spec.python_path_not_found"
+	ErrObjectConflictingWithPrefix = "spec.object_conflicting_with_prefix"
 
 	ErrS3FileNotFound = "spec.s3_file_not_found"
 	ErrS3DirNotFound  = "spec.s3_dir_not_found"
@@ -274,6 +275,13 @@ func ErrorPythonPathNotFound(pythonPath string) error {
 	})
 }
 
+func ErrorObjectConflictingWithPrefix(path string) error {
+	return errors.WithStack(&errors.Error{
+		Kind:    ErrObjectConflictingWithPrefix,
+		Message: fmt.Sprintf("%s: path prefix conflicts with object %s; remove object %s to be able to use %s as prefix for other objects", path, path, path, path),
+	})
+}
+
 func ErrorS3FileNotFound(path string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrS3FileNotFound,

pkg/types/spec/utils.go

@@ -140,6 +140,7 @@ func validateDirModels(
 		if err != nil {
 			return nil, err
 		}
+		dirPrefix = s.EnsureSuffix(dirPrefix, "/")
 
 		s3Objects, err := awsClientForBucket.ListS3PathDir(modelPath, false, nil)
 		if err != nil {
@@ -152,6 +153,7 @@ func validateDirModels(
 		if err != nil {
 			return nil, err
 		}
+		dirPrefix = s.EnsureSuffix(dirPrefix, "/")
 
 		gcsObjects, err := gcpClient.ListGCSPathDir(modelPath, nil)
 		if err != nil {
@@ -162,6 +164,12 @@ func validateDirModels(
 	if len(modelDirPaths) == 0 {
 		return nil, errorForPredictorType(dirPrefix, modelDirPaths)
 	}
+	if len(modelDirPaths) != len(slices.RemoveIfHasString(modelDirPaths, dirPrefix)) {
+		return nil, ErrorObjectConflictingWithPrefix(dirPrefix)
+	}
+	if len(modelDirPaths) != len(slices.RemoveIfHasString(modelDirPaths, strings.TrimRight(dirPrefix, "/"))) {
+		return nil, ErrorObjectConflictingWithPrefix(strings.TrimRight(dirPrefix, "/"))
+	}
 
 	modelNames := []string{}
 	modelDirPathLength := len(slices.RemoveEmpties(strings.Split(dirPrefix, "/")))
@@ -179,7 +187,13 @@ func validateDirModels(
 		}
 
 		modelPrefix := filepath.Join(dirPrefix, modelName)
+		if len(modelDirPaths) != len(slices.RemoveIfHasString(modelDirPaths, modelPrefix)) {
+			return nil, ErrorObjectConflictingWithPrefix(modelPrefix)
+		}
 		modelPrefix = s.EnsureSuffix(modelPrefix, "/")
+		if len(modelDirPaths) != len(slices.RemoveIfHasString(modelDirPaths, modelPrefix)) {
+			return nil, ErrorObjectConflictingWithPrefix(modelPrefix)
+		}
 
 		modelStructureType := determineBaseModelStructure(modelDirPaths, modelPrefix)
 		if modelStructureType == userconfig.UnknownModelStructureType {
@@ -296,6 +310,12 @@ func validateModels(
 		if len(modelPaths) == 0 {
 			return nil, errors.Wrap(errorForPredictorType(modelPrefix, modelPaths), modelNameWrapStr)
 		}
+		if len(modelPaths) != len(slices.RemoveIfHasString(modelPaths, modelPrefix)) {
+			return nil, ErrorObjectConflictingWithPrefix(modelPrefix)
+		}
+		if len(modelPaths) != len(slices.RemoveIfHasString(modelPaths, strings.TrimRight(modelPrefix, "/"))) {
+			return nil, ErrorObjectConflictingWithPrefix(strings.TrimRight(modelPrefix, "/"))
+		}
 
 		modelStructureType := determineBaseModelStructure(modelPaths, modelPrefix)
 		if modelStructureType == userconfig.UnknownModelStructureType {