Merge pull request #7 from mdevilliers/retain-max-versions

Feature : Ability to auto deleted non-aliased functions

Author: mdevilliers

aws/lambda_alias.go

@@ -1,8 +1,6 @@
 package aws
 
 import (
-	"log"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/lambda"
@@ -32,9 +30,7 @@ func updateAlias(svc *lambda.Lambda, functionName, aliasName, functionVersion st
 		FunctionVersion: aws.String(functionVersion),
 	}
 
-	resp, err := svc.UpdateAlias(req)
-
-	log.Println("UpdateAlias : ", resp, err)
+	_, err := svc.UpdateAlias(req)
 
 	if err != nil {
 		return errors.WithStack(err)
@@ -52,9 +48,7 @@ func newAlias(svc *lambda.Lambda, functionName, aliasName, functionVersion strin
 		FunctionVersion: aws.String(functionVersion),
 	}
 
-	resp, err := svc.CreateAlias(req)
-
-	log.Println("CreateAlias : ", resp, err)
+	_, err := svc.CreateAlias(req)
 
 	if err != nil {
 		return errors.WithStack(err)
@@ -71,9 +65,7 @@ func aliasExists(svc *lambda.Lambda, functionName, aliasName string) (bool, erro
 		Name:         aws.String(aliasName),
 	}
 
-	resp, err := svc.GetAlias(req)
-
-	log.Println("GetAlias : ", resp, err)
+	_, err := svc.GetAlias(req)
 
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {

aws/lambda_function.go

@@ -3,6 +3,8 @@ package aws
 import (
 	"fmt"
 	"log"
+	"sort"
+	"strconv"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -35,9 +37,7 @@ func functionExists(svc *lambda.Lambda, name string) (bool, error) {
 		FunctionName: aws.String(name),
 	}
 
-	resp, err := svc.GetFunction(req)
-
-	log.Println("GetFunction : ", resp, err)
+	_, err := svc.GetFunction(req)
 
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
@@ -62,8 +62,6 @@ func updateLambdaFunction(svc *lambda.Lambda, s3Bucket, s3Key string, metadata d
 
 	resp, err := svc.UpdateFunctionCode(req)
 
-	log.Println("UpdateFunctionCode : ", resp, err)
-
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -97,11 +95,119 @@ func newLambdaFunction(svc *lambda.Lambda, s3Bucket, s3Key, role string, metadat
 
 	resp, err := svc.CreateFunction(req)
 
-	log.Println("CreateFunction : ", resp, err)
-
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 
 	return resp, nil
 }
+
+func ReduceUnAliasedVersions(svc *lambda.Lambda, maxVersions int, metadata deployer.FunctionMetadata) error {
+
+	// get all aliased functions
+	allAliasesReq := &lambda.ListAliasesInput{
+		FunctionName: aws.String(metadata.FunctionName),
+	}
+	allAliasResp, err := svc.ListAliases(allAliasesReq)
+
+	if err != nil {
+		errors.WithStack(err)
+	}
+
+	// get all versions for a function
+	versionReq := &lambda.ListVersionsByFunctionInput{
+		FunctionName: aws.String(metadata.FunctionName),
+	}
+
+	versionResp, err := svc.ListVersionsByFunction(versionReq)
+
+	if err != nil {
+		errors.WithStack(err)
+	}
+
+	// if there are less (or equal) versions than the max versions
+	if len(versionResp.Versions) <= maxVersions {
+		return nil
+	}
+
+	// create an array to hold all versions without an active alias
+	versionsUnAliased := []*lambda.FunctionConfiguration{}
+
+	// use the code hash to build up a list of versions without aliases
+	for _, version := range versionResp.Versions {
+
+		drop := false
+
+		// $LATEST is a special poiter to the latest function
+		// helpfully it isn't returned in the list of aliases
+		// so we need a special case here
+		if *(version.Version) == "$LATEST" {
+			drop = true
+		} else {
+			// we need to loop though our list of aliases checking if that version
+			// hasn't been assigned an alias
+			for _, aliasedFunction := range allAliasResp.Aliases {
+
+				if *(aliasedFunction.FunctionVersion) == *(version.Version) {
+					drop = true
+				}
+			}
+		}
+
+		if !drop {
+			versionsUnAliased = append(versionsUnAliased, version)
+		}
+
+	}
+
+	// if the unaliased versions number less or equal to maxVersions to retain
+	if len(versionsUnAliased) <= maxVersions {
+		return nil
+	}
+
+	// order by versions
+	sort.Sort(byVersion(versionsUnAliased))
+
+	// delete all versions - the last n (maxVersions)
+	toDelete := versionsUnAliased[0 : len(versionsUnAliased)-maxVersions]
+
+	for _, version := range toDelete {
+
+		log.Println("deleting unaliased function : ", *(version.Version))
+		deleteRequest := &lambda.DeleteFunctionInput{
+			FunctionName: version.FunctionName,
+			Qualifier:    version.Version,
+		}
+		_, err := svc.DeleteFunction(deleteRequest)
+
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+	}
+
+	return nil
+}
+
+// byVersion implements sort.Interface for []*lambda.FunctionConfiguration based on
+// the Version.
+type byVersion []*lambda.FunctionConfiguration
+
+func (a byVersion) Len() int      { return len(a) }
+func (a byVersion) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byVersion) Less(i, j int) bool {
+
+	iVersion, err := strconv.Atoi(*(a[i].Version))
+
+	if err != nil {
+		panic("version not a number")
+	}
+
+	jVersion, _ := strconv.Atoi(*(a[j].Version))
+
+	if err != nil {
+		panic("version not a number")
+	}
+
+	return iVersion < jVersion
+}

cmd/deployer/main.go

@@ -21,10 +21,18 @@ func main() {
 	// DO NOTHING
 }
 
+// Policy holds information for the deployer to implement
 type Policy struct {
-	MaximumVersions int
+	// MaximumUnAliasedVersions is the maximum unaliased versions of a lambda function
+	// we want to keep. Versions with an alias are never deleted.
+	MaximumUnAliasedVersions int
+
+	// ReduceUnAliasedVersions is true if MaxUnAliasedVersions has been specified
+	ReduceUnAliasedVersions bool
 }
 
+// S3Event struct captures the JSON structure of the event passed when a new
+// object is created in S3
 type S3Event struct {
 	Records []struct {
 		EventVersion      string    `json:"eventVersion"`
@@ -62,6 +70,9 @@ type S3Event struct {
 	} `json:"Records"`
 }
 
+// Handle is called when ever an object is written to S3 via the uploader.
+// We assume this is always a lambda function zip file and that AWS Lambda will error
+// if the file is not of a correct format.
 func Handle(evt json.RawMessage, ctx *runtime.Context) (string, error) {
 
 	log.Println("deployer : ", deployer.VersionString())
@@ -73,9 +84,15 @@ func Handle(evt json.RawMessage, ctx *runtime.Context) (string, error) {
 		return "error", errors.New("DEPLOYER_FUNCTION_ROLE_ARN not set")
 	}
 
+	policy, err := loadPolicy()
+
+	if err != nil {
+		return "error", errors.Wrap(err, "error loading policy")
+	}
+
 	s3Event := S3Event{}
 
-	err := json.Unmarshal(evt, &s3Event)
+	err = json.Unmarshal(evt, &s3Event)
 
 	if err != nil {
 		return "error", errors.Wrap(err, "error un-marshaling event json")
@@ -87,36 +104,48 @@ func Handle(evt json.RawMessage, ctx *runtime.Context) (string, error) {
 		return "error", err
 	}
 
-	svc := lambda.New(session, aws.NewConfig())
+	lambdaSvc := lambda.New(session, aws.NewConfig())
+	s3Svc := s3.New(session, aws.NewConfig())
 
 	bucket := s3Event.Records[0].S3.Bucket.Name
 	key := s3Event.Records[0].S3.Object.Key
 
-	s3Svc := s3.New(session, aws.NewConfig())
 	meta, err := getMetadata(s3Svc, bucket, key)
 
 	if err != nil {
 		return "error", errors.Wrap(err, "error reading metadata from s3 object")
 	}
 
 	// create or update the lambda function
-	conf, err := aws_helper.CreateOrUpdateFunction(svc, bucket, key, role, meta)
+	conf, err := aws_helper.CreateOrUpdateFunction(lambdaSvc, bucket, key, role, meta)
 
 	if err != nil {
 		return "error", errors.Wrap(err, "error creating or updating lambda function")
 	}
 
 	// update, create the alias
-	err = aws_helper.CreateOrUpdateAlias(svc, conf, meta)
+	err = aws_helper.CreateOrUpdateAlias(lambdaSvc, conf, meta)
 
 	if err != nil {
 		return "error", errors.Wrap(err, "error creating or updating alias")
 	}
 
+	// delete unused versions if required
+	if policy.ReduceUnAliasedVersions {
+
+		err = aws_helper.ReduceUnAliasedVersions(lambdaSvc, policy.MaximumUnAliasedVersions, meta)
+
+		if err != nil {
+			return "error", errors.Wrap(err, "error deleting UnAliased versions")
+		}
+
+	}
+
 	return "ok", nil
 
 }
 
+// getMetadata parses the S3 object metadata
 func getMetadata(svc *s3.S3, s3Bucket, s3Key string) (deployer.FunctionMetadata, error) {
 
 	req := &s3.HeadObjectInput{
@@ -130,26 +159,26 @@ func getMetadata(svc *s3.S3, s3Bucket, s3Key string) (deployer.FunctionMetadata,
 		return deployer.FunctionMetadata{}, err
 	}
 
-	memorySize, err := strconv.ParseInt(*(resp.Metadata["Function-Memory-Size"]), 10, 64)
+	memorySize, err := strconv.ParseInt(*(resp.Metadata[deployer.FunctionMemorySizeTag]), 10, 64)
 
 	if err != nil {
 		return deployer.FunctionMetadata{}, errors.Wrap(err, "cannot parse function-memory-size")
 	}
 
-	timeout, err := strconv.ParseInt(*(resp.Metadata["Function-Timeout"]), 10, 64)
+	timeout, err := strconv.ParseInt(*(resp.Metadata[deployer.FunctionTimeoutTag]), 10, 64)
 
 	if err != nil {
 		return deployer.FunctionMetadata{}, errors.Wrap(err, "cannot parse function-timeout")
 	}
 
 	meta := deployer.FunctionMetadata{
-		Description:  *(resp.Metadata["Function-Description"]),
-		FunctionName: *(resp.Metadata["Function-Name"]),
-		Handler:      *(resp.Metadata["Function-Handler"]),
-		Runtime:      *(resp.Metadata["Function-Runtime"]),
+		Description:  *(resp.Metadata[deployer.FunctionDescriptionTag]),
+		FunctionName: *(resp.Metadata[deployer.FunctionNameTag]),
+		Handler:      *(resp.Metadata[deployer.FunctionHandlerTag]),
+		Runtime:      *(resp.Metadata[deployer.FunctionRuntimeTag]),
 		MemorySize:   int64(memorySize),
 		Timeout:      int64(timeout),
-		Alias:        *(resp.Metadata["Function-Alias"]),
+		Alias:        *(resp.Metadata[deployer.FunctionAliasTag]),
 		EnvVars:      map[string]interface{}{},
 	}
 
@@ -159,9 +188,33 @@ func getMetadata(svc *s3.S3, s3Bucket, s3Key string) (deployer.FunctionMetadata,
 	err = json.Unmarshal([]byte(envVars), &meta.EnvVars)
 
 	if err != nil {
-		return deployer.FunctionMetadata{}, errors.Wrap(err, "error un-marshaling envionmental vars")
+		return deployer.FunctionMetadata{}, errors.Wrap(err, "error un-marshaling environmental vars")
 	}
 
 	return meta, nil
 
 }
+
+func loadPolicy() (Policy, error) {
+
+	maxUnAliasedVersionsStr := os.Getenv("DEPLOYER_POLICY_MAX_UNALIASED_VERSIONS")
+
+	maxUnAliasedVersions := int64(0)
+	var reduceUnAliasedVersions bool
+	var err error
+
+	if maxUnAliasedVersionsStr != "" {
+
+		maxUnAliasedVersions, err = strconv.ParseInt(maxUnAliasedVersionsStr, 10, 64)
+
+		if err != nil {
+			return Policy{}, err
+		}
+		reduceUnAliasedVersions = true
+	}
+	return Policy{
+		MaximumUnAliasedVersions: int(maxUnAliasedVersions),
+		ReduceUnAliasedVersions:  reduceUnAliasedVersions,
+	}, nil
+
+}

cmd/uploader/cmd_upload.go

@@ -9,6 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	deployer "github.com/mdevilliers/lambda-deployer"
 	"github.com/spf13/cobra"
 )
 
@@ -51,13 +52,13 @@ func newUploadCommand() *cobra.Command {
 				Bucket: aws.String(_config.S3BucketName),
 				Key:    aws.String(fileName),
 				Metadata: map[string]*string{
-					"Function-Description": aws.String(_config.Description),
-					"Function-Name":        aws.String(_config.FunctionName),
-					"Function-Handler":     aws.String(_config.Handler),
-					"Function-Runtime":     aws.String(_config.Runtime),
-					"Function-Memory-Size": aws.String(fmt.Sprintf("%d", _config.MemorySize)),
-					"Function-Timeout":     aws.String(fmt.Sprintf("%d", _config.Timeout)),
-					"Function-Alias":       aws.String(_config.Alias),
+					deployer.FunctionDescriptionTag: aws.String(_config.Description),
+					deployer.FunctionNameTag:        aws.String(_config.FunctionName),
+					deployer.FunctionHandlerTag:     aws.String(_config.Handler),
+					deployer.FunctionRuntimeTag:     aws.String(_config.Runtime),
+					deployer.FunctionMemorySizeTag:  aws.String(fmt.Sprintf("%d", _config.MemorySize)),
+					deployer.FunctionTimeoutTag:     aws.String(fmt.Sprintf("%d", _config.Timeout)),
+					deployer.FunctionAliasTag:       aws.String(_config.Alias),
 				},
 			}
 

terraform/example/main.tf

@@ -85,7 +85,7 @@ module "auto_deployer" {
 
   application       = "example"
   environment       = "staging"
-  deployer_filepath = "../../cmd/deployer/deployer.zip"
+  deployer_filepath = "../../cmd/deployer/lambda-deployer.zip"
 
   function_role_arn = "${aws_iam_role.my_lambda_role.arn}"
   s3_bucket_arn     = "${aws_s3_bucket.deployment_uploads.arn}"