feat: Reduce payload size for device events (#1012)

The proposal introduces a new global environment variable, EDGEX_OPTIMIZE_EVENT_PAYLOAD, for all EdgeX services.
When EDGEX_OPTIMIZE_EVENT_PAYLOAD is set to true, the event payload will be optimized.

Signed-off-by: bruce <[email protected]>

Author: weichou1229

common/constants.go

@@ -295,8 +295,9 @@ const (
 
 // Constants for Edgex Environment variable
 const (
-	EnvEncodeAllEvents   = "EDGEX_ENCODE_ALL_EVENTS_CBOR"
-	EnvMessageCborEncode = "EDGEX_MSG_CBOR_ENCODE"
+	EnvEncodeAllEvents      = "EDGEX_ENCODE_ALL_EVENTS_CBOR"
+	EnvMessageCborEncode    = "EDGEX_MSG_CBOR_ENCODE"
+	EnvOptimizeEventPayload = "EDGEX_OPTIMIZE_EVENT_PAYLOAD"
 )
 
 // Miscellaneous constants

dtos/event.go

@@ -1,35 +1,39 @@
 //
-// Copyright (C) 2020-2023 IOTech Ltd
+// Copyright (C) 2020-2025 IOTech Ltd
 //
 // SPDX-License-Identifier: Apache-2.0
 
 package dtos
 
 import (
+	"encoding/json"
 	"encoding/xml"
+	"os"
 	"time"
 
-	"github.com/edgexfoundry/go-mod-core-contracts/v4/dtos/common"
-	"github.com/edgexfoundry/go-mod-core-contracts/v4/models"
-
+	"github.com/fxamacker/cbor/v2"
 	"github.com/google/uuid"
+
+	"github.com/edgexfoundry/go-mod-core-contracts/v4/common"
+	dtoCommon "github.com/edgexfoundry/go-mod-core-contracts/v4/dtos/common"
+	"github.com/edgexfoundry/go-mod-core-contracts/v4/models"
 )
 
 type Event struct {
-	common.Versionable `json:",inline"`
-	Id                 string        `json:"id" validate:"required,uuid"`
-	DeviceName         string        `json:"deviceName" validate:"required,edgex-dto-none-empty-string"`
-	ProfileName        string        `json:"profileName" validate:"required,edgex-dto-none-empty-string"`
-	SourceName         string        `json:"sourceName" validate:"required,edgex-dto-none-empty-string"`
-	Origin             int64         `json:"origin" validate:"required"`
-	Readings           []BaseReading `json:"readings" validate:"gt=0,dive,required"`
-	Tags               Tags          `json:"tags,omitempty"`
+	dtoCommon.Versionable `json:",inline"`
+	Id                    string        `json:"id" validate:"required,uuid"`
+	DeviceName            string        `json:"deviceName" validate:"required,edgex-dto-none-empty-string"`
+	ProfileName           string        `json:"profileName" validate:"required,edgex-dto-none-empty-string"`
+	SourceName            string        `json:"sourceName" validate:"required,edgex-dto-none-empty-string"`
+	Origin                int64         `json:"origin" validate:"required"`
+	Readings              []BaseReading `json:"readings" validate:"gt=0,dive,required"`
+	Tags                  Tags          `json:"tags,omitempty"`
 }
 
 // NewEvent creates and returns an initialized Event with no Readings
 func NewEvent(profileName, deviceName, sourceName string) Event {
 	return Event{
-		Versionable: common.NewVersionable(),
+		Versionable: dtoCommon.NewVersionable(),
 		Id:          uuid.NewString(),
 		DeviceName:  deviceName,
 		ProfileName: profileName,
@@ -51,7 +55,7 @@ func FromEventModelToDTO(event models.Event) Event {
 	}
 
 	return Event{
-		Versionable: common.NewVersionable(),
+		Versionable: dtoCommon.NewVersionable(),
 		Id:          event.Id,
 		DeviceName:  event.DeviceName,
 		ProfileName: event.ProfileName,
@@ -96,3 +100,101 @@ func (e *Event) ToXML() (string, error) {
 
 	return string(eventXml), nil
 }
+
+func (e Event) MarshalJSON() ([]byte, error) {
+	return e.marshal(json.Marshal)
+}
+
+func (e Event) MarshalCBOR() ([]byte, error) {
+	return e.marshal(cbor.Marshal)
+}
+
+func (e Event) marshal(marshal func(any) ([]byte, error)) ([]byte, error) {
+	var aux struct {
+		dtoCommon.Versionable `json:",inline"`
+		Id                    string        `json:"id" validate:"required,uuid"`
+		DeviceName            string        `json:"deviceName"`
+		ProfileName           string        `json:"profileName"`
+		SourceName            string        `json:"sourceName"`
+		Origin                int64         `json:"origin"`
+		Readings              []BaseReading `json:"readings"`
+		Tags                  Tags          `json:"tags,omitempty"`
+	}
+	aux.Versionable = e.Versionable
+	aux.Id = e.Id
+	aux.DeviceName = e.DeviceName
+	aux.ProfileName = e.ProfileName
+	aux.SourceName = e.SourceName
+	aux.Origin = e.Origin
+	aux.Tags = e.Tags
+	if len(e.Readings) > 0 {
+		aux.Readings = make([]BaseReading, len(e.Readings))
+	}
+
+	if os.Getenv(common.EnvOptimizeEventPayload) == common.ValueTrue {
+		for i, reading := range e.Readings {
+			reading.Id = ""
+			reading.DeviceName = ""
+			reading.ProfileName = ""
+			if e.Origin == reading.Origin {
+				reading.Origin = 0
+			}
+			if len(e.Readings) == 1 && e.SourceName == reading.ResourceName {
+				reading.ResourceName = ""
+			}
+			aux.Readings[i] = reading
+		}
+	} else {
+		copy(aux.Readings, e.Readings)
+	}
+
+	return marshal(aux)
+}
+
+func (e *Event) UnmarshalJSON(b []byte) error {
+	return e.unmarshal(b, json.Unmarshal)
+}
+
+func (e *Event) UnmarshalCBOR(b []byte) error {
+	return e.unmarshal(b, cbor.Unmarshal)
+}
+
+func (e *Event) unmarshal(data []byte, unmarshal func([]byte, any) error) error {
+	var aux struct {
+		dtoCommon.Versionable
+		Id          string
+		DeviceName  string
+		ProfileName string
+		SourceName  string
+		Origin      int64
+		Readings    []BaseReading
+		Tags        Tags
+	}
+	if err := unmarshal(data, &aux); err != nil {
+		return err
+	}
+
+	e.Versionable = aux.Versionable
+	e.Id = aux.Id
+	e.DeviceName = aux.DeviceName
+	e.ProfileName = aux.ProfileName
+	e.SourceName = aux.SourceName
+	e.Origin = aux.Origin
+	e.Readings = aux.Readings
+	e.Tags = aux.Tags
+
+	if os.Getenv(common.EnvOptimizeEventPayload) == common.ValueTrue {
+		// recover the reduced fields
+		for i, reading := range e.Readings {
+			e.Readings[i].DeviceName = e.DeviceName
+			e.Readings[i].ProfileName = e.ProfileName
+			if reading.Origin == 0 {
+				e.Readings[i].Origin = e.Origin
+			}
+			if len(e.Readings) == 1 && len(reading.ResourceName) == 0 {
+				e.Readings[i].ResourceName = e.SourceName
+			}
+		}
+	}
+	return nil
+}

dtos/event_test.go

@@ -1,5 +1,5 @@
 //
-// Copyright (C) 2020-2021 IOTech Ltd
+// Copyright (C) 2020-2025 IOTech Ltd
 //
 // SPDX-License-Identifier: Apache-2.0
 
@@ -8,6 +8,8 @@ package dtos
 import (
 	"encoding/json"
 	"fmt"
+	"math"
+	"os"
 	"testing"
 
 	"github.com/fxamacker/cbor/v2"
@@ -287,3 +289,34 @@ func TestEvent_MarshalNullReading(t *testing.T) {
 		})
 	}
 }
+
+func TestEvent_MarshalOptimizedEventPayload(t *testing.T) {
+	testEvent := NewEvent(TestDeviceProfileName, TestDeviceName, TestSourceName)
+	testEvent.Id = TestUUID
+	testEvent.Origin = TestTimestamp
+	err := testEvent.AddSimpleReading(TestReadingName, common.ValueTypeUint8, uint8(math.MaxUint8))
+	require.NoError(t, err)
+	testEvent.Readings[0].Id = TestUUID
+	testEvent.Readings[0].Origin = TestTimestamp
+
+	err = os.Setenv(common.EnvOptimizeEventPayload, common.ValueTrue)
+	require.NoError(t, err)
+
+	// Marshal reduce the reading fields
+	data, err := json.Marshal(testEvent)
+	require.NoError(t, err)
+	assert.JSONEq(t,
+		fmt.Sprintf(
+			`{"apiVersion":"v3", "id":"%s",
+					"deviceName":"TestDevice","profileName":"TestDeviceProfileName","sourceName":"TestSourceName","origin":%d,
+					"readings":[{"resourceName":"TestDeviceResource","valueType":"Uint8","value":"255"}]}`, TestUUID, TestTimestamp),
+		string(data))
+
+	// Unmarshal recover the reading fields
+	var res Event
+	err = json.Unmarshal(data, &res)
+	require.NoError(t, err)
+
+	testEvent.Readings[0].Id = "" // the id field will be omitted ,and it isn’t used to store in the database
+	assert.Equal(t, testEvent, res)
+}

dtos/reading.go

@@ -538,14 +538,14 @@ func (b BaseReading) MarshalCBOR() ([]byte, error) {
 
 func (b BaseReading) marshal(marshal func(any) ([]byte, error)) ([]byte, error) {
 	type reading struct {
-		Id           string `json:"id,omitempty"`
-		Origin       int64  `json:"origin"`
-		DeviceName   string `json:"deviceName"`
-		ResourceName string `json:"resourceName"`
-		ProfileName  string `json:"profileName"`
-		ValueType    string `json:"valueType"`
-		Units        string `json:"units,omitempty"`
-		Tags         Tags   `json:"tags,omitempty"`
+		Id           string `json:"id,omitempty" cbor:"id,omitempty"`
+		Origin       int64  `json:"origin,omitempty" cbor:"origin,omitempty"`
+		DeviceName   string `json:"deviceName,omitempty" cbor:"deviceName,omitempty"`
+		ResourceName string `json:"resourceName,omitempty" cbor:"resourceName,omitempty"`
+		ProfileName  string `json:"profileName,omitempty" cbor:"profileName,omitempty"`
+		ValueType    string `json:"valueType" cbor:"valueType"`
+		Units        string `json:"units,omitempty" cbor:"units,omitempty"`
+		Tags         Tags   `json:"tags,omitempty" cbor:"tags,omitempty"`
 	}
 	if b.isNull {
 		return marshal(&struct {