Create Protobuf.Presence module (#403)

* Create Protobuf.Presence module

Unifies presence tracking between encoders. Provides a public API for checking field presence

* Add test for messages, fix proto3

* Improve docs

Author: v0idpwn

lib/protobuf/encoder.ex

@@ -77,35 +77,22 @@ defmodule Protobuf.Encoder do
           "Got error when encoding #{inspect(struct_mod)}##{prop.name_atom}: #{Exception.format(:error, error)}"
   end
 
-  defp skip_field?(_syntax, [], _prop), do: true
-  defp skip_field?(_syntax, val, _prop) when is_map(val), do: map_size(val) == 0
-  defp skip_field?(:proto2, nil, %FieldProps{optional?: optional?}), do: optional?
-  defp skip_field?(:proto2, value, %FieldProps{default: value, oneof: nil}), do: true
-
-  defp skip_field?(:proto3, val, %FieldProps{proto3_optional?: true}),
-    do: is_nil(val)
-
-  defp skip_field?(:proto3, nil, _prop), do: true
-  defp skip_field?(:proto3, 0, %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, +0.0, %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, "", %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, false, %FieldProps{oneof: nil}), do: true
-
-  defp skip_field?(:proto3, value, %FieldProps{type: {:enum, enum_mod}, oneof: nil}) do
-    enum_props = enum_mod.__message_props__()
-    %{name_atom: name_atom, name: name} = enum_props.field_props[0]
-    value == name_atom or value == name
+  defp skip_field?(syntax, value, field_prop) do
+    case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do
+      :present -> false
+      # Proto2 required isn't skipped even if not present
+      :maybe -> not(syntax == :proto2 && field_prop.required?)
+      :not_present -> not(syntax == :proto2 && field_prop.required?)
+    end
   end
 
-  defp skip_field?(_syntax, _val, _prop), do: false
-
   defp do_encode_field(
          :normal,
          val,
          syntax,
          %FieldProps{encoded_fnum: fnum, type: type, repeated?: repeated?} = prop
        ) do
-    if skip_field?(syntax, val, prop) or skip_enum?(syntax, val, prop) do
+    if skip_field?(syntax, val, prop) do
       :skip
     else
       iodata = apply_or_map(val, repeated?, &[fnum | Wire.encode(type, &1)])
@@ -143,7 +130,7 @@ defmodule Protobuf.Encoder do
   end
 
   defp do_encode_field(:packed, val, syntax, %FieldProps{type: type, encoded_fnum: fnum} = prop) do
-    if skip_field?(syntax, val, prop) or skip_enum?(syntax, val, prop) do
+    if skip_field?(syntax, val, prop) do
       :skip
     else
       encoded = Enum.map(val, &Wire.encode(type, &1))
@@ -198,20 +185,6 @@ defmodule Protobuf.Encoder do
   defp apply_or_map(val, _repeated? = true, func), do: Enum.map(val, func)
   defp apply_or_map(val, _repeated? = false, func), do: func.(val)
 
-  defp skip_enum?(:proto2, _value, _prop), do: false
-  defp skip_enum?(:proto3, _value, %FieldProps{proto3_optional?: true}), do: false
-  defp skip_enum?(_syntax, _value, %FieldProps{enum?: false}), do: false
-
-  defp skip_enum?(_syntax, _value, %FieldProps{enum?: true, oneof: oneof}) when not is_nil(oneof),
-    do: false
-
-  defp skip_enum?(_syntax, _value, %FieldProps{required?: true}), do: false
-  defp skip_enum?(_syntax, value, %FieldProps{type: type}), do: enum_default?(type, value)
-
-  defp enum_default?({:enum, enum_mod}, val) when is_atom(val), do: enum_mod.value(val) == 0
-  defp enum_default?({:enum, _enum_mod}, val) when is_integer(val), do: val == 0
-  defp enum_default?({:enum, _enum_mod}, list) when is_list(list), do: false
-
   # Returns a map of %{field_name => field_value} from oneofs. For example, if you have:
   # oneof body {
   #   string a = 1;

lib/protobuf/json/encode.ex

@@ -162,7 +162,7 @@ defmodule Protobuf.JSON.Encode do
   defp encode_regular_fields(struct, %{field_props: field_props, syntax: syntax}, opts) do
     for {_field_num, %{name_atom: name, oneof: nil} = prop} <- field_props,
         %{^name => value} = struct,
-        emit?(syntax, prop, value) || opts[:emit_unpopulated] do
+        emit?(syntax, prop, value, opts[:emit_unpopulated]) do
       encode_field(prop, value, opts)
     end
   end
@@ -301,18 +301,17 @@ defmodule Protobuf.JSON.Encode do
   defp maybe_repeat(%{repeated?: false}, val, fun), do: fun.(val)
   defp maybe_repeat(%{repeated?: true}, val, fun), do: Enum.map(val, fun)
 
-  defp emit?(:proto2, %{default: value}, value), do: false
-  defp emit?(:proto2, %{optional?: true}, val), do: not is_nil(val)
-  defp emit?(:proto3, %{proto3_optional?: true}, val), do: not is_nil(val)
-  defp emit?(_syntax, _prop, +0.0), do: false
-  defp emit?(_syntax, _prop, nil), do: false
-  defp emit?(_syntax, _prop, 0), do: false
-  defp emit?(_syntax, _prop, false), do: false
-  defp emit?(_syntax, _prop, []), do: false
-  defp emit?(_syntax, _prop, ""), do: false
-  defp emit?(_syntax, _prop, %{} = map) when map_size(map) == 0, do: false
-  defp emit?(_syntax, %{type: {:enum, enum}}, key) when is_atom(key), do: enum.value(key) != 0
-  defp emit?(_syntax, _prop, _value), do: true
+  defp emit?(_syntax, _field_prop, _value, true = _emit_unpopulated?) do
+    true
+  end
+
+  defp emit?(syntax, field_prop, value, _emit_unpopulated?) do
+    case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do
+      :present -> true
+      :maybe -> false
+      :not_present -> false
+    end
+  end
 
   defp transform_module(message, module) do
     if transform_module = module.transform_module() do

lib/protobuf/presence.ex

@@ -0,0 +1,179 @@
+defmodule Protobuf.Presence do
+  @moduledoc """
+  Helpers for determining Protobuf field presence.
+  """
+
+  alias Protobuf.FieldProps
+
+  @doc """
+  Returns whether a field or oneof is present, not present, or maybe present
+
+  `:present` and `:not present` mean that a field is **explicitly** present or not,
+  respectively.
+
+  Some values may be implicitly present. For example, lists in `repeated` fields
+  always have implicit presence. In these cases, if the presence is ambiguous,
+  returns `:maybe`.
+
+  For more information about field presence tracking rules, refer to the official
+  [Field Presence docs](https://protobuf.dev/programming-guides/field_presence/).
+
+
+  ## Examples
+
+      # Non-optional proto3 field:
+      Protobuf.Presence(%MyMessage{foo: 42}, :foo)
+      #=> :present
+
+      Protobuf.Presence(%MyMessage{foo: 0}, :foo)
+      #=> :maybe
+
+      Protobuf.Presence(%MyMessage{}, :foo)
+      #=> :maybe
+
+      # Optional proto3 field:
+      Protobuf.Presence(%MyMessage{bar: 42}, :bar)
+      #=> :present
+
+      Protobuf.Presence(%MyMessage{bar: 0}, :bar)
+      #=> :present
+
+      Protobuf.Presence(%MyMessage{}, :bar)
+      #=> :not_present
+
+  """
+  @spec field_presence(message :: struct(), field :: atom()) :: :present | :not_present | :maybe
+  def field_presence(%mod{} = message, field) do
+    message_props = mod.__message_props__()
+    transformed_message = transform_module(message, mod)
+    fnum = Map.fetch!(message_props.field_tags, field)
+    field_prop = Map.fetch!(message_props.field_props, fnum)
+    value = get_oneof_value(transformed_message, message_props, field, field_prop)
+
+    transformed_value =
+      case field_prop do
+        %{embedded: true, type: mod} -> transform_module(value, mod)
+        _ -> value
+      end
+
+    get_field_presence(message_props.syntax, transformed_value, field_prop)
+  end
+
+  defp get_oneof_value(message, message_props, field, field_prop) do
+    case field_prop.oneof do
+      nil ->
+        Map.fetch!(message, field)
+
+      oneof_num ->
+        {oneof_field, _} = Enum.find(message_props.oneof, fn {_name, tag} -> tag == oneof_num end)
+
+        case Map.fetch!(message, oneof_field) do
+          {^field, value} -> value
+          _ -> nil
+        end
+    end
+  end
+
+  defp transform_module(message, module) do
+    if transform_module = module.transform_module() do
+      transform_module.encode(message, module)
+    else
+      message
+    end
+  end
+
+  # We probably want to make this public eventually, but it makes sense to hold
+  # it until we add editions support, since we definitely don't want to add
+  # `syntax` in a public API
+  @doc false
+  @spec get_field_presence(:proto2 | :proto3, term(), FieldProps.t()) :: :present | :not_present | :maybe
+  def get_field_presence(syntax, value, field_prop)
+
+  # Repeated and maps are always implicit.
+  def get_field_presence(_syntax, [], _prop) do
+    :maybe
+  end
+
+  def get_field_presence(_syntax, val, _prop) when is_map(val) do
+    if map_size(val) == 0 do
+      :maybe
+    else
+      :present
+    end
+  end
+
+  # For proto2 singular cardinality fields:
+  #
+  # - Non-one_of fields with default values have implicit presence
+  # - Others have explicit presence
+  def get_field_presence(:proto2, nil, _prop) do
+    :not_present
+  end
+
+  def get_field_presence(:proto2, value, %FieldProps{default: value, oneof: nil}) do
+    :maybe
+  end
+
+  def get_field_presence(:proto2, _value, _props) do
+    :present
+  end
+
+  # For proto3 singular cardinality fields:
+  #
+  # - Optional and Oneof fields have explicit presence tracking
+  # - Other fields have implicit presence tracking
+  def get_field_presence(:proto3, nil, %FieldProps{proto3_optional?: true}) do
+    :not_present
+  end
+
+  def get_field_presence(:proto3, _, %FieldProps{proto3_optional?: true}) do
+    :present
+  end
+
+  def get_field_presence(_syntax, value, %FieldProps{oneof: oneof}) when not is_nil(oneof) do
+    if is_nil(value) do
+      :not_present
+    else
+      :present
+    end
+  end
+
+  # Messages have explicit presence tracking in proto3
+  def get_field_presence(:proto3, nil, _prop) do
+    :not_present
+  end
+
+  # Defaults for different field types: implicit presence means they are maybe set
+  def get_field_presence(:proto3, 0, _prop) do
+    :maybe
+  end
+
+  def get_field_presence(:proto3, +0.0, _prop) do
+    :maybe
+  end
+
+  def get_field_presence(:proto3, "", _prop) do
+    :maybe
+  end
+
+  def get_field_presence(:proto3, false, _prop) do
+    :maybe
+  end
+
+  def get_field_presence(_syntax, value, %FieldProps{type: {:enum, enum_mod}}) do
+    if enum_default?(enum_mod, value) do
+      :maybe
+    else
+      :present
+    end
+  end
+
+  # Finally, everything else.
+  def get_field_presence(_syntax, _val, _prop) do
+    :present
+  end
+
+  defp enum_default?(enum_mod, val) when is_atom(val), do: enum_mod.value(val) == 0
+  defp enum_default?(_enum_mod, val) when is_integer(val), do: val == 0
+  defp enum_default?(_enum_mod, list) when is_list(list), do: false
+end

lib/protobuf/text.ex

@@ -176,28 +176,14 @@ defmodule Protobuf.Text do
     end
   end
 
-  # Copied from Protobuf.Encoder. Should it be shared?
-  defp skip_field?(_syntax, [], _prop), do: true
-  defp skip_field?(_syntax, val, _prop) when is_map(val), do: map_size(val) == 0
-  defp skip_field?(:proto2, nil, %FieldProps{optional?: optional?}), do: optional?
-  defp skip_field?(:proto2, value, %FieldProps{default: value, oneof: nil}), do: true
-  defp skip_field?(:proto3, val, %FieldProps{proto3_optional?: true}), do: is_nil(val)
-
-  defp skip_field?(:proto3, nil, _prop), do: true
-  defp skip_field?(:proto3, 0, %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, +0.0, %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, "", %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, false, %FieldProps{oneof: nil}), do: true
-
-  # This is actually new. Should it be ported to Protobuf.Encoder?
-  defp skip_field?(:proto3, value, %FieldProps{type: {:enum, enum_mod}, oneof: nil}) do
-    enum_props = enum_mod.__message_props__()
-    %{name_atom: name_atom, name: name, json_name: name_json} = enum_props.field_props[0]
-
-    value == name_atom or value == name or value == name_json
+  defp skip_field?(syntax, value, field_prop) do
+    case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do
+      :present -> false
+      # Proto2 required isn't skipped even if not present
+      :maybe -> not(syntax == :proto2 && field_prop.required?)
+      :not_present -> not(syntax == :proto2 && field_prop.required?)
+    end
   end
 
-  defp skip_field?(_, _, _), do: false
-
   defp inspect_opts(), do: %Inspect.Opts{limit: :infinity}
 end