Add support to new atom chunk format introduced with OTP-28

Atom length prefix now is a varlen int, so up to 255 codepoints can be
supported.

AtomVM will not support this. When going embedded some features should
be left out, and this is one of them.

See also:
- erlang/otp#8913
- erlang/otp#9336

Signed-off-by: Davide Bettio <[email protected]>

Author: bettio

src/libAtomVM/atom_table.c

@@ -445,26 +445,71 @@ long atom_table_ensure_atom(struct AtomTable *table, AtomString string, enum Ato
     return new_index;
 }
 
-int atom_table_ensure_atoms(
-    struct AtomTable *table, const void *atoms, int count, int *translate_table)
+static inline int read_encoded_len(const uint8_t **len_bytes)
 {
+    uint8_t byte0 = (*len_bytes)[0];
+
+    if ((byte0 & 0x8) == 0) {
+        (*len_bytes)++;
+        return byte0 >> 4;
+
+    } else if ((byte0 & 0x10) == 0) {
+        uint8_t byte1 = (*len_bytes)[1];
+        (*len_bytes) += 2;
+        return ((byte0 >> 5) << 8) | byte1;
+
+    } else {
+        return -1;
+    }
+}
+
+int atom_table_ensure_atoms(struct AtomTable *table, const void *atoms, int count,
+    int *translate_table, enum EnsureAtomsOpt opt)
+{
+    bool is_long_format = (opt & EnsureLongEncoding) != 0;
+
     SMP_WRLOCK(table);
 
     int new_atoms_count = 0;
 
     const uint8_t *current_atom = atoms;
 
     for (int i = 0; i < count; i++) {
-        struct HNode *node = get_node(table, current_atom);
+        struct HNode *node;
+        if (is_long_format) {
+            int atom_len = read_encoded_len(&current_atom);
+            if (UNLIKELY(atom_len < 0)) {
+                fprintf(stderr, "Found invalid atom len.");
+                SMP_UNLOCK(table);
+                return ATOM_TABLE_INVALID_LEN;
+            } else if (UNLIKELY(atom_len > 255)) {
+                fprintf(stderr,
+                    "Unsupported atom length %i bytes.\n"
+                    "Unlike OTP >= 28, AtomVM supports a maximum of 255 bytes"
+                    "regardeless the number of codepoints.\n"
+                    "If you are seeing this error please open an issue on GitHub:\n"
+                    "https://github.com/atomvm/AtomVM/issues\n",
+                    atom_len);
+                SMP_UNLOCK(table);
+                return ATOM_TABLE_INVALID_LEN;
+            }
+            char tmp_old_fmt[256];
+            tmp_old_fmt[0] = atom_len;
+            memcpy(tmp_old_fmt + 1, current_atom, atom_len);
+            node = get_node(table, tmp_old_fmt);
+            current_atom += atom_len;
+        } else {
+            node = get_node(table, current_atom);
+            uint8_t atom_len = current_atom[0];
+            current_atom += 1 + atom_len;
+        }
+
         if (node) {
             translate_table[i] = node->index;
         } else {
             new_atoms_count++;
             translate_table[i] = ATOM_TABLE_NOT_FOUND;
         }
-
-        uint8_t atom_len = current_atom[0];
-        current_atom += 1 + atom_len;
     }
 
     maybe_rehash(table, new_atoms_count);
@@ -473,6 +518,19 @@ int atom_table_ensure_atoms(
     int remaining_atoms = new_atoms_count;
     struct HNodeGroup *node_group = table->last_node_group;
     for (int i = 0; i < count; i++) {
+
+        const uint8_t *to_be_copied = NULL;
+        const uint8_t *next_atom = current_atom;
+        uint8_t atom_len;
+        if (is_long_format) {
+            atom_len = read_encoded_len(&next_atom);
+            to_be_copied = next_atom;
+            next_atom += atom_len;
+        } else {
+            atom_len = current_atom[0];
+            next_atom += 1 + atom_len;
+        }
+
         if (translate_table[i] == ATOM_TABLE_NOT_FOUND) {
             if (!table->last_node_group_avail) {
                 node_group = new_node_group(table, remaining_atoms);
@@ -482,7 +540,19 @@ int atom_table_ensure_atoms(
                 }
             }
 
-            unsigned long hash = sdbm_hash(current_atom, atom_string_len(current_atom));
+            if (is_long_format) {
+                uint8_t *atom_copy = malloc(atom_len + 1);
+                if (IS_NULL_PTR(atom_copy)) {
+                    // we are not going to remove atoms that have already been added up to this one
+                    SMP_UNLOCK(table);
+                    return ATOM_TABLE_ALLOC_FAIL;
+                }
+                atom_copy[0] = atom_len;
+                memcpy(atom_copy + 1, to_be_copied, atom_len);
+                current_atom = atom_copy;
+            }
+
+            unsigned long hash = sdbm_hash(current_atom, atom_len);
             unsigned long bucket_index = hash % table->capacity;
 
             translate_table[i] = insert_node(table, node_group, bucket_index, current_atom);
@@ -491,8 +561,7 @@ int atom_table_ensure_atoms(
                 break;
             }
         }
-        uint8_t atom_len = current_atom[0];
-        current_atom += 1 + atom_len;
+        current_atom = next_atom;
     }
 
     SMP_UNLOCK(table);

src/libAtomVM/atom_table.h

@@ -31,9 +31,16 @@ extern "C" {
 
 #define ATOM_TABLE_NOT_FOUND -1
 #define ATOM_TABLE_ALLOC_FAIL -2
+#define ATOM_TABLE_INVALID_LEN -3
 
 struct AtomTable;
 
+enum EnsureAtomsOpt
+{
+    EnsureAtomsNoOpts = 0,
+    EnsureLongEncoding = 1
+};
+
 enum AtomTableCopyOpt
 {
     AtomTableNoOpts = 0,
@@ -56,8 +63,8 @@ AtomString atom_table_get_atom_string(struct AtomTable *table, long index);
 
 long atom_table_get_index(struct AtomTable *table, AtomString string);
 
-int atom_table_ensure_atoms(
-    struct AtomTable *table, const void *atoms, int count, int *translate_table);
+int atom_table_ensure_atoms(struct AtomTable *table, const void *atoms, int count,
+    int *translate_table, enum EnsureAtomsOpt opts);
 
 int atom_table_cmp_using_atom_index(
     struct AtomTable *table, int t_atom_index, int other_atom_index);

src/libAtomVM/module.c

@@ -72,6 +72,13 @@ static void parse_line_table(uint16_t **line_refs, struct ModuleFilename **filen
 static enum ModuleLoadResult module_populate_atoms_table(Module *this_module, uint8_t *table_data, GlobalContext *glb)
 {
     int atoms_count = READ_32_ALIGNED(table_data + 8);
+
+    enum EnsureAtomsOpt ensure_opts = EnsureAtomsNoOpts;
+    if (atoms_count < 0) {
+        ensure_opts = EnsureLongEncoding;
+        atoms_count = -atoms_count;
+    }
+
     const char *current_atom = (const char *) table_data + 12;
 
     this_module->local_atoms_to_global_table = calloc(atoms_count + 1, sizeof(int));
@@ -81,10 +88,12 @@ static enum ModuleLoadResult module_populate_atoms_table(Module *this_module, ui
     }
 
     long ensure_result = atom_table_ensure_atoms(
-        glb->atom_table, current_atom, atoms_count, this_module->local_atoms_to_global_table + 1);
-    if (ensure_result == ATOM_TABLE_ALLOC_FAIL) {
+        glb->atom_table, current_atom, atoms_count, this_module->local_atoms_to_global_table + 1, ensure_opts);
+    if (UNLIKELY(ensure_result == ATOM_TABLE_ALLOC_FAIL)) {
         fprintf(stderr, "Cannot allocate memory while loading module (line: %i).\n", __LINE__);
         return MODULE_ERROR_FAILED_ALLOCATION;
+    } else if (UNLIKELY(ensure_result == ATOM_TABLE_INVALID_LEN)) {
+        return MODULE_ERROR_INVALID;
     }
 
     return MODULE_LOAD_OK;

src/libAtomVM/module.h

@@ -144,7 +144,8 @@ typedef struct Module Module;
 enum ModuleLoadResult
 {
     MODULE_LOAD_OK = 0,
-    MODULE_ERROR_FAILED_ALLOCATION = 1
+    MODULE_ERROR_FAILED_ALLOCATION = 1,
+    MODULE_ERROR_INVALID = 2
 };
 
 #ifdef ENABLE_ADVANCED_TRACE