api: option to skip nullability check on flatten

Add `skip_nullability_check_on_flatten` option for `insert_object`,
`insert_object_many`, `replace_object`, `replace_object_many`. `false`
by default. By setting the option to `true` one allows setting null
values to non-nullable fields, which can be useful if non-nullable field
value is generated by sequence [1].

**Warning**: there is no native support of sequences in sharded systems
since each replicaset has its own sequence. If sequence field is a part
of the sharding key (which is true by default), choosing the bucket id is
the sole responsibility of the developer (#328).

The option wasn't added to `upsert_object*` methods since upsert
operation doesn't support sequence generated fields:

```
box.schema.space.create('seq', {
    format = {
        {name = 'id', type = 'unsigned'},
        {name = 'payload', type = 'string'},
    },
})

box.schema.sequence.create('id', {if_not_exists = true})

box.space.seq:create_index('id',
    {
        parts = {{field = 'id'}},
        unique = true,
        sequence = 'id'
    }
)

box.space.seq:upsert({nil, 'payload'}, {{'=', 'payload', 'payload'}})
```

```
---
- error: 'Tuple field 1 (id) type does not match one required by operation: expected
    unsigned, got nil'
...
```

Refer to #328 discussion regarding solution design.

1. https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_schema_sequence/create_index/

Closes #328

Author: DifferentialOrange

CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added
+* `skip_nullability_check_on_flatten` option for `insert_object`,
+  `insert_object_many`, `replace_object`, `replace_object_many`.
+  `false` by default. By setting the option to `true` you allow
+  setting null values to non-nullable fields, which can be useful
+  if non-nullable field value is generated by
+  [sequence](https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_schema_sequence/create_index/).
+  **Warning**: there is no native support for sequences in sharded systems
+  since each replicaset has its own sequence. If sequence field is a part
+  of the sharding key (which is true by default), choosing the bucket id is
+  the sole responsibility of the developer (#328).
+
 ### Changed
 * Rework `NonInitialized` error message to be more helpful for
   troubleshooting (#326).

README.md

@@ -210,6 +210,15 @@ where:
   * `vshard_router` (`?string|table`) - Cartridge vshard group name or
     vshard router instance. Set this parameter if your space is not
     a part of the default vshard cluster
+  * `skip_nullability_check_on_flatten` (`?boolean`) - option for
+    `insert_object` only. `false` by default. Set this parameter to
+    `true` if you want to allow setting null values to non-nullable
+    fields, which can be useful if non-nullable field value is generated by
+    [sequence](https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_schema_sequence/create_index/).
+    **Warning**: there is no native support for sequences in sharded systems
+    since each replicaset has its own sequence. If sequence field is a part
+    of the sharding key (which is true by default), choosing the bucket id is
+    the sole responsibility of the developer
 
 Returns metadata and array contains one inserted row, error.
 
@@ -265,6 +274,15 @@ where:
   * `vshard_router` (`?string|table`) - Cartridge vshard group name or
     vshard router instance. Set this parameter if your space is not
     a part of the default vshard cluster
+  * `skip_nullability_check_on_flatten` (`?boolean`) - option for
+    `insert_object_many` only. `false` by default. Set this parameter to
+    `true` if you want to allow setting null values to non-nullable
+    fields, which can be useful if non-nullable field value is generated by
+    [sequence](https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_schema_sequence/create_index/).
+    **Warning**: there is no native support for sequences in sharded systems
+    since each replicaset has its own sequence. If sequence field is a part
+    of the sharding key (which is true by default), choosing the bucket id is
+    the sole responsibility of the developer
 
 Returns metadata and array with inserted rows, array of errors.
 Each error object can contain field `operation_data`.
@@ -510,6 +528,15 @@ where:
   * `vshard_router` (`?string|table`) - Cartridge vshard group name or
     vshard router instance. Set this parameter if your space is not
     a part of the default vshard cluster
+  * `skip_nullability_check_on_flatten` (`?boolean`) - option for
+    `replace_object` only. `false` by default. Set this parameter to
+    `true` if you want to allow setting null values to non-nullable
+    fields, which can be useful if non-nullable field value is generated by
+    [sequence](https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_schema_sequence/create_index/).
+    **Warning**: there is no native support for sequences in sharded systems
+    since each replicaset has its own sequence. If sequence field is a part
+    of the sharding key (which is true by default), choosing the bucket id is
+    the sole responsibility of the developer
 
 Returns inserted or replaced rows and metadata or nil with error.
 
@@ -565,6 +592,15 @@ where:
   * `vshard_router` (`?string|table`) - Cartridge vshard group name or
     vshard router instance. Set this parameter if your space is not
     a part of the default vshard cluster
+  * `skip_nullability_check_on_flatten` (`?boolean`) - option for
+    `replace_object_many` only. `false` by default. Set this parameter to
+    `true` if you want to allow setting null values to non-nullable
+    fields, which can be useful if non-nullable field value is generated by
+    [sequence](https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_schema_sequence/create_index/).
+    **Warning**: there is no native support for sequences in sharded systems
+    since each replicaset has its own sequence. If sequence field is a part
+    of the sharding key (which is true by default), choosing the bucket id is
+    the sole responsibility of the developer
 
 Returns metadata and array with inserted/replaced rows, array of errors.
 Each error object can contain field `operation_data`.

crud/common/utils.lua

@@ -119,18 +119,18 @@ end
 
 local flatten_functions_cache = setmetatable({}, {__mode = 'k'})
 
-function utils.flatten(object, space_format, bucket_id)
+function utils.flatten(object, space_format, bucket_id, skip_nullability_check)
     local flatten_func = flatten_functions_cache[space_format]
     if flatten_func ~= nil then
-        local data, err = flatten_func(object, bucket_id)
+        local data, err = flatten_func(object, bucket_id, skip_nullability_check)
         if err ~= nil then
             return nil, FlattenError:new(err)
         end
         return data
     end
 
     local lines = {}
-    append(lines, 'local object, bucket_id = ...')
+    append(lines, 'local object, bucket_id, skip_nullability_check = ...')
 
     append(lines, 'for k in pairs(object) do')
     append(lines, '    if fieldmap[k] == nil then')
@@ -149,8 +149,10 @@ function utils.flatten(object, space_format, bucket_id)
             append(lines, 'if object[%q] ~= nil then', field.name)
             append(lines, '    result[%d] = object[%q]', i, field.name)
             if field.is_nullable ~= true then
-                append(lines, 'else')
-                append(lines, '    return nil, \'Field %q isn\\\'t nullable\'', field.name)
+                append(lines, 'elseif skip_nullability_check ~= true then')
+                append(lines, '    return nil, \'Field %q isn\\\'t nullable' ..
+                              ' (set skip_nullability_check_on_flatten option to true to skip check)\'',
+                              field.name)
             end
             append(lines, 'end')
         else
@@ -173,7 +175,7 @@ function utils.flatten(object, space_format, bucket_id)
     flatten_func = assert(load(code, '@flatten', 't', env))
 
     flatten_functions_cache[space_format] = flatten_func
-    local data, err = flatten_func(object, bucket_id)
+    local data, err = flatten_func(object, bucket_id, skip_nullability_check)
     if err ~= nil then
         return nil, FlattenError:new(err)
     end
@@ -636,22 +638,22 @@ function utils.cut_rows(rows, metadata, field_names)
     }
 end
 
-local function flatten_obj(vshard_router, space_name, obj)
+local function flatten_obj(vshard_router, space_name, obj, skip_nullability_check)
     local space_format, err = utils.get_space_format(space_name, vshard_router:routeall())
     if err ~= nil then
         return nil, FlattenError:new("Failed to get space format: %s", err), const.NEED_SCHEMA_RELOAD
     end
 
-    local tuple, err = utils.flatten(obj, space_format)
+    local tuple, err = utils.flatten(obj, space_format, nil, skip_nullability_check)
     if err ~= nil then
         return nil, FlattenError:new("Object is specified in bad format: %s", err), const.NEED_SCHEMA_RELOAD
     end
 
     return tuple
 end
 
-function utils.flatten_obj_reload(vshard_router, space_name, obj)
-    return schema.wrap_func_reload(vshard_router, flatten_obj, space_name, obj)
+function utils.flatten_obj_reload(vshard_router, space_name, obj, skip_nullability_check)
+    return schema.wrap_func_reload(vshard_router, flatten_obj, space_name, obj, skip_nullability_check)
 end
 
 -- Merge two options map.

crud/insert.lua

@@ -190,6 +190,7 @@ function insert.object(space_name, obj, opts)
         add_space_schema_hash = '?boolean',
         fields = '?table',
         vshard_router = '?string|table',
+        skip_nullability_check_on_flatten = '?boolean',
     })
 
     opts = opts or {}
@@ -202,7 +203,8 @@ function insert.object(space_name, obj, opts)
     -- insert can fail if router uses outdated schema to flatten object
     opts = utils.merge_options(opts, {add_space_schema_hash = true})
 
-    local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj)
+    local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj,
+                                                opts.skip_nullability_check_on_flatten)
     if err ~= nil then
         return nil, InsertError:new("Failed to flatten object: %s", err)
     end

crud/insert_many.lua

@@ -251,6 +251,7 @@ function insert_many.objects(space_name, objs, opts)
         stop_on_error = '?boolean',
         rollback_on_error = '?boolean',
         vshard_router = '?string|table',
+        skip_nullability_check_on_flatten = '?boolean',
     })
 
     opts = opts or {}
@@ -268,7 +269,8 @@ function insert_many.objects(space_name, objs, opts)
 
     for _, obj in ipairs(objs) do
 
-        local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj)
+        local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj,
+                                                    opts.skip_nullability_check_on_flatten)
         if err ~= nil then
             local err_obj = InsertManyError:new("Failed to flatten object: %s", err)
             err_obj.operation_data = obj

crud/replace.lua

@@ -193,6 +193,7 @@ function replace.object(space_name, obj, opts)
         add_space_schema_hash = '?boolean',
         fields = '?table',
         vshard_router = '?string|table',
+        skip_nullability_check_on_flatten = '?boolean',
     })
 
     opts = opts or {}
@@ -205,7 +206,8 @@ function replace.object(space_name, obj, opts)
     -- replace can fail if router uses outdated schema to flatten object
     opts = utils.merge_options(opts, {add_space_schema_hash = true})
 
-    local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj)
+    local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj,
+                                                opts.skip_nullability_check_on_flatten)
     if err ~= nil then
         return nil, ReplaceError:new("Failed to flatten object: %s", err)
     end

crud/replace_many.lua

@@ -253,6 +253,7 @@ function replace_many.objects(space_name, objs, opts)
         stop_on_error = '?boolean',
         rollback_on_error = '?boolean',
         vshard_router = '?string|table',
+        skip_nullability_check_on_flatten = '?boolean',
     })
 
     opts = opts or {}
@@ -270,7 +271,8 @@ function replace_many.objects(space_name, objs, opts)
 
     for _, obj in ipairs(objs) do
 
-        local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj)
+        local tuple, err = utils.flatten_obj_reload(vshard_router, space_name, obj,
+                                                    opts.skip_nullability_check_on_flatten)
         if err ~= nil then
             local err_obj = ReplaceManyError:new("Failed to flatten object: %s", err)
             err_obj.operation_data = obj

test/entrypoint/srv_simple_operations.lua

@@ -88,6 +88,30 @@ package.preload['customers-storage'] = function()
                 unique = false,
                 if_not_exists = true,
             })
+
+            local sequence_space = box.schema.space.create('notebook', {
+                format = {
+                    {name = 'local_id', type = 'unsigned', is_nullable = false},
+                    {name = 'bucket_id', type = 'unsigned', is_nullable = false},
+                    {name = 'record', type = 'string', is_nullable = false},
+                },
+                if_not_exists = true,
+                engine = engine,
+            })
+
+            box.schema.sequence.create('local_id', {if_not_exists = true})
+
+            sequence_space:create_index('local_id', {
+                parts = { {field = 'local_id'} },
+                unique = true,
+                if_not_exists = true,
+                sequence = 'local_id',
+            })
+            sequence_space:create_index('bucket_id', {
+                parts = { {field = 'bucket_id'} },
+                unique = false,
+                if_not_exists = true,
+            })
         end,
     }
 end

test/helper.lua

@@ -141,6 +141,19 @@ function helpers.truncate_space_on_cluster(cluster, space_name)
     end
 end
 
+function helpers.reset_sequence_on_cluster(cluster, sequence_name)
+    assert(cluster ~= nil)
+    for _, server in ipairs(cluster.servers) do
+        server.net_box:eval([[
+            local sequence_name = ...
+            local sequence = box.sequence[sequence_name]
+            if sequence ~= nil and not box.cfg.read_only then
+                sequence:reset()
+            end
+        ]], {sequence_name})
+    end
+end
+
 function helpers.get_test_replicasets()
     return {
         {