Merge pull request #46 from cipherstash/CIP-916/allow-jsonb-target-columns

Allow jsonb columns as encrypted targets

Author: tobyhede

sql/021-config-functions.sql

@@ -226,20 +226,39 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
+
+--
+--
+-- Marks the currently `pending` configuration as `encrypting`.
+--
+-- Validates the database schema and raises an exception if the configured columns are not of `jsonb` or `cs_encrypted_v1` type.
+--
+-- Accepts an optional `force` parameter.
+-- If `force` is `true`, the schema validation is skipped.
+--
+-- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt.
+--
 DROP FUNCTION IF EXISTS cs_encrypt_v1();
 
-CREATE FUNCTION cs_encrypt_v1()
+CREATE FUNCTION cs_encrypt_v1(force boolean DEFAULT false)
   RETURNS boolean
 AS $$
 	BEGIN
-    IF NOT cs_ready_for_encryption_v1() THEN
-      RAISE EXCEPTION 'Some pending columns do not have an encrypted target';
+
+    IF EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting') THEN
+      RAISE EXCEPTION 'An encryption is already in progress';
     END IF;
 
 		IF NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending') THEN
 			RAISE EXCEPTION 'No pending configuration exists to encrypt';
 		END IF;
 
+    IF NOT force THEN
+      IF NOT cs_ready_for_encryption_v1() THEN
+        RAISE EXCEPTION 'Some pending columns do not have an encrypted target';
+      END IF;
+    END IF;
+
     UPDATE cs_configuration_v1 SET state = 'encrypting' WHERE state = 'pending';
 		RETURN true;
   END;

sql/030-encryptindex.sql

@@ -87,7 +87,7 @@ AS $$
   LEFT JOIN information_schema.columns s ON
     s.table_name = c.table_name AND
     (s.column_name = c.column_name OR s.column_name = c.column_name || '_encrypted') AND
-    s.domain_name = 'cs_encrypted_v1';
+    (s.domain_name = 'cs_encrypted_v1' OR s.data_type = 'jsonb');
 $$ LANGUAGE sql;
 
 

sql/README.md

@@ -0,0 +1,13 @@
+### Adding SQL
+
+
+- Never drop the configuration table as it may contain customer data and needs to live across EQL versions
+- Everything else should have a `DROP IF EXISTS`
+- Functions should be `DROP` and `CREATE`, instead of `CREATE OR REPLACE`
+ - Data types cannot be changed once created, so dropping first is more flexible
+- Keep `DROP` and `CREATE` together in the code
+- Types need to be dropped last, add to the `666-drop_types.sql`
+
+
+
+

tests/encryptindex.sql

@@ -114,7 +114,7 @@ $$ LANGUAGE plpgsql;
 
 
 -- -----------------------------------------------
--- Start encryptindexing wwith no target table
+-- Start encryptindexing with no target table
 --
 -- The schema should be validated first.
 -- Users table does not exist, so should fail.
@@ -129,7 +129,7 @@ DO $$
 
     BEGIN
       PERFORM cs_encrypt_v1();
-      RAISE NOTICE 'Missinbg users table. Encrypt should have failed.';
+      RAISE NOTICE 'Missing users table. Encrypt should have failed.';
       ASSERT false; -- skipped by exception
     EXCEPTION
       WHEN OTHERS THEN
@@ -143,6 +143,29 @@ DO $$
 $$ LANGUAGE plpgsql;
 
 
+-- -----------------------------------------------
+-- FORCE start encryptindexing with no target table
+--
+-- Schema validation is skipped
+-- -----------------------------------------------
+DROP TABLE IF EXISTS users;
+TRUNCATE TABLE cs_configuration_v1;
+
+
+DO $$
+  BEGIN
+    PERFORM cs_add_index_v1('users', 'name', 'match');
+
+    PERFORM cs_encrypt_v1(true);
+    RAISE NOTICE 'Missing users table. Encrypt should have failed.';
+
+    -- configuration state should be changed
+    ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending'));
+    ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting'));
+
+  END;
+$$ LANGUAGE plpgsql;
+
 
 -- -----------------------------------------------
 -- With existing active config
@@ -171,7 +194,7 @@ INSERT INTO cs_configuration_v1 (state, data) VALUES (
   }'::jsonb
 );
 
--- Create a table with multiple plaintext columns
+-- Create a table with plaintext and encrypted columns
 DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
@@ -195,6 +218,56 @@ DO $$
 $$ LANGUAGE plpgsql;
 
 
+-- -----------------------------------------------
+-- With existing active config and an updated schema using a raw JSONB column
+-- Start encryptindexing
+-- The active config is unchanged
+-- The pending config should now be encrypting
+-- -----------------------------------------------
+TRUNCATE TABLE cs_configuration_v1;
+
+-- create an active configuration
+INSERT INTO cs_configuration_v1 (state, data) VALUES (
+  'active',
+  '{
+    "v": 1,
+    "tables": {
+      "users": {
+        "name": {
+          "cast_as": "text",
+          "indexes": {
+            "unique": {}
+          }
+        }
+      }
+    }
+  }'::jsonb
+);
+
+-- Create a table with plaintext and jsonb column
+DROP TABLE IF EXISTS users;
+CREATE TABLE users
+(
+    id bigint GENERATED ALWAYS AS IDENTITY,
+    name TEXT,
+    name_encrypted jsonb,
+    PRIMARY KEY(id)
+);
+
+
+-- An encrypting config should exist
+DO $$
+  BEGIN
+    PERFORM cs_add_index_v1('users', 'name', 'match');
+    PERFORM cs_encrypt_v1();
+
+    ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'active'));
+    ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting'));
+    ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending'));
+  END;
+$$ LANGUAGE plpgsql;
+
+
 -- -----------------------------------------------
 -- With existing active config
 -- Activate encrypting config