Merge pull request #119 from cipherstash/remove-zero-downtime-migration-workflow

EQL: remove zero-downtime migration workflow

Author: tobyhede

README.md

@@ -98,6 +98,9 @@ SELECT eql_v2.add_column('users', 'encrypted_email');
 
 **Note:** This function allows you to encrypt and decrypt data but does not enable searchable encryption. See [Searching data with EQL](#searching-data-with-eql) for enabling searchable encryption.
 
+<!--
+NOTE: NO LONGER REQUIRED
+      DOCUMENTATION CAN BE UPDATED WHEN/IF ZERO DOWNTIME SUPPORT IS ADDED TO PROXY
 ### Activating configuration
 
 After modifying configurations, activate them by running:
@@ -106,6 +109,7 @@ After modifying configurations, activate them by running:
 SELECT eql_v2.migrate_config();
 SELECT eql_v2.activate_config();
 ```
+-->
 
 **Important:** These functions must be run after any modifications to the configuration.
 

src/config/config_test.sql

@@ -25,11 +25,11 @@ DO $$
   BEGIN
 
     -- Add indexes
-    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
     ASSERT (SELECT _search_config_exists('users', 'name', 'match'));
 
     -- Add index with cast
-    PERFORM eql_v2.add_search_config('users', 'name', 'unique', 'int');
+    PERFORM eql_v2.add_search_config('users', 'name', 'unique', 'int', migrating => true);
     ASSERT (SELECT _search_config_exists('users', 'name', 'unique'));
 
     ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
@@ -60,15 +60,15 @@ DO $$
   BEGIN
 
     -- Add indexes
-    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
     ASSERT (SELECT _search_config_exists('users', 'name', 'match'));
 
     ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
             WHERE c.state = 'pending' AND
             c.data #> array['tables', 'users', 'name', 'indexes'] ? 'match'));
 
     -- Add index with cast
-    PERFORM eql_v2.add_search_config('blah', 'vtha', 'unique', 'int');
+    PERFORM eql_v2.add_search_config('blah', 'vtha', 'unique', 'int', migrating => true);
     ASSERT (SELECT _search_config_exists('blah', 'vtha', 'unique'));
 
     ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
@@ -107,11 +107,11 @@ $$ LANGUAGE plpgsql;
 
 DO $$
   BEGIN
-    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
     ASSERT (SELECT _search_config_exists('users', 'name', 'match'));
 
     -- Pending configuration contains the path `user/name.match.option`
-    PERFORM eql_v2.modify_search_config('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb);
+    PERFORM eql_v2.modify_search_config('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb, migrating => true);
     ASSERT (SELECT _search_config_exists('users', 'name', 'match'));
 
     ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
@@ -162,7 +162,7 @@ DO $$
   BEGIN
     ASSERT (SELECT _search_config_exists('users', 'blah', 'match', 'active'));
 
-    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
 
     -- index added to name
     ASSERT (SELECT _search_config_exists('users', 'name', 'match' ));
@@ -205,15 +205,15 @@ DO $$
     -- reset the table
     PERFORM create_table_with_encrypted();
 
-    PERFORM eql_v2.add_column('encrypted', 'e');
+    PERFORM eql_v2.add_column('encrypted', 'e', migrating => true);
 
     PERFORM assert_count(
         'Pending configuration was created',
         'SELECT * FROM eql_v2_configuration c WHERE c.state = ''pending''',
         1);
 
 
-    PERFORM eql_v2.remove_column('encrypted', 'e');
+    PERFORM eql_v2.remove_column('encrypted', 'e', migrating => true);
 
     PERFORM assert_no_result(
         'Pending configuration was removed',

src/config/functions.sql

@@ -1,6 +1,8 @@
 -- REQUIRE: src/config/types.sql
 -- REQUIRE: src/config/functions_private.sql
---
+-- REQUIRE: src/encrypted/functions.sql
+
+
 -- Customer-facing configuration functions
 -- Depends on private functions for implemenation
 --
@@ -10,7 +12,7 @@
 -- Adds an index term to the configuration
 --
 
-CREATE FUNCTION eql_v2.add_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
+CREATE FUNCTION eql_v2.add_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}', migrating boolean DEFAULT false)
   RETURNS jsonb
 
 AS $$
@@ -54,6 +56,13 @@ AS $$
     DO UPDATE
       SET data = _config;
 
+    IF NOT migrating THEN
+      PERFORM eql_v2.migrate_config();
+      PERFORM eql_v2.activate_config();
+    END IF;
+
+    -- PERFORM eql_v2.add_encrypted_constraint(table_name, column_name);
+
     -- exeunt
     RETURN _config;
   END;
@@ -121,12 +130,12 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.modify_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
+CREATE FUNCTION eql_v2.modify_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}', migrating boolean DEFAULT false)
   RETURNS jsonb
 AS $$
   BEGIN
     PERFORM eql_v2.remove_search_config(table_name, column_name, index_name);
-    RETURN eql_v2.add_search_config(table_name, column_name, index_name, cast_as, opts);
+    RETURN eql_v2.add_search_config(table_name, column_name, index_name, cast_as, opts, migrating);
   END;
 $$ LANGUAGE plpgsql;
 
@@ -200,7 +209,7 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text')
+CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text', migrating boolean DEFAULT false)
   RETURNS jsonb
 AS $$
   DECLARE
@@ -231,6 +240,11 @@ AS $$
     DO UPDATE
       SET data = _config;
 
+    IF NOT migrating THEN
+      PERFORM eql_v2.migrate_config();
+      PERFORM eql_v2.activate_config();
+    END IF;
+
     PERFORM eql_v2.add_encrypted_constraint(table_name, column_name);
 
     -- exeunt
@@ -240,7 +254,7 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.remove_column(table_name text, column_name text)
+CREATE FUNCTION eql_v2.remove_column(table_name text, column_name text, migrating boolean DEFAULT false)
   RETURNS jsonb
 AS $$
   DECLARE
@@ -289,6 +303,11 @@ AS $$
 
     PERFORM eql_v2.remove_encrypted_constraint(table_name, column_name);
 
+    IF NOT migrating THEN
+      PERFORM eql_v2.migrate_config();
+      PERFORM eql_v2.activate_config();
+    END IF;
+
     -- exeunt
     RETURN _config;
 

src/encryptindex/functions_test.sql

@@ -154,16 +154,38 @@ CREATE TABLE users
 -- An encrypting config should exist
 DO $$
   BEGIN
-    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
     PERFORM eql_v2.migrate_config();
-
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
     ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
   END;
 $$ LANGUAGE plpgsql;
 
 
+-- Encrypting config without `migrating = true` is immediately active
+DO $$
+  BEGIN
+    TRUNCATE TABLE eql_v2_configuration;
+    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
+  END;
+$$ LANGUAGE plpgsql;
+
+
+-- migrate_config() should raise an exception when no pending configuration exists
+DO $$
+  BEGIN
+    TRUNCATE TABLE eql_v2_configuration;
+    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+
+    PERFORM assert_exception(
+        'eql_v2.migrate_config() should raise an exception when no pending configuration exists',
+        'SELECT eql_v2.migrate_config()'
+    );
+  END;
+$$ LANGUAGE plpgsql;
+
 -- -----------------------------------------------
 -- With existing active config and an updated schema using a raw JSONB column
 -- Start encryptindexing
@@ -204,7 +226,7 @@ CREATE TABLE users
 -- An encrypting config should exist
 DO $$
   BEGIN
-    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
     PERFORM eql_v2.migrate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
@@ -254,7 +276,7 @@ CREATE TABLE users
 -- An encrypting config should exist
 DO $$
   BEGIN
-    PERFORM eql_v2.add_search_config('users', 'name', 'match');
+    PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
 
     PERFORM eql_v2.migrate_config(); -- need to encrypt first
     PERFORM eql_v2.activate_config();