Re-adding encryption (#1558)

abuabraham-ttd · Release Workflow · vishalegbert-ttd · web-flow · commit de95e07b7631 · 2025-04-01T08:48:18.000-07:00
* Re-adding encryption, flag to toggle. 

---- 
Co-authored-by: Release Workflow &lt;unifiedid-admin+release@thetradedesk.com&gt;
Co-authored-by: Vishal Egbert &lt;vishal.egbert@thetradedesk.com&gt;
diff --git a/conf/default-config.json b/conf/default-config.json
@@ -30,6 +30,9 @@
   "salts_metadata_path": "salts/metadata.json",
   "services_metadata_path": "services/metadata.json",
   "service_links_metadata_path": "service_links/metadata.json",
+  "cloud_encryption_keys_metadata_path": "cloud_encryption_keys/metadata.json",
+  "encrypted_files": false,
+  "cloud_encryption_keys_refresh_ms": 300000,
   "optout_metadata_path": null,
   "optout_inmem_cache": false,
   "enclave_platform": null,
diff --git a/conf/docker-config.json b/conf/docker-config.json
@@ -31,6 +31,8 @@
   "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json",
   "services_metadata_path": "/com.uid2.core/test/services/metadata.json",
   "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json",
+  "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json",
+  "encrypted_files": true,
   "identity_token_expires_after_seconds": 3600,
   "optout_metadata_path": null,
   "optout_inmem_cache": false,
diff --git a/conf/integ-config.json b/conf/integ-config.json
@@ -13,6 +13,7 @@
   "core_api_token": "trusted-partner-key",
   "optout_api_token": "test-operator-key",
   "optout_api_uri": "http://localhost:8081/optout/replicate",
+  "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve",
   "salts_expired_shutdown_hours": 12,
   "operator_type": "public",
   "runtime_config_store": {
diff --git a/conf/local-config.json b/conf/local-config.json
@@ -9,6 +9,7 @@
   "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json",
   "services_metadata_path": "/com.uid2.core/test/services/metadata.json",
   "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json",
+  "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
@@ -37,6 +38,7 @@
   "client_side_token_generate_log_invalid_http_origins": true,
   "salts_expired_shutdown_hours": 12,
   "operator_type": "public",
+  "encrypted_files": true,
   "runtime_config_store": {
     "type": "file",
     "config" : {
diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json
@@ -11,6 +11,8 @@
   "keysets_metadata_path": "http://core:8088/key/keyset/refresh",
   "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh",
   "salts_metadata_path": "http://core:8088/salt/refresh",
+  "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve",
+  "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json
@@ -13,6 +13,8 @@
   "salts_metadata_path": "http://core:8088/salt/refresh",
   "services_metadata_path": "http://core:8088/services/refresh",
   "service_links_metadata_path": "http://core:8088/service_links/refresh",
+  "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve",
+  "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json
@@ -13,6 +13,8 @@
   "salts_metadata_path": "http://localhost:8088/salt/refresh",
   "services_metadata_path": "http://localhost:8088/services/refresh",
   "service_links_metadata_path": "http://localhost:8088/service_links/refresh",
+  "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve",
+  "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json
@@ -13,6 +13,8 @@
   "salts_metadata_path": "http://localhost:8088/salt/refresh",
   "services_metadata_path": "http://localhost:8088/services/refresh",
   "service_links_metadata_path": "http://localhost:8088/service_links/refresh",
+  "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve",
+  "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json
@@ -14,6 +14,8 @@
   "salts_metadata_path": "http://core:8088/salt/refresh",
   "services_metadata_path": "http://core:8088/services/refresh",
   "service_links_metadata_path": "http://core:8088/service_links/refresh",
+  "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve",
+  "encrypted_files": true,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
diff --git a/pom.xml b/pom.xml
@@ -22,7 +22,7 @@
         <enclave-aws.version>2.1.0</enclave-aws.version>
         <enclave-azure.version>2.1.13</enclave-azure.version>
         <enclave-gcp.version>2.1.0</enclave-gcp.version>
-        <uid2-shared.version>8.1.25</uid2-shared.version>
+        <uid2-shared.version>9.0.8</uid2-shared.version>
         <image.version>${project.version}</image.version>
         <maven.compiler.source>21</maven.compiler.source>
         <maven.compiler.target>21</maven.compiler.target>
diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json
@@ -10,6 +10,7 @@
   "optout_metadata_path": "https://optout.integ.euid.eu/optout/refresh",
   "core_attest_url": "https://core.integ.euid.eu/attest",
   "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate",
+  "cloud_encryption_keys_metadata_path": "https://core.integ.euid.eu/cloud_encryption_keys/retrieve",
   "optout_s3_folder": "optout/",
   "allow_legacy_api": false,
   "identity_scope": "euid"
diff --git a/scripts/aws/conf/euid-prod-config.json b/scripts/aws/conf/euid-prod-config.json
@@ -10,6 +10,7 @@
   "service_links_metadata_path": "https://core.prod.euid.eu/service_links/refresh",
   "optout_metadata_path": "https://optout.prod.euid.eu/optout/refresh",
   "core_attest_url": "https://core.prod.euid.eu/attest",
+  "cloud_encryption_keys_metadata_path": "https://core.prod.euid.eu/cloud_encryption_keys/retrieve",
   "core_api_token": "your-api-token",
   "optout_s3_path_compat": false,
   "optout_api_uri": "https://optout.prod.euid.eu/optout/replicate",
diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json
@@ -10,6 +10,7 @@
   "services_metadata_path": "https://core-integ.uidapi.com/services/refresh",
   "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh",
   "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh",
+  "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve",
   "optout_s3_folder": "uid-optout-integ/",
   "identity_scope": "uid2"
 }
diff --git a/scripts/aws/conf/uid2-prod-config.json b/scripts/aws/conf/uid2-prod-config.json
@@ -10,6 +10,7 @@
   "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh",
   "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh",
   "core_attest_url": "https://core-prod.uidapi.com/attest",
+  "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve",
   "core_api_token": "your-api-token",
   "optout_s3_path_compat": false,
   "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate",
diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json
@@ -10,6 +10,7 @@
   "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
   "core_attest_url": "https://core.uidapi.com/attest",
   "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
+  "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve",
   "optout_s3_folder": "uid-optout-integ/",
   "runtime_config_store": {
     "type": "http",
diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json
@@ -9,6 +9,7 @@
   "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
   "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
   "core_attest_url": "https://core.uidapi.com/attest",
+  "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve",
   "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
   "optout_s3_folder": "optout-v2/",
   "identity_token_expires_after_seconds": 259200,
diff --git a/scripts/gcp-oidc/conf/integ-config.json b/scripts/gcp-oidc/conf/integ-config.json
@@ -9,6 +9,7 @@
     "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
     "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
     "core_attest_url": "https://core.uidapi.com/attest",
+    "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve",
     "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
     "optout_s3_folder": "uid-optout-integ/",
     "runtime_config_store": {
diff --git a/scripts/gcp-oidc/conf/prod-config.json b/scripts/gcp-oidc/conf/prod-config.json
@@ -9,6 +9,7 @@
   "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
   "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
   "core_attest_url": "https://core.uidapi.com/attest",
+  "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve",
   "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
   "optout_s3_folder": "optout-v2/",
   "identity_token_expires_after_seconds": 259200,
diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java
@@ -20,6 +20,7 @@ public class Config extends com.uid2.shared.Const.Config {
         public static final String ValidateServiceLinks = "validate_service_links";
         public static final String OperatorTypeProp = "operator_type";
         public static final String EnclavePlatformProp = "enclave_platform";
+        public static final String EncryptedFiles = "encrypted_files";
 
         public static final String AzureVaultNameProp = "azure_vault_name";
         public static final String AzureSecretNameProp = "azure_secret_name";
diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java
diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java
diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java
diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json
diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json
diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java
diff --git a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java
