fail fast keysetkey out of sync. Currently still using fixed interval

Author: lizk886

conf/default-config.json

@@ -37,6 +37,7 @@
   "optout_inmem_cache": false,
   "enclave_platform": null,
   "failure_shutdown_wait_hours": 120,
+  "keyset_key_shutdown_hours": 2,
   "sharing_token_expiry_seconds": 2592000,
   "operator_type": "public",
   "enable_remote_config": true,

src/main/java/com/uid2/operator/Main.java

@@ -114,7 +114,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception {
         this.clientSideTokenGenerate = config.getBoolean(Const.Config.EnableClientSideTokenGenerate, false);
         this.validateServiceLinks = config.getBoolean(Const.Config.ValidateServiceLinks, false);
         this.encryptedCloudFilesEnabled = config.getBoolean(Const.Config.EncryptedFiles, false);
-        this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Clock.systemUTC(), new ShutdownService());
+        this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofHours(config.getInteger(Const.Config.KeysetKeyShutdownHours, 2)), Clock.systemUTC(), new ShutdownService());
         this.uidInstanceIdProvider = new UidInstanceIdProvider(config);
 
         String coreAttestUrl = this.config.getString(Const.Config.CoreAttestUrlProp);
@@ -243,7 +243,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception {
     }
 
     private KeyManager getKeyManager() {
-        return new KeyManager(this.keysetKeyStore, this.keysetProvider);
+        return new KeyManager(this.keysetKeyStore, this.keysetProvider, 
+            hasKeys -> shutdownHandler.handleKeysetKeyRefreshResponse(hasKeys));
     }
 
     public static void recordStartupComplete() {

src/main/java/com/uid2/operator/model/KeyManager.java

@@ -13,16 +13,19 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 public class KeyManager {
     private static final Logger LOGGER = LoggerFactory.getLogger(UIDOperatorVerticle.class);
     private final IKeysetKeyStore keysetKeyStore;
     private final RotatingKeysetProvider keysetProvider;
+    private final Consumer<Boolean> keyAvailabilityHandler;
 
-    public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider) {
+    public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider, Consumer<Boolean> keyAvailabilityHandler) {
         this.keysetKeyStore = keysetKeyStore;
         this.keysetProvider = keysetProvider;
+        this.keyAvailabilityHandler = keyAvailabilityHandler;
     }
 
     public KeyManagerSnapshot getKeyManagerSnapshot(int siteId) {
@@ -107,6 +110,7 @@ public KeysetKey getMasterKey() {
     public KeysetKey getMasterKey(Instant asOf) {
         KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.MasterKeysetId, asOf);
         if (key == null) {
+            if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false);
             throw new NoActiveKeyException(String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId));
         }
         return key;
@@ -119,6 +123,7 @@ public KeysetKey getRefreshKey() {
     public KeysetKey getRefreshKey(Instant asOf) {
         KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf);
         if (key == null) {
+            if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false);
             throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId));
         }
         return key;

src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java

@@ -2,7 +2,6 @@
 
 import com.uid2.operator.service.ShutdownService;
 import com.uid2.shared.attest.AttestationResponseCode;
-import lombok.extern.java.Log;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import software.amazon.awssdk.utils.Pair;
@@ -16,17 +15,22 @@
 public class OperatorShutdownHandler {
     private static final Logger LOGGER = LoggerFactory.getLogger(OperatorShutdownHandler.class);
     private static final int SALT_FAILURE_LOG_INTERVAL_MINUTES = 10;
+    private static final int KEYSET_KEY_FAILURE_LOG_INTERVAL_MINUTES = 10;
     private final Duration attestShutdownWaitTime;
     private final Duration saltShutdownWaitTime;
+    private final Duration keysetKeyShutdownWaitTime;
     private final AtomicReference<Instant> attestFailureStartTime = new AtomicReference<>(null);
     private final AtomicReference<Instant> saltFailureStartTime = new AtomicReference<>(null);
+    private final AtomicReference<Instant> keysetKeyFailureStartTime = new AtomicReference<>(null);
     private final AtomicReference<Instant> lastSaltFailureLogTime = new AtomicReference<>(null);
+    private final AtomicReference<Instant> lastKeysetKeyFailureLogTime = new AtomicReference<>(null);
     private final Clock clock;
     private final ShutdownService shutdownService;
 
-    public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Clock clock, ShutdownService shutdownService) {
+    public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Duration keysetKeyShutdownWaitTime, Clock clock, ShutdownService shutdownService) {
         this.attestShutdownWaitTime = attestShutdownWaitTime;
         this.saltShutdownWaitTime = saltShutdownWaitTime;
+        this.keysetKeyShutdownWaitTime = keysetKeyShutdownWaitTime;
         this.clock = clock;
         this.shutdownService = shutdownService;
     }
@@ -54,6 +58,29 @@ public void logSaltFailureAtInterval() {
         }
     }
 
+    public void handleKeysetKeyRefreshResponse(Boolean success) {
+        if (success) {
+            keysetKeyFailureStartTime.set(null);
+        } else {
+            logKeysetKeyFailureAtInterval();
+            Instant t = keysetKeyFailureStartTime.get();
+            if (t == null) {
+                keysetKeyFailureStartTime.set(clock.instant());
+            } else if (Duration.between(t, clock.instant()).compareTo(this.keysetKeyShutdownWaitTime) > 0) {
+                LOGGER.error("keyset keys have been failing to sync for too long. shutting down operator");
+                this.shutdownService.Shutdown(1);
+            }
+        }
+    }
+
+    public void logKeysetKeyFailureAtInterval() {
+        Instant t = lastKeysetKeyFailureLogTime.get();
+        if (t == null || clock.instant().isAfter(t.plus(KEYSET_KEY_FAILURE_LOG_INTERVAL_MINUTES, ChronoUnit.MINUTES))) {
+            LOGGER.error("keyset keys sync failing");
+            lastKeysetKeyFailureLogTime.set(Instant.now());
+        }
+    }
+
     public void handleAttestResponse(Pair<AttestationResponseCode, String> response) {
         if (response.left() == AttestationResponseCode.AttestationFailure) {
             LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", response.right());