Disable store auto disable and reenablement (#2078)

Author: ruhan1

addons/schedule/common/src/test/java/org/commonjava/indy/schedule/ScheduleTest.java

@@ -83,11 +83,11 @@ public void testScheduleExpired() throws Exception
         final String jobName = "org/jboss";
         final String payload = "json_string";
 
-        Long timeout = Long.valueOf( 10 );
+        Long timeout = Long.valueOf( 5 );
 
         scheduleDB.createSchedule( storeKey, JobType.CONTENT.getJobType(), jobName, payload, timeout );
 
-        Thread.sleep( 15 * 1000 );
+        Thread.sleep( 10 * 1000 );
 
         DtxSchedule schedule = scheduleDB.querySchedule( storeKey, jobName );
 

api/src/main/java/org/commonjava/indy/conf/InternalFeatureConfig.java

@@ -39,6 +39,13 @@ public class InternalFeatureConfig implements IndyConfigInfo {
 
     private boolean mavenMetadataCacheEnabled = TRUE;
 
+    /**
+     * Indy disables a remote store when transfer error happens, and try to re-enable it
+     * after a timeout specified either by global 'storeDisableTimeoutSeconds' or store specific timeout.
+     * Auto disable-and-re-enabling a store may not be very useful. Thus, false by default.
+     */
+    private boolean storeAutoDisableAndReEnable;
+
     private final Logger logger = LoggerFactory.getLogger( getClass() );
 
     public InternalFeatureConfig() {
@@ -85,4 +92,13 @@ public boolean isMavenMetadataCacheEnabled() {
     public void setMavenMetadataCacheEnabled(boolean mavenMetadataCacheEnabled) {
         this.mavenMetadataCacheEnabled = mavenMetadataCacheEnabled;
     }
+
+    public boolean isStoreAutoDisableAndReEnable() {
+        return storeAutoDisableAndReEnable;
+    }
+
+    @ConfigName("store.auto.disable.reenable")
+    public void setStoreAutoDisableAndReEnable(boolean storeAutoDisableAndReEnable) {
+        this.storeAutoDisableAndReEnable = storeAutoDisableAndReEnable;
+    }
 }

core/src/main/java/org/commonjava/indy/core/change/StoreEnablementManager.java

@@ -21,6 +21,7 @@
 import org.commonjava.indy.change.event.ArtifactStorePostUpdateEvent;
 import org.commonjava.indy.change.event.IndyStoreErrorEvent;
 import org.commonjava.indy.conf.IndyConfiguration;
+import org.commonjava.indy.conf.InternalFeatureConfig;
 import org.commonjava.indy.core.expire.IndySchedulerException;
 import org.commonjava.indy.core.expire.ScheduleManager;
 import org.commonjava.indy.core.expire.SchedulerEvent;
@@ -63,6 +64,9 @@ public class StoreEnablementManager
     @Inject
     private IndyConfiguration config;
 
+    @Inject
+    private InternalFeatureConfig internalFeatureConfig;
+
     @Measure
     public void onStoreEnablementChange( @Observes ArtifactStoreEnablementEvent event )
     {
@@ -78,6 +82,13 @@ public void onStoreError( @Observes IndyStoreErrorEvent evt )
         Logger logger = LoggerFactory.getLogger( getClass() );
         StoreKey key = evt.getStoreKey();
         Throwable error = evt.getError();
+
+        if ( !internalFeatureConfig.isStoreAutoDisableAndReEnable() )
+        {
+            logger.info( "Store auto-disable-reenable is not enabled. Skipping {}", evt );
+            return;
+        }
+
         try
         {
             ArtifactStore store = storeDataManager.getArtifactStore( key );
@@ -87,16 +98,14 @@ public void onStoreError( @Observes IndyStoreErrorEvent evt )
                 return;
             }
 
-            store = store.copyOf();
-
             int disableTimeout = store.getDisableTimeout();
             if ( disableTimeout <= TIMEOUT_NEVER_DISABLE )
             {
                 logger.debug( "Disable-timeout set to {}, will never disable the repo", disableTimeout );
-                store.setDisabled( false );
             }
             else
             {
+                store = store.copyOf();
                 store.setDisabled( true );
                 final ChangeSummary changeSummary = new ChangeSummary( ChangeSummary.SYSTEM_USER, String.format(
                         "Disabling %s due to error: %s\n\nStack Trace:\n  %s", key, error,

ftests/common/src/main/java/org/commonjava/indy/ftest/core/AbstractIndyFunctionalTest.java

@@ -311,7 +311,7 @@ protected void initBaseTestConfig( CoreServerFixture fixture )
         {
             writeConfigFile( "conf.d/scheduledb.conf", readTestResource( "default-test-scheduledb.conf" ) );
             writeConfigFile( "conf.d/threadpools.conf", "[threadpools]\nenabled=false" );
-            writeConfigFile( "conf.d/internal-features.conf", "[_internal]\nstore.validation.enabled=false" );
+            writeConfigFile( "conf.d/internal-features.conf", "[_internal]\nstore.validation.enabled=false\nstore.auto.disable.reenable=true\n" );
             writeConfigFile( "conf.d/durable-state.conf", readTestResource( "default-durable-state.conf" ) );
         }
         else

ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/RemoteRepoTimeoutDisablesStoreAndShowsInDisabledTimeoutsMapTest.java

@@ -67,7 +67,6 @@ protected void initTestConfig( CoreServerFixture fixture )
             throws IOException
     {
         writeConfigFile( "conf.d/indexer.conf", "[indexer]\nenabled=false" );
-        writeConfigFile( "conf.d/internal-features.conf", "[_internal]\nstore.validation.enabled=false" );
     }
 
     @Override

ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/RemoteRepoTimeoutDisablesThenReEnabledTest.java

@@ -47,7 +47,6 @@ protected void initTestConfig( CoreServerFixture fixture )
     {
         writeConfigFile( "main.conf", "store.disable.timeout=2\n\nInclude conf.d/*.conf\n" );
         writeConfigFile( "conf.d/scheduler.conf", readTestResource( "default-scheduler.conf" ) );
-        writeConfigFile( "conf.d/internal-features.conf", "[_internal]\nstore.validation.enabled=false" );
     }
 
     @Test