Add support for Vault KV Engine version 2.

Add support for Vault 1+ operations.
Add support for Namespaces.
Add tests for these changes.
Increase test coverage.

Author: jarrodcodes

build.gradle

@@ -8,8 +8,8 @@ version '3.1.0'
 ext.isReleaseVersion = !version.endsWith('SNAPSHOT')
 
 compileJava {
-    sourceCompatibility = 1.7
-    targetCompatibility = 1.7
+    sourceCompatibility = 1.8
+    targetCompatibility = 1.8
 }
 
 compileTestJava {
@@ -22,21 +22,21 @@ repositories {
 }
 
 dependencies {
-    compileOnly('org.projectlombok:lombok:1.16.20')
+    compileOnly('org.projectlombok:lombok:1.18.4')
 
     testCompile('junit:junit:4.12')
-    testCompile('org.mockito:mockito-core:2.13.0')
-    testCompile('org.testcontainers:testcontainers:1.5.1')
-    testCompile('org.eclipse.jetty:jetty-server:9.4.8.v20171121')
+    testCompile('org.mockito:mockito-core:2.23.4')
+    testCompile('org.testcontainers:testcontainers:1.6.0')
+    testCompile('org.eclipse.jetty:jetty-server:9.4.14.v20181114')
     testCompile('org.slf4j:slf4j-api:1.7.25')
-    testCompile('org.bouncycastle:bcprov-jdk15on:1.59')
-    testCompile('org.bouncycastle:bcpkix-jdk15on:1.59')
+    testCompile('org.bouncycastle:bcprov-jdk15on:1.60')
+    testCompile('org.bouncycastle:bcpkix-jdk15on:1.60')
 
     testRuntime('org.slf4j:slf4j-simple:1.7.25')
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '4.2.1'
+    gradleVersion = '4.10.2'
 }
 
 task javadocJar(type: Jar, dependsOn: javadoc) {

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip

src/main/java/com/bettercloud/vault/EnvironmentLoader.java

@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
@@ -25,7 +26,7 @@ public String loadVariable(final String name) {
                 try {
                     final byte[] bytes = Files.readAllBytes(
                             Paths.get(System.getProperty("user.home")).resolve(".vault-token"));
-                    value = new String(bytes, "UTF-8").trim();
+                    value = new String(bytes, StandardCharsets.UTF_8).trim();
                 } catch (IOException e) {
                     // No-op... there simply isn't a token value available
                 }

src/main/java/com/bettercloud/vault/SslConfig.java

@@ -19,6 +19,7 @@
 import java.io.InputStreamReader;
 import java.io.ObjectInputStream;
 import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
 import java.security.KeyFactory;
 import java.security.KeyManagementException;
 import java.security.KeyStore;
@@ -538,7 +539,7 @@ private SSLContext buildSslContextFromPem() throws VaultException {
                 final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
                 // Convert the trusted servers PEM data into an X509Certificate
                 X509Certificate certificate;
-                try (final ByteArrayInputStream pem = new ByteArrayInputStream(pemUTF8.getBytes("UTF-8"))) {
+                try (final ByteArrayInputStream pem = new ByteArrayInputStream(pemUTF8.getBytes(StandardCharsets.UTF_8))) {
                     certificate = (X509Certificate) certificateFactory.generateCertificate(pem);
                 }
                 // Build a truststore
@@ -553,7 +554,7 @@ private SSLContext buildSslContextFromPem() throws VaultException {
                 final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
                 // Convert the client certificate PEM data into an X509Certificate
                 X509Certificate clientCertificate;
-                try (final ByteArrayInputStream pem = new ByteArrayInputStream(clientPemUTF8.getBytes("UTF-8"))) {
+                try (final ByteArrayInputStream pem = new ByteArrayInputStream(clientPemUTF8.getBytes(StandardCharsets.UTF_8))) {
                     clientCertificate = (X509Certificate) certificateFactory.generateCertificate(pem);
                 }
 
@@ -613,7 +614,7 @@ private KeyStore inputStreamToKeyStore(final InputStream inputStream, final Stri
      * @throws IOException
      */
     private static String inputStreamToUTF8(final InputStream input) throws IOException {
-        final BufferedReader in = new BufferedReader(new InputStreamReader(input, "UTF-8"));
+        final BufferedReader in = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
         final StringBuilder utf8 = new StringBuilder("");
         String str;
         while ((str = in.readLine()) != null) {

src/main/java/com/bettercloud/vault/Vault.java

@@ -6,6 +6,16 @@
 import com.bettercloud.vault.api.Logical;
 import com.bettercloud.vault.api.Seal;
 import com.bettercloud.vault.api.pki.Pki;
+import com.bettercloud.vault.json.Json;
+import com.bettercloud.vault.json.JsonObject;
+import com.bettercloud.vault.json.JsonValue;
+import com.bettercloud.vault.rest.Rest;
+import com.bettercloud.vault.rest.RestException;
+import com.bettercloud.vault.rest.RestResponse;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * <p>The Vault driver class, the primary interface through which dependent applications will access Vault.</p>
@@ -50,16 +60,63 @@ public class Vault {
      * Construct a Vault driver instance with the provided config settings.
      *
      * @param vaultConfig Configuration settings for Vault interaction (e.g. server address, token, etc)
+     *                    If the VaultConfig Engine version path map is not supplied in the config, default to global KV
+     *                    engine version 2.
      */
     public Vault(final VaultConfig vaultConfig) {
         this.vaultConfig = vaultConfig;
+        if (this.vaultConfig.getSecretsEnginePathMap().isEmpty() && this.vaultConfig.getGlobalEngineVersion() == null) {
+            System.out.println("Constructing a Vault instance with no provided Engine version, defaulting to version 2.");
+            this.vaultConfig.setEngineVersion(2);
+        }
+    }
+
+    /**
+     * Construct a Vault driver instance with the provided config settings, and use the provided global KV Engine version for all secrets.
+     */
+    public Vault(final VaultConfig vaultConfig, final Integer engineVersion) {
+        if (engineVersion < 1 || engineVersion > 2) {
+            throw new IllegalArgumentException("The Engine version must be '1' or '2', the version supplied was: '"
+                    + engineVersion + "'.");
+        }
+        vaultConfig.setEngineVersion(engineVersion);
+        this.vaultConfig = vaultConfig;
+    }
+
+    /**
+     * Construct a Vault driver instance with the provided config settings.
+     *
+     * @param vaultConfig             Configuration settings for Vault interaction (e.g. server address, token, etc)
+     *                                If the Secrets engine version path map is not provided, or does not contain the
+     *                                requested secret, fall back to the global version supplied.
+     * @param useSecretsEnginePathMap Whether to use a provided KV Engine version map from the Vault config, or generate one.
+     *                                If a secrets KV Engine version map is not supplied, use Vault APIs to determine the
+     *                                KV Engine version for each secret. This call requires admin rights.
+     * @param globalFallbackVersion   The Integer version of the KV Engine to use as a global fallback.
+     */
+    public Vault(final VaultConfig vaultConfig, final Boolean useSecretsEnginePathMap, final Integer globalFallbackVersion)
+            throws VaultException {
+        this.vaultConfig = vaultConfig;
+        this.vaultConfig.setEngineVersion(globalFallbackVersion);
+        if (useSecretsEnginePathMap && this.vaultConfig.getSecretsEnginePathMap().isEmpty()) {
+            try {
+                System.out.println("No secrets Engine version map was supplied, attempting to generate one.");
+                final Map<String, String> secretsEnginePathMap = collectSecretEngineVersions();
+                assert secretsEnginePathMap != null;
+                this.vaultConfig.getSecretsEnginePathMap().putAll(secretsEnginePathMap);
+            } catch (Exception e) {
+                throw new VaultException(String.format("An Engine KV version map was not supplied, and unable to determine " +
+                        "KV Engine " +
+                        "version, " + "due to exception: %s", e.getMessage() + ". Do you have admin rights?"));
+            }
+        }
     }
 
     /**
      * This method is chained ahead of endpoints (e.g. <code>logical()</code>, <code>auth()</code>,
      * etc... to specify retry rules for any API operations invoked on that endpoint.
      *
-     * @param maxRetries The number of times that API operations will be retried when a failure occurs
+     * @param maxRetries                The number of times that API operations will be retried when a failure occurs
      * @param retryIntervalMilliseconds The number of milliseconds that the driver will wait in between retries
      * @return This object, with maxRetries and retryIntervalMilliseconds populated
      */
@@ -146,4 +203,60 @@ public Debug debug() {
     public Seal seal() {
         return new Seal(vaultConfig);
     }
+
+    /**
+     * Makes a REST call to Vault, to collect information on which secret engine version (if any) is used by each available
+     * mount point.  Possibilities are:
+     *
+     * <ul>
+     * <li>"2" - A mount point running on Vault 0.10 or higher, configured to use the engine 2 API</li>
+     * <li>"1" - A mount point running on Vault 0.10 or higher, configured to use the engine 1 API</li>
+     * <li>"unknown" - A mount point running on an older version of Vault.  Can more or less be treated as "1".</li>
+     * </ul>
+     * <p>
+     * IMPORTANT:  Whichever authentication mechanism is being used with the <code>VaultConfig</code> object, that principal
+     * needs permission to access the <code>/v1/sys/mounts</code> REST endpoint.
+     *
+     * @return A map of mount points (e.g. "/secret") to secret engine version numbers (e.g. "2")
+     */
+    private Map<String, String> collectSecretEngineVersions() {
+        try {
+            final RestResponse restResponse = new Rest()//NOPMD
+                    .url(vaultConfig.getAddress() + "/v1/sys/mounts")
+                    .header("X-Vault-Token", vaultConfig.getToken())
+                    .connectTimeoutSeconds(vaultConfig.getOpenTimeout())
+                    .readTimeoutSeconds(vaultConfig.getReadTimeout())
+                    .sslVerification(vaultConfig.getSslConfig().isVerify())
+                    .sslContext(vaultConfig.getSslConfig().getSslContext())
+                    .get();
+            if (restResponse.getStatus() != 200) {
+                return null;
+            }
+
+            final String jsonString = new String(restResponse.getBody(), StandardCharsets.UTF_8);
+            final Map<String, String> data = new HashMap<>();
+            final JsonObject jsonData = Json.parse(jsonString).asObject().get("data").asObject();
+            for (JsonObject.Member member : jsonData) {
+                final String name = member.getName();
+                String version = "unknown";
+
+                final JsonValue options = member.getValue().asObject().get("options");
+                if (options != null && options.isObject()) {
+                    final JsonValue ver = options.asObject().get("version");
+                    if (ver != null && ver.isString()) {
+                        version = ver.asString();
+                    }
+                }
+                data.put(name, version);
+            }
+            return data;
+        } catch (RestException e) {
+            System.err.print(String.format("Unable to retrieve the KV Engine secrets, due to exception: %s", e.getMessage()));
+            return null;
+        }
+    }
+
+    public Map<String, String> getSecretEngineVersions() {
+        return this.collectSecretEngineVersions();
+    }
 }

src/main/java/com/bettercloud/vault/VaultConfig.java

@@ -3,6 +3,8 @@
 import lombok.Getter;
 
 import java.io.Serializable;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * <p>A container for the configuration settings needed to initialize a <code>Vault</code> driver instance.</p>
@@ -30,13 +32,24 @@ public class VaultConfig implements Serializable {
     private static final String VAULT_OPEN_TIMEOUT = "VAULT_OPEN_TIMEOUT";
     private static final String VAULT_READ_TIMEOUT = "VAULT_READ_TIMEOUT";
 
-    @Getter private String address;
-    @Getter private String token;
-    @Getter private SslConfig sslConfig;
-    @Getter private Integer openTimeout;
-    @Getter private Integer readTimeout;
-    @Getter private int maxRetries;
-    @Getter private int retryIntervalMilliseconds;
+    @Getter
+    private Map<String, String> secretsEnginePathMap = new ConcurrentHashMap<>();
+    @Getter
+    private String address;
+    @Getter
+    private String token;
+    @Getter
+    private SslConfig sslConfig;
+    @Getter
+    private Integer openTimeout;
+    @Getter
+    private Integer readTimeout;
+    @Getter
+    private int maxRetries;
+    @Getter
+    private int retryIntervalMilliseconds;
+    @Getter
+    private Integer globalEngineVersion;
     private EnvironmentLoader environmentLoader;
 
     /**
@@ -48,7 +61,7 @@ public class VaultConfig implements Serializable {
      * constructor.  This method's access level was therefore originally set to <code>protected</code>, but was bumped
      * up to <code>public</code> due to community request for the ability to disable environment loading altogether
      * (see https://github.com/BetterCloud/vault-java-driver/issues/77).
-     *
+     * <p>
      * Note that if you do override this, however, then obviously all of the environment checking discussed in the
      * documentation becomes disabled.
      *
@@ -60,6 +73,20 @@ public VaultConfig environmentLoader(final EnvironmentLoader environmentLoader)
         return this;
     }
 
+    /**
+     * <p>Sets the KV Secrets Engine version of the Vault server instance.
+     *
+     * <p>If no version is explicitly set, it will be defaulted to version 2, the current version.</p>
+     *
+     * @param globalEngineVersion The Vault KV Secrets Engine version
+     * @return This object, with KV Secrets Engine version populated, ready for additional builder-pattern method calls or else
+     * finalization with the build() method
+     */
+    public VaultConfig engineVersion(final Integer globalEngineVersion) {
+        this.globalEngineVersion = globalEngineVersion;
+        return this;
+    }
+
     /**
      * <p>Sets the address (URL) of the Vault server instance to which API calls should be sent.
      * E.g. <code>http://127.0.0.1:8200</code>.</p>
@@ -95,6 +122,20 @@ public VaultConfig token(final String token) {
         return this;
     }
 
+    /**
+     * <p>Sets the secrets Engine paths used by Vault.</p>
+     *
+     * @param secretEngineVersions paths to use for accessing Vault secrets.
+     *                             Key: secret path, value: Engine version to use.
+     *                             Example map: "/secret/foo" , "1",
+     *                             "/secret/bar", "2"
+     * @return This object, with secrets paths populated, ready for additional builder-pattern method calls or else finalization with the build() method
+     */
+    VaultConfig secretsEnginePathMap(final Map<String, String> secretEngineVersions) {
+        this.secretsEnginePathMap = secretEngineVersions;
+        return this;
+    }
+
     /**
      * <p>A container for SSL-related configuration options (e.g. certificates).</p>
      *
@@ -150,7 +191,7 @@ public VaultConfig readTimeout(final Integer readTimeout) {
      *
      * @param maxRetries The number of times that API operations will be retried when a failure occurs.
      */
-    protected void setMaxRetries(final int maxRetries) {
+    void setMaxRetries(final int maxRetries) {
         this.maxRetries = maxRetries;
     }
 
@@ -164,10 +205,20 @@ protected void setMaxRetries(final int maxRetries) {
      *
      * @param retryIntervalMilliseconds The number of milliseconds that the driver will wait in between retries.
      */
-    protected void setRetryIntervalMilliseconds(final int retryIntervalMilliseconds) {
+    void setRetryIntervalMilliseconds(final int retryIntervalMilliseconds) {
         this.retryIntervalMilliseconds = retryIntervalMilliseconds;
     }
 
+    /**
+     * <p>Sets the global Engine version for this Vault Config instance. If no KV Engine version map is provided, use this version
+     * globally.</p>
+     * If the provided KV Engine version map does not contain a requested secret, or when writing new secrets, fall back to this version.
+     *
+     * @param engineVersion The version of the Vault KV Engine to use globally.
+     */
+    void setEngineVersion(final Integer engineVersion) {
+        this.globalEngineVersion = engineVersion;
+    }
 
     /**
      * <p>This is the terminating method in the builder pattern.  The method that validates all of the fields that