diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/build.gradle b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/build.gradle index 641a93a44bc..a8a35e05a8a 100644 --- a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/build.gradle +++ b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/build.gradle @@ -15,8 +15,11 @@ dependencies { api group: 'org.msgpack', name: 'jackson-dataformat-msgpack', version: '0.9.6' api group: 'org.xmlunit', name: 'xmlunit-core', version: '2.10.3' - compileOnly(libs.junit.jupiter) - compileOnly(libs.bundles.groovy) - compileOnly(libs.bundles.spock) + api(libs.bundles.junit5) + api(libs.tabletest) } +// civisibility-test-fixtures is a test-support module — every consumer pulls it on their test +// classpath. Production-code-quality gates like forbidden APIs don't apply here. +tasks.named('forbiddenApisMain').configure { enabled = false } + diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy deleted file mode 100644 index 0eaeb2e755a..00000000000 --- a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy +++ /dev/null @@ -1,214 +0,0 @@ -package datadog.trace.civisibility - -import datadog.trace.api.Config -import datadog.trace.api.civisibility.config.TestFQN -import datadog.trace.api.config.CiVisibilityConfig -import datadog.trace.api.config.GeneralConfig -import datadog.trace.api.config.TraceInstrumentationConfig -import datadog.trace.api.config.TracerConfig -import java.nio.file.Paths -import spock.lang.Specification -import spock.lang.TempDir - -import java.nio.file.Path - -import static datadog.trace.util.ConfigStrings.propertyNameToSystemPropertyName - -abstract class CiVisibilitySmokeTest extends Specification { - static final List SMOKE_IGNORED_TAGS = ["content.meta.['_dd.integration']", "content.meta.['_dd.svc_src']"] - - protected static final String AGENT_JAR = System.getProperty("datadog.smoketest.agent.shadowJar.path") - protected static final String TEST_ENVIRONMENT_NAME = "integration-test" - protected static final String JAVAC_PLUGIN_VERSION = Config.get().ciVisibilityCompilerPluginVersion - protected static final String JACOCO_PLUGIN_VERSION = Config.get().ciVisibilityJacocoPluginVersion - - private static final Map DEFAULT_TRACER_CONFIG = defaultJvmArguments() - - @TempDir - protected Path prefsDir - - protected static String buildJavaHome() { - def javaHome = System.getProperty("java.home") - def javacPath = Paths.get(javaHome, "bin", "javac").toFile() - if (javacPath.exists()) { - return javaHome - } - // In CI for JDK 8, java.home may point to the JRE directory (e.g., /usr/lib/jvm/8/jre) - // The JDK with javac is in the parent directory - def parentDir = new File(javaHome).getParentFile() - def parentJavacPath = new File(parentDir, Paths.get("bin", "javac").toString()) - if (parentJavacPath.exists()) { - return parentDir.getAbsolutePath() - } - // Fallback to java.home and let callers handle the error if javac is not found - return javaHome - } - - protected static String javaPath() { - final String separator = System.getProperty("file.separator") - return "${buildJavaHome()}${separator}bin${separator}java" - } - - protected static String javacPath() { - final String separator = System.getProperty("file.separator") - return "${buildJavaHome()}${separator}bin${separator}javac" - } - - private static Map defaultJvmArguments() { - Map argMap = new HashMap<>() - argMap.put(GeneralConfig.TRACE_DEBUG, "true") - argMap.put(GeneralConfig.ENV, TEST_ENVIRONMENT_NAME) - argMap.put(CiVisibilityConfig.CIVISIBILITY_ENABLED, "true") - argMap.put(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED, "true") - argMap.put(CiVisibilityConfig.CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED, "false") - argMap.put(CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED, "false") - argMap.put(CiVisibilityConfig.CIVISIBILITY_GIT_CLIENT_ENABLED, "false") - argMap.put(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES, "true") - argMap.put(CiVisibilityConfig.CIVISIBILITY_COMPILER_PLUGIN_VERSION, JAVAC_PLUGIN_VERSION) - argMap.put(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, "false") - return argMap - } - - private static Map buildJvmArgMap(String mockBackendIntakeUrl, String serviceName, Map additionalArgs) { - Map argMap = new HashMap<>(DEFAULT_TRACER_CONFIG) - argMap.put(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL, mockBackendIntakeUrl) - argMap.put(CiVisibilityConfig.CIVISIBILITY_INTAKE_AGENTLESS_URL, mockBackendIntakeUrl) - argMap.put(TracerConfig.TRACE_AGENT_URL, mockBackendIntakeUrl) - argMap.putAll(additionalArgs) - - if (serviceName != null) { - argMap.put(GeneralConfig.SERVICE_NAME, serviceName) - } - - return argMap - } - - protected List buildJvmArguments(String mockBackendIntakeUrl, String serviceName, Map additionalArgs) { - List arguments = ["-Xms256m", "-Xmx256m"] - - arguments += preventJulPrefsFileLock() - - Map argMap = buildJvmArgMap(mockBackendIntakeUrl, serviceName, additionalArgs) - - // for convenience when debugging locally - if (System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_PARENT") != null) { - arguments += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005" - } - if (System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_CHILD") != null) { - argMap.put(CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT, "5055") - } - - String agentArgs = argMap.collect { k, v -> "${propertyNameToSystemPropertyName(k)}=${v}" }.join(",") - arguments += "-javaagent:${AGENT_JAR}=${agentArgs}".toString() - - return arguments - } - - /** - * Trick to prevent jul Prefs file lock issue on forked processes, in particular in CI which - * runs on Linux and have competing processes trying to write to it, including the Gradle daemon. - * - * 

-   * Couldn't flush user prefs: java.util.prefs.BackingStoreException: Couldn't get file lock.
-   *
- * - * Note, some tests can setup arguments on spec level, so `prefsDir` will be `null` during - * `setupSpec()`. - */ - protected String preventJulPrefsFileLock() { - String prefsPath = (prefsDir ?: tempUserPrefsPath()).toAbsolutePath() - return "-Djava.util.prefs.userRoot=$prefsPath".toString() - } - - private static Path tempUserPrefsPath() { - String uniqueId = "${System.currentTimeMillis()}_${System.nanoTime()}_${Thread.currentThread().id}" - Path prefsPath = Paths.get(System.getProperty("java.io.tmpdir"), "gradle-test-userPrefs", uniqueId) - return prefsPath - } - - protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, List> events, List> coverages, List additionalDynamicTags = []) { - def additionalReplacements = ["content.meta.['test.toolchain']": "$toolchain:$toolchainVersion"] - - if (System.getenv("GENERATE_TEST_FIXTURES") != null) { - def baseTemplatesPath = CiVisibilitySmokeTest.classLoader.getResource(projectName).toURI().schemeSpecificPart.replace('build/resources/test', 'src/test/resources') - CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements.keySet() + additionalDynamicTags, SMOKE_IGNORED_TAGS) - } else { - CiVisibilityTestUtils.assertData(projectName, events, coverages, additionalReplacements, SMOKE_IGNORED_TAGS, additionalDynamicTags) - } - } - - protected test(String suiteName, String testName) { - return new TestFQN(suiteName, testName) - } - - protected verifyTestOrder(List> events, List expectedOrder) { - CiVisibilityTestUtils.assertTestsOrder(events, expectedOrder) - } - - /** - * This is a basic sanity check for telemetry metrics. - * It only checks that the reported number of events created and finished is as expected. - *

- * Currently the check is not performed for Gradle builds: - * Gradle daemon started with Gradle TestKit outlives the test, so the final telemetry flush happens after the assertions. - */ - protected verifyTelemetryMetrics(List> receivedTelemetryMetrics, List> receivedTelemetryDistributions, int expectedEventsCount) { - int eventsCreated = 0, eventsFinished = 0 - for (Map metric : receivedTelemetryMetrics) { - if (metric["metric"] == "event_created") { - for (def point : metric["points"]) { - eventsCreated += point[1] - } - } - if (metric["metric"] == "event_finished") { - for (def point : metric["points"]) { - eventsFinished += point[1] - } - } - } - assert eventsCreated == expectedEventsCount - assert eventsFinished == expectedEventsCount - - // an even more basic smoke check for distributions: assert that we received some - assert !receivedTelemetryDistributions.isEmpty() - } - - protected verifyCoverageReports(String projectName, List reports, Map replacements) { - CiVisibilityTestUtils.assertData(projectName, reports, replacements) - } - - protected static verifySnapshotLogs(List> receivedLogs, int expectedProbes, int expectedSnapshots) { - def logsPerProbe = 3 // 3 probe statuses per probe -> received, installed, emitting - - assert receivedLogs.size() == logsPerProbe * expectedProbes + expectedSnapshots - - def probeStatusLogs = receivedLogs.findAll { it.containsKey("message") } - def snapshotLogs = receivedLogs.findAll { !it.containsKey("message") } - - verifyProbeStatuses(probeStatusLogs, expectedProbes) - verifySnapshots(snapshotLogs, expectedSnapshots) - } - - private static verifyProbeStatuses(List> logs, int expectedCount) { - assert logs.findAll { log -> ((String) log.message).startsWith("Received probe") }.size() == expectedCount - assert logs.findAll { log -> ((String) log.message).startsWith("Installed probe") }.size() == expectedCount - assert logs.findAll { log -> ((String) log.message).endsWith("is emitting.") }.size() == expectedCount - } - - protected static verifySnapshots(List> logs, expectedCount) { - assert logs.size() == expectedCount - - def requiredLogFields = ["logger.name", "logger.method", "dd.spanid", "dd.traceid"] - def requiredSnapshotFields = ["captures", "exceptionId", "probe", "stack"] - - logs.each { log -> - requiredLogFields.each { field -> log.containsKey(field) } - - Map debuggerMap = log.debugger as Map - Map snapshotContent = debuggerMap.snapshot as Map - - assert snapshotContent != null - requiredSnapshotFields.each { field -> snapshotContent.containsKey(field) } - } - } -} diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy deleted file mode 100644 index c12ae929675..00000000000 --- a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy +++ /dev/null @@ -1,397 +0,0 @@ -package datadog.trace.civisibility - -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.SerializationFeature -import com.jayway.jsonpath.Configuration -import com.jayway.jsonpath.JsonPath -import com.jayway.jsonpath.Option -import com.jayway.jsonpath.ReadContext -import com.jayway.jsonpath.WriteContext -import datadog.trace.api.DDSpanTypes -import datadog.trace.api.civisibility.config.LibraryCapability -import datadog.trace.api.civisibility.config.TestFQN -import datadog.trace.core.DDSpan -import freemarker.core.Environment -import freemarker.core.InvalidReferenceException -import freemarker.template.Template -import freemarker.template.TemplateException -import freemarker.template.TemplateExceptionHandler -import org.opentest4j.AssertionFailedError -import org.skyscreamer.jsonassert.JSONAssert -import org.skyscreamer.jsonassert.JSONCompareMode -import org.w3c.dom.Document -import org.xmlunit.builder.DiffBuilder -import org.xmlunit.builder.Input -import org.xmlunit.diff.Diff -import org.xmlunit.util.Convert - -import javax.xml.parsers.DocumentBuilderFactory -import java.nio.file.Files -import java.nio.file.Paths -import java.util.regex.Pattern -import java.util.stream.Collectors - -import static org.junit.jupiter.api.Assertions.assertEquals - -abstract class CiVisibilityTestUtils { - - static final List EVENT_DYNAMIC_PATHS = [ - path("content.trace_id"), - path("content.span_id"), - path("content.parent_id"), - path("content.test_session_id"), - path("content.test_module_id"), - path("content.test_suite_id"), - path("content.metrics.process_id"), - path("content.meta.['os.architecture']"), - path("content.meta.['os.platform']"), - path("content.meta.['os.version']"), - path("content.meta.['runtime.name']"), - path("content.meta.['runtime.vendor']"), - path("content.meta.['runtime.version']"), - path("content.meta.['ci.workspace_path']"), - path("content.meta.['error.message']"), - path("content.meta.library_version"), - path("content.meta.runtime-id"), - path("content.meta.['_dd.tracer_host']"), - // Different events might or might not have the same start or duration. - // Regardless, the values of these fields should be treated as different - path("content.start", false), - path("content.duration", false), - path("content.metrics.['_dd.host.vcpu_count']", false), - path("content.meta.['_dd.p.tid']", false), - path("content.meta.['error.stack']", false), - ] - - // ignored tags on assertion and fixture build - static final List IGNORED_TAGS = LibraryCapability.values().toList().stream().map(c -> "content.meta.['${c.asTag()}']").collect(Collectors.toList()) + - ["content.meta.['_dd.integration']", "content.meta.['_dd.svc_src']"] - - static final List COVERAGE_DYNAMIC_PATHS = [path("test_session_id"), path("test_suite_id"), path("span_id"),] - - private static final Comparator> EVENT_RESOURCE_COMPARATOR = Comparator., String> comparing((Map m) -> { - def content = (Map) m.get("content") - return content.get("resource") - }).thenComparing(Comparator., String> comparing((Map m) -> { - // module and session have the same resource name in headless mode - return m.get("type") - }).reversed()) - - /** - * Use this method to generate expected data templates - */ - static void generateTemplates(String baseTemplatesPath, List> events, List> coverages, Collection additionalDynamicPaths, List ignoredTags = []) { - if (!ignoredTags.empty) { - events = removeTags(events, ignoredTags) - } - events.sort(EVENT_RESOURCE_COMPARATOR) - - def templateGenerator = new TemplateGenerator(new LabelGenerator()) - def compiledAdditionalReplacements = compile(additionalDynamicPaths) - - Files.createDirectories(Paths.get(baseTemplatesPath)) - Files.write(Paths.get(baseTemplatesPath, "events.ftl"), templateGenerator.generateTemplate(events, EVENT_DYNAMIC_PATHS + compiledAdditionalReplacements).bytes) - Files.write(Paths.get(baseTemplatesPath, "coverages.ftl"), templateGenerator.generateTemplate(coverages, COVERAGE_DYNAMIC_PATHS + compiledAdditionalReplacements).bytes) - } - - static void assertData(String baseTemplatesPath, List reports, Map replacements) { - def expectedReportEvent = getFreemarkerTemplate(baseTemplatesPath + "/coverage_report_event.ftl", replacements) - def actualReportEvent = JSON_MAPPER.writeValueAsString(reports[0].event) - - compareJson(expectedReportEvent, actualReportEvent) - - def expectedReport = getFreemarkerTemplate(baseTemplatesPath + "/coverage_report.ftl", replacements) - def actualReport = reports[0].report - - if (expectedReport.contains(" assertData(String baseTemplatesPath, List> events, List> coverages, Map additionalReplacements, List ignoredTags, List additionalDynamicPaths = []) { - events.sort(EVENT_RESOURCE_COMPARATOR) - - def labelGenerator = new LabelGenerator() - def templateGenerator = new TemplateGenerator(labelGenerator) - - def replacementMap - replacementMap = templateGenerator.generateReplacementMap(events, EVENT_DYNAMIC_PATHS + compile(additionalDynamicPaths)) - replacementMap = templateGenerator.generateReplacementMap(coverages, COVERAGE_DYNAMIC_PATHS) - - for (Map.Entry e : additionalReplacements.entrySet()) { - replacementMap.put(labelGenerator.forKey(e.key), "\"$e.value\"") - } - - // ignore provided tags - events = removeTags(events, ignoredTags) - - def expectedEvents = getFreemarkerTemplate(baseTemplatesPath + "/events.ftl", replacementMap, events) - def actualEvents = JSON_MAPPER.writeValueAsString(events) - - compareJson(expectedEvents, actualEvents) - - def expectedCoverages = getFreemarkerTemplate(baseTemplatesPath + "/coverages.ftl", replacementMap, coverages) - def actualCoverages = JSON_MAPPER.writeValueAsString(coverages) - compareJson(expectedCoverages, actualCoverages) - - return replacementMap - } - - private static void compareJson(String expectedJson, String actualJson) { - def environment = System.getenv() - def ciRun = environment.get("GITHUB_ACTION") != null || environment.get("GITLAB_CI") != null - def comparisonMode = ciRun ? JSONCompareMode.LENIENT : JSONCompareMode.NON_EXTENSIBLE - - try { - JSONAssert.assertEquals(expectedJson, actualJson, comparisonMode) - } catch (AssertionError e) { - if (ciRun) { - // When running in CI the assertion error message does not contain the actual diff, - // so we print the events to the console to help debug the issue - println "Expected JSON: $expectedJson" - println "Actual JSON: $actualJson" - } - throw new AssertionFailedError("Expected and actual JSON mismatch", expectedJson, actualJson, e) - } - } - - static boolean assertTestsOrder(List> events, List expectedOrder) { - def identifiers = getTestIdentifiers(events) - if (identifiers != expectedOrder) { - throw new AssertionError("Expected order: $expectedOrder, but got: $identifiers") - } - return true - } - - static List getTestIdentifiers(List> events) { - events.sort(Comparator.comparing { - it['content']['start'] as Long - }) - def testIdentifiers = [] - for (Map event : events) { - if (event['content']['meta']['test.name']) { - testIdentifiers.add(new TestFQN(event['content']['meta']['test.suite'] as String, event['content']['meta']['test.name'] as String)) - } - } - return testIdentifiers - } - - static List> removeTags(List> events, List tags) { - def filteredEvents = [] - - for (Map event : events) { - ReadContext ctx = JsonPath.parse(event, JSON_PATH_CONFIG) - for (String tag : tags) { - ctx.delete(path(tag).path) - } - filteredEvents.add(ctx.json()) - } - - return filteredEvents - } - - // Will sort traces in the following order: TEST -> SUITE -> MODULE -> SESSION - static class SortTracesByType implements Comparator> { - @Override - int compare(List o1, List o2) { - return Integer.compare(rootSpanTypeToVal(o1), rootSpanTypeToVal(o2)) - } - - int rootSpanTypeToVal(List trace) { - assert !trace.isEmpty() - def spanType = trace.get(0).getSpanType() - switch (spanType) { - case DDSpanTypes.TEST: - return 0 - case DDSpanTypes.TEST_SUITE_END: - return 1 - case DDSpanTypes.TEST_MODULE_END: - return 2 - case DDSpanTypes.TEST_SESSION_END: - return 3 - default: - return 4 - } - } - } - - static final Configuration JSON_PATH_CONFIG = Configuration.builder() - .options(Option.SUPPRESS_EXCEPTIONS) - .build() - - static final ObjectMapper JSON_MAPPER = new ObjectMapper() { { - enable(SerializationFeature.INDENT_OUTPUT) - enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) - } - } - - static final TemplateExceptionHandler SUPPRESS_EXCEPTION_HANDLER = new TemplateExceptionHandler() { - @Override - void handleTemplateException(TemplateException e, Environment environment, Writer writer) throws TemplateException { - if (e instanceof InvalidReferenceException) { - writer.write('""') - } else { - throw e - } - } - } - - static final freemarker.template.Configuration FREEMARKER = new freemarker.template.Configuration(freemarker.template.Configuration.VERSION_2_3_30) { { - setClassLoaderForTemplateLoading(CiVisibilityTestUtils.classLoader, "") - setDefaultEncoding("UTF-8") - setTemplateExceptionHandler(SUPPRESS_EXCEPTION_HANDLER) - setLogTemplateExceptions(false) - setWrapUncheckedExceptions(true) - setFallbackOnNullLoopVariable(false) - setNumberFormat("0.######") - } - } - - static String getFreemarkerTemplate(String templatePath, Map replacements, List> replacementsSource = []) { - try { - Template coveragesTemplate = FREEMARKER.getTemplate(templatePath) - StringWriter coveragesOut = new StringWriter() - coveragesTemplate.process(replacements, coveragesOut) - return coveragesOut.toString() - } catch (Exception e) { - throw new RuntimeException("Could not get Freemarker template " + templatePath + "; replacements map: " + replacements + "; replacements source: " + replacementsSource, e) - } - } - - private static final class TemplateGenerator { - private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\"(\\\$\\{.*?\\})\"") - - private final Map uniqueValues = new HashMap<>() - private final Map nonUniqueValues = new HashMap<>() - private final LabelGenerator label - - TemplateGenerator(LabelGenerator label) { - this.label = label - } - - String generateTemplate(Collection> objects, List dynamicPaths) { - for (Map object : objects) { - WriteContext ctx = JsonPath.parse(object, JSON_PATH_CONFIG) - for (DynamicPath dynamicPath : dynamicPaths) { - ctx.map(dynamicPath.path, (currentValue, config) -> { - if (dynamicPath.unique) { - return uniqueValues.computeIfAbsent(currentValue, (k) -> label.forTemplateKey(dynamicPath.rawPath)) - } - - return label.forTemplateKey(dynamicPath.rawPath) - }) - } - } - return JSON_MAPPER - .writeValueAsString(objects) - .replaceAll(PLACEHOLDER_PATTERN, "\$1") // remove quotes around placeholders - } - - Map generateReplacementMap(Collection> objects, List dynamicPaths) { - for (Map object : objects) { - ReadContext ctx = JsonPath.parse(object, JSON_PATH_CONFIG) - for (DynamicPath dynamicPath : dynamicPaths) { - def value = ctx.read(dynamicPath.path) - if (value != null) { - if (value instanceof String) { - value = '"' + // restore quotes around string values - value.replace('"', '\\"') + // escape quotes inside string values - '"' // restore quotes around string values - } - if (dynamicPath.unique) { - uniqueValues.computeIfAbsent(value, (k) -> label.forKey(dynamicPath.rawPath)) - } else { - nonUniqueValues.put(label.forKey(dynamicPath.rawPath), value) - } - } - } - } - return invert(uniqueValues) + nonUniqueValues - } - } - - private static final class LabelGenerator { - private static final Pattern ERASED_CHARS = Pattern.compile("[\\[\\]']") - private static final Pattern REPLACED_CHARS = Pattern.compile("[.-]") - - private final Map usageCounters = new HashMap<>() - - String forTemplateKey(String key) { - return "\${" + forKey(key) + "}" - } - - String forKey(String key) { - def usages = usageCounters.merge(key, 1, Integer::sum) - def sanitizedKey = key.replaceAll(ERASED_CHARS, "").replaceAll(REPLACED_CHARS, "_") - return sanitizedKey + (usages == 1 ? "" : "_${usages}") - } - } - - private static Map invert(Map map) { - Map inverted = new HashMap(map.size()) - for (Map.Entry e : map.entrySet()) { - inverted.put(e.value, e.key) - } - return inverted - } - - private static List compile(Iterable rawPaths) { - def compiledPaths = [] - for (String rawPath : rawPaths) { - compiledPaths += path(rawPath) - } - return compiledPaths - } - - private static DynamicPath path(String rawPath, boolean unique = true) { - return new DynamicPath(rawPath, JsonPath.compile(rawPath), unique) - } - - private static final class DynamicPath { - private final String rawPath - private final JsonPath path - // if true, same values are replaced with same placeholders; - // otherwise every path gets its own placeholder, even if its value is non-unique - private final boolean unique - - DynamicPath(String rawPath, JsonPath path, boolean unique) { - this.rawPath = rawPath - this.path = path - this.unique = unique - } - } - - static final class CoverageReport { - final Map event - final String report - - CoverageReport(Map event, String report) { - this.event = event - this.report = report - } - } -} diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilitySmokeTest.java b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilitySmokeTest.java new file mode 100644 index 00000000000..5bab6cc7ef6 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilitySmokeTest.java @@ -0,0 +1,303 @@ +package datadog.trace.civisibility; + +import static datadog.trace.util.ConfigStrings.propertyNameToSystemPropertyName; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import datadog.trace.api.Config; +import datadog.trace.api.civisibility.config.TestFQN; +import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.api.config.TraceInstrumentationConfig; +import datadog.trace.api.config.TracerConfig; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.io.TempDir; + +public abstract class CiVisibilitySmokeTest { + + public static final List SMOKE_IGNORED_TAGS = + Collections.unmodifiableList( + Arrays.asList("content.meta.['_dd.integration']", "content.meta.['_dd.svc_src']")); + + protected static final String AGENT_JAR = + System.getProperty("datadog.smoketest.agent.shadowJar.path"); + protected static final String TEST_ENVIRONMENT_NAME = "integration-test"; + protected static final String JAVAC_PLUGIN_VERSION = + Config.get().getCiVisibilityCompilerPluginVersion(); + protected static final String JACOCO_PLUGIN_VERSION = + Config.get().getCiVisibilityJacocoPluginVersion(); + + private static final Map DEFAULT_TRACER_CONFIG = defaultJvmArguments(); + + @TempDir protected Path prefsDir; + + protected static String buildJavaHome() { + String javaHome = System.getProperty("java.home"); + File javacPath = Paths.get(javaHome, "bin", "javac").toFile(); + if (javacPath.exists()) { + return javaHome; + } + // In CI for JDK 8, java.home may point to the JRE directory (e.g., /usr/lib/jvm/8/jre). + // The JDK with javac is in the parent directory. + File parentDir = new File(javaHome).getParentFile(); + File parentJavacPath = new File(parentDir, Paths.get("bin", "javac").toString()); + if (parentJavacPath.exists()) { + return parentDir.getAbsolutePath(); + } + // Fallback to java.home and let callers handle the error if javac is not found. + return javaHome; + } + + protected static String javaPath() { + String separator = System.getProperty("file.separator"); + return buildJavaHome() + separator + "bin" + separator + "java"; + } + + protected static String javacPath() { + String separator = System.getProperty("file.separator"); + return buildJavaHome() + separator + "bin" + separator + "javac"; + } + + private static Map defaultJvmArguments() { + Map argMap = new HashMap<>(); + argMap.put(GeneralConfig.TRACE_DEBUG, "true"); + argMap.put(GeneralConfig.ENV, TEST_ENVIRONMENT_NAME); + argMap.put(CiVisibilityConfig.CIVISIBILITY_ENABLED, "true"); + argMap.put(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED, "true"); + argMap.put(CiVisibilityConfig.CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED, "false"); + argMap.put(CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED, "false"); + argMap.put(CiVisibilityConfig.CIVISIBILITY_GIT_CLIENT_ENABLED, "false"); + argMap.put(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES, "true"); + argMap.put(CiVisibilityConfig.CIVISIBILITY_COMPILER_PLUGIN_VERSION, JAVAC_PLUGIN_VERSION); + argMap.put(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, "false"); + return argMap; + } + + private static Map buildJvmArgMap( + String mockBackendIntakeUrl, String serviceName, Map additionalArgs) { + Map argMap = new HashMap<>(DEFAULT_TRACER_CONFIG); + argMap.put(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL, mockBackendIntakeUrl); + argMap.put(CiVisibilityConfig.CIVISIBILITY_INTAKE_AGENTLESS_URL, mockBackendIntakeUrl); + argMap.put(TracerConfig.TRACE_AGENT_URL, mockBackendIntakeUrl); + argMap.putAll(additionalArgs); + + if (serviceName != null) { + argMap.put(GeneralConfig.SERVICE_NAME, serviceName); + } + + return argMap; + } + + protected List buildJvmArguments( + String mockBackendIntakeUrl, String serviceName, Map additionalArgs) { + List arguments = new ArrayList<>(Arrays.asList("-Xms256m", "-Xmx256m")); + + arguments.add(preventJulPrefsFileLock()); + + Map argMap = buildJvmArgMap(mockBackendIntakeUrl, serviceName, additionalArgs); + + // Convenience switches for local debugging. Set as JVM system properties (e.g. via + // `-Ddatadog.civisibility.smoketest.debug.parent=1`) rather than env vars, to keep the + // config-inversion-linter happy (it forbids unregistered `DD_…` env-var literals in + // `src/main/java`) and to avoid `System.getenv` in main sources. + if (System.getProperty("datadog.civisibility.smoketest.debug.parent") != null) { + arguments.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"); + } + if (System.getProperty("datadog.civisibility.smoketest.debug.child") != null) { + argMap.put(CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT, "5055"); + } + + String agentArgs = + argMap.entrySet().stream() + .map(e -> propertyNameToSystemPropertyName(e.getKey()) + "=" + e.getValue()) + .collect(Collectors.joining(",")); + arguments.add("-javaagent:" + AGENT_JAR + "=" + agentArgs); + + return arguments; + } + + /** + * Trick to prevent jul Prefs file lock issue on forked processes, in particular in CI which runs + * on Linux and have competing processes trying to write to it, including the Gradle daemon. + * + * 

{@code
+   * Couldn't flush user prefs: java.util.prefs.BackingStoreException: Couldn't get file lock.
+   * }
+ * + * Note, some tests can setup arguments on spec level, so {@code prefsDir} will be {@code null} + * during {@code @BeforeAll}. + */ + protected String preventJulPrefsFileLock() { + Path resolved = prefsDir != null ? prefsDir : tempUserPrefsPath(); + return "-Djava.util.prefs.userRoot=" + resolved.toAbsolutePath(); + } + + private static Path tempUserPrefsPath() { + String uniqueId = + System.currentTimeMillis() + "_" + System.nanoTime() + "_" + Thread.currentThread().getId(); + return Paths.get(System.getProperty("java.io.tmpdir"), "gradle-test-userPrefs", uniqueId); + } + + protected void verifyEventsAndCoverages( + String projectName, + String toolchain, + String toolchainVersion, + List> events, + List> coverages) { + verifyEventsAndCoverages( + projectName, toolchain, toolchainVersion, events, coverages, Collections.emptyList()); + } + + protected void verifyEventsAndCoverages( + String projectName, + String toolchain, + String toolchainVersion, + List> events, + List> coverages, + List additionalDynamicTags) { + Map additionalReplacements = new HashMap<>(); + additionalReplacements.put( + "content.meta.['test.toolchain']", toolchain + ":" + toolchainVersion); + + if (System.getenv("GENERATE_TEST_FIXTURES") != null) { + String baseTemplatesPath; + try { + baseTemplatesPath = + CiVisibilitySmokeTest.class + .getClassLoader() + .getResource(projectName) + .toURI() + .getSchemeSpecificPart() + .replace("build/resources/test", "src/test/resources"); + } catch (Exception e) { + throw new RuntimeException(e); + } + List dynamicPaths = new ArrayList<>(additionalReplacements.keySet()); + dynamicPaths.addAll(additionalDynamicTags); + CiVisibilityTestUtils.generateTemplates( + baseTemplatesPath, events, coverages, dynamicPaths, SMOKE_IGNORED_TAGS); + } else { + CiVisibilityTestUtils.assertData( + projectName, + events, + coverages, + additionalReplacements, + SMOKE_IGNORED_TAGS, + additionalDynamicTags); + } + } + + protected TestFQN test(String suiteName, String testName) { + return new TestFQN(suiteName, testName); + } + + protected void verifyTestOrder(List> events, List expectedOrder) { + CiVisibilityTestUtils.assertTestsOrder(events, expectedOrder); + } + + /** + * This is a basic sanity check for telemetry metrics. It only checks that the reported number of + * events created and finished is as expected. + * + *

Currently the check is not performed for Gradle builds: Gradle daemon started with Gradle + * TestKit outlives the test, so the final telemetry flush happens after the assertions. + */ + protected void verifyTelemetryMetrics( + List> receivedTelemetryMetrics, + List> receivedTelemetryDistributions, + int expectedEventsCount) { + int eventsCreated = 0; + int eventsFinished = 0; + for (Map metric : receivedTelemetryMetrics) { + if ("event_created".equals(metric.get("metric"))) { + for (Object point : (List) metric.get("points")) { + eventsCreated += ((Number) ((List) point).get(1)).intValue(); + } + } + if ("event_finished".equals(metric.get("metric"))) { + for (Object point : (List) metric.get("points")) { + eventsFinished += ((Number) ((List) point).get(1)).intValue(); + } + } + } + assertEquals(expectedEventsCount, eventsCreated); + assertEquals(expectedEventsCount, eventsFinished); + + // an even more basic smoke check for distributions: assert that we received some + assertFalse(receivedTelemetryDistributions.isEmpty()); + } + + protected void verifyCoverageReports( + String projectName, + List reports, + Map replacements) { + CiVisibilityTestUtils.assertData(projectName, reports, replacements); + } + + protected static void verifySnapshotLogs( + List> receivedLogs, int expectedProbes, int expectedSnapshots) { + int logsPerProbe = 3; // 3 probe statuses per probe -> received, installed, emitting + + assertEquals(logsPerProbe * expectedProbes + expectedSnapshots, receivedLogs.size()); + + List> probeStatusLogs = new ArrayList<>(); + List> snapshotLogs = new ArrayList<>(); + for (Map log : receivedLogs) { + if (log.containsKey("message")) { + probeStatusLogs.add(log); + } else { + snapshotLogs.add(log); + } + } + + verifyProbeStatuses(probeStatusLogs, expectedProbes); + verifySnapshots(snapshotLogs, expectedSnapshots); + } + + private static void verifyProbeStatuses(List> logs, int expectedCount) { + long received = + logs.stream() + .filter(log -> ((String) log.get("message")).startsWith("Received probe")) + .count(); + long installed = + logs.stream() + .filter(log -> ((String) log.get("message")).startsWith("Installed probe")) + .count(); + long emitting = + logs.stream().filter(log -> ((String) log.get("message")).endsWith("is emitting.")).count(); + assertEquals(expectedCount, received); + assertEquals(expectedCount, installed); + assertEquals(expectedCount, emitting); + } + + protected static void verifySnapshots(List> logs, int expectedCount) { + assertEquals(expectedCount, logs.size()); + + List requiredLogFields = + Arrays.asList("logger.name", "logger.method", "dd.spanid", "dd.traceid"); + List requiredSnapshotFields = + Arrays.asList("captures", "exceptionId", "probe", "stack"); + + for (Map log : logs) { + requiredLogFields.forEach(log::containsKey); + + @SuppressWarnings("unchecked") + Map debuggerMap = (Map) log.get("debugger"); + @SuppressWarnings("unchecked") + Map snapshotContent = (Map) debuggerMap.get("snapshot"); + + assertNotNull(snapshotContent, "snapshot must not be null"); + requiredSnapshotFields.forEach(snapshotContent::containsKey); + } + } +} diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTableTestConverters.java b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTableTestConverters.java new file mode 100644 index 00000000000..d4a1bfc567c --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTableTestConverters.java @@ -0,0 +1,17 @@ +package datadog.trace.civisibility; + +import datadog.trace.api.civisibility.config.TestFQN; +import org.tabletest.junit.TypeConverter; + +/** Shared {@code @TableTest} converters for CiVisibility smoke tests. */ +public final class CiVisibilityTableTestConverters { + + private CiVisibilityTableTestConverters() {} + + /** Parses a {@code suite:name} string into a {@link TestFQN}. */ + @TypeConverter + public static TestFQN toTestFQN(String value) { + int colon = value.indexOf(':'); + return new TestFQN(value.substring(0, colon), value.substring(colon + 1)); + } +} diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java new file mode 100644 index 00000000000..fb9ff951a88 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java @@ -0,0 +1,546 @@ +package datadog.trace.civisibility; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.ReadContext; +import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.civisibility.config.LibraryCapability; +import datadog.trace.api.civisibility.config.TestFQN; +import datadog.trace.core.DDSpan; +import freemarker.core.Environment; +import freemarker.core.InvalidReferenceException; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import freemarker.template.TemplateExceptionHandler; +import java.io.Serializable; +import java.io.StringWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.xml.parsers.DocumentBuilderFactory; +import org.junit.jupiter.api.Assertions; +import org.opentest4j.AssertionFailedError; +import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompareMode; +import org.w3c.dom.Document; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.builder.Input; +import org.xmlunit.diff.Diff; +import org.xmlunit.util.Convert; + +public abstract class CiVisibilityTestUtils { + + public static final List EVENT_DYNAMIC_PATHS = + Collections.unmodifiableList( + Arrays.asList( + path("content.trace_id"), + path("content.span_id"), + path("content.parent_id"), + path("content.test_session_id"), + path("content.test_module_id"), + path("content.test_suite_id"), + path("content.metrics.process_id"), + path("content.meta.['os.architecture']"), + path("content.meta.['os.platform']"), + path("content.meta.['os.version']"), + path("content.meta.['runtime.name']"), + path("content.meta.['runtime.vendor']"), + path("content.meta.['runtime.version']"), + path("content.meta.['ci.workspace_path']"), + path("content.meta.['error.message']"), + path("content.meta.library_version"), + path("content.meta.runtime-id"), + path("content.meta.['_dd.tracer_host']"), + // Different events might or might not have the same start or duration. Regardless, + // the values of these fields should be treated as different. + path("content.start", false), + path("content.duration", false), + path("content.metrics.['_dd.host.vcpu_count']", false), + path("content.meta.['_dd.p.tid']", false), + path("content.meta.['error.stack']", false))); + + // ignored tags on assertion and fixture build + public static final List IGNORED_TAGS; + + static { + List ignored = + Arrays.stream(LibraryCapability.values()) + .map(c -> "content.meta.['" + c.asTag() + "']") + .collect(Collectors.toList()); + ignored.add("content.meta.['_dd.integration']"); + ignored.add("content.meta.['_dd.svc_src']"); + IGNORED_TAGS = Collections.unmodifiableList(ignored); + } + + public static final List COVERAGE_DYNAMIC_PATHS = + Collections.unmodifiableList( + Arrays.asList(path("test_session_id"), path("test_suite_id"), path("span_id"))); + + private static final Comparator> EVENT_RESOURCE_COMPARATOR = + Comparator., String>comparing( + m -> { + Map content = (Map) m.get("content"); + return (String) content.get("resource"); + }) + .thenComparing( + Comparator., String>comparing( + // module and session have the same resource name in headless mode + m -> (String) m.get("type")) + .reversed()); + + /** Use this method to generate expected data templates. */ + public static void generateTemplates( + String baseTemplatesPath, + List> events, + List> coverages, + Collection additionalDynamicPaths) { + generateTemplates( + baseTemplatesPath, events, coverages, additionalDynamicPaths, Collections.emptyList()); + } + + public static void generateTemplates( + String baseTemplatesPath, + List> events, + List> coverages, + Collection additionalDynamicPaths, + List ignoredTags) { + List> mutableEvents = new ArrayList<>(events); + if (!ignoredTags.isEmpty()) { + mutableEvents = removeTags(mutableEvents, ignoredTags); + } + mutableEvents.sort(EVENT_RESOURCE_COMPARATOR); + + TemplateGenerator templateGenerator = new TemplateGenerator(new LabelGenerator()); + List compiledAdditionalReplacements = compile(additionalDynamicPaths); + + try { + Files.createDirectories(Paths.get(baseTemplatesPath)); + List eventPaths = new ArrayList<>(EVENT_DYNAMIC_PATHS); + eventPaths.addAll(compiledAdditionalReplacements); + Files.write( + Paths.get(baseTemplatesPath, "events.ftl"), + templateGenerator + .generateTemplate(mutableEvents, eventPaths) + .getBytes(StandardCharsets.UTF_8)); + + List coveragePaths = new ArrayList<>(COVERAGE_DYNAMIC_PATHS); + coveragePaths.addAll(compiledAdditionalReplacements); + Files.write( + Paths.get(baseTemplatesPath, "coverages.ftl"), + templateGenerator + .generateTemplate(coverages, coveragePaths) + .getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static void assertData( + String baseTemplatesPath, List reports, Map replacements) { + try { + String expectedReportEvent = + getFreemarkerTemplate(baseTemplatesPath + "/coverage_report_event.ftl", replacements); + String actualReportEvent = JSON_MAPPER.writeValueAsString(reports.get(0).event); + + compareJson(expectedReportEvent, actualReportEvent); + + String expectedReport = + getFreemarkerTemplate(baseTemplatesPath + "/coverage_report.ftl", replacements); + String actualReport = reports.get(0).report; + + if (expectedReport.contains(" assertData( + String baseTemplatesPath, + List> events, + List> coverages, + Map additionalReplacements, + List ignoredTags) { + return assertData( + baseTemplatesPath, + events, + coverages, + additionalReplacements, + ignoredTags, + Collections.emptyList()); + } + + public static Map assertData( + String baseTemplatesPath, + List> events, + List> coverages, + Map additionalReplacements, + List ignoredTags, + List additionalDynamicPaths) { + List> mutableEvents = new ArrayList<>(events); + mutableEvents.sort(EVENT_RESOURCE_COMPARATOR); + + LabelGenerator labelGenerator = new LabelGenerator(); + TemplateGenerator templateGenerator = new TemplateGenerator(labelGenerator); + + List eventPaths = new ArrayList<>(EVENT_DYNAMIC_PATHS); + eventPaths.addAll(compile(additionalDynamicPaths)); + templateGenerator.generateReplacementMap(mutableEvents, eventPaths); + Map replacementMap = + templateGenerator.generateReplacementMap(coverages, COVERAGE_DYNAMIC_PATHS); + + // Tolerate Groovy callers passing GString values: convert each value to String via + // String.valueOf before storing it in the replacement map. + for (Map.Entry e : additionalReplacements.entrySet()) { + replacementMap.put( + labelGenerator.forKey(e.getKey()), "\"" + String.valueOf(e.getValue()) + "\""); + } + + // ignore provided tags + mutableEvents = removeTags(mutableEvents, ignoredTags); + + try { + String expectedEvents = + getFreemarkerTemplate(baseTemplatesPath + "/events.ftl", replacementMap, mutableEvents); + String actualEvents = JSON_MAPPER.writeValueAsString(mutableEvents); + + compareJson(expectedEvents, actualEvents); + + String expectedCoverages = + getFreemarkerTemplate(baseTemplatesPath + "/coverages.ftl", replacementMap, coverages); + String actualCoverages = JSON_MAPPER.writeValueAsString(coverages); + compareJson(expectedCoverages, actualCoverages); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return replacementMap; + } + + private static void compareJson(String expectedJson, String actualJson) { + Map environment = System.getenv(); + boolean ciRun = + environment.get("GITHUB_ACTION") != null || environment.get("GITLAB_CI") != null; + JSONCompareMode comparisonMode = + ciRun ? JSONCompareMode.LENIENT : JSONCompareMode.NON_EXTENSIBLE; + + try { + JSONAssert.assertEquals(expectedJson, actualJson, comparisonMode); + } catch (org.json.JSONException jsonException) { + throw new RuntimeException(jsonException); + } catch (AssertionError e) { + if (ciRun) { + // When running in CI the assertion error message does not contain the actual diff, + // so we print the events to the console to help debug the issue. + System.out.println("Expected JSON: " + expectedJson); + System.out.println("Actual JSON: " + actualJson); + } + throw new AssertionFailedError( + "Expected and actual JSON mismatch", expectedJson, actualJson, e); + } + } + + public static boolean assertTestsOrder( + List> events, List expectedOrder) { + List identifiers = getTestIdentifiers(events); + if (!identifiers.equals(expectedOrder)) { + throw new AssertionError("Expected order: " + expectedOrder + ", but got: " + identifiers); + } + return true; + } + + public static List getTestIdentifiers(List> events) { + List> sorted = new ArrayList<>(events); + sorted.sort( + Comparator.comparing( + it -> ((Number) ((Map) it.get("content")).get("start")).longValue())); + List testIdentifiers = new ArrayList<>(); + for (Map event : sorted) { + Map content = (Map) event.get("content"); + Map meta = (Map) content.get("meta"); + Object testName = meta.get("test.name"); + if (testName != null) { + testIdentifiers.add(new TestFQN((String) meta.get("test.suite"), (String) testName)); + } + } + return testIdentifiers; + } + + public static List> removeTags(List> events, List tags) { + List> filteredEvents = new ArrayList<>(); + for (Map event : events) { + DocumentContext ctx = JsonPath.parse(event, JSON_PATH_CONFIG); + for (String tag : tags) { + ctx.delete(path(tag).path); + } + filteredEvents.add(ctx.json()); + } + return filteredEvents; + } + + // Sort traces in the following order: TEST -> SUITE -> MODULE -> SESSION + public static class SortTracesByType implements Comparator>, Serializable { + private static final long serialVersionUID = 1L; + + @Override + public int compare(List o1, List o2) { + return Integer.compare(rootSpanTypeToVal(o1), rootSpanTypeToVal(o2)); + } + + public int rootSpanTypeToVal(List trace) { + assert !trace.isEmpty(); + CharSequence spanType = trace.get(0).getSpanType(); + if (spanType == null) { + return 4; + } + if (DDSpanTypes.TEST.contentEquals(spanType)) { + return 0; + } + if (DDSpanTypes.TEST_SUITE_END.contentEquals(spanType)) { + return 1; + } + if (DDSpanTypes.TEST_MODULE_END.contentEquals(spanType)) { + return 2; + } + if (DDSpanTypes.TEST_SESSION_END.contentEquals(spanType)) { + return 3; + } + return 4; + } + } + + public static final Configuration JSON_PATH_CONFIG = + Configuration.builder().options(Option.SUPPRESS_EXCEPTIONS).build(); + + public static final ObjectMapper JSON_MAPPER = createJsonMapper(); + + private static ObjectMapper createJsonMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + mapper.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + return mapper; + } + + public static final TemplateExceptionHandler SUPPRESS_EXCEPTION_HANDLER = + new TemplateExceptionHandler() { + @Override + public void handleTemplateException( + TemplateException e, Environment environment, Writer writer) throws TemplateException { + if (e instanceof InvalidReferenceException) { + try { + writer.write("\"\""); + } catch (java.io.IOException ioe) { + throw new TemplateException(ioe, environment); + } + } else { + throw e; + } + } + }; + + public static final freemarker.template.Configuration FREEMARKER = createFreemarker(); + + private static freemarker.template.Configuration createFreemarker() { + freemarker.template.Configuration config = + new freemarker.template.Configuration(freemarker.template.Configuration.VERSION_2_3_30); + config.setClassLoaderForTemplateLoading(CiVisibilityTestUtils.class.getClassLoader(), ""); + config.setDefaultEncoding("UTF-8"); + config.setTemplateExceptionHandler(SUPPRESS_EXCEPTION_HANDLER); + config.setLogTemplateExceptions(false); + config.setWrapUncheckedExceptions(true); + config.setFallbackOnNullLoopVariable(false); + config.setNumberFormat("0.######"); + return config; + } + + public static String getFreemarkerTemplate(String templatePath, Map replacements) { + return getFreemarkerTemplate(templatePath, replacements, Collections.emptyList()); + } + + public static String getFreemarkerTemplate( + String templatePath, + Map replacements, + List> replacementsSource) { + try { + Template template = FREEMARKER.getTemplate(templatePath); + StringWriter out = new StringWriter(); + template.process(replacements, out); + return out.toString(); + } catch (Exception e) { + throw new RuntimeException( + "Could not get Freemarker template " + + templatePath + + "; replacements map: " + + replacements + + "; replacements source: " + + replacementsSource, + e); + } + } + + private static final class TemplateGenerator { + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\"(\\$\\{.*?\\})\""); + + private final Map uniqueValues = new HashMap<>(); + private final Map nonUniqueValues = new HashMap<>(); + private final LabelGenerator label; + + TemplateGenerator(LabelGenerator label) { + this.label = label; + } + + String generateTemplate(Collection> objects, List dynamicPaths) + throws Exception { + for (Map object : objects) { + DocumentContext ctx = JsonPath.parse(object, JSON_PATH_CONFIG); + for (DynamicPath dynamicPath : dynamicPaths) { + ctx.map( + dynamicPath.path, + (currentValue, config) -> { + if (dynamicPath.unique) { + return uniqueValues.computeIfAbsent( + String.valueOf(currentValue), k -> label.forTemplateKey(dynamicPath.rawPath)); + } + return label.forTemplateKey(dynamicPath.rawPath); + }); + } + } + // remove quotes around placeholders + return PLACEHOLDER_PATTERN.matcher(JSON_MAPPER.writeValueAsString(objects)).replaceAll("$1"); + } + + Map generateReplacementMap( + Collection> objects, List dynamicPaths) { + for (Map object : objects) { + ReadContext ctx = JsonPath.parse(object, JSON_PATH_CONFIG); + for (DynamicPath dynamicPath : dynamicPaths) { + Object value = ctx.read(dynamicPath.path); + if (value != null) { + String stringValue; + if (value instanceof String) { + stringValue = "\"" + ((String) value).replace("\"", "\\\"") + "\""; + } else { + stringValue = String.valueOf(value); + } + if (dynamicPath.unique) { + uniqueValues.computeIfAbsent(stringValue, k -> label.forKey(dynamicPath.rawPath)); + } else { + nonUniqueValues.put(label.forKey(dynamicPath.rawPath), stringValue); + } + } + } + } + Map result = new LinkedHashMap<>(invert(uniqueValues)); + result.putAll(nonUniqueValues); + return result; + } + } + + private static final class LabelGenerator { + private static final Pattern ERASED_CHARS = Pattern.compile("[\\[\\]']"); + private static final Pattern REPLACED_CHARS = Pattern.compile("[.-]"); + + private final Map usageCounters = new HashMap<>(); + + String forTemplateKey(String key) { + return "${" + forKey(key) + "}"; + } + + String forKey(String key) { + int usages = usageCounters.merge(key, 1, Integer::sum); + String sanitizedKey = ERASED_CHARS.matcher(key).replaceAll(""); + sanitizedKey = REPLACED_CHARS.matcher(sanitizedKey).replaceAll("_"); + return sanitizedKey + (usages == 1 ? "" : "_" + usages); + } + } + + private static Map invert(Map map) { + Map inverted = new HashMap<>(map.size()); + for (Map.Entry e : map.entrySet()) { + inverted.put(e.getValue(), e.getKey()); + } + return inverted; + } + + private static List compile(Iterable rawPaths) { + List compiledPaths = new ArrayList<>(); + for (String rawPath : rawPaths) { + compiledPaths.add(path(rawPath)); + } + return compiledPaths; + } + + private static DynamicPath path(String rawPath) { + return path(rawPath, true); + } + + private static DynamicPath path(String rawPath, boolean unique) { + return new DynamicPath(rawPath, JsonPath.compile(rawPath), unique); + } + + public static final class DynamicPath { + private final String rawPath; + private final JsonPath path; + // if true, same values are replaced with same placeholders; + // otherwise every path gets its own placeholder, even if its value is non-unique + private final boolean unique; + + DynamicPath(String rawPath, JsonPath path, boolean unique) { + this.rawPath = rawPath; + this.path = path; + this.unique = unique; + } + } + + public static final class CoverageReport { + public final Map event; + public final String report; + + public CoverageReport(Map event, String report) { + this.event = event; + this.report = report; + } + } +} diff --git a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/AbstractGradleTest.groovy b/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/AbstractGradleTest.groovy deleted file mode 100644 index 5328b4bacac..00000000000 --- a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/AbstractGradleTest.groovy +++ /dev/null @@ -1,143 +0,0 @@ -package datadog.smoketest - -import datadog.environment.JavaVirtualMachine -import datadog.trace.civisibility.CiVisibilitySmokeTest -import datadog.trace.util.ComparableVersion -import org.gradle.internal.impldep.org.apache.commons.io.FileUtils -import org.junit.jupiter.api.Assumptions -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.TempDir -import spock.util.environment.Jvm - -import java.nio.charset.StandardCharsets -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes -import java.util.regex.Matcher -import java.util.regex.Pattern - -class AbstractGradleTest extends CiVisibilitySmokeTest { - - static final String LATEST_GRADLE_VERSION = getLatestGradleVersion() - - // test resources use this instead of ".gradle" to avoid unwanted evaluation - private static final String GRADLE_TEST_RESOURCE_EXTENSION = ".gradleTest" - private static final String GRADLE_REGULAR_EXTENSION = ".gradle" - - private static final ComparableVersion GRADLE_9 = new ComparableVersion("9.0.0") - - @TempDir - protected Path projectFolder - - @Shared - @AutoCleanup - protected MockBackend mockBackend = new MockBackend() - - def setup() { - mockBackend.reset() - } - - private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile('\\$\\{(.+?)\\}') - - protected void givenGradleProjectFiles(String projectFilesSources, Map> replacementsByFileName = [:]) { - def projectResourcesUri = this.getClass().getClassLoader().getResource(projectFilesSources).toURI() - def projectResourcesPath = Paths.get(projectResourcesUri) - FileUtils.copyDirectory(projectResourcesPath.toFile(), projectFolder.toFile()) - - Files.walkFileTree(projectFolder, new SimpleFileVisitor() { - @Override - FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - def replacements = replacementsByFileName.get(file.getFileName().toString()) - if (replacements != null) { - def fileContents = new String(Files.readAllBytes(file), StandardCharsets.UTF_8) - Matcher matcher = PLACEHOLDER_PATTERN.matcher(fileContents) - - StringBuffer result = new StringBuffer() - while (matcher.find()) { - String propertyName = matcher.group(1) - String replacement = replacements.getOrDefault(propertyName, matcher.group(0)) - matcher.appendReplacement(result, Matcher.quoteReplacement(replacement)) - } - matcher.appendTail(result) - - Files.write(file, result.toString().getBytes(StandardCharsets.UTF_8)) - } - - if (file.toString().endsWith(GRADLE_TEST_RESOURCE_EXTENSION)) { - def fileWithFixedExtension = Paths.get(file.toString().replace(GRADLE_TEST_RESOURCE_EXTENSION, GRADLE_REGULAR_EXTENSION)) - Files.move(file, fileWithFixedExtension) - } - - return FileVisitResult.CONTINUE - } - }) - - // creating empty .git directory so that the tracer could detect projectFolder as repo root - Files.createDirectory(projectFolder.resolve(".git")) - } - - protected void givenGradleVersionIsCompatibleWithCurrentJvm(String gradleVersion) { - Assumptions.assumeTrue(isSupported(new ComparableVersion(gradleVersion)), - "Current JVM " + Jvm.current.javaVersion + " does not support Gradle version " + gradleVersion) - } - - private static boolean isSupported(ComparableVersion gradleVersion) { - // https://docs.gradle.org/current/userguide/compatibility.html - if (Jvm.current.isJavaVersionCompatible(26)) { - return gradleVersion.compareTo(new ComparableVersion("9.4")) >= 0 - } else if (Jvm.current.isJavaVersionCompatible(25)) { - return gradleVersion.compareTo(new ComparableVersion("9.1")) >= 0 - } else if (Jvm.current.isJavaVersionCompatible(24)) { - return gradleVersion.compareTo(new ComparableVersion("8.14")) >= 0 - } else if (Jvm.current.java21Compatible) { - return gradleVersion.compareTo(new ComparableVersion("8.4")) >= 0 - } else if (Jvm.current.java20) { - return gradleVersion.compareTo(new ComparableVersion("8.1")) >= 0 - } else if (Jvm.current.java19) { - return gradleVersion.compareTo(new ComparableVersion("7.6")) >= 0 - } else if (Jvm.current.java18) { - return gradleVersion.compareTo(new ComparableVersion("7.5")) >= 0 - } else if (Jvm.current.java17) { - return gradleVersion.compareTo(new ComparableVersion("7.3")) >= 0 - } else if (Jvm.current.java16) { - return gradleVersion.isWithin(new ComparableVersion("7.0"), GRADLE_9) - } else if (Jvm.current.java15) { - return gradleVersion.isWithin(new ComparableVersion("6.7"), GRADLE_9) - } else if (Jvm.current.java14) { - return gradleVersion.isWithin(new ComparableVersion("6.3"), GRADLE_9) - } else if (Jvm.current.java13) { - return gradleVersion.isWithin(new ComparableVersion("6.0"), GRADLE_9) - } else if (Jvm.current.java12) { - return gradleVersion.isWithin(new ComparableVersion("5.4"), GRADLE_9) - } else if (Jvm.current.java11) { - return gradleVersion.isWithin(new ComparableVersion("5.0"), GRADLE_9) - } else if (Jvm.current.java10) { - return gradleVersion.isWithin(new ComparableVersion("4.7"), GRADLE_9) - } else if (Jvm.current.java9) { - return gradleVersion.isWithin(new ComparableVersion("4.3"), GRADLE_9) - } else if (Jvm.current.java8) { - return gradleVersion.isWithin(new ComparableVersion("2.0"), GRADLE_9) - } - return false - } - - protected void givenConfigurationCacheIsCompatibleWithCurrentPlatform(boolean configurationCacheEnabled) { - if (configurationCacheEnabled) { - Assumptions.assumeFalse(JavaVirtualMachine.isIbm8(), "Configuration cache is not compatible with IBM 8") - } - } - - private static String getLatestGradleVersion() { - def properties = new Properties() - def stream = AbstractGradleTest.classLoader.getResourceAsStream("latest-tool-versions.properties") - if (stream == null) { - throw new IllegalStateException("Could not find latest-tool-versions.properties on classpath") - } - stream.withCloseable { properties.load(it) } - return properties.getProperty("gradle.version") - } -} diff --git a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy b/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy deleted file mode 100644 index 2b240f47ab9..00000000000 --- a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy +++ /dev/null @@ -1,261 +0,0 @@ -package datadog.smoketest - -import datadog.environment.JavaVirtualMachine -import datadog.trace.api.config.CiVisibilityConfig -import datadog.trace.api.config.GeneralConfig -import datadog.trace.api.config.TraceInstrumentationConfig -import java.nio.file.Files -import java.nio.file.Path -import org.gradle.testkit.runner.BuildResult -import org.gradle.testkit.runner.GradleRunner -import org.gradle.testkit.runner.TaskOutcome -import org.gradle.util.DistributionLocator -import org.gradle.util.GradleVersion -import org.gradle.wrapper.Download -import org.gradle.wrapper.GradleUserHomeLookup -import org.gradle.wrapper.Install -import org.gradle.wrapper.PathAssembler -import org.gradle.wrapper.WrapperConfiguration -import spock.lang.IgnoreIf -import spock.lang.Shared -import spock.lang.TempDir - -class GradleDaemonSmokeTest extends AbstractGradleTest { - - private static final String TEST_SERVICE_NAME = "test-gradle-service" - - private static final int GRADLE_DISTRIBUTION_NETWORK_TIMEOUT = 30_000 // Gradle's default timeout is 10s - - @Shared - @TempDir - Path testKitFolder - - @IgnoreIf(reason = "Jacoco plugin does not work with OpenJ9 in older Gradle versions", value = { - JavaVirtualMachine.isJ9() - }) - def "test legacy #projectName, v#gradleVersion"() { - runGradleTest(gradleVersion, projectName, false, successExpected, false, expectedTraces, expectedCoverages) - - where: - gradleVersion | projectName | successExpected | expectedTraces | expectedCoverages - "3.5" | "test-succeed-old-gradle" | true | 5 | 1 - "7.6.4" | "test-succeed-legacy-instrumentation" | true | 5 | 1 - "7.6.4" | "test-succeed-multi-module-legacy-instrumentation" | true | 7 | 2 - "7.6.4" | "test-succeed-multi-forks-legacy-instrumentation" | true | 6 | 2 - "7.6.4" | "test-skip-legacy-instrumentation" | true | 2 | 0 - "7.6.4" | "test-failed-legacy-instrumentation" | false | 4 | 0 - "7.6.4" | "test-corrupted-config-legacy-instrumentation" | false | 1 | 0 - } - - def "test #projectName, v#gradleVersion, configCache: #configurationCache"() { - runGradleTest(gradleVersion, projectName, configurationCache, successExpected, flakyRetries, expectedTraces, expectedCoverages) - - where: - gradleVersion | projectName | configurationCache | successExpected | flakyRetries | expectedTraces | expectedCoverages - "8.3" | "test-succeed-new-instrumentation" | false | true | false | 5 | 1 - "8.9" | "test-succeed-new-instrumentation" | false | true | false | 5 | 1 - LATEST_GRADLE_VERSION | "test-succeed-new-instrumentation" | false | true | false | 5 | 1 - "8.3" | "test-succeed-new-instrumentation" | true | true | false | 5 | 1 - "8.9" | "test-succeed-new-instrumentation" | true | true | false | 5 | 1 - LATEST_GRADLE_VERSION | "test-succeed-new-instrumentation" | true | true | false | 5 | 1 - LATEST_GRADLE_VERSION | "test-succeed-multi-module-new-instrumentation" | false | true | false | 7 | 2 - LATEST_GRADLE_VERSION | "test-succeed-multi-forks-new-instrumentation" | false | true | false | 6 | 2 - LATEST_GRADLE_VERSION | "test-skip-new-instrumentation" | false | true | false | 2 | 0 - LATEST_GRADLE_VERSION | "test-failed-new-instrumentation" | false | false | false | 4 | 0 - LATEST_GRADLE_VERSION | "test-corrupted-config-new-instrumentation" | false | false | false | 1 | 0 - LATEST_GRADLE_VERSION | "test-succeed-junit-5" | false | true | false | 5 | 1 - LATEST_GRADLE_VERSION | "test-failed-flaky-retries" | false | false | true | 8 | 0 - // TODO: add back LATEST_GRADLE_VERSION after fixing in Gradle 9.4.0 - "9.3.1" | "test-succeed-gradle-plugin-test" | false | true | false | 5 | 0 - } - - def "test junit4 class ordering v#gradleVersion"() { - givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion) - givenGradleProjectFiles(projectName) - givenGradleProjectProperties() - ensureDependenciesDownloaded(gradleVersion) - - mockBackend.givenKnownTests(true) - for (flakyTest in flakyTests) { - mockBackend.givenFlakyTest(":test", flakyTest.getSuite(), flakyTest.getName()) - mockBackend.givenKnownTest(":test", flakyTest.getSuite(), flakyTest.getName()) - } - - BuildResult buildResult = runGradleTests(gradleVersion, true, false) - assertBuildSuccessful(buildResult) - - verifyTestOrder(mockBackend.waitForEvents(eventsNumber), expectedOrder) - - where: - gradleVersion | projectName | flakyTests | expectedOrder | eventsNumber - "7.6.4" | "test-succeed-junit-4-class-ordering" | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedC", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another") - ] | 15 - // TODO: add back LATEST_GRADLE_VERSION after fixing ordering on Gradle 9.3.0 - "9.2.1" | "test-succeed-junit-4-class-ordering" | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedC", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another") - ] | 15 - } - - private runGradleTest(String gradleVersion, String projectName, boolean configurationCache, boolean successExpected, boolean flakyRetries, int expectedTraces, int expectedCoverages) { - givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion) - givenConfigurationCacheIsCompatibleWithCurrentPlatform(configurationCache) - givenGradleProjectFiles(projectName) - givenGradleProjectProperties() - ensureDependenciesDownloaded(gradleVersion) - - mockBackend.givenFlakyRetries(flakyRetries) - mockBackend.givenFlakyTest(":test", "datadog.smoke.TestFailed", "test_failed") - - mockBackend.givenTestsSkipping(true) - mockBackend.givenSkippableTest(":test", "datadog.smoke.TestSucceed", "test_to_skip_with_itr", [:]) - - BuildResult buildResult = runGradleTests(gradleVersion, successExpected, configurationCache) - - if (successExpected) { - assertBuildSuccessful(buildResult) - } - - verifyEventsAndCoverages(projectName, "gradle", gradleVersion, mockBackend.waitForEvents(expectedTraces), mockBackend.waitForCoverages(expectedCoverages)) - - if (configurationCache) { - // if configuration cache is enabled, run the build one more time - // to verify that building with existing configuration cache entry works - BuildResult buildResultWithConfigCacheEntry = runGradleTests(gradleVersion, successExpected, configurationCache) - - assertBuildSuccessful(buildResultWithConfigCacheEntry) - verifyEventsAndCoverages(projectName, "gradle", gradleVersion, mockBackend.waitForEvents(expectedTraces), mockBackend.waitForCoverages(expectedCoverages)) - } - } - - private void givenGradleProjectProperties() { - assert new File(AGENT_JAR).isFile() - - def ddApiKeyPath = testKitFolder.resolve(".dd.api.key") - Files.write(ddApiKeyPath, "dummy".getBytes()) - - def additionalArgs = [ - (GeneralConfig.API_KEY_FILE) : ddApiKeyPath.toAbsolutePath().toString(), - (CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION): JACOCO_PLUGIN_VERSION, - /* - * Some of the smoke tests (in particular the one with the Gradle plugin), are using Gradle Test Kit for their tests. - * Gradle Test Kit needs to do a "chmod" when starting a Gradle Daemon. - * This "chmod" operation is traced by datadog.trace.instrumentation.java.lang.ProcessImplInstrumentation and is reported as a span. - * The problem is that the "chmod" only happens when running in CI (could be due to differences in OS or FS permissions), - * so when running the tests locally, the "chmod" span is not there. - * This causes the tests to fail because the number of reported traces is different. - * To avoid this discrepancy between local and CI runs, we disable tracing instrumentations. - */ - (TraceInstrumentationConfig.TRACE_ENABLED) : "false" - ] - def arguments = buildJvmArguments(mockBackend.intakeUrl, TEST_SERVICE_NAME, additionalArgs) - - def gradleProperties = "org.gradle.jvmargs=${arguments.join(" ")}".toString() - // Write to projectFolder (per-test) instead of testKitFolder (shared), so each - // Gradle daemon gets its own unique preference directory - Files.write(projectFolder.resolve("gradle.properties"), gradleProperties.getBytes()) - } - - private BuildResult runGradleTests(String gradleVersion, boolean successExpected = true, boolean configurationCache = false) { - def arguments = ["test", "--stacktrace"] - if (gradleVersion > "4.5") { - // warning mode available starting from Gradle 4.5 - arguments += ["--warning-mode", "all"] - } - if (configurationCache) { - arguments += ["--configuration-cache", "--rerun-tasks"] - } - BuildResult buildResult = runGradle(gradleVersion, arguments, successExpected) - buildResult - } - - /** - * Sometimes Gradle Test Kit fails because it cannot download the required Gradle distribution - * due to intermittent network issues. - * This method performs the download manually (if needed) with increased timeout (30s vs default 10s). - * Retry logic (3 retries) is already present in org.gradle.wrapper.Install - */ - private ensureDependenciesDownloaded(String gradleVersion) { - try { - println "${new Date()}: $specificationContext.currentIteration.displayName - Starting dependencies download" - - def logger = new org.gradle.wrapper.Logger(false) - def download = new Download(logger, "Gradle Tooling API", GradleVersion.current().getVersion(), GRADLE_DISTRIBUTION_NETWORK_TIMEOUT) - - def userHomeDir = GradleUserHomeLookup.gradleUserHome() - def projectDir = projectFolder.toFile() - def install = new Install(logger, download, new PathAssembler(userHomeDir, projectDir)) - - def configuration = new WrapperConfiguration() - def distribution = new DistributionLocator().getDistributionFor(GradleVersion.version(gradleVersion)) - configuration.setDistribution(distribution) - configuration.setNetworkTimeout(GRADLE_DISTRIBUTION_NETWORK_TIMEOUT) - - // this will download distribution (if not downloaded yet to userHomeDir) and verify its SHA - install.createDist(configuration) - - println "${new Date()}: $specificationContext.currentIteration.displayName - Finished dependencies download" - } catch (Exception e) { - println "${new Date()}: $specificationContext.currentIteration.displayName " + - "- Failed to install Gradle distribution, will proceed to run test kit hoping for the best: $e" - } - } - - private runGradle(String gradleVersion, List arguments, boolean successExpected) { - def buildEnv = ["GRADLE_VERSION": gradleVersion] - - def mavenRepositoryProxy = System.getenv("MAVEN_REPOSITORY_PROXY") - if (mavenRepositoryProxy != null) { - buildEnv += ["MAVEN_REPOSITORY_PROXY": System.getenv("MAVEN_REPOSITORY_PROXY")] - } - - GradleRunner gradleRunner = GradleRunner.create() - .withTestKitDir(testKitFolder.toFile()) - .withProjectDir(projectFolder.toFile()) - .withGradleVersion(gradleVersion) - .withArguments(arguments) - .withEnvironment(buildEnv) - .forwardOutput() - - println "${new Date()}: $specificationContext.currentIteration.displayName - Starting Gradle run" - try { - def buildResult = successExpected ? gradleRunner.build() : gradleRunner.buildAndFail() - println "${new Date()}: $specificationContext.currentIteration.displayName - Finished Gradle run" - return buildResult - } catch (Exception e) { - def daemonLog = Files.list(testKitFolder.resolve("test-kit-daemon/" + gradleVersion)).filter(p -> p.toString().endsWith("log")).findAny().orElse(null) - if (daemonLog != null) { - println "==============================================================" - println "${new Date()}: $specificationContext.currentIteration.displayName - Gradle Daemon log:\n${new String(Files.readAllBytes(daemonLog))}" - println "==============================================================" - } - throw e - } - } - - private void assertBuildSuccessful(buildResult) { - assert buildResult.tasks != null - assert buildResult.tasks.size() > 0 - for (def task : buildResult.tasks) { - assert task.outcome != TaskOutcome.FAILED - } - } -} diff --git a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleLauncherSmokeTest.groovy b/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleLauncherSmokeTest.groovy deleted file mode 100644 index 5d11606774a..00000000000 --- a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleLauncherSmokeTest.groovy +++ /dev/null @@ -1,111 +0,0 @@ -package datadog.smoketest - -import datadog.communication.util.IOUtils -import datadog.trace.civisibility.utils.ShellCommandExecutor -import org.opentest4j.AssertionFailedError -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -/** - * This test runs Gradle Launcher with the Java Tracer injected - * and verifies that the tracer is injected into the Gradle Daemon. - */ -class GradleLauncherSmokeTest extends AbstractGradleTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(GradleLauncherSmokeTest) - - private static final int GRADLE_BUILD_TIMEOUT_MILLIS = 90_000 - private static final int GRADLE_WRAPPER_RETRIES = 3 - - private static final String JAVA_HOME = buildJavaHome() - - def "test Gradle Launcher injects tracer into Gradle Daemon: v#gradleVersion, cmd line - #gradleDaemonCmdLineParams"() { - given: - givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion) - givenGradleProjectFiles("test-gradle-wrapper", ["gradle-wrapper.properties": ["gradle-version": gradleVersion]]) - givenGradleWrapper(gradleVersion) // we want to check that instrumentation works with different wrapper versions too - - when: - def output = whenRunningGradleLauncherWithJavaTracerInjected(gradleDaemonCmdLineParams) - - then: - gradleDaemonStartCommandContains(output, - // verify that the javaagent is injected into the Gradle Daemon start command - "-javaagent:${AGENT_JAR}", - // verify that existing Gradle Daemon JVM args are preserved: - // org.gradle.jvmargs provided on the command line, if present, - // otherwise org.gradle.jvmargs from gradle.properties file - // ("user.country" is used, as Gradle will filter out properties it is not aware of) - gradleDaemonCmdLineParams ? gradleDaemonCmdLineParams : "-Duser.country=VALUE_FROM_GRADLE_PROPERTIES_FILE") - - where: - gradleVersion | gradleDaemonCmdLineParams - "6.6.1" | null - "6.6.1" | "-Duser.country=VALUE_FROM_CMD_LINE" - "7.6.4" | null - "7.6.4" | "-Duser.country=VALUE_FROM_CMD_LINE" - "8.11.1" | null - "8.11.1" | "-Duser.country=VALUE_FROM_CMD_LINE" - LATEST_GRADLE_VERSION | null - LATEST_GRADLE_VERSION | "-Duser.country=VALUE_FROM_CMD_LINE" - } - - private void givenGradleWrapper(String gradleVersion) { - def shellCommandExecutor = new ShellCommandExecutor( - projectFolder.toFile(), - GRADLE_BUILD_TIMEOUT_MILLIS, - [ - "JAVA_HOME": JAVA_HOME, - "GRADLE_OPTS": "" // avoids inheriting CI's GRADLE_OPTS which might be incompatible with the tested JVM - ]) - - for (int attempt = 0; attempt < GRADLE_WRAPPER_RETRIES; attempt++) { - try { - shellCommandExecutor.executeCommand(IOUtils::readFully, "./gradlew", "wrapper", "--gradle-version", gradleVersion) - return - } catch (ShellCommandExecutor.ShellCommandFailedException e) { - LOGGER.warn("Failed gradle wrapper resolution with exception: ", e) - Thread.sleep(2000) // small delay for rapid retries on network issues - } - } - throw new AssertionError((Object) "Tried $GRADLE_WRAPPER_RETRIES times to execute gradle wrapper command and failed") - } - - private String whenRunningGradleLauncherWithJavaTracerInjected(String gradleDaemonCmdLineParams) { - def shellCommandExecutor = new ShellCommandExecutor(projectFolder.toFile(), GRADLE_BUILD_TIMEOUT_MILLIS, [ - "JAVA_HOME" : JAVA_HOME, - "GRADLE_OPTS" : "-javaagent:${AGENT_JAR}".toString(), - "DD_CIVISIBILITY_ENABLED" : "true", - "DD_CIVISIBILITY_AGENTLESS_ENABLED" : "true", - "DD_CIVISIBILITY_AGENTLESS_URL" : "${mockBackend.intakeUrl}".toString(), - "DD_CIVISIBILITY_GIT_UPLOAD_ENABLED": "false", - "DD_CIVISIBILITY_GIT_CLIENT_ENABLED": "false", - "DD_CODE_ORIGIN_FOR_SPANS_ENABLED" : "false", - "DD_CIVISIBILITY_CODE_COVERAGE_ENABLED": "false", - "DD_API_KEY" : "dummy" - ]) - String[] command = ["./gradlew", "--no-daemon", "--info"] - if (gradleDaemonCmdLineParams) { - command += "-Dorg.gradle.jvmargs=$gradleDaemonCmdLineParams".toString() - } - - try { - return shellCommandExecutor.executeCommand(IOUtils::readFully, command) - } catch (Exception e) { - println "==============================================================" - println "${new Date()}: $specificationContext.currentIteration.displayName - Gradle Launcher execution failed with exception:\n ${e.message}" - println "==============================================================" - throw e - } - } - - private static boolean gradleDaemonStartCommandContains(String buildOutput, String... tokens) { - def daemonStartCommandLog = buildOutput.split("\n").find { it.contains("Starting process 'Gradle build daemon'") } - for (String token : tokens) { - if (!daemonStartCommandLog.contains(token)) { - throw new AssertionFailedError("Gradle Daemon start command does not contain " + token, token, daemonStartCommandLog) - } - } - return true - } -} diff --git a/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/AbstractGradleTest.java b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/AbstractGradleTest.java new file mode 100644 index 00000000000..7ba519bf935 --- /dev/null +++ b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/AbstractGradleTest.java @@ -0,0 +1,186 @@ +package datadog.smoketest; + +import datadog.environment.JavaVirtualMachine; +import datadog.trace.civisibility.CiVisibilitySmokeTest; +import datadog.trace.util.ComparableVersion; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.gradle.internal.impldep.org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class AbstractGradleTest extends CiVisibilitySmokeTest { + + protected static final String LATEST_GRADLE_VERSION = getLatestGradleVersion(); + + // test resources use this instead of ".gradle" to avoid unwanted evaluation + private static final String GRADLE_TEST_RESOURCE_EXTENSION = ".gradleTest"; + private static final String GRADLE_REGULAR_EXTENSION = ".gradle"; + + private static final ComparableVersion GRADLE_9 = new ComparableVersion("9.0.0"); + + @TempDir protected Path projectFolder; + + protected final MockBackend mockBackend = new MockBackend(); + + @BeforeEach + void resetMockBackend() { + mockBackend.reset(); + } + + @AfterAll + void closeMockBackend() throws Exception { + mockBackend.close(); + } + + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\$\\{(.+?)\\}"); + + protected void givenGradleProjectFiles(String projectFilesSources) throws IOException { + givenGradleProjectFiles(projectFilesSources, Collections.emptyMap()); + } + + protected void givenGradleProjectFiles( + String projectFilesSources, Map> replacementsByFileName) + throws IOException { + Path projectResourcesPath; + try { + projectResourcesPath = + Paths.get(this.getClass().getClassLoader().getResource(projectFilesSources).toURI()); + } catch (Exception e) { + throw new RuntimeException(e); + } + FileUtils.copyDirectory(projectResourcesPath.toFile(), projectFolder.toFile()); + + Files.walkFileTree( + projectFolder, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Map replacements = + replacementsByFileName.get(file.getFileName().toString()); + if (replacements != null) { + String fileContents = new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + Matcher matcher = PLACEHOLDER_PATTERN.matcher(fileContents); + + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + String propertyName = matcher.group(1); + String replacement = replacements.getOrDefault(propertyName, matcher.group(0)); + matcher.appendReplacement(result, Matcher.quoteReplacement(replacement)); + } + matcher.appendTail(result); + + Files.write(file, result.toString().getBytes(StandardCharsets.UTF_8)); + } + + if (file.toString().endsWith(GRADLE_TEST_RESOURCE_EXTENSION)) { + Path fileWithFixedExtension = + Paths.get( + file.toString() + .replace(GRADLE_TEST_RESOURCE_EXTENSION, GRADLE_REGULAR_EXTENSION)); + Files.move(file, fileWithFixedExtension); + } + + return FileVisitResult.CONTINUE; + } + }); + + // creating empty .git directory so that the tracer could detect projectFolder as repo root + Files.createDirectory(projectFolder.resolve(".git")); + } + + protected void givenGradleVersionIsCompatibleWithCurrentJvm(String gradleVersion) { + Assumptions.assumeTrue( + isSupported(new ComparableVersion(gradleVersion)), + "Current JVM does not support Gradle version " + gradleVersion); + } + + private static boolean isSupported(ComparableVersion gradleVersion) { + // https://docs.gradle.org/current/userguide/compatibility.html + if (JavaVirtualMachine.isJavaVersionAtLeast(26)) { + return gradleVersion.compareTo(new ComparableVersion("9.4")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(25)) { + return gradleVersion.compareTo(new ComparableVersion("9.1")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(24)) { + return gradleVersion.compareTo(new ComparableVersion("8.14")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(21)) { + return gradleVersion.compareTo(new ComparableVersion("8.4")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(20)) { + return gradleVersion.compareTo(new ComparableVersion("8.1")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(19)) { + return gradleVersion.compareTo(new ComparableVersion("7.6")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(18)) { + return gradleVersion.compareTo(new ComparableVersion("7.5")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(17)) { + return gradleVersion.compareTo(new ComparableVersion("7.3")) >= 0; + } else if (JavaVirtualMachine.isJavaVersionAtLeast(16)) { + return isWithin(gradleVersion, new ComparableVersion("7.0"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(15)) { + return isWithin(gradleVersion, new ComparableVersion("6.7"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(14)) { + return isWithin(gradleVersion, new ComparableVersion("6.3"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(13)) { + return isWithin(gradleVersion, new ComparableVersion("6.0"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(12)) { + return isWithin(gradleVersion, new ComparableVersion("5.4"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(11)) { + return isWithin(gradleVersion, new ComparableVersion("5.0"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(10)) { + return isWithin(gradleVersion, new ComparableVersion("4.7"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(9)) { + return isWithin(gradleVersion, new ComparableVersion("4.3"), GRADLE_9); + } else if (JavaVirtualMachine.isJavaVersionAtLeast(8)) { + return isWithin(gradleVersion, new ComparableVersion("2.0"), GRADLE_9); + } + return false; + } + + private static boolean isWithin( + ComparableVersion version, + ComparableVersion lowerInclusive, + ComparableVersion upperExclusive) { + return version.compareTo(lowerInclusive) >= 0 && version.compareTo(upperExclusive) < 0; + } + + protected void givenConfigurationCacheIsCompatibleWithCurrentPlatform( + boolean configurationCacheEnabled) { + if (configurationCacheEnabled) { + Assumptions.assumeFalse( + JavaVirtualMachine.isIbm8(), "Configuration cache is not compatible with IBM 8"); + } + } + + private static String getLatestGradleVersion() { + Properties properties = new Properties(); + try (InputStream stream = + AbstractGradleTest.class + .getClassLoader() + .getResourceAsStream("latest-tool-versions.properties")) { + if (stream == null) { + throw new IllegalStateException( + "Could not find latest-tool-versions.properties on classpath"); + } + properties.load(stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + return properties.getProperty("gradle.version"); + } +} diff --git a/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java new file mode 100644 index 00000000000..4d1f4997bd6 --- /dev/null +++ b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java @@ -0,0 +1,321 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.environment.JavaVirtualMachine; +import datadog.trace.api.civisibility.config.TestFQN; +import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.api.config.TraceInstrumentationConfig; +import datadog.trace.civisibility.CiVisibilityTableTestConverters; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.gradle.testkit.runner.BuildResult; +import org.gradle.testkit.runner.BuildTask; +import org.gradle.testkit.runner.GradleRunner; +import org.gradle.testkit.runner.TaskOutcome; +import org.gradle.util.GradleVersion; +import org.gradle.util.internal.DistributionLocator; +import org.gradle.wrapper.Download; +import org.gradle.wrapper.GradleUserHomeLookup; +import org.gradle.wrapper.Install; +import org.gradle.wrapper.PathAssembler; +import org.gradle.wrapper.WrapperConfiguration; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.tabletest.junit.TableTest; +import org.tabletest.junit.TypeConverterSources; + +@TypeConverterSources(CiVisibilityTableTestConverters.class) +class GradleDaemonSmokeTest extends AbstractGradleTest { + + private static final String TEST_SERVICE_NAME = "test-gradle-service"; + + // Gradle's default timeout is 10s + private static final int GRADLE_DISTRIBUTION_NETWORK_TIMEOUT = 30_000; + + @TempDir static Path testKitFolder; + + @TableTest({ + "scenario | gradleVersion | projectName | successExpected | expectedTraces | expectedCoverages", + "succeed-old-gradle-3.5 | 3.5 | test-succeed-old-gradle | true | 5 | 1 ", + "succeed-legacy | 7.6.4 | test-succeed-legacy-instrumentation | true | 5 | 1 ", + "succeed-multi-module-legacy | 7.6.4 | test-succeed-multi-module-legacy-instrumentation | true | 7 | 2 ", + "succeed-multi-forks-legacy | 7.6.4 | test-succeed-multi-forks-legacy-instrumentation | true | 6 | 2 ", + "skip-legacy | 7.6.4 | test-skip-legacy-instrumentation | true | 2 | 0 ", + "failed-legacy | 7.6.4 | test-failed-legacy-instrumentation | false | 4 | 0 ", + "corrupted-config-legacy | 7.6.4 | test-corrupted-config-legacy-instrumentation | false | 1 | 0 " + }) + @ParameterizedTest + void testLegacy( + String gradleVersion, + String projectName, + boolean successExpected, + int expectedTraces, + int expectedCoverages) + throws IOException { + // Jacoco plugin does not work with OpenJ9 in older Gradle versions + Assumptions.assumeFalse( + JavaVirtualMachine.isJ9(), + "Jacoco plugin does not work with OpenJ9 in older Gradle versions"); + runGradleTest( + gradleVersion, + projectName, + false, + successExpected, + false, + expectedTraces, + expectedCoverages); + } + + @TableTest({ + "scenario | gradleVersion | projectName | configurationCache | successExpected | flakyRetries | expectedTraces | expectedCoverages", + "succeed-new-8.3 | 8.3 | test-succeed-new-instrumentation | {false, true} | true | false | 5 | 1 ", + "succeed-new-8.9 | 8.9 | test-succeed-new-instrumentation | {false, true} | true | false | 5 | 1 ", + "succeed-new-latest | latest | test-succeed-new-instrumentation | {false, true} | true | false | 5 | 1 ", + "succeed-multi-module-new | latest | test-succeed-multi-module-new-instrumentation | false | true | false | 7 | 2 ", + "succeed-multi-forks-new | latest | test-succeed-multi-forks-new-instrumentation | false | true | false | 6 | 2 ", + "skip-new | latest | test-skip-new-instrumentation | false | true | false | 2 | 0 ", + "failed-new | latest | test-failed-new-instrumentation | false | false | false | 4 | 0 ", + "corrupted-config-new | latest | test-corrupted-config-new-instrumentation | false | false | false | 1 | 0 ", + "succeed-junit-5 | latest | test-succeed-junit-5 | false | true | false | 5 | 1 ", + "failed-flaky-retries | latest | test-failed-flaky-retries | false | false | true | 8 | 0 ", + "succeed-gradle-plugin-test | 9.3.1 | test-succeed-gradle-plugin-test | false | true | false | 5 | 0 " + }) + @ParameterizedTest + void testNew( + String gradleVersion, + String projectName, + boolean configurationCache, + boolean successExpected, + boolean flakyRetries, + int expectedTraces, + int expectedCoverages) + throws IOException { + String resolvedGradleVersion = resolveLatest(gradleVersion); + runGradleTest( + resolvedGradleVersion, + projectName, + configurationCache, + successExpected, + flakyRetries, + expectedTraces, + expectedCoverages); + } + + // TODO: add back LATEST_GRADLE_VERSION after fixing ordering on Gradle 9.3.0 + @TableTest({ + "scenario | gradleVersion | projectName | flakyTests | expectedOrder | eventsNumber", + "junit4-ordering-7.6.4 | 7.6.4 | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 ", + "junit4-ordering-9.2.1 | 9.2.1 | test-succeed-junit-4-class-ordering | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 " + }) + @ParameterizedTest + void testJunit4ClassOrdering( + String gradleVersion, + String projectName, + List flakyTests, + List expectedOrder, + int eventsNumber) + throws IOException { + givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion); + givenGradleProjectFiles(projectName); + givenGradleProjectProperties(); + ensureDependenciesDownloaded(gradleVersion); + + mockBackend.givenKnownTests(true); + for (TestFQN flakyTest : flakyTests) { + mockBackend.givenFlakyTest(":test", flakyTest.getSuite(), flakyTest.getName()); + mockBackend.givenKnownTest(":test", flakyTest.getSuite(), flakyTest.getName()); + } + + BuildResult buildResult = runGradleTests(gradleVersion, true, false); + assertBuildSuccessful(buildResult); + + verifyTestOrder(mockBackend.waitForEvents(eventsNumber), expectedOrder); + } + + private static String resolveLatest(String gradleVersion) { + return "latest".equals(gradleVersion) ? LATEST_GRADLE_VERSION : gradleVersion; + } + + private void runGradleTest( + String gradleVersion, + String projectName, + boolean configurationCache, + boolean successExpected, + boolean flakyRetries, + int expectedTraces, + int expectedCoverages) + throws IOException { + givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion); + givenConfigurationCacheIsCompatibleWithCurrentPlatform(configurationCache); + givenGradleProjectFiles(projectName); + givenGradleProjectProperties(); + ensureDependenciesDownloaded(gradleVersion); + + mockBackend.givenFlakyRetries(flakyRetries); + mockBackend.givenFlakyTest(":test", "datadog.smoke.TestFailed", "test_failed"); + + mockBackend.givenTestsSkipping(true); + mockBackend.givenSkippableTest( + ":test", "datadog.smoke.TestSucceed", "test_to_skip_with_itr", Collections.emptyMap()); + + BuildResult buildResult = runGradleTests(gradleVersion, successExpected, configurationCache); + + if (successExpected) { + assertBuildSuccessful(buildResult); + } + + verifyEventsAndCoverages( + projectName, + "gradle", + gradleVersion, + mockBackend.waitForEvents(expectedTraces), + mockBackend.waitForCoverages(expectedCoverages)); + + if (configurationCache) { + // If configuration cache is enabled, run the build one more time to verify that building + // with an existing configuration cache entry works. + BuildResult buildResultWithConfigCacheEntry = + runGradleTests(gradleVersion, successExpected, configurationCache); + + assertBuildSuccessful(buildResultWithConfigCacheEntry); + verifyEventsAndCoverages( + projectName, + "gradle", + gradleVersion, + mockBackend.waitForEvents(expectedTraces), + mockBackend.waitForCoverages(expectedCoverages)); + } + } + + private void givenGradleProjectProperties() throws IOException { + assertTrue(new java.io.File(AGENT_JAR).isFile()); + + Path ddApiKeyPath = testKitFolder.resolve(".dd.api.key"); + Files.write(ddApiKeyPath, "dummy".getBytes()); + + Map additionalArgs = new HashMap<>(); + additionalArgs.put(GeneralConfig.API_KEY_FILE, ddApiKeyPath.toAbsolutePath().toString()); + additionalArgs.put( + CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION, JACOCO_PLUGIN_VERSION); + /* + * Some of the smoke tests (in particular the one with the Gradle plugin), are using Gradle Test Kit for their tests. + * Gradle Test Kit needs to do a "chmod" when starting a Gradle Daemon. + * This "chmod" operation is traced by datadog.trace.instrumentation.java.lang.ProcessImplInstrumentation and is reported as a span. + * The problem is that the "chmod" only happens when running in CI (could be due to differences in OS or FS permissions), + * so when running the tests locally, the "chmod" span is not there. + * This causes the tests to fail because the number of reported traces is different. + * To avoid this discrepancy between local and CI runs, we disable tracing instrumentations. + */ + additionalArgs.put(TraceInstrumentationConfig.TRACE_ENABLED, "false"); + List arguments = + buildJvmArguments(mockBackend.getIntakeUrl(), TEST_SERVICE_NAME, additionalArgs); + + String gradleProperties = "org.gradle.jvmargs=" + String.join(" ", arguments); + // Write to projectFolder (per-test) instead of testKitFolder (shared), so each + // Gradle daemon gets its own unique preference directory. + Files.write(projectFolder.resolve("gradle.properties"), gradleProperties.getBytes()); + } + + private BuildResult runGradleTests( + String gradleVersion, boolean successExpected, boolean configurationCache) + throws IOException { + List arguments = new java.util.ArrayList<>(Arrays.asList("test", "--stacktrace")); + if (gradleVersion.compareTo("4.5") > 0) { + // warning mode available starting from Gradle 4.5 + arguments.addAll(Arrays.asList("--warning-mode", "all")); + } + if (configurationCache) { + arguments.addAll(Arrays.asList("--configuration-cache", "--rerun-tasks")); + } + return runGradle(gradleVersion, arguments, successExpected); + } + + /** + * Sometimes Gradle Test Kit fails because it cannot download the required Gradle distribution due + * to intermittent network issues. This method performs the download manually (if needed) with + * increased timeout (30s vs default 10s). Retry logic (3 retries) is already present in {@code + * org.gradle.wrapper.Install}. + */ + private void ensureDependenciesDownloaded(String gradleVersion) { + try { + org.gradle.wrapper.Logger logger = new org.gradle.wrapper.Logger(false); + Download download = + new Download( + logger, + "Gradle Tooling API", + GradleVersion.current().getVersion(), + GRADLE_DISTRIBUTION_NETWORK_TIMEOUT); + + java.io.File userHomeDir = GradleUserHomeLookup.gradleUserHome(); + java.io.File projectDir = projectFolder.toFile(); + Install install = new Install(logger, download, new PathAssembler(userHomeDir, projectDir)); + + WrapperConfiguration configuration = new WrapperConfiguration(); + configuration.setDistribution( + new DistributionLocator().getDistributionFor(GradleVersion.version(gradleVersion))); + configuration.setNetworkTimeout(GRADLE_DISTRIBUTION_NETWORK_TIMEOUT); + + // This will download distribution (if not downloaded yet to userHomeDir) and verify its SHA. + install.createDist(configuration); + } catch (Exception e) { + System.out.println( + "Failed to install Gradle distribution, will proceed to run test kit hoping for the best: " + + e); + } + } + + private BuildResult runGradle( + String gradleVersion, List arguments, boolean successExpected) throws IOException { + Map buildEnv = new HashMap<>(); + buildEnv.put("GRADLE_VERSION", gradleVersion); + + String mavenRepositoryProxy = System.getenv("MAVEN_REPOSITORY_PROXY"); + if (mavenRepositoryProxy != null) { + buildEnv.put("MAVEN_REPOSITORY_PROXY", mavenRepositoryProxy); + } + + GradleRunner gradleRunner = + GradleRunner.create() + .withTestKitDir(testKitFolder.toFile()) + .withProjectDir(projectFolder.toFile()) + .withGradleVersion(gradleVersion) + .withArguments(arguments) + .withEnvironment(buildEnv) + .forwardOutput(); + + try { + return successExpected ? gradleRunner.build() : gradleRunner.buildAndFail(); + } catch (Exception e) { + Path daemonLog = + Files.list(testKitFolder.resolve("test-kit-daemon/" + gradleVersion)) + .filter(p -> p.toString().endsWith("log")) + .findAny() + .orElse(null); + if (daemonLog != null) { + System.out.println("=============================================================="); + System.out.println("Gradle Daemon log:\n" + new String(Files.readAllBytes(daemonLog))); + System.out.println("=============================================================="); + } + throw new RuntimeException(e); + } + } + + private void assertBuildSuccessful(BuildResult buildResult) { + assertNotNull(buildResult.getTasks()); + assertFalse(buildResult.getTasks().isEmpty(), "build produced zero tasks"); + for (BuildTask task : buildResult.getTasks()) { + assertFalse(task.getOutcome() == TaskOutcome.FAILED, "task " + task.getPath() + " failed"); + } + } +} diff --git a/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleLauncherSmokeTest.java b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleLauncherSmokeTest.java new file mode 100644 index 00000000000..56c0fbb777d --- /dev/null +++ b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleLauncherSmokeTest.java @@ -0,0 +1,143 @@ +package datadog.smoketest; + +import datadog.communication.util.IOUtils; +import datadog.trace.civisibility.utils.ShellCommandExecutor; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.params.ParameterizedTest; +import org.opentest4j.AssertionFailedError; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tabletest.junit.TableTest; + +/** + * This test runs Gradle Launcher with the Java Tracer injected and verifies that the tracer is + * injected into the Gradle Daemon. + */ +class GradleLauncherSmokeTest extends AbstractGradleTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(GradleLauncherSmokeTest.class); + + private static final int GRADLE_BUILD_TIMEOUT_MILLIS = 90_000; + private static final int GRADLE_WRAPPER_RETRIES = 3; + + private static final String JAVA_HOME = buildJavaHome(); + + @TableTest({ + "scenario | gradleVersion | gradleDaemonCmdLineParams ", + "6.6.1-from-gradle-props | 6.6.1 | ", + "6.6.1-from-cmd-line | 6.6.1 | -Duser.country=VALUE_FROM_CMD_LINE", + "7.6.4-from-gradle-props | 7.6.4 | ", + "7.6.4-from-cmd-line | 7.6.4 | -Duser.country=VALUE_FROM_CMD_LINE", + "8.11.1-from-gradle-props | 8.11.1 | ", + "8.11.1-from-cmd-line | 8.11.1 | -Duser.country=VALUE_FROM_CMD_LINE", + "latest-from-gradle-props | latest | ", + "latest-from-cmd-line | latest | -Duser.country=VALUE_FROM_CMD_LINE" + }) + @ParameterizedTest + void testGradleLauncherInjectsTracerIntoGradleDaemon( + String gradleVersion, String gradleDaemonCmdLineParams) throws Exception { + String resolvedGradleVersion = + "latest".equals(gradleVersion) ? LATEST_GRADLE_VERSION : gradleVersion; + String cmdLineParams = + (gradleDaemonCmdLineParams == null || gradleDaemonCmdLineParams.isEmpty()) + ? null + : gradleDaemonCmdLineParams; + + givenGradleVersionIsCompatibleWithCurrentJvm(resolvedGradleVersion); + Map> replacements = new HashMap<>(); + Map versionMap = new HashMap<>(); + versionMap.put("gradle-version", resolvedGradleVersion); + replacements.put("gradle-wrapper.properties", versionMap); + givenGradleProjectFiles("test-gradle-wrapper", replacements); + // we want to check that instrumentation works with different wrapper versions too + givenGradleWrapper(resolvedGradleVersion); + + String output = whenRunningGradleLauncherWithJavaTracerInjected(cmdLineParams); + + gradleDaemonStartCommandContains( + output, + // Verify that the javaagent is injected into the Gradle Daemon start command. + "-javaagent:" + AGENT_JAR, + // Verify that existing Gradle Daemon JVM args are preserved: org.gradle.jvmargs provided + // on the command line (if present), otherwise org.gradle.jvmargs from gradle.properties. + // "user.country" is used, as Gradle will filter out properties it is not aware of. + cmdLineParams != null ? cmdLineParams : "-Duser.country=VALUE_FROM_GRADLE_PROPERTIES_FILE"); + } + + private void givenGradleWrapper(String gradleVersion) throws Exception { + Map env = new HashMap<>(); + env.put("JAVA_HOME", JAVA_HOME); + // Avoid inheriting CI's GRADLE_OPTS which might be incompatible with the tested JVM. + env.put("GRADLE_OPTS", ""); + ShellCommandExecutor shellCommandExecutor = + new ShellCommandExecutor(projectFolder.toFile(), GRADLE_BUILD_TIMEOUT_MILLIS, env); + + for (int attempt = 0; attempt < GRADLE_WRAPPER_RETRIES; attempt++) { + try { + shellCommandExecutor.executeCommand( + IOUtils::readFully, "./gradlew", "wrapper", "--gradle-version", gradleVersion); + return; + } catch (ShellCommandExecutor.ShellCommandFailedException e) { + LOGGER.warn("Failed gradle wrapper resolution with exception: ", e); + Thread.sleep(2000); // small delay for rapid retries on network issues + } + } + throw new AssertionError( + "Tried " + GRADLE_WRAPPER_RETRIES + " times to execute gradle wrapper command and failed"); + } + + private String whenRunningGradleLauncherWithJavaTracerInjected(String gradleDaemonCmdLineParams) + throws Exception { + Map env = new HashMap<>(); + env.put("JAVA_HOME", JAVA_HOME); + env.put("GRADLE_OPTS", "-javaagent:" + AGENT_JAR); + env.put("DD_CIVISIBILITY_ENABLED", "true"); + env.put("DD_CIVISIBILITY_AGENTLESS_ENABLED", "true"); + env.put("DD_CIVISIBILITY_AGENTLESS_URL", mockBackend.getIntakeUrl()); + env.put("DD_CIVISIBILITY_GIT_UPLOAD_ENABLED", "false"); + env.put("DD_CIVISIBILITY_GIT_CLIENT_ENABLED", "false"); + env.put("DD_CODE_ORIGIN_FOR_SPANS_ENABLED", "false"); + env.put("DD_CIVISIBILITY_CODE_COVERAGE_ENABLED", "false"); + env.put("DD_API_KEY", "dummy"); + + ShellCommandExecutor shellCommandExecutor = + new ShellCommandExecutor(projectFolder.toFile(), GRADLE_BUILD_TIMEOUT_MILLIS, env); + + List command = + new java.util.ArrayList<>(Arrays.asList("./gradlew", "--no-daemon", "--info")); + if (gradleDaemonCmdLineParams != null) { + command.add("-Dorg.gradle.jvmargs=" + gradleDaemonCmdLineParams); + } + + try { + return shellCommandExecutor.executeCommand( + IOUtils::readFully, command.toArray(new String[0])); + } catch (Exception e) { + System.out.println("=============================================================="); + System.out.println("Gradle Launcher execution failed with exception:\n " + e.getMessage()); + System.out.println("=============================================================="); + throw e; + } + } + + private static void gradleDaemonStartCommandContains(String buildOutput, String... tokens) { + String daemonStartCommandLog = null; + for (String line : buildOutput.split("\n")) { + if (line.contains("Starting process 'Gradle build daemon'")) { + daemonStartCommandLog = line; + break; + } + } + for (String token : tokens) { + if (daemonStartCommandLog == null || !daemonStartCommandLog.contains(token)) { + throw new AssertionFailedError( + "Gradle Daemon start command does not contain " + token, + token, + String.valueOf(daemonStartCommandLog)); + } + } + } +} diff --git a/dd-smoke-tests/junit-console/src/test/groovy/datadog/smoketest/JUnitConsoleSmokeTest.groovy b/dd-smoke-tests/junit-console/src/test/groovy/datadog/smoketest/JUnitConsoleSmokeTest.groovy deleted file mode 100644 index 22fb515c1af..00000000000 --- a/dd-smoke-tests/junit-console/src/test/groovy/datadog/smoketest/JUnitConsoleSmokeTest.groovy +++ /dev/null @@ -1,250 +0,0 @@ -package datadog.smoketest - - -import datadog.trace.api.config.CiVisibilityConfig -import datadog.trace.api.config.DebuggerConfig -import datadog.trace.api.config.GeneralConfig -import datadog.trace.civisibility.CiVisibilitySmokeTest -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes -import java.util.concurrent.TimeUnit -import java.util.concurrent.TimeoutException -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.TempDir - -class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest { - // CodeNarc incorrectly thinks ".class" is unnecessary in getLogger - @SuppressWarnings('UnnecessaryDotClass') - private static final Logger LOGGER = LoggerFactory.getLogger(JUnitConsoleSmokeTest.class) - - private static final String TEST_SERVICE_NAME = "test-headless-service" - - private static final int PROCESS_TIMEOUT_SECS = 60 - private static final String JUNIT_CONSOLE_JAR_PATH = System.getProperty("datadog.smoketest.junit.console.jar.path") - private static final String JAVA_HOME = buildJavaHome() - - @TempDir - Path projectHome - - @Shared - @AutoCleanup - MockBackend mockBackend = new MockBackend() - - def setup() { - mockBackend.reset() - } - - def "test headless failed test replay"() { - givenProjectFiles(projectName) - - mockBackend.givenFlakyRetries(true) - mockBackend.givenFlakyTest("test-headless-service", "com.example.TestFailed", "test_failed") - mockBackend.givenFailedTestReplay(true) - - def compileCode = compileTestProject() - assert compileCode == 0 - - def exitCode = whenRunningJUnitConsole([ - (CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_COUNT): "3", - (GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL): mockBackend.intakeUrl, - (DebuggerConfig.DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL): "999999" // avoid possible race conditions on shutdown - ], - [:]) - assert exitCode == 1 - - def additionalDynamicTags = ["content.meta.['_dd.debug.error.6.snapshot_id']", "content.meta.['_dd.debug.error.exception_id']"] - verifyEventsAndCoverages(projectName, "junit-console", "headless", mockBackend.waitForEvents(7), mockBackend.waitForCoverages(0), additionalDynamicTags) - verifySnapshots(mockBackend.waitForLogs(2), 2) - - where: - projectName = "test_junit_console_failed_test_replay" - } - - private void givenProjectFiles(String projectFilesSources) { - def projectResourcesUri = this.getClass().getClassLoader().getResource(projectFilesSources).toURI() - def projectResourcesPath = Paths.get(projectResourcesUri) - copyFolder(projectResourcesPath, projectHome) - } - - private void copyFolder(Path src, Path dest) throws IOException { - Files.walkFileTree(src, new SimpleFileVisitor() { - @Override - FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - Files.createDirectories(dest.resolve(src.relativize(dir))) - return FileVisitResult.CONTINUE - } - - @Override - FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - Files.copy(file, dest.resolve(src.relativize(file))) - return FileVisitResult.CONTINUE - } - }) - - // creating empty .git directory so that the tracer could detect projectFolder as repo root - Files.createDirectory(projectHome.resolve(".git")) - } - - private int compileTestProject() { - def srcDir = projectHome.resolve("src/main/java") - def testSrcDir = projectHome.resolve("src/test/java") - def classesDir = projectHome.resolve("target/classes") - def testClassesDir = projectHome.resolve("target/test-classes") - - Files.createDirectories(classesDir) - Files.createDirectories(testClassesDir) - - // Compile main classes if they exist - if (Files.exists(srcDir)) { - def mainJavaFiles = findJavaFiles(srcDir) - if (!mainJavaFiles.isEmpty()) { - def result = runProcess(createCompilerProcessBuilder(classesDir.toString(), mainJavaFiles).start()) - if (result != 0) { - LOGGER.error("Error compiling source classes for JUnit Console smoke test") - return result - } - } - } - - // Compile test classes - def testJavaFiles = findJavaFiles(testSrcDir) - if (!testJavaFiles.isEmpty()) { - def result = runProcess(createCompilerProcessBuilder(testClassesDir.toString(), testJavaFiles, [classesDir.toString()]).start()) - if (result != 0) { - LOGGER.error("Error compiling source classes for JUnit Console smoke test") - return result - } - } - - return 0 - } - - private ProcessBuilder createCompilerProcessBuilder(String targetDir, List files, List additionalDeps = []) { - assert new File(JUNIT_CONSOLE_JAR_PATH).isFile() - - List deps = [JUNIT_CONSOLE_JAR_PATH] - deps.addAll(additionalDeps) - - List command = new ArrayList<>() - command.add(javacPath()) - command.addAll(["-cp", deps.join(":")]) - command.addAll(["-d", targetDir]) - command.addAll(files) - - ProcessBuilder processBuilder = new ProcessBuilder(command) - - processBuilder.directory(projectHome.toFile()) - processBuilder.environment().put("JAVA_HOME", JAVA_HOME) - - return processBuilder - } - - private static List findJavaFiles(Path directory) { - if (!Files.exists(directory)) { - return [] - } - - List javaFiles = [] - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (file.toString().endsWith(".java")) { - javaFiles.add(file.toString()) - } - return FileVisitResult.CONTINUE - } - }) - - return javaFiles - } - - private int whenRunningJUnitConsole(Map additionalAgentArgs, Map additionalEnvVars) { - def processBuilder = createConsoleProcessBuilder(["execute"], additionalAgentArgs, additionalEnvVars) - - processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF") - - return runProcess(processBuilder.start()) - } - - private static runProcess(Process p, int timeoutSecs = PROCESS_TIMEOUT_SECS) { - StreamConsumer errorGobbler = new StreamConsumer(p.getErrorStream(), "ERROR") - StreamConsumer outputGobbler = new StreamConsumer(p.getInputStream(), "OUTPUT") - outputGobbler.start() - errorGobbler.start() - - if (!p.waitFor(timeoutSecs, TimeUnit.SECONDS)) { - p.destroyForcibly() - throw new TimeoutException("Instrumented process failed to exit within $timeoutSecs seconds") - } - - return p.exitValue() - } - - ProcessBuilder createConsoleProcessBuilder(List consoleCommand, Map additionalAgentArgs, Map additionalEnvVars) { - assert new File(JUNIT_CONSOLE_JAR_PATH).isFile() - - List command = new ArrayList<>() - command.add(javaPath()) - command.add("-Ddatadog.slf4j.simpleLogger.defaultLogLevel=DEBUG") - command.addAll((String[]) ["-jar", JUNIT_CONSOLE_JAR_PATH]) - command.addAll(consoleCommand) - command.addAll([ - "--class-path", - [projectHome.resolve("target/classes").toString(), projectHome.resolve("target/test-classes")].join(":") - ]) - command.add("--scan-class-path") - - ProcessBuilder processBuilder = new ProcessBuilder(command) - processBuilder.directory(projectHome.toFile()) - - processBuilder.environment().put("JAVA_HOME", JAVA_HOME) - processBuilder.environment().put("JAVA_TOOL_OPTIONS", javaToolOptions(additionalAgentArgs)) - for (envVar in additionalEnvVars) { - processBuilder.environment().put(envVar.key, envVar.value) - } - - def mavenRepositoryProxy = System.getenv("MAVEN_REPOSITORY_PROXY") - if (mavenRepositoryProxy != null) { - processBuilder.environment().put("MAVEN_REPOSITORY_PROXY", mavenRepositoryProxy) - } - - return processBuilder - } - - String javaToolOptions(Map additionalAgentArgs) { - additionalAgentArgs.put(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED, "false") - return buildJvmArguments(mockBackend.intakeUrl, TEST_SERVICE_NAME, additionalAgentArgs).join(" ") - } - - private static class StreamConsumer extends Thread { - final InputStream is - final String messagePrefix - - StreamConsumer(InputStream is, String messagePrefix) { - this.is = is - this.messagePrefix = messagePrefix - } - - @Override - void run() { - try { - BufferedReader br = new BufferedReader(new InputStreamReader(is)) - String line - while ((line = br.readLine()) != null) { - System.out.println(messagePrefix + ": " + line) - } - } catch (IOException e) { - e.printStackTrace() - } - } - } -} diff --git a/dd-smoke-tests/junit-console/src/test/java/datadog/smoketest/JUnitConsoleSmokeTest.java b/dd-smoke-tests/junit-console/src/test/java/datadog/smoketest/JUnitConsoleSmokeTest.java new file mode 100644 index 00000000000..939a81c633a --- /dev/null +++ b/dd-smoke-tests/junit-console/src/test/java/datadog/smoketest/JUnitConsoleSmokeTest.java @@ -0,0 +1,303 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.DebuggerConfig; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.civisibility.CiVisibilitySmokeTest; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest { + private static final Logger LOGGER = LoggerFactory.getLogger(JUnitConsoleSmokeTest.class); + + private static final String TEST_SERVICE_NAME = "test-headless-service"; + + private static final int PROCESS_TIMEOUT_SECS = 60; + private static final String JUNIT_CONSOLE_JAR_PATH = + System.getProperty("datadog.smoketest.junit.console.jar.path"); + private static final String JAVA_HOME = buildJavaHome(); + + @TempDir Path projectHome; + + static final MockBackend mockBackend = new MockBackend(); + + @BeforeEach + void resetMockBackend() { + mockBackend.reset(); + } + + @AfterAll + static void closeMockBackend() throws Exception { + mockBackend.close(); + } + + @Test + void testHeadlessFailedTestReplay() throws Exception { + String projectName = "test_junit_console_failed_test_replay"; + givenProjectFiles(projectName); + + mockBackend.givenFlakyRetries(true); + mockBackend.givenFlakyTest("test-headless-service", "com.example.TestFailed", "test_failed"); + mockBackend.givenFailedTestReplay(true); + + int compileCode = compileTestProject(); + assertEquals(0, compileCode); + + Map agentArgs = new HashMap<>(); + agentArgs.put(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_COUNT, "3"); + agentArgs.put(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL, mockBackend.getIntakeUrl()); + // avoid possible race conditions on shutdown + agentArgs.put(DebuggerConfig.DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL, "999999"); + + int exitCode = whenRunningJUnitConsole(agentArgs, Collections.emptyMap()); + assertEquals(1, exitCode); + + List additionalDynamicTags = + Arrays.asList( + "content.meta.['_dd.debug.error.6.snapshot_id']", + "content.meta.['_dd.debug.error.exception_id']"); + verifyEventsAndCoverages( + projectName, + "junit-console", + "headless", + mockBackend.waitForEvents(7), + mockBackend.waitForCoverages(0), + additionalDynamicTags); + verifySnapshots(mockBackend.waitForLogs(2), 2); + } + + private void givenProjectFiles(String projectFilesSources) throws Exception { + Path projectResourcesPath = + Paths.get(this.getClass().getClassLoader().getResource(projectFilesSources).toURI()); + copyFolder(projectResourcesPath, projectHome); + } + + private void copyFolder(Path src, Path dest) throws IOException { + Files.walkFileTree( + src, + new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + Files.createDirectories(dest.resolve(src.relativize(dir))); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.copy(file, dest.resolve(src.relativize(file))); + return FileVisitResult.CONTINUE; + } + }); + + // creating empty .git directory so that the tracer could detect projectFolder as repo root + Files.createDirectory(projectHome.resolve(".git")); + } + + private int compileTestProject() throws Exception { + Path srcDir = projectHome.resolve("src/main/java"); + Path testSrcDir = projectHome.resolve("src/test/java"); + Path classesDir = projectHome.resolve("target/classes"); + Path testClassesDir = projectHome.resolve("target/test-classes"); + + Files.createDirectories(classesDir); + Files.createDirectories(testClassesDir); + + // Compile main classes if they exist + if (Files.exists(srcDir)) { + List mainJavaFiles = findJavaFiles(srcDir); + if (!mainJavaFiles.isEmpty()) { + int result = + runProcess( + createCompilerProcessBuilder( + classesDir.toString(), mainJavaFiles, Collections.emptyList()) + .start(), + PROCESS_TIMEOUT_SECS); + if (result != 0) { + LOGGER.error("Error compiling source classes for JUnit Console smoke test"); + return result; + } + } + } + + // Compile test classes + List testJavaFiles = findJavaFiles(testSrcDir); + if (!testJavaFiles.isEmpty()) { + int result = + runProcess( + createCompilerProcessBuilder( + testClassesDir.toString(), + testJavaFiles, + Collections.singletonList(classesDir.toString())) + .start(), + PROCESS_TIMEOUT_SECS); + if (result != 0) { + LOGGER.error("Error compiling source classes for JUnit Console smoke test"); + return result; + } + } + + return 0; + } + + private ProcessBuilder createCompilerProcessBuilder( + String targetDir, List files, List additionalDeps) { + assertTrue(new File(JUNIT_CONSOLE_JAR_PATH).isFile()); + + List deps = new ArrayList<>(); + deps.add(JUNIT_CONSOLE_JAR_PATH); + deps.addAll(additionalDeps); + + List command = new ArrayList<>(); + command.add(javacPath()); + command.addAll(Arrays.asList("-cp", String.join(":", deps))); + command.addAll(Arrays.asList("-d", targetDir)); + command.addAll(files); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(projectHome.toFile()); + processBuilder.environment().put("JAVA_HOME", JAVA_HOME); + return processBuilder; + } + + private static List findJavaFiles(Path directory) throws IOException { + if (!Files.exists(directory)) { + return Collections.emptyList(); + } + + List javaFiles = new ArrayList<>(); + Files.walkFileTree( + directory, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + if (file.toString().endsWith(".java")) { + javaFiles.add(file.toString()); + } + return FileVisitResult.CONTINUE; + } + }); + + return javaFiles; + } + + private int whenRunningJUnitConsole( + Map additionalAgentArgs, Map additionalEnvVars) + throws Exception { + ProcessBuilder processBuilder = + createConsoleProcessBuilder( + Collections.singletonList("execute"), additionalAgentArgs, additionalEnvVars); + processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF"); + return runProcess(processBuilder.start(), PROCESS_TIMEOUT_SECS); + } + + private static int runProcess(Process p, int timeoutSecs) throws Exception { + StreamConsumer errorGobbler = new StreamConsumer(p.getErrorStream(), "ERROR"); + StreamConsumer outputGobbler = new StreamConsumer(p.getInputStream(), "OUTPUT"); + outputGobbler.start(); + errorGobbler.start(); + + if (!p.waitFor(timeoutSecs, TimeUnit.SECONDS)) { + p.destroyForcibly(); + throw new TimeoutException( + "Instrumented process failed to exit within " + timeoutSecs + " seconds"); + } + + return p.exitValue(); + } + + ProcessBuilder createConsoleProcessBuilder( + List consoleCommand, + Map additionalAgentArgs, + Map additionalEnvVars) { + assertTrue(new File(JUNIT_CONSOLE_JAR_PATH).isFile()); + + List command = new ArrayList<>(); + command.add(javaPath()); + command.add("-Ddatadog.slf4j.simpleLogger.defaultLogLevel=DEBUG"); + command.addAll(Arrays.asList("-jar", JUNIT_CONSOLE_JAR_PATH)); + command.addAll(consoleCommand); + command.addAll( + Arrays.asList( + "--class-path", + projectHome.resolve("target/classes") + + ":" + + projectHome.resolve("target/test-classes"))); + command.add("--scan-class-path"); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(projectHome.toFile()); + + processBuilder.environment().put("JAVA_HOME", JAVA_HOME); + processBuilder.environment().put("JAVA_TOOL_OPTIONS", javaToolOptions(additionalAgentArgs)); + for (Map.Entry envVar : additionalEnvVars.entrySet()) { + processBuilder.environment().put(envVar.getKey(), envVar.getValue()); + } + + String mavenRepositoryProxy = System.getenv("MAVEN_REPOSITORY_PROXY"); + if (mavenRepositoryProxy != null) { + processBuilder.environment().put("MAVEN_REPOSITORY_PROXY", mavenRepositoryProxy); + } + + return processBuilder; + } + + String javaToolOptions(Map additionalAgentArgs) { + additionalAgentArgs.put(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED, "false"); + return buildJvmArguments(mockBackend.getIntakeUrl(), TEST_SERVICE_NAME, additionalAgentArgs) + .stream() + .collect(Collectors.joining(" ")); + } + + private static class StreamConsumer extends Thread { + final InputStream is; + final String messagePrefix; + + StreamConsumer(InputStream is, String messagePrefix) { + this.is = is; + this.messagePrefix = messagePrefix; + } + + @Override + public void run() { + try { + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line; + while ((line = br.readLine()) != null) { + System.out.println(messagePrefix + ": " + line); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } +} diff --git a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy deleted file mode 100644 index 4686c959cea..00000000000 --- a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy +++ /dev/null @@ -1,463 +0,0 @@ -package datadog.smoketest - -import datadog.environment.JavaVirtualMachine -import datadog.trace.api.civisibility.CIConstants -import datadog.trace.api.config.CiVisibilityConfig -import datadog.trace.api.config.GeneralConfig -import datadog.trace.civisibility.CiVisibilitySmokeTest -import org.apache.maven.wrapper.MavenWrapperMain -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import spock.lang.AutoCleanup -import spock.lang.IgnoreIf -import spock.lang.Shared -import spock.lang.TempDir -import spock.util.environment.Jvm - -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes -import java.util.concurrent.TimeUnit -import java.util.concurrent.TimeoutException - -import static org.junit.jupiter.api.Assumptions.assumeTrue - -@IgnoreIf(reason = "IBM8 has flaky AES-GCM TLS failures when downloading Maven artifacts", value = { - JavaVirtualMachine.isIbm8() -}) -class MavenSmokeTest extends CiVisibilitySmokeTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(MavenSmokeTest) - - private static final String LATEST_MAVEN_VERSION = getLatestMavenVersion() - - private static final String TEST_SERVICE_NAME = "test-maven-service" - - private static final int DEPENDENCIES_DOWNLOAD_TIMEOUT_SECS = 120 - private static final int PROCESS_TIMEOUT_SECS = 60 - private static final int DEPENDENCIES_DOWNLOAD_RETRIES = 5 - - @TempDir - Path projectHome - - @Shared - @AutoCleanup - MockBackend mockBackend = new MockBackend() - - def setup() { - mockBackend.reset() - } - - def "test #projectName, v#mavenVersion"() { - println "Starting: ${projectName} ${mavenVersion}" - assumeTrue(Jvm.current.isJavaVersionCompatible(minSupportedJavaVersion), - "Current JVM " + Jvm.current.javaVersion + " is not compatible with minimum required version " + minSupportedJavaVersion) - - givenWrapperPropertiesFile(mavenVersion) - givenMavenProjectFiles(projectName) - givenMavenDependenciesAreLoaded(projectName, mavenVersion) - - mockBackend.givenFlakyRetries(flakyRetries) - mockBackend.givenFlakyTest("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_failed") - - mockBackend.givenTestsSkipping(testsSkipping) - mockBackend.givenSkippableTest("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestSucceed", "test_to_skip_with_itr", ["src/main/java/datadog/smoke/Calculator.java": bits(9)]) - - mockBackend.givenImpactedTestsDetection(true) - - def coverageReportExpected = jacocoCoverage && CiVisibilitySmokeTest.classLoader.getResource(projectName + "/coverage_report_event.ftl") != null - if (coverageReportExpected) { - mockBackend.givenCodeCoverageReportUpload(true) - } - - def agentArgs = jacocoCoverage ? [(CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION): JACOCO_PLUGIN_VERSION] : [:] - def exitCode = whenRunningMavenBuild(agentArgs, commandLineParams, [:]) - - if (expectSuccess) { - assert exitCode == 0 - } else { - assert exitCode != 0 - } - - verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(expectedEvents), mockBackend.waitForCoverages(expectedCoverages)) - verifyTelemetryMetrics(mockBackend.getAllReceivedTelemetryMetrics(), mockBackend.getAllReceivedTelemetryDistributions(), expectedEvents) - - if (coverageReportExpected) { - def reports = mockBackend.waitForCoverageReports(1) - def realProjectHome = projectHome.toRealPath().toString() - verifyCoverageReports(projectName, reports, ["ci_workspace_path": realProjectHome]) - } - - where: - projectName | mavenVersion | expectedEvents | expectedCoverages | expectSuccess | testsSkipping | flakyRetries | jacocoCoverage | commandLineParams | minSupportedJavaVersion - "test_successful_maven_run" | "3.5.4" | 5 | 1 | true | true | false | true | [] | 8 - "test_successful_maven_run" | "3.6.3" | 5 | 1 | true | true | false | true | [] | 8 - "test_successful_maven_run" | "3.8.8" | 5 | 1 | true | true | false | true | [] | 8 - "test_successful_maven_run" | "3.9.9" | 5 | 1 | true | true | false | true | [] | 8 - "test_successful_maven_run_surefire_3_0_0" | "3.9.9" | 5 | 1 | true | true | false | true | [] | 8 - "test_successful_maven_run_surefire_3_0_0" | LATEST_MAVEN_VERSION | 5 | 1 | true | true | false | true | [] | 17 - "test_successful_maven_run_surefire_3_5_0" | "3.9.9" | 5 | 1 | true | true | false | true | [] | 8 - "test_successful_maven_run_surefire_3_5_0" | LATEST_MAVEN_VERSION | 5 | 1 | true | true | false | true | [] | 17 - "test_successful_maven_run_builtin_coverage" | "3.9.9" | 5 | 1 | true | true | false | false | [] | 8 - "test_successful_maven_run_with_jacoco_and_argline" | "3.9.9" | 5 | 1 | true | true | false | true | [] | 8 - // "expectedEvents" count for this test case does not include the spans that correspond to Cucumber steps - "test_successful_maven_run_with_cucumber" | "3.9.9" | 4 | 1 | true | false | false | true | [] | 8 - "test_failed_maven_run_flaky_retries" | "3.9.9" | 8 | 5 | false | false | true | true | [] | 8 - "test_successful_maven_run_junit_platform_runner" | "3.9.9" | 4 | 0 | true | false | false | false | [] | 8 - "test_successful_maven_run_with_arg_line_property" | "3.9.9" | 4 | 0 | true | false | false | false | ["-DargLine='-Dmy-custom-property=provided-via-command-line'"] | 8 - "test_successful_maven_run_multiple_forks" | "3.9.9" | 5 | 1 | true | true | false | true | [] | 8 - "test_successful_maven_run_multiple_forks" | LATEST_MAVEN_VERSION | 5 | 1 | true | true | false | true | [] | 17 - } - - def "test test management"() { - givenWrapperPropertiesFile(mavenVersion) - givenMavenProjectFiles(projectName) - givenMavenDependenciesAreLoaded(projectName, mavenVersion) - - mockBackend.givenTestManagement(true) - mockBackend.givenAttemptToFixRetries(5) - - mockBackend.givenQuarantinedTests("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_failed") - - mockBackend.givenDisabledTests("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestSucceeded", "test_succeeded") - - mockBackend.givenAttemptToFixTests("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestSucceeded", "test_another_succeeded") - - def exitCode = whenRunningMavenBuild([:], [], [:]) - assert exitCode == 0 - - verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(11), mockBackend.waitForCoverages(3)) - - where: - projectName | mavenVersion - "test_successful_maven_run_test_management" | "3.9.9" - } - - def "test junit4 class ordering #testcaseName"() { - def additionalEnvVars = ["SMOKE_TEST_SUREFIRE_VERSION": surefireVersion] - - givenWrapperPropertiesFile(mavenVersion) - givenMavenProjectFiles(projectName) - givenMavenDependenciesAreLoaded(projectName, mavenVersion, additionalEnvVars) - - for (flakyTest in flakyTests) { - mockBackend.givenFlakyTest("Maven Smoke Tests Project maven-surefire-plugin default-test", flakyTest.getSuite(), flakyTest.getName()) - } - - mockBackend.givenKnownTests(true) - for (knownTest in knownTests) { - mockBackend.givenKnownTest("Maven Smoke Tests Project maven-surefire-plugin default-test", knownTest.getSuite(), knownTest.getName()) - } - - def exitCode = whenRunningMavenBuild([(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER): CIConstants.FAIL_FAST_TEST_ORDER], [], additionalEnvVars) - assert exitCode == 0 - - verifyTestOrder(mockBackend.waitForEvents(eventsNumber), expectedOrder) - - where: - testcaseName | projectName | mavenVersion | surefireVersion | flakyTests | knownTests | expectedOrder | eventsNumber - "junit4-provider" | "test_successful_maven_run_junit4_class_ordering" | "3.9.9" | "3.0.0" | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedC", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another") - ] | 15 - "junit47-provider" | "test_successful_maven_run_junit4_class_ordering_parallel" | "3.9.9" | "3.0.0" | [test("datadog.smoke.TestSucceedC", "test_succeed")] | [ - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | 12 - "junit4-provider-latest-surefire" | "test_successful_maven_run_junit4_class_ordering" | "3.9.9" | getLatestMavenSurefireVersion() | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedC", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed_another"), - test("datadog.smoke.TestSucceedA", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedB", "test_succeed_another") - ] | 15 - "junit47-provider-latest-surefire" | "test_successful_maven_run_junit4_class_ordering_parallel" | "3.9.9" | getLatestMavenSurefireVersion() | [test("datadog.smoke.TestSucceedC", "test_succeed")] | [ - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | [ - test("datadog.smoke.TestSucceedB", "test_succeed"), - test("datadog.smoke.TestSucceedC", "test_succeed"), - test("datadog.smoke.TestSucceedA", "test_succeed") - ] | 12 - } - - def "test service name is propagated to child processes"() { - givenWrapperPropertiesFile(mavenVersion) - givenMavenProjectFiles(projectName) - givenMavenDependenciesAreLoaded(projectName, mavenVersion) - - def exitCode = whenRunningMavenBuild([:], [], [:], false) - assert exitCode == 0 - - def additionalDynamicPaths = ["content.service"] - verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(5), mockBackend.waitForCoverages(1), additionalDynamicPaths) - - where: - projectName | mavenVersion - "test_successful_maven_run_child_service_propagation" | "3.9.9" - } - - def "test failed test replay"() { - givenWrapperPropertiesFile(mavenVersion) - givenMavenProjectFiles(projectName) - givenMavenDependenciesAreLoaded(projectName, mavenVersion) - - mockBackend.givenFlakyRetries(true) - mockBackend.givenFlakyTest("Maven Smoke Tests Project maven-surefire-plugin default-test", "com.example.TestFailed", "test_failed") - mockBackend.givenFailedTestReplay(true) - - def exitCode = whenRunningMavenBuild([ - (CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_COUNT): "3", - (GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL): mockBackend.intakeUrl - ], - [], - [:]) - assert exitCode == 1 - - def additionalDynamicTags = ["content.meta.['_dd.debug.error.3.snapshot_id']", "content.meta.['_dd.debug.error.exception_id']"] - verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(7), mockBackend.waitForCoverages(0), additionalDynamicTags) - verifySnapshots(mockBackend.waitForLogs(2), 2) - - where: - projectName | mavenVersion - "test_failed_maven_failed_test_replay" | "3.9.9" - } - - private void givenWrapperPropertiesFile(String mavenVersion) { - def distributionUrl = "https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${mavenVersion}/apache-maven-${mavenVersion}-bin.zip" - - def properties = new Properties() - properties.setProperty("distributionUrl", distributionUrl) - - def propertiesFile = projectHome.resolve("maven/wrapper/maven-wrapper.properties") - Files.createDirectories(propertiesFile.getParent()) - new FileOutputStream(propertiesFile.toFile()).withCloseable { - properties.store(it, "") - } - } - - private void givenMavenProjectFiles(String projectFilesSources) { - def projectResourcesUri = this.getClass().getClassLoader().getResource(projectFilesSources).toURI() - def projectResourcesPath = Paths.get(projectResourcesUri) - copyFolder(projectResourcesPath, projectHome) - - def sharedSettingsPath = Paths.get(this.getClass().getClassLoader().getResource("settings.mirror.xml").toURI()) - Files.copy(sharedSettingsPath, projectHome.resolve("settings.mirror.xml")) - } - - private void copyFolder(Path src, Path dest) throws IOException { - Files.walkFileTree(src, new SimpleFileVisitor() { - @Override - FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - Files.createDirectories(dest.resolve(src.relativize(dir))) - return FileVisitResult.CONTINUE - } - - @Override - FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - Files.copy(file, dest.resolve(src.relativize(file))) - return FileVisitResult.CONTINUE - } - }) - - // creating empty .git directory so that the tracer could detect projectFolder as repo root - Files.createDirectory(projectHome.resolve(".git")) - } - - /** - * Sometimes Maven has problems downloading project dependencies because of intermittent network issues. - * Here, in order to reduce flakiness, we ensure that all of the dependencies are loaded (retrying if necessary), - * before proceeding with running the build - */ - private void givenMavenDependenciesAreLoaded(String projectName, String mavenVersion, Map additionalEnvVars = [:]) { - if (LOADED_DEPENDENCIES.add("$projectName:$mavenVersion")) { - retryUntilSuccessfulOrNoAttemptsLeft(["org.apache.maven.plugins:maven-dependency-plugin:3.6.1:go-offline"], additionalEnvVars) - } - // dependencies below are download separately - // because they are not declared in the project, - // but are added at runtime by the tracer - if (LOADED_DEPENDENCIES.add("com.datadoghq:dd-javac-plugin:$JAVAC_PLUGIN_VERSION")) { - retryUntilSuccessfulOrNoAttemptsLeft([ - "org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get", - "-Dartifact=com.datadoghq:dd-javac-plugin:$JAVAC_PLUGIN_VERSION".toString() - ], additionalEnvVars) - } - if (LOADED_DEPENDENCIES.add("org.jacoco:jacoco-maven-plugin:$JACOCO_PLUGIN_VERSION")) { - retryUntilSuccessfulOrNoAttemptsLeft([ - "org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get", - "-Dartifact=org.jacoco:jacoco-maven-plugin:$JACOCO_PLUGIN_VERSION".toString() - ], additionalEnvVars) - } - } - - private static final Collection LOADED_DEPENDENCIES = new HashSet<>() - - private void retryUntilSuccessfulOrNoAttemptsLeft(List mvnCommand, Map additionalEnvVars = [:]) { - def processBuilder = createProcessBuilder(mvnCommand, false, false, [:], additionalEnvVars) - for (int attempt = 0; attempt < DEPENDENCIES_DOWNLOAD_RETRIES; attempt++) { - try { - def exitCode = runProcess(processBuilder.start(), DEPENDENCIES_DOWNLOAD_TIMEOUT_SECS) - if (exitCode == 0) { - return - } - } catch (TimeoutException e) { - LOGGER.warn("Failed dependency resolution with exception: ", e) - } - } - throw new AssertionError((Object) "Tried $DEPENDENCIES_DOWNLOAD_RETRIES times to execute $mvnCommand and failed") - } - - private int whenRunningMavenBuild(Map additionalAgentArgs, List additionalCommandLineParams, Map additionalEnvVars, boolean setServiceName = true) { - def processBuilder = createProcessBuilder(["-B", "test"] + additionalCommandLineParams, true, setServiceName, additionalAgentArgs, additionalEnvVars) - - processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF") - - return runProcess(processBuilder.start()) - } - - private static runProcess(Process p, int timeoutSecs = PROCESS_TIMEOUT_SECS) { - StreamConsumer errorGobbler = new StreamConsumer(p.getErrorStream(), "ERROR") - StreamConsumer outputGobbler = new StreamConsumer(p.getInputStream(), "OUTPUT") - outputGobbler.start() - errorGobbler.start() - - if (!p.waitFor(timeoutSecs, TimeUnit.SECONDS)) { - p.destroyForcibly() - throw new TimeoutException("Instrumented process failed to exit within $timeoutSecs seconds") - } - - return p.exitValue() - } - - ProcessBuilder createProcessBuilder(List mvnCommand, boolean runWithAgent, boolean setServiceName, Map additionalAgentArgs, Map additionalEnvVars) { - String mavenRunnerShadowJar = System.getProperty("datadog.smoketest.maven.jar.path") - assert new File(mavenRunnerShadowJar).isFile() - - List command = new ArrayList<>() - command.add(javaPath()) - command.addAll(jvmArguments(runWithAgent, setServiceName, additionalAgentArgs)) - command.addAll((String[]) ["-jar", mavenRunnerShadowJar]) - command.addAll(programArguments()) - - if (System.getenv().get("MAVEN_REPOSITORY_PROXY") != null) { - command.addAll(["-s", "${projectHome.toAbsolutePath()}/settings.mirror.xml".toString()]) - } - command.addAll(mvnCommand) - - ProcessBuilder processBuilder = new ProcessBuilder(command) - processBuilder.directory(projectHome.toFile()) - - processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")) - for (envVar in additionalEnvVars) { - processBuilder.environment().put(envVar.key, envVar.value) - } - - def mavenRepositoryProxy = System.getenv("MAVEN_REPOSITORY_PROXY") - if (mavenRepositoryProxy != null) { - processBuilder.environment().put("MAVEN_REPOSITORY_PROXY", mavenRepositoryProxy) - } - - return processBuilder - } - - List jvmArguments(boolean runWithAgent, boolean setServiceName, Map additionalAgentArgs) { - def arguments = [ - "-D${MavenWrapperMain.MVNW_VERBOSE}=true".toString(), - "-Duser.dir=${projectHome.toAbsolutePath()}".toString(), - "-Dmaven.mainClass=org.apache.maven.cli.MavenCli".toString(), - "-Dmaven.multiModuleProjectDirectory=${projectHome.toAbsolutePath()}".toString(), - "-Dmaven.artifact.threads=10", - ] - if (runWithAgent) { - arguments += buildJvmArguments(mockBackend.intakeUrl, setServiceName ? TEST_SERVICE_NAME : null, additionalAgentArgs) - } - return arguments - } - - List programArguments() { - return [projectHome.toAbsolutePath().toString()] - } - - private static class StreamConsumer extends Thread { - final InputStream is - final String messagePrefix - - StreamConsumer(InputStream is, String messagePrefix) { - this.is = is - this.messagePrefix = messagePrefix - } - - @Override - void run() { - try { - BufferedReader br = new BufferedReader(new InputStreamReader(is)) - String line - while ((line = br.readLine()) != null) { - System.out.println(messagePrefix + ": " + line) - } - } catch (IOException e) { - e.printStackTrace() - } - } - } - - private static Properties loadLatestToolVersions() { - def properties = new Properties() - def stream = MavenSmokeTest.classLoader.getResourceAsStream("latest-tool-versions.properties") - if (stream == null) { - throw new IllegalStateException("Could not find latest-tool-versions.properties on classpath") - } - stream.withCloseable { properties.load(it) } - return properties - } - - private static String getLatestMavenVersion() { - def version = loadLatestToolVersions().getProperty("maven.version") - LOGGER.info("Will run the 'latest' tests with Maven version ${version}") - return version - } - - private static String getLatestMavenSurefireVersion() { - def version = loadLatestToolVersions().getProperty("maven-surefire.version") - LOGGER.info("Will run the 'latest' tests with Maven Surefire version ${version}") - return version - } - - private static BitSet bits(int ... indices) { - BitSet bitSet = new BitSet() - for (int i : indices) { - bitSet.set(i) - } - return bitSet - } -} diff --git a/dd-smoke-tests/maven/src/test/java/datadog/smoketest/MavenSmokeTest.java b/dd-smoke-tests/maven/src/test/java/datadog/smoketest/MavenSmokeTest.java new file mode 100644 index 00000000000..50504d0f7c5 --- /dev/null +++ b/dd-smoke-tests/maven/src/test/java/datadog/smoketest/MavenSmokeTest.java @@ -0,0 +1,610 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import datadog.environment.JavaVirtualMachine; +import datadog.trace.api.civisibility.CIConstants; +import datadog.trace.api.civisibility.config.TestFQN; +import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.civisibility.CiVisibilitySmokeTest; +import datadog.trace.civisibility.CiVisibilityTableTestConverters; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.maven.wrapper.MavenWrapperMain; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.condition.DisabledIf; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tabletest.junit.TableTest; +import org.tabletest.junit.TypeConverterSources; + +@DisabledIf( + value = "disabledOnIbm8", + disabledReason = "IBM8 has flaky AES-GCM TLS failures when downloading Maven artifacts") +@TypeConverterSources(CiVisibilityTableTestConverters.class) +class MavenSmokeTest extends CiVisibilitySmokeTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(MavenSmokeTest.class); + + private static final String LATEST_MAVEN_VERSION = getLatestMavenVersion(); + + private static final String TEST_SERVICE_NAME = "test-maven-service"; + + private static final int DEPENDENCIES_DOWNLOAD_TIMEOUT_SECS = 120; + private static final int PROCESS_TIMEOUT_SECS = 60; + private static final int DEPENDENCIES_DOWNLOAD_RETRIES = 5; + + public static boolean disabledOnIbm8() { + return JavaVirtualMachine.isIbm8(); + } + + @TempDir Path projectHome; + + static final MockBackend mockBackend = new MockBackend(); + + @BeforeEach + void resetMockBackend() { + mockBackend.reset(); + } + + @AfterAll + static void closeMockBackend() throws Exception { + mockBackend.close(); + } + + @TableTest({ + "scenario | projectName | mavenVersion | expectedEvents | expectedCoverages | expectSuccess | testsSkipping | flakyRetries | jacocoCoverage | commandLineParams | minSupportedJavaVersion", + "succeed-base | test_successful_maven_run | {3.5.4, 3.6.3, 3.8.8, 3.9.9} | 5 | 1 | true | true | false | true | [] | 8 ", + "succeed-surefire-3.0.0-j8 | test_successful_maven_run_surefire_3_0_0 | 3.9.9 | 5 | 1 | true | true | false | true | [] | 8 ", + "succeed-surefire-3.0.0-j17 | test_successful_maven_run_surefire_3_0_0 | latest | 5 | 1 | true | true | false | true | [] | 17 ", + "succeed-surefire-3.5.0-j8 | test_successful_maven_run_surefire_3_5_0 | 3.9.9 | 5 | 1 | true | true | false | true | [] | 8 ", + "succeed-surefire-3.5.0-j17 | test_successful_maven_run_surefire_3_5_0 | latest | 5 | 1 | true | true | false | true | [] | 17 ", + "succeed-builtin-coverage | test_successful_maven_run_builtin_coverage | 3.9.9 | 5 | 1 | true | true | false | false | [] | 8 ", + "succeed-jacoco-argline | test_successful_maven_run_with_jacoco_and_argline | 3.9.9 | 5 | 1 | true | true | false | true | [] | 8 ", + "succeed-cucumber | test_successful_maven_run_with_cucumber | 3.9.9 | 4 | 1 | true | false | false | true | [] | 8 ", + "failed-flaky-retries | test_failed_maven_run_flaky_retries | 3.9.9 | 8 | 5 | false | false | true | true | [] | 8 ", + "succeed-junit-platform | test_successful_maven_run_junit_platform_runner | 3.9.9 | 4 | 0 | true | false | false | false | [] | 8 ", + "succeed-arg-line-property | test_successful_maven_run_with_arg_line_property | 3.9.9 | 4 | 0 | true | false | false | false | [\"-DargLine='-Dmy-custom-property=provided-via-command-line'\"] | 8 ", + "succeed-multi-forks-j8 | test_successful_maven_run_multiple_forks | 3.9.9 | 5 | 1 | true | true | false | true | [] | 8 ", + "succeed-multi-forks-j17 | test_successful_maven_run_multiple_forks | latest | 5 | 1 | true | true | false | true | [] | 17 " + }) + @ParameterizedTest + void testMavenRun( + String projectName, + String mavenVersion, + int expectedEvents, + int expectedCoverages, + boolean expectSuccess, + boolean testsSkipping, + boolean flakyRetries, + boolean jacocoCoverage, + List commandLineParams, + int minSupportedJavaVersion) + throws Exception { + mavenVersion = resolveLatestMaven(mavenVersion); + System.out.println("Starting: " + projectName + " " + mavenVersion); + assumeTrue( + JavaVirtualMachine.isJavaVersionAtLeast(minSupportedJavaVersion), + "Current JVM is not compatible with minimum required version " + minSupportedJavaVersion); + + givenWrapperPropertiesFile(mavenVersion); + givenMavenProjectFiles(projectName); + givenMavenDependenciesAreLoaded(projectName, mavenVersion); + + mockBackend.givenFlakyRetries(flakyRetries); + mockBackend.givenFlakyTest( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + "datadog.smoke.TestFailed", + "test_failed"); + + mockBackend.givenTestsSkipping(testsSkipping); + mockBackend.givenSkippableTest( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + "datadog.smoke.TestSucceed", + "test_to_skip_with_itr", + Collections.singletonMap("src/main/java/datadog/smoke/Calculator.java", bits(9))); + + mockBackend.givenImpactedTestsDetection(true); + + boolean coverageReportExpected = + jacocoCoverage + && CiVisibilitySmokeTest.class + .getClassLoader() + .getResource(projectName + "/coverage_report_event.ftl") + != null; + if (coverageReportExpected) { + mockBackend.givenCodeCoverageReportUpload(true); + } + + Map agentArgs = + jacocoCoverage + ? Collections.singletonMap( + CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION, JACOCO_PLUGIN_VERSION) + : Collections.emptyMap(); + int exitCode = + whenRunningMavenBuild(agentArgs, commandLineParams, Collections.emptyMap(), true); + + if (expectSuccess) { + assertEquals(0, exitCode); + } else { + assertNotEquals(0, exitCode); + } + + verifyEventsAndCoverages( + projectName, + "maven", + mavenVersion, + mockBackend.waitForEvents(expectedEvents), + mockBackend.waitForCoverages(expectedCoverages)); + verifyTelemetryMetrics( + mockBackend.getAllReceivedTelemetryMetrics(), + mockBackend.getAllReceivedTelemetryDistributions(), + expectedEvents); + + if (coverageReportExpected) { + List reports = + mockBackend.waitForCoverageReports(1); + String realProjectHome = projectHome.toRealPath().toString(); + verifyCoverageReports( + projectName, reports, Collections.singletonMap("ci_workspace_path", realProjectHome)); + } + } + + @TableTest({ + "scenario | projectName | mavenVersion", + "test-management | test_successful_maven_run_test_management | 3.9.9 " + }) + @ParameterizedTest + void testTestManagement(String projectName, String mavenVersion) throws Exception { + givenWrapperPropertiesFile(mavenVersion); + givenMavenProjectFiles(projectName); + givenMavenDependenciesAreLoaded(projectName, mavenVersion); + + mockBackend.givenTestManagement(true); + mockBackend.givenAttemptToFixRetries(5); + + mockBackend.givenQuarantinedTests( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + "datadog.smoke.TestFailed", + "test_failed"); + + mockBackend.givenDisabledTests( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + "datadog.smoke.TestSucceeded", + "test_succeeded"); + + mockBackend.givenAttemptToFixTests( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + "datadog.smoke.TestSucceeded", + "test_another_succeeded"); + + int exitCode = + whenRunningMavenBuild( + Collections.emptyMap(), Collections.emptyList(), Collections.emptyMap(), true); + assertEquals(0, exitCode); + + verifyEventsAndCoverages( + projectName, + "maven", + mavenVersion, + mockBackend.waitForEvents(11), + mockBackend.waitForCoverages(3)); + } + + @TableTest({ + "scenario | projectName | mavenVersion | surefireVersion | flakyTests | knownTests | expectedOrder | eventsNumber", + "junit4-provider | test_successful_maven_run_junit4_class_ordering | 3.9.9 | 3.0.0 | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 ", + "junit47-provider | test_successful_maven_run_junit4_class_ordering_parallel | 3.9.9 | 3.0.0 | ['datadog.smoke.TestSucceedC:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedA:test_succeed'] | 12 ", + "junit4-provider-latest-surefire | test_successful_maven_run_junit4_class_ordering | 3.9.9 | latest-maven-surefire | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed_another', 'datadog.smoke.TestSucceedA:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedB:test_succeed_another'] | 15 ", + "junit47-provider-latest-surefire | test_successful_maven_run_junit4_class_ordering_parallel | 3.9.9 | latest-maven-surefire | ['datadog.smoke.TestSucceedC:test_succeed'] | ['datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedA:test_succeed'] | ['datadog.smoke.TestSucceedB:test_succeed', 'datadog.smoke.TestSucceedC:test_succeed', 'datadog.smoke.TestSucceedA:test_succeed'] | 12 " + }) + @ParameterizedTest + void testJunit4ClassOrdering( + String projectName, + String mavenVersion, + String surefireVersion, + List flakyTests, + List knownTests, + List expectedOrder, + int eventsNumber) + throws Exception { + surefireVersion = + "latest-maven-surefire".equals(surefireVersion) + ? getLatestMavenSurefireVersion() + : surefireVersion; + Map additionalEnvVars = new HashMap<>(); + additionalEnvVars.put("SMOKE_TEST_SUREFIRE_VERSION", surefireVersion); + + givenWrapperPropertiesFile(mavenVersion); + givenMavenProjectFiles(projectName); + givenMavenDependenciesAreLoaded(projectName, mavenVersion, additionalEnvVars); + + for (TestFQN flakyTest : flakyTests) { + mockBackend.givenFlakyTest( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + flakyTest.getSuite(), + flakyTest.getName()); + } + + mockBackend.givenKnownTests(true); + for (TestFQN knownTest : knownTests) { + mockBackend.givenKnownTest( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + knownTest.getSuite(), + knownTest.getName()); + } + + int exitCode = + whenRunningMavenBuild( + Collections.singletonMap( + CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, CIConstants.FAIL_FAST_TEST_ORDER), + Collections.emptyList(), + additionalEnvVars, + true); + assertEquals(0, exitCode); + + verifyTestOrder(mockBackend.waitForEvents(eventsNumber), expectedOrder); + } + + @TableTest({ + "scenario | projectName | mavenVersion", + "child-service-propagation | test_successful_maven_run_child_service_propagation | 3.9.9 " + }) + @ParameterizedTest + void testServiceNamePropagation(String projectName, String mavenVersion) throws Exception { + givenWrapperPropertiesFile(mavenVersion); + givenMavenProjectFiles(projectName); + givenMavenDependenciesAreLoaded(projectName, mavenVersion); + + int exitCode = + whenRunningMavenBuild( + Collections.emptyMap(), Collections.emptyList(), Collections.emptyMap(), false); + assertEquals(0, exitCode); + + List additionalDynamicPaths = Collections.singletonList("content.service"); + verifyEventsAndCoverages( + projectName, + "maven", + mavenVersion, + mockBackend.waitForEvents(5), + mockBackend.waitForCoverages(1), + additionalDynamicPaths); + } + + @TableTest({ + "scenario | projectName | mavenVersion", + "failed-test-replay | test_failed_maven_failed_test_replay | 3.9.9 " + }) + @ParameterizedTest + void testFailedTestReplay(String projectName, String mavenVersion) throws Exception { + givenWrapperPropertiesFile(mavenVersion); + givenMavenProjectFiles(projectName); + givenMavenDependenciesAreLoaded(projectName, mavenVersion); + + mockBackend.givenFlakyRetries(true); + mockBackend.givenFlakyTest( + "Maven Smoke Tests Project maven-surefire-plugin default-test", + "com.example.TestFailed", + "test_failed"); + mockBackend.givenFailedTestReplay(true); + + Map agentArgs = new HashMap<>(); + agentArgs.put(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_COUNT, "3"); + agentArgs.put(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL, mockBackend.getIntakeUrl()); + + int exitCode = + whenRunningMavenBuild(agentArgs, Collections.emptyList(), Collections.emptyMap(), true); + assertEquals(1, exitCode); + + List additionalDynamicTags = + Arrays.asList( + "content.meta.['_dd.debug.error.3.snapshot_id']", + "content.meta.['_dd.debug.error.exception_id']"); + verifyEventsAndCoverages( + projectName, + "maven", + mavenVersion, + mockBackend.waitForEvents(7), + mockBackend.waitForCoverages(0), + additionalDynamicTags); + verifySnapshots(mockBackend.waitForLogs(2), 2); + } + + private void givenWrapperPropertiesFile(String mavenVersion) throws IOException { + String distributionUrl = + "https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/" + + mavenVersion + + "/apache-maven-" + + mavenVersion + + "-bin.zip"; + + Properties properties = new Properties(); + properties.setProperty("distributionUrl", distributionUrl); + + Path propertiesFile = projectHome.resolve("maven/wrapper/maven-wrapper.properties"); + Files.createDirectories(propertiesFile.getParent()); + try (FileOutputStream fos = new FileOutputStream(propertiesFile.toFile())) { + properties.store(fos, ""); + } + } + + private void givenMavenProjectFiles(String projectFilesSources) throws Exception { + Path projectResourcesPath = + Paths.get(this.getClass().getClassLoader().getResource(projectFilesSources).toURI()); + copyFolder(projectResourcesPath, projectHome); + + Path sharedSettingsPath = + Paths.get(this.getClass().getClassLoader().getResource("settings.mirror.xml").toURI()); + Files.copy(sharedSettingsPath, projectHome.resolve("settings.mirror.xml")); + } + + private void copyFolder(Path src, Path dest) throws IOException { + Files.walkFileTree( + src, + new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + Files.createDirectories(dest.resolve(src.relativize(dir))); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.copy(file, dest.resolve(src.relativize(file))); + return FileVisitResult.CONTINUE; + } + }); + + // creating empty .git directory so that the tracer could detect projectFolder as repo root + Files.createDirectory(projectHome.resolve(".git")); + } + + /** + * Sometimes Maven has problems downloading project dependencies because of intermittent network + * issues. Here, in order to reduce flakiness, we ensure that all of the dependencies are loaded + * (retrying if necessary), before proceeding with running the build. + */ + private void givenMavenDependenciesAreLoaded(String projectName, String mavenVersion) + throws Exception { + givenMavenDependenciesAreLoaded(projectName, mavenVersion, Collections.emptyMap()); + } + + private void givenMavenDependenciesAreLoaded( + String projectName, String mavenVersion, Map additionalEnvVars) + throws Exception { + if (LOADED_DEPENDENCIES.add(projectName + ":" + mavenVersion)) { + retryUntilSuccessfulOrNoAttemptsLeft( + Collections.singletonList( + "org.apache.maven.plugins:maven-dependency-plugin:3.6.1:go-offline"), + additionalEnvVars); + } + // Dependencies below are downloaded separately because they are not declared in the project, + // but are added at runtime by the tracer. + if (LOADED_DEPENDENCIES.add("com.datadoghq:dd-javac-plugin:" + JAVAC_PLUGIN_VERSION)) { + retryUntilSuccessfulOrNoAttemptsLeft( + Arrays.asList( + "org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get", + "-Dartifact=com.datadoghq:dd-javac-plugin:" + JAVAC_PLUGIN_VERSION), + additionalEnvVars); + } + if (LOADED_DEPENDENCIES.add("org.jacoco:jacoco-maven-plugin:" + JACOCO_PLUGIN_VERSION)) { + retryUntilSuccessfulOrNoAttemptsLeft( + Arrays.asList( + "org.apache.maven.plugins:maven-dependency-plugin:3.6.1:get", + "-Dartifact=org.jacoco:jacoco-maven-plugin:" + JACOCO_PLUGIN_VERSION), + additionalEnvVars); + } + } + + private static final Collection LOADED_DEPENDENCIES = new HashSet<>(); + + private void retryUntilSuccessfulOrNoAttemptsLeft( + List mvnCommand, Map additionalEnvVars) throws Exception { + ProcessBuilder processBuilder = + createProcessBuilder(mvnCommand, false, false, Collections.emptyMap(), additionalEnvVars); + for (int attempt = 0; attempt < DEPENDENCIES_DOWNLOAD_RETRIES; attempt++) { + try { + int exitCode = runProcess(processBuilder.start(), DEPENDENCIES_DOWNLOAD_TIMEOUT_SECS); + if (exitCode == 0) { + return; + } + } catch (TimeoutException e) { + LOGGER.warn("Failed dependency resolution with exception: ", e); + } + } + throw new AssertionError( + "Tried " + + DEPENDENCIES_DOWNLOAD_RETRIES + + " times to execute " + + mvnCommand + + " and failed"); + } + + private int whenRunningMavenBuild( + Map additionalAgentArgs, + List additionalCommandLineParams, + Map additionalEnvVars, + boolean setServiceName) + throws Exception { + List mvnCommand = new ArrayList<>(); + mvnCommand.add("-B"); + mvnCommand.add("test"); + mvnCommand.addAll(additionalCommandLineParams); + + ProcessBuilder processBuilder = + createProcessBuilder( + mvnCommand, true, setServiceName, additionalAgentArgs, additionalEnvVars); + + processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF"); + + return runProcess(processBuilder.start(), PROCESS_TIMEOUT_SECS); + } + + private static int runProcess(Process p, int timeoutSecs) throws Exception { + StreamConsumer errorGobbler = new StreamConsumer(p.getErrorStream(), "ERROR"); + StreamConsumer outputGobbler = new StreamConsumer(p.getInputStream(), "OUTPUT"); + outputGobbler.start(); + errorGobbler.start(); + + if (!p.waitFor(timeoutSecs, TimeUnit.SECONDS)) { + p.destroyForcibly(); + throw new TimeoutException( + "Instrumented process failed to exit within " + timeoutSecs + " seconds"); + } + + return p.exitValue(); + } + + ProcessBuilder createProcessBuilder( + List mvnCommand, + boolean runWithAgent, + boolean setServiceName, + Map additionalAgentArgs, + Map additionalEnvVars) { + String mavenRunnerShadowJar = System.getProperty("datadog.smoketest.maven.jar.path"); + assertTrue(new File(mavenRunnerShadowJar).isFile()); + + List command = new ArrayList<>(); + command.add(javaPath()); + command.addAll(jvmArguments(runWithAgent, setServiceName, additionalAgentArgs)); + command.addAll(Arrays.asList("-jar", mavenRunnerShadowJar)); + command.addAll(programArguments()); + + if (System.getenv().get("MAVEN_REPOSITORY_PROXY") != null) { + command.addAll(Arrays.asList("-s", projectHome.toAbsolutePath() + "/settings.mirror.xml")); + } + command.addAll(mvnCommand); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(projectHome.toFile()); + + processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")); + for (Map.Entry envVar : additionalEnvVars.entrySet()) { + processBuilder.environment().put(envVar.getKey(), envVar.getValue()); + } + + String mavenRepositoryProxy = System.getenv("MAVEN_REPOSITORY_PROXY"); + if (mavenRepositoryProxy != null) { + processBuilder.environment().put("MAVEN_REPOSITORY_PROXY", mavenRepositoryProxy); + } + + return processBuilder; + } + + List jvmArguments( + boolean runWithAgent, boolean setServiceName, Map additionalAgentArgs) { + List arguments = new ArrayList<>(); + arguments.add("-D" + MavenWrapperMain.MVNW_VERBOSE + "=true"); + arguments.add("-Duser.dir=" + projectHome.toAbsolutePath()); + arguments.add("-Dmaven.mainClass=org.apache.maven.cli.MavenCli"); + arguments.add("-Dmaven.multiModuleProjectDirectory=" + projectHome.toAbsolutePath()); + arguments.add("-Dmaven.artifact.threads=10"); + if (runWithAgent) { + arguments.addAll( + buildJvmArguments( + mockBackend.getIntakeUrl(), + setServiceName ? TEST_SERVICE_NAME : null, + additionalAgentArgs)); + } + return arguments; + } + + List programArguments() { + return Collections.singletonList(projectHome.toAbsolutePath().toString()); + } + + private static class StreamConsumer extends Thread { + final InputStream is; + final String messagePrefix; + + StreamConsumer(InputStream is, String messagePrefix) { + this.is = is; + this.messagePrefix = messagePrefix; + } + + @Override + public void run() { + try { + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line; + while ((line = br.readLine()) != null) { + System.out.println(messagePrefix + ": " + line); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private static Properties loadLatestToolVersions() { + Properties properties = new Properties(); + try (InputStream stream = + MavenSmokeTest.class + .getClassLoader() + .getResourceAsStream("latest-tool-versions.properties")) { + if (stream == null) { + throw new IllegalStateException( + "Could not find latest-tool-versions.properties on classpath"); + } + properties.load(stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + return properties; + } + + private static String getLatestMavenVersion() { + String version = loadLatestToolVersions().getProperty("maven.version"); + LOGGER.info("Will run the 'latest' tests with Maven version {}", version); + return version; + } + + private static String getLatestMavenSurefireVersion() { + String version = loadLatestToolVersions().getProperty("maven-surefire.version"); + LOGGER.info("Will run the 'latest' tests with Maven Surefire version {}", version); + return version; + } + + private static BitSet bits(int... indices) { + BitSet bitSet = new BitSet(); + for (int i : indices) { + bitSet.set(i); + } + return bitSet; + } + + private static String resolveLatestMaven(String mavenVersion) { + return "latest".equals(mavenVersion) ? LATEST_MAVEN_VERSION : mavenVersion; + } +}