diff --git a/docs/plugins.md b/docs/plugins.md index a3e89fc04..b3f16510d 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -569,6 +569,49 @@ More config options are available: * `config` (optional, default `{}`) +## junitReporter + +Generates a JUnit-compatible XML report after a test run. + +Unlike Mocha's `mocha-junit-reporter`, this plugin understands CodeceptJS steps and substeps. +For every `` it includes: + +* `` — the test's meta information: every `meta` key from `Scenario('...', { meta })`, plus its `tags` and `retries` +* `` — an indented step/substep log (substeps are nested under their meta step); only failed steps are marked +* `` — for failed tests: the error message, type, stack trace and (optionally) the step trace + +The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions test reporters, etc. + +#### Configuration + +```js +"plugins": { + "junitReporter": { + "enabled": true + } + } +``` + +Possible config options: + +* `outputName`: file name for the report. Default: `report.xml`. +* `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. +* `testGroupName`: value of the `name` attribute on the root `` element. Default: `CodeceptJS`. +* `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as ``. Default: true. +* `attachSteps`: add the step/substep log as ``. Default: true. +* `stepsInFailure`: append the step trace to the `` body. Default: true. + +CLI examples: + + npx codeceptjs run -p junitReporter + npx codeceptjs run -p junitReporter:outputName=junit.xml + +> ℹ When running with `run-workers`, steps are serialized between processes and substep nesting is flattened. + +### Parameters + +* `config` **any** (optional, default `{}`) + ## pageInfo Collects information from web page after each failed test and adds it to the test as an artifact. diff --git a/lib/plugin/junitReporter.js b/lib/plugin/junitReporter.js new file mode 100644 index 000000000..025676234 --- /dev/null +++ b/lib/plugin/junitReporter.js @@ -0,0 +1,303 @@ +import fs from 'fs' +import path from 'path' +import os from 'os' +import { mkdirp } from 'mkdirp' +import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom' + +import event from '../event.js' +import store from '../store.js' +import output from '../output.js' + +const defaultConfig = { + outputName: 'report.xml', + output: null, + testGroupName: 'CodeceptJS', + attachSteps: true, + attachMeta: true, + stepsInFailure: true, +} + +const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\uFFFE\\uFFFF]', 'g') + +/** + * + * Generates a JUnit-compatible XML report after a test run. + * + * Unlike Mocha's `mocha-junit-reporter`, this plugin understands CodeceptJS steps and substeps. + * For every `` it includes: + * + * * `` — the test's meta information: every `meta` key from `Scenario('...', { meta })`, plus its `tags` and `retries` + * * `` — an indented step/substep log (substeps are nested under their meta step); only failed steps are marked + * * `` — for failed tests: the error message, type, stack trace and (optionally) the step trace + * + * The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions test reporters, etc. + * + * #### Configuration + * + * ```js + * "plugins": { + * "junitReporter": { + * "enabled": true + * } + * } + * ``` + * + * Possible config options: + * + * * `outputName`: file name for the report. Default: `report.xml`. + * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. + * * `testGroupName`: value of the `name` attribute on the root `` element. Default: `CodeceptJS`. + * * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as ``. Default: true. + * * `attachSteps`: add the step/substep log as ``. Default: true. + * * `stepsInFailure`: append the step trace to the `` body. Default: true. + * + * CLI examples: + * + * ``` + * npx codeceptjs run -p junitReporter + * npx codeceptjs run -p junitReporter:outputName=junit.xml + * ``` + * + * > ℹ When running with `run-workers`, steps are serialized between processes and substep nesting is flattened. + * + * @param {*} config + */ +export default function (config = {}) { + config = Object.assign({}, defaultConfig, config) + + let written = false + + const writeReport = result => { + if (written) return + if (!result || !Array.isArray(result.tests)) return + written = true + + const dir = config.output ? path.resolve(store.codeceptDir || process.cwd(), config.output) : store.outputDir || process.cwd() + mkdirp.sync(dir) + const file = path.join(dir, config.outputName) + + fs.writeFileSync(file, buildXml(result, config)) + output.plugin('junitReporter', `JUnit report saved to ${file}`) + } + + event.dispatcher.on(event.all.result, writeReport) + event.dispatcher.on(event.workers.result, writeReport) +} + +function buildXml(result, config) { + const doc = new DOMImplementation().createDocument(null, null, null) + const suites = groupBySuite(result.tests) + + const root = doc.createElement('testsuites') + setAttr(root, 'name', config.testGroupName) + setAttr(root, 'tests', result.tests.length) + setAttr(root, 'failures', countState(result.tests, 'failed')) + setAttr(root, 'skipped', countSkipped(result.tests)) + setAttr(root, 'errors', 0) + setAttr(root, 'time', toSeconds(sumDuration(result.tests))) + setAttr(root, 'timestamp', toIso(result.stats && result.stats.start)) + doc.appendChild(root) + + suites.forEach((tests, index) => { + const suite = tests[0] && tests[0].parent + const suiteName = (suite && suite.title) || 'Tests' + const suiteFile = (suite && suite.file) || (tests[0] && tests[0].file) || '' + + const suiteEl = doc.createElement('testsuite') + setAttr(suiteEl, 'name', suiteName) + setAttr(suiteEl, 'id', index) + setAttr(suiteEl, 'tests', tests.length) + setAttr(suiteEl, 'failures', countState(tests, 'failed')) + setAttr(suiteEl, 'skipped', countSkipped(tests)) + setAttr(suiteEl, 'errors', 0) + setAttr(suiteEl, 'time', toSeconds(sumDuration(tests))) + setAttr(suiteEl, 'timestamp', toIso(suite && suite.startedAt)) + setAttr(suiteEl, 'hostname', os.hostname()) + if (suiteFile) setAttr(suiteEl, 'file', suiteFile) + root.appendChild(suiteEl) + + for (const test of tests) { + suiteEl.appendChild(buildTestCase(doc, test, suiteName, config)) + } + }) + + return '\n' + new XMLSerializer().serializeToString(doc) + '\n' +} + +function buildTestCase(doc, test, suiteName, config) { + const testEl = doc.createElement('testcase') + setAttr(testEl, 'name', test.title || '(no title)') + setAttr(testEl, 'classname', suiteName) + setAttr(testEl, 'time', toSeconds(test.duration || 0)) + const file = test.file || (test.parent && test.parent.file) + if (file) setAttr(testEl, 'file', file) + + if (config.attachMeta) { + const properties = metaProperties(test) + if (properties.length) { + const propertiesEl = doc.createElement('properties') + for (const [name, value] of properties) { + const prop = doc.createElement('property') + setAttr(prop, 'name', name) + setAttr(prop, 'value', value) + propertiesEl.appendChild(prop) + } + testEl.appendChild(propertiesEl) + } + } + + const flat = flattenSteps(Array.isArray(test.steps) ? test.steps : []) + + if (test.state === 'skipped' || test.state === 'pending') { + const skipped = doc.createElement('skipped') + const reason = skipReason(test) + if (reason) setAttr(skipped, 'message', reason) + testEl.appendChild(skipped) + } else if (test.state === 'failed') { + const err = test.err || {} + const failure = doc.createElement('failure') + setAttr(failure, 'message', err.message || 'Test failed') + setAttr(failure, 'type', err.name || 'Error') + let body = err.stack || err.message || 'Test failed' + if (config.stepsInFailure && flat.length) { + body += '\n\nSteps:\n' + flat.map(stepLogLine).join('\n') + } + failure.appendChild(doc.createTextNode(cleanText(body))) + testEl.appendChild(failure) + } + + if (config.attachSteps && flat.length) { + const out = doc.createElement('system-out') + out.appendChild(doc.createTextNode(cleanText(flat.map(stepLogLine).join('\n')))) + testEl.appendChild(out) + } + + return testEl +} + +function metaProperties(test) { + const props = [] + const meta = test.meta || {} + for (const key of Object.keys(meta)) { + if (meta[key] === undefined || meta[key] === null) continue + props.push([key, stringifyMeta(meta[key])]) + } + if (Array.isArray(test.tags) && test.tags.length) { + props.push(['tags', test.tags.join(' ')]) + } + if (test.retries > 0 || test.retryNum > 0) { + props.push(['retries', String(test.retryNum || test.retries)]) + } + return props +} + +function stringifyMeta(value) { + if (typeof value === 'string') return value + if (typeof value === 'number' || typeof value === 'boolean') return String(value) + try { + return JSON.stringify(value) + } catch (err) { + return String(value) + } +} + +function flattenSteps(steps) { + const out = [] + let prevChain = [] + + for (const step of steps) { + const chain = metaChain(step) + + let common = 0 + while (common < chain.length && common < prevChain.length && chain[common].key === prevChain[common].key) common++ + + for (let d = common; d < chain.length; d++) { + out.push({ depth: d, step: chain[d].step }) + } + out.push({ depth: chain.length, step }) + prevChain = chain + } + + return out +} + +function metaChain(step) { + const chain = [] + let meta = step && step.metaStep + while (meta) { + chain.unshift({ step: meta, key: meta }) + meta = meta.metaStep + } + if (!chain.length && step && step.parent && step.parent.title) { + chain.push({ step: { title: step.parent.title, status: step.status }, key: `meta:${step.parent.title}` }) + } + return chain +} + +function stepLogLine(entry) { + const indent = ' '.repeat(entry.depth) + const mark = entry.step && entry.step.status === 'failed' ? '[FAILED] ' : '' + return `${indent}${mark}${stepText(entry.step)} (${stepDuration(entry.step)}ms)` +} + +function stepText(step) { + if (step && typeof step.toString === 'function' && step.toString !== Object.prototype.toString) return step.toString() + return (step && (step.title || step.name)) || 'step' +} + +function stepDuration(step) { + if (!step) return 0 + if (typeof step.duration === 'number' && step.duration >= 0) return step.duration + if (step.startTime && step.endTime) return Math.max(0, step.endTime - step.startTime) + return 0 +} + +function groupBySuite(tests) { + const groups = [] + const byKey = new Map() + for (const test of tests) { + const key = test.parent || test + if (!byKey.has(key)) { + const list = [] + byKey.set(key, list) + groups.push(list) + } + byKey.get(key).push(test) + } + return groups +} + +function skipReason(test) { + if (test.opts && test.opts.skipInfo && test.opts.skipInfo.message) return test.opts.skipInfo.message + if (test.meta && test.meta.skipReason) return test.meta.skipReason + return '' +} + +function countState(tests, state) { + return tests.filter(t => t.state === state).length +} + +function countSkipped(tests) { + return tests.filter(t => t.state === 'skipped' || t.state === 'pending').length +} + +function sumDuration(tests) { + return tests.reduce((sum, t) => sum + (t.duration || 0), 0) +} + +function toSeconds(ms) { + return (Math.max(0, ms) / 1000).toFixed(3) +} + +function toIso(value) { + const date = value ? new Date(value) : new Date() + return Number.isNaN(date.getTime()) ? new Date().toISOString() : date.toISOString() +} + +function cleanText(text) { + return String(text == null ? '' : text).replace(INVALID_XML_CHARS, '') +} + +function setAttr(el, name, value) { + el.setAttribute(name, cleanText(value)) +} diff --git a/test/unit/junitReporter_test.js b/test/unit/junitReporter_test.js new file mode 100644 index 000000000..d0c6929ca --- /dev/null +++ b/test/unit/junitReporter_test.js @@ -0,0 +1,201 @@ +import fs from 'fs' +import os from 'os' +import path from 'path' +import { assert, expect } from 'chai' +import { describe, it, beforeEach, afterEach } from 'mocha' +import xml2js from 'xml2js' + +import junitReporter from '../../lib/plugin/junitReporter.js' +import event from '../../lib/event.js' +import store from '../../lib/store.js' +import Step from '../../lib/step/base.js' +import MetaStep from '../../lib/step/meta.js' + +function step(name, args, status) { + const s = new Step(name) + s.args = args + s.status = status + s.startTime = 1000 + s.endTime = 1042 + return s +} + +function metaStep(actor, method, args, status) { + const m = new MetaStep(actor, method) + m.args = args + m.status = status + m.startTime = 1000 + m.endTime = 1310 + return m +} + +function makeResult() { + const loginSuite = { title: 'Login', file: '/tests/login_test.js', startedAt: Date.now() } + const dashSuite = { title: 'Dashboard', file: '/tests/dashboard_test.js', startedAt: Date.now() } + + const login = metaStep('user', 'login', ['joe'], 'success') + const passed = { + title: 'logs in successfully', + state: 'passed', + duration: 1234, + uid: 'pass-1', + parent: loginSuite, + file: loginSuite.file, + tags: ['@important', '@smoke'], + meta: { severity: 'critical', owner: 'qa-team' }, + steps: [step('amOnPage', ['/login'], 'success'), step('fillField', ['login', 'joe'], 'success'), step('click', ['Sign in'], 'success'), step('see', ['Welcome'], 'success')], + } + passed.steps[1].setMetaStep(login) + passed.steps[2].setMetaStep(login) + + const failLogin = metaStep('user', 'login', ['bad'], 'failed') + const failed = { + title: 'fails to log in', + state: 'failed', + duration: 567, + uid: 'fail-1', + parent: loginSuite, + file: loginSuite.file, + meta: {}, + err: new Error('expected dashboard to be visible'), + steps: [step('amOnPage', ['/login'], 'success'), step('fillField', ['login', 'bad'], 'failed')], + } + failed.steps[1].setMetaStep(failLogin) + + const skipped = { + title: 'shows widgets', + state: 'skipped', + uid: 'skip-1', + parent: dashSuite, + file: dashSuite.file, + meta: {}, + steps: [], + } + + return { + tests: [passed, failed, skipped], + stats: { start: new Date(), passes: 1, failures: 1, pending: 1, tests: 3 }, + duration: 4321, + } +} + +function parseReport(dir) { + return new xml2js.Parser().parseStringPromise(fs.readFileSync(path.join(dir, 'report.xml'), 'utf8')) +} + +describe('JUnit Reporter Plugin', () => { + let tmpDir + let prevOutputDir + + beforeEach(() => { + prevOutputDir = store._outputDir + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cjs-junit-')) + store.outputDir = tmpDir + event.dispatcher.removeAllListeners(event.all.result) + event.dispatcher.removeAllListeners(event.workers.result) + }) + + afterEach(() => { + event.dispatcher.removeAllListeners(event.all.result) + event.dispatcher.removeAllListeners(event.workers.result) + store._outputDir = prevOutputDir + fs.rmSync(tmpDir, { recursive: true, force: true }) + }) + + it('exports a default function', () => { + assert.isFunction(junitReporter, 'junitReporter should export a default function') + }) + + it('writes a well-formed JUnit XML report on event.all.result', async () => { + junitReporter({}) + event.dispatcher.emit(event.all.result, makeResult()) + + const reportFile = path.join(tmpDir, 'report.xml') + assert.isTrue(fs.existsSync(reportFile), 'report.xml should be created') + + const xml = fs.readFileSync(reportFile, 'utf8') + assert.match(xml, /^<\?xml version="1.0" encoding="UTF-8"\?>/) + + const parsed = await parseReport(tmpDir) + expect(parsed.testsuites.$.name).to.equal('CodeceptJS') + expect(parsed.testsuites.$.tests).to.equal('3') + expect(parsed.testsuites.$.failures).to.equal('1') + expect(parsed.testsuites.$.skipped).to.equal('1') + expect(parsed.testsuites.testsuite).to.have.length(2) + }) + + it('respects a custom outputName', () => { + junitReporter({ outputName: 'junit-results.xml' }) + event.dispatcher.emit(event.all.result, makeResult()) + assert.isTrue(fs.existsSync(path.join(tmpDir, 'junit-results.xml'))) + }) + + it('puts the test meta information into , not steps', async () => { + junitReporter({}) + event.dispatcher.emit(event.all.result, makeResult()) + + const parsed = await parseReport(tmpDir) + const loginSuite = parsed.testsuites.testsuite.find(s => s.$.name === 'Login') + const passedCase = loginSuite.testcase.find(c => c.$.name === 'logs in successfully') + + const props = {} + for (const p of passedCase.properties[0].property) props[p.$.name] = p.$.value + expect(props).to.deep.include({ severity: 'critical', owner: 'qa-team', tags: '@important @smoke' }) + expect(Object.keys(props)).to.not.include('step.1') + expect(passedCase.properties[0].property.some(p => /^step\./.test(p.$.name))).to.be.false + }) + + it('writes the step/substep log to without [passed] markers', async () => { + junitReporter({}) + event.dispatcher.emit(event.all.result, makeResult()) + + const parsed = await parseReport(tmpDir) + const loginSuite = parsed.testsuites.testsuite.find(s => s.$.name === 'Login') + const passedCase = loginSuite.testcase.find(c => c.$.name === 'logs in successfully') + + const systemOut = passedCase['system-out'][0] + expect(systemOut).to.contain('am on page') + expect(systemOut).to.contain('fill field') + expect(systemOut).to.not.contain('[passed]') + expect(systemOut).to.not.contain('[FAILED]') + + const lines = systemOut.split('\n') + const fillFieldLine = lines.find(l => l.includes('fill field')) + expect(fillFieldLine).to.match(/^\s{2,}/, 'substeps should be indented under their meta step') + }) + + it('records failures with message, stack and a step trace marking the failed step', async () => { + junitReporter({}) + event.dispatcher.emit(event.all.result, makeResult()) + + const parsed = await parseReport(tmpDir) + const loginSuite = parsed.testsuites.testsuite.find(s => s.$.name === 'Login') + const failedCase = loginSuite.testcase.find(c => c.$.name === 'fails to log in') + + expect(failedCase.failure).to.have.length(1) + expect(failedCase.failure[0].$.message).to.contain('expected dashboard to be visible') + expect(failedCase.failure[0].$.type).to.equal('Error') + expect(failedCase.failure[0]._).to.contain('Steps:') + expect(failedCase.failure[0]._).to.contain('[FAILED]') + }) + + it('marks skipped tests with a element', async () => { + junitReporter({}) + event.dispatcher.emit(event.all.result, makeResult()) + + const parsed = await parseReport(tmpDir) + const dashSuite = parsed.testsuites.testsuite.find(s => s.$.name === 'Dashboard') + const skippedCase = dashSuite.testcase.find(c => c.$.name === 'shows widgets') + + expect(skippedCase.skipped).to.exist + }) + + it('does not write the report twice when both result events fire', () => { + junitReporter({}) + event.dispatcher.emit(event.all.result, makeResult()) + const firstMtime = fs.statSync(path.join(tmpDir, 'report.xml')).mtimeMs + event.dispatcher.emit(event.workers.result, makeResult()) + const secondMtime = fs.statSync(path.join(tmpDir, 'report.xml')).mtimeMs + expect(secondMtime).to.equal(firstMtime) + }) +})