♻️ Refactor reporter code (#31)

* Add example test

* Refactor reporter code

* Fix verbosity levels again

* Fix eslint warnings

Author: tolauwae

.eslint.config.mjs

@@ -6,15 +6,17 @@ import stylisticJs from '@stylistic/eslint-plugin-js'
 
 /** @type {import('eslint').Linter.Config[]} */
 export default [
+  pluginJs.configs.recommended,
+  ...tseslint.configs.recommended,
   { plugins: {
       '@stylistic/js': stylisticJs
     }, rules: {
       '@stylistic/js/indent': ['error', 4],
+      '@typescript-eslint/no-wrapper-object-types': 'off'
     }
   },
   {files: ["**/*.{js,mjs,cjs,ts}"]},
   {files: ["**/*.js"], languageOptions: {sourceType: "commonjs"}},
   {languageOptions: { globals: globals.browser }},
-  pluginJs.configs.recommended,
-  ...tseslint.configs.recommended,
+
 ];

package.json

@@ -23,6 +23,7 @@
         "test:prebuild": "npm run build && npm run build:tests",
         "test:all": "npm run test:prebuild && npm run test:ava",
         "test:ava": "ava",
+        "test:example": "npx ts-node ./tests/examples/example.ts",
         "coverage:test:ava": "c8 --src src/ --all ava"
     },
     "dependencies": {

src/framework/Framework.ts

@@ -3,10 +3,11 @@ import {HybridScheduler, Scheduler} from './Scheduler';
 import {TestScenario} from './scenario/TestScenario';
 
 import {TestbedSpecification} from '../testbeds/TestbedSpecification';
-import {Reporter, SuiteResults} from '../reporter/Reporter';
 
 import {StyleType} from '../reporter';
 import {styling} from '../reporter/Style';
+import {SuiteResult} from '../reporter/Results';
+import {Reporter} from '../reporter/Reporter';
 
 export interface Suite {
 
@@ -90,10 +91,10 @@ export class Framework {
         for (const suite of suites) {
             for (const testee of suite.testees) {
                 const order: TestScenario[] = suite.scheduler.sequential(suite);
-                const result: SuiteResults = new SuiteResults(suite, testee);
+                const result: SuiteResult = new SuiteResult(suite);
 
                 const first: TestScenario = order[0];
-                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
+                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));
 
                 await this.runSuite(result, testee, order);
                 this.reporter.report(result);
@@ -114,10 +115,10 @@ export class Framework {
         await Promise.all(suites.map(async (suite: Suite) => {
             await Promise.all(suite.testees.map(async (testee: Testee) => {
                 const order: TestScenario[] = suite.scheduler.sequential(suite);
-                const result: SuiteResults = new SuiteResults(suite, testee);
+                const result: SuiteResult = new SuiteResult(suite);
 
                 const first: TestScenario = order[0];
-                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
+                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));
 
                 await this.runSuite(result, testee, order);
                 this.reporter.report(result);
@@ -139,10 +140,10 @@ export class Framework {
             const order: TestScenario[][] = suite.scheduler.parallel(suite, suite.testees.length);
             await Promise.all(suite.testees.map(async (testee: Testee, i: number) => {
                 // console.log(`scheduling on ${testee.name}`)
-                const result: SuiteResults = new SuiteResults(suite, testee);
+                const result: SuiteResult = new SuiteResult(suite);
 
                 const first: TestScenario = order[i][0];
-                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
+                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));
 
                 for (let j = i; j < order.length; j += suite.testees.length) {
                     await this.runSuite(result, testee, order[j]);
@@ -159,7 +160,7 @@ export class Framework {
         this.reporter.results(t1 - t0);
     }
 
-    private async runSuite(result: SuiteResults, testee: Testee, order: TestScenario[]) {
+    private async runSuite(result: SuiteResult, testee: Testee, order: TestScenario[]) {
         for (const test of order) {
             await testee.describe(test, result, this.runs);
         }

src/framework/Testee.ts

@@ -9,10 +9,11 @@ import {TestScenario} from './scenario/TestScenario';
 import {OutofPlaceSpecification, PlatformType, TestbedSpecification} from '../testbeds/TestbedSpecification';
 import {CompileOutput, CompilerFactory} from '../manage/Compiler';
 import {WABT} from '../util/env';
-import {Completion, expect, ScenarioResult, SuiteResults} from '../reporter/Reporter';
+import {Outcome} from '../reporter/describers/Describer';
 import {WASM} from '../sourcemap/Wasm';
 import {DummyProxy} from '../testbeds/Emulator';
-import {Result} from '../reporter/Result';
+import {ScenarioResult, Skipped, StepOutcome, SuiteResult} from '../reporter/Results';
+import {Verifier} from './Verifier';
 
 export function timeout<T>(label: string, time: number, promise: Promise<T>): Promise<T> {
     if (time === 0) {
@@ -46,14 +47,14 @@ export function getValue(object: any, field: string): any {
 }
 
 export enum Target {
-    supervisor,
-    proxy
+    supervisor = 'supervisor',
+    proxy = 'proxy'
 }
 
 export class Testee { // TODO unified with testbed interface
 
     /** The current state for each described test */
-    private states: Map<string, Result> = new Map<string, Result>();
+    private states: Map<string, StepOutcome> = new Map<string, StepOutcome>();
 
     /** Factory to establish new connections to VMs */
     public readonly connector: TestbedFactory;
@@ -132,16 +133,16 @@ export class Testee { // TODO unified with testbed interface
     }
 
     private run(name: string, limit: number, fn: () => Promise<any>) {
-        return timeout<Object | void>(name, limit, fn());
+        return timeout<object | void>(name, limit, fn());
     }
 
     private step(name: string, limit: number, fn: () => Promise<any>) {
-        return timeout<Object | void>(name, limit, fn());
+        return timeout<object | void>(name, limit, fn());
     }
 
-    public async describe(description: TestScenario, suiteResult: SuiteResults, runs: number = 1) {
+    public async describe(description: TestScenario, suiteResult: SuiteResult, runs: number = 1) {
         const testee = this;
-        const scenarioResult: ScenarioResult = new ScenarioResult(description, testee);
+        const scenarioResult: ScenarioResult = new ScenarioResult(description);
 
         if (description.skip) {
             return;
@@ -154,11 +155,11 @@ export class Testee { // TODO unified with testbed interface
         await this.run('Check for failing dependencies', testee.timeout, async function () {
             const failedDependencies: TestScenario[] = testee.failedDependencies(description);
             if (failedDependencies.length > 0) {
-                testee.states.set(description.title, new Result('Skipping', 'Test has failing dependencies', Completion.skipped));
+                testee.states.set(description.title, new Skipped('Skipping', 'Test has failing dependencies'));
                 throw new Error(`Skipped: failed dependent tests: ${failedDependencies.map(dependence => dependence.title)}`);
             }
         }).catch((e: Error) => {
-            scenarioResult.error = e;
+            scenarioResult.error(e.message);
         });
 
         await this.run('Compile and upload program', testee.connector.timeout, async function () {
@@ -169,31 +170,31 @@ export class Testee { // TODO unified with testbed interface
 
             const compiled: CompileOutput = await new CompilerFactory(WABT).pickCompiler(description.program).compile(description.program);
             try {
-                await timeout<Object | void>(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e));
+                await timeout<object | void>(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e));
                 testee.current = description.program;
             } catch (e) {
                 await testee.initialize(description.program, description.args ?? []).catch((o) => Promise.reject(o));
             }
         }).catch((e: Error | string) => {
             if (typeof e === 'string') {
-                scenarioResult.error = new Error(e);
+                scenarioResult.error(e);
             } else {
-                scenarioResult.error = e;
+                scenarioResult.error(e.toString());
             }
         });
 
         await this.run('Get source mapping', testee.connector.timeout, async function () {
             map = await testee.mapper.map(description.program);
         }).catch((e: Error | string) => {
             if (typeof e === 'string') {
-                scenarioResult.error = new Error(e);
+                scenarioResult.error(e);
             } else {
-                scenarioResult.error = e;
+                scenarioResult.error(e.toString());
             }
         });
 
-        if (scenarioResult.error) {
-            suiteResult.scenarios.push(scenarioResult);
+        if (scenarioResult.outcome === Outcome.error) {
+            suiteResult.add(scenarioResult);
             return;
         }
 
@@ -205,62 +206,60 @@ export class Testee { // TODO unified with testbed interface
                 await this.run('resetting before retry', testee.timeout, async function () {
                     await testee.reset(testee.testbed);
                 }).catch((e: Error) => {
-                    scenarioResult.error = e;
+                    scenarioResult.error(e.toString());
                 });
             }
 
             for (const step of description.steps ?? []) {
                 /** Perform the step and check if expectations were met */
                 await this.step(step.title, testee.timeout, async function () {
-                    let result: Result = new Result(step.title, 'incomplete');
+                    const verifier: Verifier = new Verifier(step);
                     if (testee.bed(step.target ?? Target.supervisor) === undefined) {
-                        testee.states.set(description.title, result);
-                        result.error('Cannot run test: no debugger connection.');
-                        testee.states.set(description.title, result);
+                        testee.states.set(description.title, verifier.error('Cannot run test: no debugger connection.'));
                         return;
                     }
 
-                    let actual: Object | void;
+                    let actual: object | void;
                     if (step.instruction.kind === Kind.Action) {
-                        actual = await timeout<Object | void>(`performing action . ${step.title}`, testee.timeout,
+                        actual = await timeout<object | void>(`performing action . ${step.title}`, testee.timeout,
                             step.instruction.value.act(testee)).catch((err) => {
-                            result.error(err);
+                            testee.states.set(description.title, verifier.error(err));
+                            return;
                         });
                     } else {
                         actual = await testee.recoverable(testee, step.instruction.value, map,
-                            (testee, req, map) => timeout<Object | void>(`sending instruction ${req.type}`, testee.timeout,
+                            (testee, req, map) => timeout<object | void>(`sending instruction ${req.type}`, testee.timeout,
                                 testee.bed(step.target ?? Target.supervisor)!.sendRequest(map, req)),
                             (testee) => testee.run(`Recover: re-initialize ${testee.testbed?.name}`, testee.connector.timeout, async function () {
                                 await testee.initialize(description.program, description.args ?? []).catch((o) => {
                                     return Promise.reject(o)
                                 });
                             }), 1).catch((e: Error) => {
-                            result.completion = (e.message.includes('timeout')) ? Completion.timedout : Completion.error;
-                            result.description = e.message;
+                            const result = new StepOutcome(step);
+                            testee.states.set(description.title, result.update((e.message.includes('timeout')) ? Outcome.timedout : Outcome.error, e.message));
                         });
                     }
 
-                    if (result.completion === Completion.uncommenced) {
-                        result = expect(step, actual, previous);
-                    }
+                    const result = verifier.verify(actual, previous);
 
                     if (actual !== undefined) {
                         previous = actual;
                     }
 
                     testee.states.set(description.title, result);
-                    scenarioResult.results.push(result);
+                    scenarioResult.add(result);
                 });
             }
-            suiteResult.scenarios.push(scenarioResult);
+            suiteResult.add(scenarioResult);
         }
     }
 
+    /* eslint @typescript-eslint/no-explicit-any: off */
     private async recoverable(testee: Testee, step: Request<any>, map: SourceMap.Mapping,
-        attempt: (t: Testee, req: Request<any>, m: SourceMap.Mapping) => Promise<Object | void>,
+        attempt: (t: Testee, req: Request<any>, m: SourceMap.Mapping) => Promise<object | void>,
         recover: (t: Testee) => Promise<any>,
-        retries: number = 0): Promise<Object | void> {
-        let result: Object | void = undefined;
+        retries: number = 0): Promise<object | void> {
+        let result: object | void = undefined;
         let error;
         while (0 <= retries && result === undefined) {
             result = await attempt(testee, step, map).catch(async (err) => {
@@ -281,7 +280,7 @@ export class Testee { // TODO unified with testbed interface
         if (instance === undefined) {
             this.framework.reporter.error('Cannot run test: no debugger connection.'); // todo
         } else {
-            await timeout<Object | void>('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset));
+            await timeout<object | void>('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset));
         }
     }
 
@@ -293,8 +292,8 @@ export class Testee { // TODO unified with testbed interface
     private failedDependencies(description: TestScenario): TestScenario[] {
         return (description?.dependencies ?? []).filter(dependence => {
             if (this.states.get(dependence.title)) {
-                const c = this.states.get(dependence.title)!.completion;
-                return !(c === Completion.succeeded || c === Completion.uncommenced);
+                const c = this.states.get(dependence.title)!.outcome;
+                return !(c === Outcome.succeeded || c === Outcome.uncommenced);
             } else {
                 return false;
             }