From 67bc21f1f44567a3ba41d7a3d8d0bec0e74f4a9e Mon Sep 17 00:00:00 2001 From: Thorsten Kober Date: Fri, 15 Nov 2024 13:42:38 -0500 Subject: [PATCH] Runner Cleanup: SuiteRunner & TestRunner classes (#452) --- resources/benchmark-runner.mjs | 154 +------------------------------ resources/suite-runner.mjs | 104 +++++++++++++++++++++ resources/test-runner.mjs | 79 ++++++++++++++++ tests/benchmark-runner-tests.mjs | 27 ++++-- 4 files changed, 204 insertions(+), 160 deletions(-) create mode 100644 resources/suite-runner.mjs create mode 100644 resources/test-runner.mjs diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index 8b84c7fda..3ab04f227 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -1,6 +1,6 @@ import { Metric } from "./metric.mjs"; import { params } from "./params.mjs"; -import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs"; +import { SUITE_RUNNER_LOOKUP } from "./suite-runner.mjs"; const performance = globalThis.performance; @@ -223,7 +223,7 @@ function geomeanToScore(geomean) { // The WarmupSuite is used to make sure all runner helper functions and // classes are compiled, to avoid unnecessary pauses due to delayed // compilation of runner methods in the middle of the measuring cycle. -const WarmupSuite = { +export const WarmupSuite = { name: "Warmup", url: "warmup/index.html", async prepare(page) { @@ -410,7 +410,7 @@ export class BenchmarkRunner { // FIXME: Encapsulate more state in the SuiteRunner. // FIXME: Return and use measured values from SuiteRunner. const suiteRunnerClass = SUITE_RUNNER_LOOKUP[suite.type ?? "default"]; - const suiteRunner = new suiteRunnerClass(this._measuredValues, this._frame, this._page, this._client, suite); + const suiteRunner = new suiteRunnerClass(this._frame, this._page, params, suite, this._client, this._measuredValues); await suiteRunner.run(); } @@ -484,151 +484,3 @@ export class BenchmarkRunner { metric.computeAggregatedMetrics(); } } - -// FIXME: Create AsyncSuiteRunner subclass. -// FIXME: Create RemoteSuiteRunner subclass. -export class SuiteRunner { - constructor(measuredValues, frame, page, client, suite) { - // FIXME: Create SuiteRunner-local measuredValues. - this._suiteResults = measuredValues.tests[suite.name]; - if (!this._suiteResults) { - this._suiteResults = { tests: {}, total: 0 }; - measuredValues.tests[suite.name] = this._suiteResults; - } - this._measuredValues = measuredValues; - this._frame = frame; - this._page = page; - this._client = client; - this._suite = suite; - } - - async run() { - await this._prepareSuite(); - await this._runSuite(); - } - - async _prepareSuite() { - const suiteName = this._suite.name; - const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`; - const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`; - - performance.mark(suitePrepareStartLabel); - await this._loadFrame(); - await this._suite.prepare(this._page); - performance.mark(suitePrepareEndLabel); - - performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); - } - - async _runSuite() { - const suiteName = this._suite.name; - const suiteStartLabel = `suite-${suiteName}-start`; - const suiteEndLabel = `suite-${suiteName}-end`; - - performance.mark(suiteStartLabel); - for (const test of this._suite.tests) - await this._runTestAndRecordResults(test); - performance.mark(suiteEndLabel); - - performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel); - this._validateSuiteTotal(); - } - - _validateSuiteTotal() { - // When the test is fast and the precision is low (for example with Firefox' - // privacy.resistFingerprinting preference), it's possible that the measured - // total duration for an entire is 0. - const suiteTotal = this._suiteResults.total; - if (suiteTotal === 0) - throw new Error(`Got invalid 0-time total for suite ${this._suite.name}: ${suiteTotal}`); - } - - async _loadFrame() { - return new Promise((resolve, reject) => { - const frame = this._page._frame; - frame.onload = () => resolve(); - frame.onerror = () => reject(); - frame.src = this._suite.url; - }); - } - - async _runTestAndRecordResults(test) { - if (this._client?.willRunTest) - await this._client.willRunTest(this._suite, test); - - // Prepare all mark labels outside the measuring loop. - const suiteName = this._suite.name; - const testName = test.name; - const startLabel = `${suiteName}.${testName}-start`; - const syncEndLabel = `${suiteName}.${testName}-sync-end`; - const asyncStartLabel = `${suiteName}.${testName}-async-start`; - const asyncEndLabel = `${suiteName}.${testName}-async-end`; - - let syncTime; - let asyncStartTime; - let asyncTime; - const runSync = () => { - if (params.warmupBeforeSync) { - performance.mark("warmup-start"); - const startTime = performance.now(); - // Infinite loop for the specified ms. - while (performance.now() - startTime < params.warmupBeforeSync) - continue; - performance.mark("warmup-end"); - } - performance.mark(startLabel); - const syncStartTime = performance.now(); - test.run(this._page); - const syncEndTime = performance.now(); - performance.mark(syncEndLabel); - - syncTime = syncEndTime - syncStartTime; - - performance.mark(asyncStartLabel); - asyncStartTime = performance.now(); - }; - const measureAsync = () => { - // Some browsers don't immediately update the layout for paint. - // Force the layout here to ensure we're measuring the layout time. - const height = this._frame.contentDocument.body.getBoundingClientRect().height; - const asyncEndTime = performance.now(); - asyncTime = asyncEndTime - asyncStartTime; - this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination. - performance.mark(asyncEndLabel); - if (params.warmupBeforeSync) - performance.measure("warmup", "warmup-start", "warmup-end"); - const suiteName = this._suite.name; - const testName = test.name; - performance.measure(`${suiteName}.${testName}-sync`, startLabel, syncEndLabel); - performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel); - }; - - const report = () => this._recordTestResults(test, syncTime, asyncTime); - const invokerClass = TEST_INVOKER_LOOKUP[params.measurementMethod]; - const invoker = new invokerClass(runSync, measureAsync, report, params); - - return invoker.start(); - } - - async _recordTestResults(test, syncTime, asyncTime) { - // Skip reporting updates for the warmup suite. - if (this._suite === WarmupSuite) - return; - - const total = syncTime + asyncTime; - this._suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; - this._suiteResults.total += total; - - if (this._client?.didRunTest) - await this._client.didRunTest(this._suite, test); - } -} - -// FIXME: implement remote steps -class RemoteSuiteRunner extends SuiteRunner {} - -const SUITE_RUNNER_LOOKUP = { - __proto__: null, - default: SuiteRunner, - remote: RemoteSuiteRunner, -}; diff --git a/resources/suite-runner.mjs b/resources/suite-runner.mjs new file mode 100644 index 000000000..16bc2cb51 --- /dev/null +++ b/resources/suite-runner.mjs @@ -0,0 +1,104 @@ +import { TestRunner } from "./test-runner.mjs"; +import { WarmupSuite } from "./benchmark-runner.mjs"; + +// FIXME: Create AsyncSuiteRunner subclass. +// FIXME: Create RemoteSuiteRunner subclass. +export class SuiteRunner { + #frame; + #page; + #params; + #suite; + #client; + #suiteResults; + + constructor(frame, page, params, suite, client, measuredValues) { + // FIXME: Create SuiteRunner-local measuredValues. + this.#suiteResults = measuredValues.tests[suite.name]; + if (!this.#suiteResults) { + this.#suiteResults = { tests: {}, total: 0 }; + measuredValues.tests[suite.name] = this.#suiteResults; + } + this.#frame = frame; + this.#page = page; + this.#client = client; + this.#suite = suite; + this.#params = params; + } + + async run() { + await this._prepareSuite(); + await this._runSuite(); + } + + async _prepareSuite() { + const suiteName = this.#suite.name; + const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`; + const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`; + + performance.mark(suitePrepareStartLabel); + await this._loadFrame(); + await this.#suite.prepare(this.#page); + performance.mark(suitePrepareEndLabel); + + performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); + } + + async _runSuite() { + const suiteName = this.#suite.name; + const suiteStartLabel = `suite-${suiteName}-start`; + const suiteEndLabel = `suite-${suiteName}-end`; + + performance.mark(suiteStartLabel); + for (const test of this.#suite.tests) { + if (this.#client?.willRunTest) + await this.#client.willRunTest(this.#suite, test); + + const testRunner = new TestRunner(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults); + await testRunner.runTest(); + } + performance.mark(suiteEndLabel); + + performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel); + this._validateSuiteTotal(); + } + + _validateSuiteTotal() { + // When the test is fast and the precision is low (for example with Firefox' + // privacy.resistFingerprinting preference), it's possible that the measured + // total duration for an entire is 0. + const suiteTotal = this.#suiteResults.total; + if (suiteTotal === 0) + throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`); + } + + async _loadFrame() { + return new Promise((resolve, reject) => { + const frame = this.#frame; + frame.onload = () => resolve(); + frame.onerror = () => reject(); + frame.src = this.#suite.url; + }); + } + + _recordTestResults = async (test, syncTime, asyncTime) => { + // Skip reporting updates for the warmup suite. + if (this.#suite === WarmupSuite) + return; + + const total = syncTime + asyncTime; + this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; + this.#suiteResults.total += total; + + if (this.#client?.didRunTest) + await this.#client.didRunTest(this.#suite, test); + }; +} + +// FIXME: implement remote steps +class RemoteSuiteRunner extends SuiteRunner {} + +export const SUITE_RUNNER_LOOKUP = { + __proto__: null, + default: SuiteRunner, + remote: RemoteSuiteRunner, +}; diff --git a/resources/test-runner.mjs b/resources/test-runner.mjs new file mode 100644 index 000000000..1b5c2598e --- /dev/null +++ b/resources/test-runner.mjs @@ -0,0 +1,79 @@ +import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs"; + +export class TestRunner { + #frame; + #page; + #params; + #suite; + #test; + #callback; + + constructor(frame, page, params, suite, test, callback) { + this.#suite = suite; + this.#test = test; + this.#params = params; + this.#callback = callback; + + this.#page = page; + this.#frame = frame; + } + + async runTest() { + // Prepare all mark labels outside the measuring loop. + const suiteName = this.#suite.name; + const testName = this.#test.name; + const syncStartLabel = `${suiteName}.${testName}-start`; + const syncEndLabel = `${suiteName}.${testName}-sync-end`; + const asyncStartLabel = `${suiteName}.${testName}-async-start`; + const asyncEndLabel = `${suiteName}.${testName}-async-end`; + + let syncTime; + let asyncStartTime; + let asyncTime; + + const runSync = () => { + if (this.#params.warmupBeforeSync) { + performance.mark("warmup-start"); + const startTime = performance.now(); + // Infinite loop for the specified ms. + while (performance.now() - startTime < this.#params.warmupBeforeSync) + continue; + performance.mark("warmup-end"); + } + performance.mark(syncStartLabel); + const syncStartTime = performance.now(); + this.#test.run(this.#page); + const syncEndTime = performance.now(); + performance.mark(syncEndLabel); + + syncTime = syncEndTime - syncStartTime; + + performance.mark(asyncStartLabel); + asyncStartTime = performance.now(); + }; + const measureAsync = () => { + const bodyReference = this.#frame ? this.#frame.contentDocument.body : document.body; + const windowReference = this.#frame ? this.#frame.contentWindow : window; + // Some browsers don't immediately update the layout for paint. + // Force the layout here to ensure we're measuring the layout time. + const height = bodyReference.getBoundingClientRect().height; + windowReference._unusedHeightValue = height; // Prevent dead code elimination. + + const asyncEndTime = performance.now(); + performance.mark(asyncEndLabel); + + asyncTime = asyncEndTime - asyncStartTime; + + if (this.#params.warmupBeforeSync) + performance.measure("warmup", "warmup-start", "warmup-end"); + performance.measure(`${suiteName}.${testName}-sync`, syncStartLabel, syncEndLabel); + performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel); + }; + + const report = () => this.#callback(this.#test, syncTime, asyncTime); + const invokerClass = TEST_INVOKER_LOOKUP[this.#params.measurementMethod]; + const invoker = new invokerClass(runSync, measureAsync, report, this.#params); + + return invoker.start(); + } +} diff --git a/tests/benchmark-runner-tests.mjs b/tests/benchmark-runner-tests.mjs index 600516f1c..2edd1e897 100644 --- a/tests/benchmark-runner-tests.mjs +++ b/tests/benchmark-runner-tests.mjs @@ -1,4 +1,6 @@ -import { BenchmarkRunner, SuiteRunner } from "../resources/benchmark-runner.mjs"; +import { BenchmarkRunner } from "../resources/benchmark-runner.mjs"; +import { SuiteRunner } from "../resources/suite-runner.mjs"; +import { TestRunner } from "../resources/test-runner.mjs"; import { defaultParams } from "../resources/params.mjs"; function TEST_FIXTURE(name) { @@ -148,14 +150,14 @@ describe("BenchmarkRunner", () => { }); describe("runSuite", () => { - let _prepareSuiteSpy, _loadFrameStub, _runTestAndRecordResultsStub, _validateSuiteTotalStub, _suitePrepareSpy, performanceMarkSpy; + let _prepareSuiteSpy, _loadFrameStub, _runTestStub, _validateSuiteTotalStub, _suitePrepareSpy, performanceMarkSpy; const suite = SUITES_FIXTURE[0]; before(async () => { _prepareSuiteSpy = spy(SuiteRunner.prototype, "_prepareSuite"); _loadFrameStub = stub(SuiteRunner.prototype, "_loadFrame").callsFake(async () => null); - _runTestAndRecordResultsStub = stub(SuiteRunner.prototype, "_runTestAndRecordResults").callsFake(async () => null); + _runTestStub = stub(TestRunner.prototype, "runTest").callsFake(async () => null); _validateSuiteTotalStub = stub(SuiteRunner.prototype, "_validateSuiteTotal").callsFake(async () => null); performanceMarkSpy = spy(window.performance, "mark"); _suitePrepareSpy = spy(suite, "prepare"); @@ -170,7 +172,7 @@ describe("BenchmarkRunner", () => { }); it("should run and record results for every test in suite", async () => { - assert.calledThrice(_runTestAndRecordResultsStub); + assert.calledThrice(_runTestStub); assert.calledOnce(_validateSuiteTotalStub); assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-start"); assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-end"); @@ -186,13 +188,14 @@ describe("BenchmarkRunner", () => { let performanceMarkSpy; const suite = SUITES_FIXTURE[0]; + const params = { measurementMethod: "raf" }; before(async () => { runner._suite = suite; await runner._appendFrame(); performanceMarkSpy = spy(window.performance, "mark"); - const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, suite); - await suiteRunner._runTestAndRecordResults(suite.tests[0]); + const suiteRunner = new SuiteRunner(runner._frame, runner._page, params, suite, runner._client, runner._measuredValues); + await suiteRunner._runSuite(); }); it("should run client pre and post hooks if present", () => { @@ -205,7 +208,11 @@ describe("BenchmarkRunner", () => { assert.calledWith(performanceMarkSpy, "Suite 1.Test 1-sync-end"); assert.calledWith(performanceMarkSpy, "Suite 1.Test 1-async-start"); assert.calledWith(performanceMarkSpy, "Suite 1.Test 1-async-end"); - expect(performanceMarkSpy.callCount).to.equal(4); + + // SuiteRunner adds 2 marks. + // Suite used here contains 3 tests. + // Each Testrunner adds 4 marks. + expect(performanceMarkSpy.callCount).to.equal(14); }); }); @@ -218,6 +225,8 @@ describe("BenchmarkRunner", () => { const asyncStart = 12000; const asyncEnd = 13000; + const params = { measurementMethod: "raf" }; + before(async () => { stub(runner, "_measuredValues").value({ tests: {}, @@ -226,8 +235,8 @@ describe("BenchmarkRunner", () => { stubPerformanceNowCalls(syncStart, syncEnd, asyncStart, asyncEnd); // instantiate recorded test results - const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, suite); - await suiteRunner._runTestAndRecordResults(suite.tests[0]); + const suiteRunner = new SuiteRunner(runner._frame, runner._page, params, suite, runner._client, runner._measuredValues); + await suiteRunner._runSuite(); await runner._finalize(); });