improve memory leak test (#2621)

2 files changed, 40 insertions, 55 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index da5f9dd8a..42212452f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -208,8 +208,8 @@ jobs:
 
       - name: Memory Leak Test
         run: |
-          node ./scripts/memory/mk.js
-          node ./scripts/memory/index.js
+          node ./scripts/memory/mk.js 
+          node --expose-gc ./scripts/memory/index.js --ci
 
 
   # Changelog can only run _after_ build.
diff --git a/scripts/memory/index.js b/scripts/memory/index.js
index 29b20832d..067abdc7c 100644
--- a/scripts/memory/index.js
+++ b/scripts/memory/index.js
@@ -1,42 +1,18 @@
-import { execa } from 'execa';
 import { fileURLToPath } from 'url';
 import v8 from 'v8';
 import dev from '../../packages/astro/dist/core/dev/index.js';
 import { loadConfig } from '../../packages/astro/dist/core/config.js';
 import prettyBytes from 'pretty-bytes';
 
-/** URL directory containing the entire project. */
-const projDir = new URL('./project/', import.meta.url);
-
-function mean(numbers) {
-	var total = 0,
-		i;
-	for (i = 0; i < numbers.length; i += 1) {
-		total += numbers[i];
-	}
-	return total / numbers.length;
+if (!global.gc) {
+	console.error('ERROR: Node must be run with --expose-gc');
+	process.exit(1);
 }
 
-function median(numbers) {
-	// median of [3, 5, 4, 4, 1, 1, 2, 3] = 3
-	var median = 0,
-		numsLen = numbers.length;
-	numbers.sort();
+const isCI = process.argv.includes('--ci');
 
-	if (
-		numsLen % 2 ===
-		0 // is even
-	) {
-		// average of two middle numbers
-		median = (numbers[numsLen / 2 - 1] + numbers[numsLen / 2]) / 2;
-	} else {
-		// is odd
-		// middle number only
-		median = numbers[(numsLen - 1) / 2];
-	}
-
-	return median;
-}
+/** URL directory containing the entire project. */
+const projDir = new URL('./project/', import.meta.url);
 
 let config = await loadConfig({
 	cwd: fileURLToPath(projDir),
@@ -44,44 +20,53 @@ let config = await loadConfig({
 
 config.buildOptions.experimentalStaticBuild = true;
 
-const server = await dev(config, { logging: 'error' });
+const server = await dev(config, { logging: { level: 'error' } });
 
 // Prime the server so initial memory is created
 await fetch(`http://localhost:3000/page-0`);
 
-const sizes = [];
-
-function addSize() {
-	sizes.push(v8.getHeapStatistics().total_heap_size);
-}
-
 async function run() {
-	addSize();
 	for (let i = 0; i < 100; i++) {
 		let path = `/page-${i}`;
 		await fetch(`http://localhost:3000${path}`);
 	}
-	addSize();
 }
 
+global.gc();
+const startSize = v8.getHeapStatistics().used_heap_size;
+
+// HUMAN mode: Runs forever. Optimized for accurate results on each snapshot Slower than CI.
+if (!isCI) {
+	console.log(`Greetings, human. This test will run forever. Run with the "--ci" flag to finish with a result.`);
+	let i = 1;
+	while (i++) {
+		await run();
+		global.gc();
+		const checkpoint = v8.getHeapStatistics().used_heap_size;
+		console.log(`Snapshot ${String(i).padStart(3, '0')}: ${(checkpoint / startSize) * 100}%`);
+	}
+}
+
+// CI mode: Runs 100 times. Optimized for speed with an accurate final result.
 for (let i = 0; i < 100; i++) {
 	await run();
+	const checkpoint = v8.getHeapStatistics().used_heap_size;
+	console.log(`Estimate ${String(i).padStart(3, '0')}/100: ${(checkpoint / startSize) * 100}%`);
 }
 
-let lastThirthy = sizes.slice(sizes.length - 30);
-let averageOfLastThirty = mean(lastThirthy);
-let medianOfAll = median(sizes);
+console.log(`Test complete. Running final garbage collection...`);
+global.gc();
+const endSize = v8.getHeapStatistics().used_heap_size;
 
 // If the trailing average is higher than the median, see if it's more than 5% higher
-if (averageOfLastThirty > medianOfAll) {
-	let percentage = Math.abs(averageOfLastThirty - medianOfAll) / medianOfAll;
-	if (percentage > 0.1) {
-		throw new Error(
-			`The average towards the end (${prettyBytes(averageOfLastThirty)}) is more than 10% higher than the median of all runs (${prettyBytes(
-				medianOfAll
-			)}). This tells us that memory continues to grow and a leak is likely.`
-		);
-	}
-}
-
+let percentage = endSize / startSize;
+const TEST_THRESHOLD = 1.2;
+const isPass = percentage < TEST_THRESHOLD;
+console.log(``);
+console.log(`Result: ${isPass ? 'PASS' : 'FAIL'} (${percentage * 100}%)`);
+console.log(`Memory usage began at ${prettyBytes(startSize)} and finished at ${prettyBytes(endSize)}.`);
+console.log(`The threshold for a probable memory leak is ${TEST_THRESHOLD * 100}%`);
+console.log(``);
+console.log(`Exiting...`);
 await server.stop();
+process.exit(isPass ? 0 : 1);