Adds memory leak test in CI (#2616)

* Add a memory leak test

* Add memory leak smoke test to CI

* Update the latest compiler

* Update yarn.lock

* Remove entries optimization

* Calculate as 5% higher

9 files changed, 109 insertions, 5 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3a483b017..aabf4feb3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -211,6 +211,11 @@ jobs:
         if: ${{ matrix.os == 'windows-latest' }}
         run: node ./scripts/smoke/index.js
 
+      - name: Memory Leak Test
+        run: |
+          node ./scripts/memory/mk.js
+          node ./scripts/memory/index.js
+
 
   # Changelog can only run _after_ build.
   # We download all `dist/` artifacts from GitHub to skip the build process.
diff --git a/.gitignore b/.gitignore
index 7b9a4ce26..670518047 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@ dist/
 .vercel
 _site/
 scripts/smoke/*-main/
+scripts/memory/project/src/pages/
 *.log
 package-lock.json
 .turbo/
diff --git a/package.json b/package.json
index 85c4c0fc7..c0c260f3d 100644
--- a/package.json
+++ b/package.json
@@ -51,6 +51,7 @@
     "eslint-plugin-prettier": "^4.0.0",
     "execa": "^6.0.0",
     "prettier": "^2.4.1",
+    "pretty-bytes": "^6.0.0",
     "tiny-glob": "^0.2.8",
     "turbo": "^1.0.0",
     "typescript": "4.5.2"
diff --git a/packages/astro/package.json b/packages/astro/package.json
index 08560d8e8..e2f4d9b4d 100644
--- a/packages/astro/package.json
+++ b/packages/astro/package.json
@@ -56,7 +56,7 @@
     "test:match": "mocha --timeout 15000 -g"
   },
   "dependencies": {
-    "@astrojs/compiler": "^0.11.0",
+    "@astrojs/compiler": "^0.11.3",
     "@astrojs/language-server": "^0.8.6",
     "@astrojs/markdown-remark": "^0.6.1-next.2",
     "@astrojs/prism": "0.4.0",
diff --git a/scripts/memory/index.js b/scripts/memory/index.js
new file mode 100644
index 000000000..8beda6e44
--- /dev/null
+++ b/scripts/memory/index.js
@@ -0,0 +1,81 @@
+import { execa } from 'execa';
+import { fileURLToPath } from 'url';
+import v8 from 'v8';
+import dev from '../../packages/astro/dist/core/dev/index.js';
+import { loadConfig } from '../../packages/astro/dist/core/config.js';
+import prettyBytes from 'pretty-bytes';
+
+/** URL directory containing the entire project. */
+const projDir = new URL('./project/', import.meta.url);
+
+function mean(numbers) {
+	var total = 0, i;
+	for (i = 0; i < numbers.length; i += 1) {
+			total += numbers[i];
+	}
+	return total / numbers.length;
+}
+
+function median(numbers) {
+	// median of [3, 5, 4, 4, 1, 1, 2, 3] = 3
+	var median = 0, numsLen = numbers.length;
+	numbers.sort();
+
+	if (
+			numsLen % 2 === 0 // is even
+	) {
+			// average of two middle numbers
+			median = (numbers[numsLen / 2 - 1] + numbers[numsLen / 2]) / 2;
+	} else { // is odd
+			// middle number only
+			median = numbers[(numsLen - 1) / 2];
+	}
+
+	return median;
+}
+
+let config = await loadConfig({
+	cwd: fileURLToPath(projDir)
+});
+
+config.buildOptions.experimentalStaticBuild = true;
+
+const server = await dev(config, { logging: 'error'});
+
+// Prime the server so initial memory is created
+await fetch(`http://localhost:3000/page-0`);
+
+const sizes = [];
+
+function addSize() {
+	sizes.push(v8.getHeapStatistics().total_heap_size);
+}
+
+async function run() {
+	addSize();
+	for(let i = 0; i < 100; i++) {
+		let path = `/page-${i}`;
+		await fetch(`http://localhost:3000${path}`);
+	}
+	addSize();
+}
+
+for(let i = 0; i < 100; i++) {
+	await run();
+}
+
+let lastThirthy = sizes.slice(sizes.length - 30);
+let averageOfLastThirty = mean(lastThirthy);
+let medianOfAll = median(sizes);
+
+// If the trailing average is higher than the median, see if it's more than 5% higher
+if(averageOfLastThirty > medianOfAll) {
+	let percentage = Math.abs(averageOfLastThirty - medianOfAll) / medianOfAll;
+	if(percentage > .05) {
+		throw new Error(`The average towards the end (${prettyBytes(averageOfLastThirty)}) is more than 5% higher than the median of all runs (${prettyBytes(medianOfAll)}). This tells us that memory continues to grow and a leak is likely.`)
+	}
+}
+
+
+
+await server.stop();
diff --git a/scripts/memory/mk.js b/scripts/memory/mk.js
new file mode 100644
index 000000000..6e6e74637
--- /dev/null
+++ b/scripts/memory/mk.js
@@ -0,0 +1,11 @@
+import fs from 'fs';
+
+const pages = new URL('./project/src/pages/', import.meta.url);
+
+for(let i = 0; i < 100; i++) {
+	let content = `---
+const i = ${i};
+---
+<span>{i}</span>`;
+	await fs.promises.writeFile(new URL(`./page-${i}.astro`, pages), content, 'utf-8');
+}
diff --git a/scripts/memory/project/.gitkeep b/scripts/memory/project/.gitkeep
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/scripts/memory/project/.gitkeep
diff --git a/scripts/memory/project/src/pages/.gitkeep b/scripts/memory/project/src/pages/.gitkeep
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/scripts/memory/project/src/pages/.gitkeep
diff --git a/yarn.lock b/yarn.lock
index 95d4cffc0..bf7de3cef 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -130,10 +130,10 @@
     jsonpointer "^5.0.0"
     leven "^3.1.0"
 
-"@astrojs/compiler@^0.11.0":
-  version "0.11.0"
-  resolved "https://registry.yarnpkg.com/@astrojs/compiler/-/compiler-0.11.0.tgz#8a15b0dcca8fba3343e3c6aad1f58c035aa1aa46"
-  integrity sha512-mCrY+e74YXnPsCERLk7Vgm/XIccalcMsSrEE+LOTRKsTU5R3eHEXrUc32bpsIiWO7Rbspf14uYtFQ6ErrGybsA==
+"@astrojs/compiler@^0.11.3":
+  version "0.11.3"
+  resolved "https://registry.yarnpkg.com/@astrojs/compiler/-/compiler-0.11.3.tgz#804de4ce6c073049175ba6e0eec470e8dabee81d"
+  integrity sha512-mg0hRk8uzk5LQjtHpAFtwmkCq7YM40Inev49WPqdxDnDaK056I1dbRCm3E/h2SJhLdUyI3KRML79N6DTmL5wRg==
   dependencies:
     typescript "^4.3.5"
 
@@ -6648,6 +6648,11 @@ pretty-bytes@^5.3.0, pretty-bytes@^5.6.0:
   resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb"
   integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==
 
+pretty-bytes@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-6.0.0.tgz#928be2ad1f51a2e336add8ba764739f9776a8140"
+  integrity sha512-6UqkYefdogmzqAZWzJ7laYeJnaXDy2/J+ZqiiMtS7t7OfpXWTlaeGMwX8U6EFvPV/YWWEKRkS8hKS4k60WHTOg==
+
 pretty-format@^3.8.0:
   version "3.8.0"
   resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-3.8.0.tgz#bfbed56d5e9a776645f4b1ff7aa1a3ac4fa3c385"