Add benchmarking for dev and build (#240)

* Add benchmarking for the dev server

This adds benchmarking for the dev server.

* Use fs.rm instead

* Only rimraf if the folder exists

* Make uncached match CI

* Change the cached time too

* Don't run benchmark in CI

* Switch back test command

* Make tests be within 10 percent

* Use yarn to run multiple things

* Turn benchmark into uvu tests

* Debugging benchmark

* Print chunk for testing

* Ignore benchmark folder in uvu

* Add build benchmarking

* Update benchmark numbers

11 files changed, 255 insertions, 4 deletions

diff --git a/contributing.md b/contributing.md
index dda573d33..5d0e9324b 100644
--- a/contributing.md
+++ b/contributing.md
@@ -38,3 +38,22 @@ Commit and push these changes, then run an npm publish for each of the packages
 cd packages/astro
 npm publish
 ```
+
+## Running benchmarks
+
+We have benchmarks to keep performance under control. You can run these by running (from the project root):
+
+```shell
+yarn workspace astro run benchmark
+```
+
+Which will fail if the performance has regressed by __10%__ or more.
+
+To update the times cd into the `packages/astro` folder and run the following:
+
+```shell
+node test/benchmark/build.bench.js --save
+node test/benchmark/dev.bench.js --save
+```
+
+Which will update the build and dev benchmarks.
\ No newline at end of file
diff --git a/packages/astro/package.json b/packages/astro/package.json
index 180824aea..c602036ad 100644
--- a/packages/astro/package.json
+++ b/packages/astro/package.json
@@ -28,7 +28,8 @@
     "build": "astro-scripts build \"src/*.ts\" \"src/compiler/index.ts\" \"src/frontend/**/*.ts\" && tsc",
     "postbuild": "astro-scripts copy \"src/**/*.astro\"",
     "dev": "astro-scripts dev \"src/**/*.ts\"",
-    "test": "uvu test -i fixtures -i test-utils.js"
+    "benchmark": "node test/benchmark/dev.bench.js && node test/benchmark/build.bench.js",
+    "test": "uvu test -i fixtures -i benchmark -i test-utils.js"
   },
   "dependencies": {
     "@babel/code-frame": "^7.12.13",
diff --git a/packages/astro/src/build.ts b/packages/astro/src/build.ts
index e4c80717f..be9ec8c7b 100644
--- a/packages/astro/src/build.ts
+++ b/packages/astro/src/build.ts
@@ -20,7 +20,7 @@ import { getDistPath, stopTimer } from './build/util.js';
 import { debug, defaultLogDestination, error, info, warn, trapWarn } from './logger.js';
 import { createRuntime } from './runtime.js';
 
-const logging: LogOptions = {
+const defaultLogging: LogOptions = {
   level: 'debug',
   dest: defaultLogDestination,
 };
@@ -39,7 +39,7 @@ function isRemote(url: string) {
 }
 
 /** The primary build action */
-export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
+export async function build(astroConfig: AstroConfig, logging: LogOptions = defaultLogging): Promise<0 | 1> {
   const { projectRoot, astroRoot } = astroConfig;
   const dist = new URL(astroConfig.dist + '/', projectRoot);
   const pageRoot = new URL('./pages/', astroRoot);
diff --git a/packages/astro/test/benchmark/benchmark.js b/packages/astro/test/benchmark/benchmark.js
new file mode 100644
index 000000000..a043ccac4
--- /dev/null
+++ b/packages/astro/test/benchmark/benchmark.js
@@ -0,0 +1,68 @@
+import { promises as fsPromises, existsSync } from 'fs';
+import { performance } from 'perf_hooks';
+
+const MUST_BE_AT_LEAST_PERC_OF = 90;
+
+const shouldSave = process.argv.includes('--save');
+
+export class Benchmark {
+  constructor(options) {
+    this.options = options;
+    this.setup = options.setup || Function.prototype;
+  }
+
+  async execute() {
+    const { run } = this.options;
+    const start = performance.now();
+    const end = await run(this.options);
+    const time = Math.floor(end - start);
+    return time;
+  }
+
+  async run() {
+    const { file } = this.options;
+
+    await this.setup();
+    const time = await this.execute();
+  
+    if(existsSync(file)) {
+      const raw = await fsPromises.readFile(file, 'utf-8');
+      const data = JSON.parse(raw);
+      if(Math.floor(data.time / time * 100) > MUST_BE_AT_LEAST_PERC_OF) {
+        this.withinPreviousRuns = true;
+      } else {
+        this.withinPreviousRuns = false;
+      }
+    }
+    this.time = time;
+  }
+
+  report() {
+    const { name } = this.options;
+    console.log(name, 'took', this.time, 'ms');
+  }
+
+  check() {
+    if(this.withinPreviousRuns === false) {
+      throw new Error(`${this.options.name} ran too slowly`);
+    }
+  }
+
+  async save() {
+    const { file, name } = this.options;
+    const data = JSON.stringify({
+      name,
+      time: this.time
+    }, null, '  ');
+    await fsPromises.writeFile(file, data, 'utf-8');
+  }
+
+  async test() {
+    await this.run();
+    if(shouldSave) {
+      await this.save();
+    }
+    this.report();
+    this.check();
+  }
+}
diff --git a/packages/astro/test/benchmark/build-cached.json b/packages/astro/test/benchmark/build-cached.json
new file mode 100644
index 000000000..8c83f39a5
--- /dev/null
+++ b/packages/astro/test/benchmark/build-cached.json
@@ -0,0 +1,4 @@
+{
+  "name": "Snowpack Example Build Cached",
+  "time": 10484
+}
\ No newline at end of file
diff --git a/packages/astro/test/benchmark/build-uncached.json b/packages/astro/test/benchmark/build-uncached.json
new file mode 100644
index 000000000..73df89698
--- /dev/null
+++ b/packages/astro/test/benchmark/build-uncached.json
@@ -0,0 +1,4 @@
+{
+  "name": "Snowpack Example Build Uncached",
+  "time": 19629
+}
\ No newline at end of file
diff --git a/packages/astro/test/benchmark/build.bench.js b/packages/astro/test/benchmark/build.bench.js
new file mode 100644
index 000000000..3cd1919fe
--- /dev/null
+++ b/packages/astro/test/benchmark/build.bench.js
@@ -0,0 +1,86 @@
+import { fileURLToPath } from 'url';
+import { performance } from 'perf_hooks';
+import { build as astroBuild } from '#astro/build';
+import { loadConfig } from '#astro/config';
+import { Benchmark } from './benchmark.js';
+import del from 'del';
+import { Writable } from 'stream';
+import { format as utilFormat } from 'util';
+
+const snowpackExampleRoot = new URL('../../../../examples/snowpack/', import.meta.url);
+
+export const errorWritable = new Writable({
+  objectMode: true,
+  write(event, _, callback) {
+    let dest = process.stderr;
+    dest.write(utilFormat(...event.args));
+    dest.write('\n');
+
+    callback();
+  },
+});
+
+let build;
+async function setupBuild() {
+  const astroConfig = await loadConfig(fileURLToPath(snowpackExampleRoot));
+
+  const logging = {
+    level: 'error',
+    dest: errorWritable,
+  };
+
+  build = () => astroBuild(astroConfig, logging);
+}
+
+async function runBuild() {
+  await build();
+  return performance.now();
+}
+
+const benchmarks = [
+  new Benchmark({
+    name: 'Snowpack Example Build Uncached',
+    root: snowpackExampleRoot,
+    file: new URL('./build-uncached.json', import.meta.url),
+    async setup() {
+      process.chdir(new URL('../../../../', import.meta.url).pathname);
+      const spcache = new URL('../../node_modules/.cache/', import.meta.url);
+      await Promise.all([
+        del(spcache.pathname, { force: true }),
+        setupBuild()
+      ]);
+    },
+    run: runBuild
+  }),
+  new Benchmark({
+    name: 'Snowpack Example Build Cached',
+    root: snowpackExampleRoot,
+    file: new URL('./build-cached.json', import.meta.url),
+    async setup() {
+      process.chdir(new URL('../../../../', import.meta.url).pathname);
+      await setupBuild();
+      await this.execute();
+    },
+    run: runBuild
+  }),
+  /*new Benchmark({
+    name: 'Snowpack Example Dev Server Cached',
+    root: snowpackExampleRoot,
+    file: new URL('./dev-server-cached.json', import.meta.url),
+    async setup() {
+      // Execute once to make sure Snowpack is cached.
+      await this.execute();
+    }
+  })*/
+];
+
+async function run() {
+  for(const b of benchmarks) {
+    await b.test();
+  }
+}
+
+run().catch(err => {
+  console.error(err);
+  process.exit(1);
+});
\ No newline at end of file
diff --git a/packages/astro/test/benchmark/dev-server-cached.json b/packages/astro/test/benchmark/dev-server-cached.json
new file mode 100644
index 000000000..3a5dffb68
--- /dev/null
+++ b/packages/astro/test/benchmark/dev-server-cached.json
@@ -0,0 +1,4 @@
+{
+  "name": "Snowpack Example Dev Server Cached",
+  "time": 1868
+}
\ No newline at end of file
diff --git a/packages/astro/test/benchmark/dev-server-uncached.json b/packages/astro/test/benchmark/dev-server-uncached.json
new file mode 100644
index 000000000..915037f91
--- /dev/null
+++ b/packages/astro/test/benchmark/dev-server-uncached.json
@@ -0,0 +1,4 @@
+{
+  "name": "Snowpack Example Dev Server Uncached",
+  "time": 9803
+}
\ No newline at end of file
diff --git a/packages/astro/test/benchmark/dev.bench.js b/packages/astro/test/benchmark/dev.bench.js
new file mode 100644
index 000000000..e2c4d3dcd
--- /dev/null
+++ b/packages/astro/test/benchmark/dev.bench.js
@@ -0,0 +1,61 @@
+import { performance } from 'perf_hooks';
+import { Benchmark } from './benchmark.js';
+import { runDevServer } from '../helpers.js';
+import del from 'del';
+
+const snowpackExampleRoot = new URL('../../../../examples/snowpack/', import.meta.url);
+
+async function runToStarted(root) {
+  const args = [];
+  const process = runDevServer(root, args);
+
+  let started = null;
+  process.stdout.setEncoding('utf8');
+  for await (const chunk of process.stdout) {
+    if (/Server started/.test(chunk)) {
+      started = performance.now();
+      break;
+    }
+  }
+
+  process.kill();
+  return started;
+}
+
+const benchmarks = [
+  new Benchmark({
+    name: 'Snowpack Example Dev Server Uncached',
+    root: snowpackExampleRoot,
+    file: new URL('./dev-server-uncached.json', import.meta.url),
+    async setup() {
+      const spcache = new URL('../../node_modules/.cache/', import.meta.url);
+      await del(spcache.pathname);
+    },
+    run({ root }) {
+      return runToStarted(root);
+    }
+  }),
+  new Benchmark({
+    name: 'Snowpack Example Dev Server Cached',
+    root: snowpackExampleRoot,
+    file: new URL('./dev-server-cached.json', import.meta.url),
+    async setup() {
+      // Execute once to make sure Snowpack is cached.
+      await this.execute();
+    },
+    run({ root }) {
+      return runToStarted(root);
+    }
+  })
+];
+
+async function run() {
+  for(const b of benchmarks) {
+    await b.test();
+  }
+}
+
+run().catch(err => {
+  console.error(err);
+  process.exit(1);
+});
\ No newline at end of file
diff --git a/packages/astro/test/helpers.js b/packages/astro/test/helpers.js
index 108383d2e..f3bb0a7e0 100644
--- a/packages/astro/test/helpers.js
+++ b/packages/astro/test/helpers.js
@@ -48,7 +48,7 @@ export function setupBuild(Suite, fixturePath) {
       dest: process.stderr,
     };
 
-    build = (...args) => astroBuild(astroConfig, ...args);
+    build = () => astroBuild(astroConfig, logging);
     context.build = build;
     context.readFile = async (path) => {
       const resolved = fileURLToPath(new URL(`${fixturePath}/${astroConfig.dist}${path}`, import.meta.url));