Web Streams API (#176)

* [bun.js] `WritableStream`, `ReadableStream`, `TransformStream`, `WritableStreamDefaultController`, `ReadableStreamDefaultController` & more

* Implement `Blob.stream()`

* Update streams.test.js

* Fix sourcemaps crash

* [TextEncoder] 3x faster in hot loops

* reading almost works

* start to implement native streams

* Implement `Blob.stream()`

* Implement `Bun.file(pathOrFd).stream()`

* Add an extra function

* [fs.readFile] Improve performance

* make jsc bindings a little easier to work with

* fix segfault

* faster async/await + readablestream optimizations

* WebKit updates

* More WebKit updates

* Add releaseWEakrefs binding

* `bun:jsc`

* More streams

* Update streams.test.js

* Update Makefile

* Update mimalloc

* Update WebKit

* Create bun-jsc.test.js

* Faster ReadableStream

* Fix off by one & exceptions

* Handle empty files/blobs

* Update streams.test.js

* Move streams to it's own file

* temp

* impl #1

* take two

* good enough for now

* Implement `readableStreamToArray`, `readableStreamToArrayBuffer`, `concatArrayBuffers`

* jsxOptimizationInlining

* Fix crash

* Add `jsxOptimizationInline` to Bun.Transpiler

* Update Transpiler types

* Update js_ast.zig

* Automatically choose production mode when NODE_ENV="production"

* Update cli.zig

* [jsx] Handle defaultProps when inlining

* Update transpiler.test.js

* uncomment some tests

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

6 files changed, 531 insertions, 13 deletions

diff --git a/integration/bunjs-only-snippets/bun-jsc.test.js b/integration/bunjs-only-snippets/bun-jsc.test.js
new file mode 100644
index 000000000..e329bc092
--- /dev/null
+++ b/integration/bunjs-only-snippets/bun-jsc.test.js
@@ -0,0 +1,94 @@
+import { describe, expect, it } from "bun:test";
+import {
+  describe as jscDescribe,
+  describeArray,
+  gcAndSweep,
+  fullGC,
+  edenGC,
+  heapSize,
+  heapStats,
+  memoryUsage,
+  getRandomSeed,
+  setRandomSeed,
+  isRope,
+  callerSourceOrigin,
+  noFTL,
+  noOSRExitFuzzing,
+  optimizeNextInvocation,
+  numberOfDFGCompiles,
+  releaseWeakRefs,
+  totalCompileTime,
+  reoptimizationRetryCount,
+  drainMicrotasks,
+} from "bun:jsc";
+
+describe("bun:jsc", () => {
+  function count() {
+    var j = 0;
+    for (var i = 0; i < 999999; i++) {
+      j += i + 2;
+    }
+
+    return j;
+  }
+
+  it("describe", () => {
+    jscDescribe([]);
+  });
+  it("describeArray", () => {
+    describeArray([1, 2, 3]);
+  });
+  it("gcAndSweep", () => {
+    gcAndSweep();
+  });
+  it("fullGC", () => {
+    fullGC();
+  });
+  it("edenGC", () => {
+    edenGC();
+  });
+  it("heapSize", () => {
+    expect(heapSize() > 0).toBe(true);
+  });
+  it("heapStats", () => {
+    heapStats();
+  });
+  it("memoryUsage", () => {
+    memoryUsage();
+  });
+  it("getRandomSeed", () => {
+    getRandomSeed(2);
+  });
+  it("setRandomSeed", () => {
+    setRandomSeed(2);
+  });
+  it("isRope", () => {
+    expect(isRope("a" + 123 + "b")).toBe(true);
+    expect(isRope("abcdefgh")).toBe(false);
+  });
+  it("callerSourceOrigin", () => {
+    expect(callerSourceOrigin()).toBe(import.meta.url);
+  });
+  it("noFTL", () => {});
+  it("noOSRExitFuzzing", () => {});
+  it("optimizeNextInvocation", () => {
+    count();
+    optimizeNextInvocation(count);
+    count();
+  });
+  it("numberOfDFGCompiles", () => {
+    expect(numberOfDFGCompiles(count)).toBe(3);
+  });
+  it("releaseWeakRefs", () => {
+    releaseWeakRefs();
+  });
+  it("totalCompileTime", () => {
+    totalCompileTime(count);
+  });
+  it("reoptimizationRetryCount", () => {
+    reoptimizationRetryCount(count);
+  });
+  it("drainMicrotasks", () => {
+    drainMicrotasks();
+  });
+});
diff --git a/integration/bunjs-only-snippets/concat.test.js b/integration/bunjs-only-snippets/concat.test.js
new file mode 100644
index 000000000..9f3e1f257
--- /dev/null
+++ b/integration/bunjs-only-snippets/concat.test.js
@@ -0,0 +1,46 @@
+import { describe, it, expect } from "bun:test";
+import { gcTick } from "./gc";
+import { concat } from "bun";
+
+describe("concat", () => {
+  function polyfill(chunks) {
+    var size = 0;
+    for (const chunk of chunks) {
+      size += chunk.byteLength;
+    }
+    var buffer = new ArrayBuffer(size);
+    var view = new Uint8Array(buffer);
+    var offset = 0;
+    for (const chunk of chunks) {
+      view.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
+    return buffer;
+  }
+
+  function concatToString(chunks) {
+    return Array.from(new Uint8Array(concat(chunks))).join("");
+  }
+
+  function polyfillToString(chunks) {
+    return Array.from(new Uint8Array(polyfill(chunks))).join("");
+  }
+
+  it("works with one element", () => {
+    expect(concatToString([new Uint8Array([123])])).toBe(
+      polyfillToString([new Uint8Array([123])])
+    );
+  });
+
+  it("works with two elements", () => {
+    expect(
+      concatToString([Uint8Array.from([123]), Uint8Array.from([456])])
+    ).toBe(polyfillToString([Uint8Array.from([123]), Uint8Array.from([456])]));
+  });
+
+  it("works with mix of ArrayBuffer and TypedArray elements", () => {
+    expect(
+      concatToString([Uint8Array.from([123]).buffer, Uint8Array.from([456])])
+    ).toBe(polyfillToString([Uint8Array.from([123]), Uint8Array.from([456])]));
+  });
+});
diff --git a/integration/bunjs-only-snippets/escapeHTML.test.js b/integration/bunjs-only-snippets/escapeHTML.test.js
new file mode 100644
index 000000000..ecfcc5e7c
--- /dev/null
+++ b/integration/bunjs-only-snippets/escapeHTML.test.js
@@ -0,0 +1,105 @@
+import { describe, it, expect } from "bun:test";
+import { gcTick } from "./gc";
+import { escapeHTML } from "bun";
+
+describe("escapeHTML", () => {
+  // The matrix of cases we need to test for:
+  // 1. Works with short strings
+  // 2. Works with long strings
+  // 3. Works with latin1 strings
+  // 4. Works with utf16 strings
+  // 5. Works when the text to escape is somewhere in the middle
+  // 6. Works when the text to escape is in the beginning
+  // 7. Works when the text to escape is in the end
+  // 8. Returns the same string when there's no need to escape
+  it("works", () => {
+    expect(escapeHTML("absolutely nothing to do here")).toBe(
+      "absolutely nothing to do here"
+    );
+    expect(escapeHTML("<script>alert(1)</script>")).toBe(
+      "&lt;script&gt;alert(1)&lt;/script&gt;"
+    );
+    expect(escapeHTML("<")).toBe("&lt;");
+    expect(escapeHTML(">")).toBe("&gt;");
+    expect(escapeHTML("&")).toBe("&amp;");
+    expect(escapeHTML("'")).toBe("&#x27;");
+    expect(escapeHTML('"')).toBe("&quot;");
+    expect(escapeHTML("\n")).toBe("\n");
+    expect(escapeHTML("\r")).toBe("\r");
+    expect(escapeHTML("\t")).toBe("\t");
+    expect(escapeHTML("\f")).toBe("\f");
+    expect(escapeHTML("\v")).toBe("\v");
+    expect(escapeHTML("\b")).toBe("\b");
+    expect(escapeHTML("\u00A0")).toBe("\u00A0");
+    expect(escapeHTML("<script>ab")).toBe("&lt;script&gt;ab");
+    expect(escapeHTML("<script>")).toBe("&lt;script&gt;");
+    expect(escapeHTML("<script><script>")).toBe("&lt;script&gt;&lt;script&gt;");
+
+    expect(escapeHTML("lalala" + "<script>alert(1)</script>" + "lalala")).toBe(
+      "lalala&lt;script&gt;alert(1)&lt;/script&gt;lalala"
+    );
+
+    expect(escapeHTML("<script>alert(1)</script>" + "lalala")).toBe(
+      "&lt;script&gt;alert(1)&lt;/script&gt;lalala"
+    );
+    expect(escapeHTML("lalala" + "<script>alert(1)</script>")).toBe(
+      "lalala" + "&lt;script&gt;alert(1)&lt;/script&gt;"
+    );
+
+    expect(escapeHTML("What does 😊 mean?")).toBe("What does 😊 mean?");
+    const output = escapeHTML("<What does 😊");
+    expect(output).toBe("&lt;What does 😊");
+    expect(escapeHTML("<div>What does 😊 mean in text?")).toBe(
+      "&lt;div&gt;What does 😊 mean in text?"
+    );
+
+    expect(
+      escapeHTML(
+        ("lalala" + "<script>alert(1)</script>" + "lalala").repeat(900)
+      )
+    ).toBe("lalala&lt;script&gt;alert(1)&lt;/script&gt;lalala".repeat(900));
+    expect(
+      escapeHTML(("<script>alert(1)</script>" + "lalala").repeat(900))
+    ).toBe("&lt;script&gt;alert(1)&lt;/script&gt;lalala".repeat(900));
+    expect(
+      escapeHTML(("lalala" + "<script>alert(1)</script>").repeat(900))
+    ).toBe(("lalala" + "&lt;script&gt;alert(1)&lt;/script&gt;").repeat(900));
+
+    // the positions of the unicode codepoint are important
+    // our simd code for U16 is at 8 bytes, so we need to especially check the boundaries
+    expect(
+      escapeHTML("😊lalala" + "<script>alert(1)</script>" + "lalala")
+    ).toBe("😊lalala&lt;script&gt;alert(1)&lt;/script&gt;lalala");
+    expect(escapeHTML("<script>😊alert(1)</script>" + "lalala")).toBe(
+      "&lt;script&gt;😊alert(1)&lt;/script&gt;lalala"
+    );
+    expect(escapeHTML("<script>alert(1)😊</script>" + "lalala")).toBe(
+      "&lt;script&gt;alert(1)😊&lt;/script&gt;lalala"
+    );
+    expect(escapeHTML("<script>alert(1)</script>" + "😊lalala")).toBe(
+      "&lt;script&gt;alert(1)&lt;/script&gt;😊lalala"
+    );
+    expect(escapeHTML("<script>alert(1)</script>" + "lal😊ala")).toBe(
+      "&lt;script&gt;alert(1)&lt;/script&gt;lal😊ala"
+    );
+    expect(
+      escapeHTML("<script>alert(1)</script>" + "lal😊ala".repeat(10))
+    ).toBe("&lt;script&gt;alert(1)&lt;/script&gt;" + "lal😊ala".repeat(10));
+
+    for (let i = 1; i < 10; i++)
+      expect(escapeHTML("<script>alert(1)</script>" + "la😊".repeat(i))).toBe(
+        "&lt;script&gt;alert(1)&lt;/script&gt;" + "la😊".repeat(i)
+      );
+
+    expect(escapeHTML("la😊" + "<script>alert(1)</script>")).toBe(
+      "la😊" + "&lt;script&gt;alert(1)&lt;/script&gt;"
+    );
+    expect(
+      escapeHTML(("lalala" + "<script>alert(1)</script>😊").repeat(1))
+    ).toBe(("lalala" + "&lt;script&gt;alert(1)&lt;/script&gt;😊").repeat(1));
+
+    expect(escapeHTML("😊".repeat(100))).toBe("😊".repeat(100));
+    expect(escapeHTML("😊<".repeat(100))).toBe("😊&lt;".repeat(100));
+    expect(escapeHTML("<😊>".repeat(100))).toBe("&lt;😊&gt;".repeat(100));
+  });
+});
diff --git a/integration/bunjs-only-snippets/ffi.test.fixture.callback.c b/integration/bunjs-only-snippets/ffi.test.fixture.callback.c
index 36949e158..3a557e7d5 100644
--- a/integration/bunjs-only-snippets/ffi.test.fixture.callback.c
+++ b/integration/bunjs-only-snippets/ffi.test.fixture.callback.c
@@ -258,6 +258,9 @@ void* JSFunctionCall(void* globalObject, void* callFrame);
 bool my_callback_function(void* arg0);
 
 bool my_callback_function(void* arg0) {
+#ifdef INJECT_BEFORE
+INJECT_BEFORE;
+#endif
   EncodedJSValue arguments[1] = {
     PTR_TO_JSVALUE(arg0)
   };
diff --git a/integration/bunjs-only-snippets/streams.test.js b/integration/bunjs-only-snippets/streams.test.js
new file mode 100644
index 000000000..d694be1ba
--- /dev/null
+++ b/integration/bunjs-only-snippets/streams.test.js
@@ -0,0 +1,186 @@
+import {
+  file,
+  gc,
+  readableStreamToArrayBuffer,
+  readableStreamToArray,
+} from "bun";
+import { expect, it } from "bun:test";
+import { writeFileSync } from "node:fs";
+
+it("exists globally", () => {
+  expect(typeof ReadableStream).toBe("function");
+  expect(typeof ReadableStreamBYOBReader).toBe("function");
+  expect(typeof ReadableStreamBYOBRequest).toBe("function");
+  expect(typeof ReadableStreamDefaultController).toBe("function");
+  expect(typeof ReadableStreamDefaultReader).toBe("function");
+  expect(typeof TransformStream).toBe("function");
+  expect(typeof TransformStreamDefaultController).toBe("function");
+  expect(typeof WritableStream).toBe("function");
+  expect(typeof WritableStreamDefaultController).toBe("function");
+  expect(typeof WritableStreamDefaultWriter).toBe("function");
+  expect(typeof ByteLengthQueuingStrategy).toBe("function");
+  expect(typeof CountQueuingStrategy).toBe("function");
+});
+
+it("ReadableStream (bytes)", async () => {
+  var stream = new ReadableStream({
+    start(controller) {
+      controller.enqueue(Buffer.from("abdefgh"));
+    },
+    pull(controller) {},
+    cancel() {},
+    type: "bytes",
+  });
+  const chunks = [];
+  const chunk = await stream.getReader().read();
+  chunks.push(chunk.value);
+  expect(chunks[0].join("")).toBe(Buffer.from("abdefgh").join(""));
+});
+
+it("ReadableStream (default)", async () => {
+  var stream = new ReadableStream({
+    start(controller) {
+      controller.enqueue(Buffer.from("abdefgh"));
+      controller.close();
+    },
+    pull(controller) {},
+    cancel() {},
+  });
+  const chunks = [];
+  const chunk = await stream.getReader().read();
+  chunks.push(chunk.value);
+  expect(chunks[0].join("")).toBe(Buffer.from("abdefgh").join(""));
+});
+
+it("readableStreamToArray", async () => {
+  var queue = [Buffer.from("abdefgh")];
+  var stream = new ReadableStream({
+    pull(controller) {
+      var chunk = queue.shift();
+      if (chunk) {
+        controller.enqueue(chunk);
+      } else {
+        controller.close();
+      }
+    },
+    cancel() {},
+    type: "bytes",
+  });
+
+  const chunks = await readableStreamToArray(stream);
+
+  expect(chunks[0].join("")).toBe(Buffer.from("abdefgh").join(""));
+});
+
+it("readableStreamToArrayBuffer (bytes)", async () => {
+  var queue = [Buffer.from("abdefgh")];
+  var stream = new ReadableStream({
+    pull(controller) {
+      var chunk = queue.shift();
+      if (chunk) {
+        controller.enqueue(chunk);
+      } else {
+        controller.close();
+      }
+    },
+    cancel() {},
+    type: "bytes",
+  });
+  const buffer = await readableStreamToArrayBuffer(stream);
+  expect(new TextDecoder().decode(new Uint8Array(buffer))).toBe("abdefgh");
+});
+
+it("readableStreamToArrayBuffer (default)", async () => {
+  var queue = [Buffer.from("abdefgh")];
+  var stream = new ReadableStream({
+    pull(controller) {
+      var chunk = queue.shift();
+      if (chunk) {
+        controller.enqueue(chunk);
+      } else {
+        controller.close();
+      }
+    },
+    cancel() {},
+  });
+
+  const buffer = await readableStreamToArrayBuffer(stream);
+  expect(new TextDecoder().decode(new Uint8Array(buffer))).toBe("abdefgh");
+});
+
+it("ReadableStream for Blob", async () => {
+  var blob = new Blob(["abdefgh", "ijklmnop"]);
+  expect(await blob.text()).toBe("abdefghijklmnop");
+  var stream = blob.stream();
+  const chunks = [];
+  var reader = stream.getReader();
+  while (true) {
+    const chunk = await reader.read();
+    if (chunk.done) break;
+    chunks.push(new TextDecoder().decode(chunk.value));
+  }
+  expect(chunks.join("")).toBe(
+    new TextDecoder().decode(Buffer.from("abdefghijklmnop"))
+  );
+});
+
+it("ReadableStream for File", async () => {
+  var blob = file(import.meta.dir + "/fetch.js.txt");
+  var stream = blob.stream(24);
+  const chunks = [];
+  var reader = stream.getReader();
+  stream = undefined;
+  while (true) {
+    const chunk = await reader.read();
+    gc(true);
+    if (chunk.done) break;
+    chunks.push(chunk.value);
+    expect(chunk.value.byteLength <= 24).toBe(true);
+    gc(true);
+  }
+  reader = undefined;
+  const output = new Uint8Array(await blob.arrayBuffer()).join("");
+  const input = chunks.map((a) => a.join("")).join("");
+  expect(output).toBe(input);
+  gc(true);
+});
+
+it("ReadableStream for File errors", async () => {
+  try {
+    var blob = file(import.meta.dir + "/fetch.js.txt.notfound");
+    blob.stream().getReader();
+    throw new Error("should not reach here");
+  } catch (e) {
+    expect(e.code).toBe("ENOENT");
+    expect(e.syscall).toBe("open");
+  }
+});
+
+it("ReadableStream for empty blob closes immediately", async () => {
+  var blob = new Blob([]);
+  var stream = blob.stream();
+  const chunks = [];
+  var reader = stream.getReader();
+  while (true) {
+    const chunk = await reader.read();
+    if (chunk.done) break;
+    chunks.push(chunk.value);
+  }
+
+  expect(chunks.length).toBe(0);
+});
+
+it("ReadableStream for empty file closes immediately", async () => {
+  writeFileSync("/tmp/bun-empty-file-123456", "");
+  var blob = file("/tmp/bun-empty-file-123456");
+  var stream = blob.stream();
+  const chunks = [];
+  var reader = stream.getReader();
+  while (true) {
+    const chunk = await reader.read();
+    if (chunk.done) break;
+    chunks.push(chunk.value);
+  }
+
+  expect(chunks.length).toBe(0);
+});
diff --git a/integration/bunjs-only-snippets/transpiler.test.js b/integration/bunjs-only-snippets/transpiler.test.js
index dd0ffdc8f..30fc2afde 100644
--- a/integration/bunjs-only-snippets/transpiler.test.js
+++ b/integration/bunjs-only-snippets/transpiler.test.js
@@ -291,6 +291,89 @@ describe("Bun.Transpiler", () => {
     );
   });
 
+  describe("inline JSX", () => {
+    const inliner = new Bun.Transpiler({
+      loader: "tsx",
+      define: {
+        "process.env.NODE_ENV": JSON.stringify("production"),
+        user_undefined: "undefined",
+      },
+      platform: "bun",
+      jsxOptimizationInline: true,
+      treeShaking: false,
+    });
+
+    it("inlines static JSX into object literals", () => {
+      expect(
+        inliner
+          .transformSync(
+            `
+export var hi = <div>{123}</div>
+export var hiWithKey = <div key="hey">{123}</div>
+export var hiWithRef = <div ref={foo}>{123}</div>
+
+export var ComponentThatChecksDefaultProps = <Hello></Hello>
+export var ComponentThatChecksDefaultPropsAndHasChildren = <Hello>my child</Hello>
+export var ComponentThatHasSpreadCausesDeopt = <Hello {...spread} />
+
+`.trim()
+          )
+          .trim()
+      ).toBe(
+        `var $$typeof = Symbol.for("react.element");
+export var hi = {
+  $$typeof,
+  type: "div",
+  key: null,
+  ref: null,
+  props: {
+    children: 123
+  },
+  _owner: null
+};
+export var hiWithKey = {
+  $$typeof,
+  type: "div",
+  key: "hey",
+  ref: null,
+  props: {
+    children: 123
+  },
+  _owner: null
+};
+export var hiWithRef = jsx("div", {
+  ref: foo,
+  children: 123
+});
+export var ComponentThatChecksDefaultProps = {
+  $$typeof,
+  type: Hello,
+  key: null,
+  ref: null,
+  props: Hello.defaultProps || {},
+  _owner: null
+};
+export var ComponentThatChecksDefaultPropsAndHasChildren = {
+  $$typeof,
+  type: Hello,
+  key: null,
+  ref: null,
+  props: !Hello.defaultProps ? {
+    children: "my child"
+  } : {
+    ...Hello.defaultProps,
+    children: "my child"
+  },
+  _owner: null
+};
+export var ComponentThatHasSpreadCausesDeopt = jsx(Hello, {
+  ...spread
+});
+`.trim()
+      );
+    });
+  });
+
   it("require with a dynamic non-string expression", () => {
     var nodeTranspiler = new Bun.Transpiler({ platform: "node" });
     expect(nodeTranspiler.transformSync("require('hi' + bar)")).toBe(
@@ -1355,21 +1438,22 @@ class Foo {
       expectPrinted("'a' != '\\x62'", "true");
       expectPrinted("'a' != 'abc'", "true");
 
-      // TODO: string simplification
-      // expectPrinted("'a' + 'b'", '"ab"');
-      // expectPrinted("'a' + 'bc'", '"abc"');
-      // expectPrinted("'ab' + 'c'", '"abc"');
-      // expectPrinted("x + 'a' + 'b'", 'x + "ab"');
-      // expectPrinted("x + 'a' + 'bc'", 'x + "abc"');
-      // expectPrinted("x + 'ab' + 'c'", 'x + "abc"');
-      // expectPrinted("'a' + 1", '"a" + 1;');
-      // expectPrinted("x * 'a' + 'b'", 'x * "a" + "b"');
-
-      // TODO: string simplification
-      // expectPrinted("'string' + `template`", "`stringtemplate`");
+      expectPrinted("'a' + 'b'", '"ab"');
+      expectPrinted("'a' + 'bc'", '"abc"');
+      expectPrinted("'ab' + 'c'", '"abc"');
+      expectPrinted("x + 'a' + 'b'", 'x + "ab"');
+      expectPrinted("x + 'a' + 'bc'", 'x + "abc"');
+      expectPrinted("x + 'ab' + 'c'", 'x + "abc"');
+      expectPrinted("'a' + 1", '"a" + 1');
+      expectPrinted("x * 'a' + 'b'", 'x * "a" + "b"');
+
+      expectPrinted("'string' + `template`", `"stringtemplate"`);
+
+      expectPrinted("`template` + 'string'", "`templatestring`");
+
+      // TODO: string template simplification
       // expectPrinted("'string' + `a${foo}b`", "`stringa${foo}b`");
       // expectPrinted("'string' + tag`template`", '"string" + tag`template`;');
-      // expectPrinted("`template` + 'string'", "`templatestring`");
       // expectPrinted("`a${foo}b` + 'string'", "`a${foo}bstring`");
       // expectPrinted("tag`template` + 'string'", 'tag`template` + "string"');
       // expectPrinted("`template` + `a${foo}b`", "`templatea${foo}b`");