Faster UTF16 -> UTF8 and UTF8 -> UTF16 (#1552)

* Fix freezing test

* Add SIMDUTF

* More micro bench snippets

* Update .gitattributes

* Update .gitattributes

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

5 files changed, 182 insertions, 0 deletions

diff --git a/bench/snippets/arraybuffersink.mjs b/bench/snippets/arraybuffersink.mjs
new file mode 100644
index 000000000..0e787dd56
--- /dev/null
+++ b/bench/snippets/arraybuffersink.mjs
@@ -0,0 +1,30 @@
+import { ArrayBufferSink } from "bun";
+import { bench, run } from "mitata";
+
+var short = "Hello World!";
+var shortUTF16 = "Hello World 💕💕💕";
+var long = "Hello World!".repeat(1024);
+var longUTF16 = "Hello World 💕💕💕".repeat(1024);
+var encoder = new ArrayBufferSink({ stream: true, highWaterMark: 512 });
+
+bench(`${short.length} ascii`, () => {
+  encoder.write(short);
+  encoder.start();
+});
+
+bench(`${short.length} utf8`, () => {
+  encoder.write(shortUTF16);
+  encoder.start();
+});
+
+bench(`${long.length} ascii`, () => {
+  encoder.write(long);
+  encoder.start();
+});
+
+bench(`${longUTF16.length} utf8`, () => {
+  encoder.write(longUTF16);
+  encoder.start();
+});
+
+await run();
diff --git a/bench/snippets/read-file.mjs b/bench/snippets/read-file.mjs
new file mode 100644
index 000000000..3e42e6a73
--- /dev/null
+++ b/bench/snippets/read-file.mjs
@@ -0,0 +1,45 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { bench, run } from "mitata";
+
+var short = (function () {
+  const text = "Hello World!";
+  const path = "/tmp/bun-bench-short.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+var shortUTF16 = (function () {
+  const text = "Hello World 💕💕💕";
+  const path = "/tmp/bun-bench-shortUTF16.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+var long = (function () {
+  const text = "Hello World!".repeat(1024);
+  const path = "/tmp/bun-bench-long.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+var longUTF16 = (function () {
+  const text = "Hello World 💕💕💕".repeat(1024);
+  const path = "/tmp/bun-bench-longUTF16.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+
+bench(`${short.length} ascii`, () => {
+  readFileSync(short.path, "utf-8");
+});
+
+bench(`${short.length} utf8`, () => {
+  readFileSync(shortUTF16.path, "utf-8");
+});
+
+bench(`${long.length} ascii`, () => {
+  readFileSync(long.path, "utf-8");
+});
+
+bench(`${longUTF16.length} utf8`, () => {
+  readFileSync(longUTF16.path, "utf-8");
+});
+
+await run();
diff --git a/bench/snippets/text-decoder.mjs b/bench/snippets/text-decoder.mjs
new file mode 100644
index 000000000..b1d669f02
--- /dev/null
+++ b/bench/snippets/text-decoder.mjs
@@ -0,0 +1,49 @@
+import { bench, run } from "../node_modules/mitata/src/cli.mjs";
+
+var short = new TextEncoder().encode("Hello World!");
+var shortUTF16 = new TextEncoder().encode("Hello World 💕💕💕");
+var long = new TextEncoder().encode("Hello World!".repeat(1024));
+var longUTF16 = new TextEncoder().encode("Hello World 💕💕💕".repeat(1024));
+bench(`${short.length} ascii`, () => {
+  var decoder = new TextDecoder();
+  decoder.decode(short);
+});
+
+bench(`${short.length} utf8`, () => {
+  var decoder = new TextDecoder();
+  decoder.decode(shortUTF16);
+});
+
+bench(`${long.length} ascii`, () => {
+  var decoder = new TextDecoder();
+  decoder.decode(long);
+});
+
+bench(`${longUTF16.length} utf8`, () => {
+  var decoder = new TextDecoder();
+  decoder.decode(longUTF16);
+});
+
+if ("Buffer" in globalThis) {
+  const buffer_short = Buffer.from(short);
+  bench(`Buffer ${buffer_short.length} ascii`, () => {
+    buffer_short.toString("ascii");
+  });
+
+  const buffer_shortUTF16 = Buffer.from(short);
+  bench(`Buffer ${buffer_shortUTF16.length} utf8`, () => {
+    buffer_shortUTF16.toString("utf8");
+  });
+
+  const buffer_long = Buffer.from(long);
+  bench(`Buffer ${buffer_long.length} ascii`, () => {
+    buffer_long.toString("ascii");
+  });
+
+  const buffer_longUTF16 = Buffer.from(longUTF16);
+  bench(`Buffer ${buffer_longUTF16.length} utf8`, () => {
+    buffer_longUTF16.toString("utf8");
+  });
+}
+
+await run();
diff --git a/bench/snippets/text-encoder.mjs b/bench/snippets/text-encoder.mjs
new file mode 100644
index 000000000..cee84bf25
--- /dev/null
+++ b/bench/snippets/text-encoder.mjs
@@ -0,0 +1,33 @@
+import { bench, run } from "mitata";
+
+var short = "Hello World!";
+var shortUTF16 = "Hello World 💕💕💕";
+var long = "Hello World!".repeat(1024);
+var longUTF16 = "Hello World 💕💕💕".repeat(1024);
+var encoder = new TextEncoder();
+
+bench(`4 ascii`, () => {
+  encoder.encode("heyo");
+});
+
+bench(`4 utf8`, () => {
+  encoder.encode("💕💕");
+});
+
+bench(`${short.length} ascii`, () => {
+  encoder.encode(short);
+});
+
+bench(`${short.length} utf8`, () => {
+  encoder.encode(shortUTF16);
+});
+
+bench(`${long.length} ascii`, () => {
+  encoder.encode(long);
+});
+
+bench(`${longUTF16.length} utf8`, () => {
+  encoder.encode(longUTF16);
+});
+
+await run();
diff --git a/bench/snippets/write-file.mjs b/bench/snippets/write-file.mjs
new file mode 100644
index 000000000..b13104bfc
--- /dev/null
+++ b/bench/snippets/write-file.mjs
@@ -0,0 +1,25 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { bench, run } from "mitata";
+
+var short = "Hello World!";
+var shortUTF16 = "Hello World 💕💕💕";
+var long = "Hello World!".repeat(1024);
+var longUTF16 = "Hello World 💕💕💕".repeat(1024);
+
+bench(`${short.length} ascii`, () => {
+  writeFileSync("/tmp/bun.bench-out.txt", short);
+});
+
+bench(`${short.length} utf8`, () => {
+  writeFileSync("/tmp/bun.bench-out.txt", shortUTF16);
+});
+
+bench(`${long.length} ascii`, () => {
+  writeFileSync("/tmp/bun.bench-out.txt", long);
+});
+
+bench(`${longUTF16.length} utf8`, () => {
+  writeFileSync("/tmp/bun.bench-out.txt", longUTF16);
+});
+
+await run();