Faster UTF16 -> UTF8 and UTF8 -> UTF16 (#1552)

* Fix freezing test

* Add SIMDUTF

* More micro bench snippets

* Update .gitattributes

* Update .gitattributes

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

1 files changed, 45 insertions, 0 deletions

diff --git a/bench/snippets/read-file.mjs b/bench/snippets/read-file.mjs
new file mode 100644
index 000000000..3e42e6a73
--- /dev/null
+++ b/bench/snippets/read-file.mjs
@@ -0,0 +1,45 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { bench, run } from "mitata";
+
+var short = (function () {
+  const text = "Hello World!";
+  const path = "/tmp/bun-bench-short.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+var shortUTF16 = (function () {
+  const text = "Hello World 💕💕💕";
+  const path = "/tmp/bun-bench-shortUTF16.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+var long = (function () {
+  const text = "Hello World!".repeat(1024);
+  const path = "/tmp/bun-bench-long.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+var longUTF16 = (function () {
+  const text = "Hello World 💕💕💕".repeat(1024);
+  const path = "/tmp/bun-bench-longUTF16.text";
+  writeFileSync(path, text, "utf8");
+  return { path, length: text.length };
+})();
+
+bench(`${short.length} ascii`, () => {
+  readFileSync(short.path, "utf-8");
+});
+
+bench(`${short.length} utf8`, () => {
+  readFileSync(shortUTF16.path, "utf-8");
+});
+
+bench(`${long.length} ascii`, () => {
+  readFileSync(long.path, "utf-8");
+});
+
+bench(`${longUTF16.length} utf8`, () => {
+  readFileSync(longUTF16.path, "utf-8");
+});
+
+await run();