Make readdir() async, fix crash in large directory trees (#3838)

* Fix unsafe GC behavior on large arrays returned by fs

* Fix crash in large arrays of strings

* async readdir

* Add tests for large number of files returned by readdir

* Move this down

* Fix encoding edgecase in path.join

* Async stat & lstat

* add test

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

1 files changed, 81 insertions, 0 deletions

diff --git a/test/js/node/fs/fs.test.ts b/test/js/node/fs/fs.test.ts
index f86c9fdce..76d06560e 100644
--- a/test/js/node/fs/fs.test.ts
+++ b/test/js/node/fs/fs.test.ts
@@ -159,6 +159,81 @@ it("readdirSync on import.meta.dir", () => {
   expect(match).toBe(true);
 });
 
+it("promises.readdir on a large folder", async () => {
+  const huge = join(tmpdir(), "huge-folder-" + Math.random().toString(32));
+  rmSync(huge, { force: true, recursive: true });
+  mkdirSync(huge, { recursive: true });
+  for (let i = 0; i < 128; i++) {
+    writeFileSync(join(huge, "file-" + i), "");
+  }
+  for (let j = 0; j < 4; j++) {
+    const promises = await Promise.all([
+      fs.promises.readdir(huge),
+      fs.promises.readdir(huge),
+      fs.promises.readdir(huge),
+      fs.promises.readdir(huge),
+    ]);
+
+    for (let chunk of promises) {
+      expect(chunk).toHaveLength(128);
+      chunk.sort();
+
+      let count = 0;
+      for (let i = 0; i < 128; i++) {
+        const current = chunk[i];
+        if (!current.startsWith("file-")) {
+          throw new Error("invalid file name");
+        }
+
+        const num = parseInt(current.slice(5));
+        // @ts-expect-error
+        count += !!(num >= 0 && num < 128);
+      }
+
+      expect(count).toBe(128);
+    }
+  }
+  rmSync(huge, { force: true, recursive: true });
+});
+
+it("promises.readdir on a large folder withFileTypes", async () => {
+  const huge = join(tmpdir(), "huge-folder-" + Math.random().toString(32));
+  rmSync(huge, { force: true, recursive: true });
+  mkdirSync(huge, { recursive: true });
+  let withFileTypes = { withFileTypes: true } as const;
+  for (let i = 0; i < 128; i++) {
+    writeFileSync(join(huge, "file-" + i), "");
+  }
+  for (let j = 0; j < 4; j++) {
+    const promises = await Promise.all([
+      fs.promises.readdir(huge, withFileTypes),
+      fs.promises.readdir(huge, withFileTypes),
+      fs.promises.readdir(huge, withFileTypes),
+      fs.promises.readdir(huge, withFileTypes),
+    ]);
+
+    for (let chunk of promises) {
+      expect(chunk).toHaveLength(128);
+      chunk.sort();
+
+      let count = 0;
+      for (let i = 0; i < 128; i++) {
+        const current = chunk[i].name;
+        if (!current.startsWith("file-")) {
+          throw new Error("invalid file name");
+        }
+
+        const num = parseInt(current.slice(5));
+        // @ts-expect-error
+        count += !!(num >= 0 && num < 128);
+      }
+
+      expect(count).toBe(128);
+    }
+  }
+  rmSync(huge, { force: true, recursive: true });
+});
+
 it("statSync throwIfNoEntry", () => {
   expect(statSync("/tmp/404/not-found/ok", { throwIfNoEntry: false })).toBeUndefined();
   expect(lstatSync("/tmp/404/not-found/ok", { throwIfNoEntry: false })).toBeUndefined();
@@ -171,6 +246,12 @@ it("statSync throwIfNoEntry: true", () => {
   expect(() => lstatSync("/tmp/404/not-found/ok")).toThrow("No such file or directory");
 });
 
+it("stat == statSync", async () => {
+  const sync = statSync(import.meta.path);
+  const async = await promises.stat(import.meta.path);
+  expect(Object.entries(sync)).toEqual(Object.entries(async));
+});
+
 // https://github.com/oven-sh/bun/issues/1887
 it("mkdtempSync, readdirSync, rmdirSync and unlinkSync with non-ascii", () => {
   const tempdir = mkdtempSync(`${tmpdir()}/emoji-fruit-🍇 🍈 🍉 🍊 🍋`);