Make readdir() async, fix crash in large directory trees (#3838)

* Fix unsafe GC behavior on large arrays returned by fs

* Fix crash in large arrays of strings

* async readdir

* Add tests for large number of files returned by readdir

* Move this down

* Fix encoding edgecase in path.join

* Async stat & lstat

* add test

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

4 files changed, 42 insertions, 36 deletions

diff --git a/src/js/node/fs.js b/src/js/node/fs.js
index 5e72d6e27..f847cf179 100644
--- a/src/js/node/fs.js
+++ b/src/js/node/fs.js
@@ -122,9 +122,6 @@ export var access = function access(...args) {
   link = function link(...args) {
     callbackify(fs.linkSync, args);
   },
-  lstat = function lstat(...args) {
-    callbackify(fs.lstatSync, args);
-  },
   mkdir = function mkdir(...args) {
     callbackify(fs.mkdirSync, args);
   },
@@ -141,7 +138,13 @@ export var access = function access(...args) {
     callbackify(fs.writeSync, args);
   },
   readdir = function readdir(...args) {
-    callbackify(fs.readdirSync, args);
+    const callback = args[args.length - 1];
+    if (typeof callback !== "function") {
+      // TODO: set code
+      throw new TypeError("Callback must be a function");
+    }
+
+    fs.readdir(...args).then(result => callback(null, result), callback);
   },
   readFile = function readFile(...args) {
     callbackify(fs.readFileSync, args);
@@ -158,8 +161,23 @@ export var access = function access(...args) {
   rename = function rename(...args) {
     callbackify(fs.renameSync, args);
   },
+  lstat = function lstat(...args) {
+    const callback = args[args.length - 1];
+    if (typeof callback !== "function") {
+      // TODO: set code
+      throw new TypeError("Callback must be a function");
+    }
+
+    fs.lstat(...args).then(result => callback(null, result), callback);
+  },
   stat = function stat(...args) {
-    callbackify(fs.statSync, args);
+    const callback = args[args.length - 1];
+    if (typeof callback !== "function") {
+      // TODO: set code
+      throw new TypeError("Callback must be a function");
+    }
+
+    fs.stat(...args).then(result => callback(null, result), callback);
   },
   symlink = function symlink(...args) {
     callbackify(fs.symlinkSync, args);
diff --git a/src/js/node/fs.promises.ts b/src/js/node/fs.promises.ts
index 12278ef53..1826586ae 100644
--- a/src/js/node/fs.promises.ts
+++ b/src/js/node/fs.promises.ts
@@ -11,30 +11,9 @@ var fs = Bun.fs();
 const notrace = "::bunternal::";
 var promisify = {
   [notrace]: fsFunction => {
-    // TODO: remove variadic arguments
-    // we can use new Function() here instead
-    // based on fsFucntion.length
-    var func = {
-      [notrace]: function (resolve, reject, args) {
-        var result;
-        try {
-          result = fsFunction.apply(fs, args);
-          args = undefined;
-        } catch (err) {
-          args = undefined;
-          reject(err);
-          return;
-        }
-
-        resolve(result);
-      },
-    }[notrace];
-
     return async function (...args) {
-      // we await it so that the stack is captured
-      return await new Promise((resolve, reject) => {
-        process.nextTick(func, resolve, reject, args);
-      });
+      await 1;
+      return fsFunction.apply(fs, args);
     };
   },
 }[notrace];
@@ -104,19 +83,19 @@ export var access = promisify(fs.accessSync),
   lchmod = promisify(fs.lchmodSync),
   lchown = promisify(fs.lchownSync),
   link = promisify(fs.linkSync),
-  lstat = promisify(fs.lstatSync),
+  lstat = fs.lstat.bind(fs),
   mkdir = promisify(fs.mkdirSync),
   mkdtemp = promisify(fs.mkdtempSync),
   open = promisify(fs.openSync),
   read = promisify(fs.readSync),
   write = promisify(fs.writeSync),
-  readdir = promisify(fs.readdirSync),
+  readdir = fs.readdir.bind(fs),
   readFile = promisify(fs.readFileSync),
   writeFile = promisify(fs.writeFileSync),
   readlink = promisify(fs.readlinkSync),
   realpath = promisify(fs.realpathSync),
   rename = promisify(fs.renameSync),
-  stat = promisify(fs.statSync),
+  stat = fs.stat.bind(fs),
   symlink = promisify(fs.symlinkSync),
   truncate = promisify(fs.truncateSync),
   unlink = promisify(fs.unlinkSync),
diff --git a/src/js/out/modules/node/fs.js b/src/js/out/modules/node/fs.js
index b7457f104..947eaf826 100644
--- a/src/js/out/modules/node/fs.js
+++ b/src/js/out/modules/node/fs.js
@@ -97,8 +97,6 @@ var access = function access2(...args) {
   callbackify(fs.lchownSync, args);
 }, link = function link2(...args) {
   callbackify(fs.linkSync, args);
-}, lstat = function lstat2(...args) {
-  callbackify(fs.lstatSync, args);
 }, mkdir = function mkdir2(...args) {
   callbackify(fs.mkdirSync, args);
 }, mkdtemp = function mkdtemp2(...args) {
@@ -110,7 +108,10 @@ var access = function access2(...args) {
 }, write = function write2(...args) {
   callbackify(fs.writeSync, args);
 }, readdir = function readdir2(...args) {
-  callbackify(fs.readdirSync, args);
+  const callback = args[args.length - 1];
+  if (typeof callback !== "function")
+    throw new TypeError("Callback must be a function");
+  fs.readdir(...args).then((result) => callback(null, result), callback);
 }, readFile = function readFile2(...args) {
   callbackify(fs.readFileSync, args);
 }, writeFile = function writeFile2(...args) {
@@ -121,8 +122,16 @@ var access = function access2(...args) {
   callbackify(fs.realpathSync, args);
 }, rename = function rename2(...args) {
   callbackify(fs.renameSync, args);
+}, lstat = function lstat2(...args) {
+  const callback = args[args.length - 1];
+  if (typeof callback !== "function")
+    throw new TypeError("Callback must be a function");
+  fs.lstat(...args).then((result) => callback(null, result), callback);
 }, stat = function stat2(...args) {
-  callbackify(fs.statSync, args);
+  const callback = args[args.length - 1];
+  if (typeof callback !== "function")
+    throw new TypeError("Callback must be a function");
+  fs.stat(...args).then((result) => callback(null, result), callback);
 }, symlink = function symlink2(...args) {
   callbackify(fs.symlinkSync, args);
 }, truncate = function truncate2(...args) {
diff --git a/src/js/out/modules/node/fs.promises.js b/src/js/out/modules/node/fs.promises.js
index 9ac3c6f65..185955dd5 100644
--- a/src/js/out/modules/node/fs.promises.js
+++ b/src/js/out/modules/node/fs.promises.js
@@ -1 +1 @@
-var r=(B)=>{return import.meta.require(B)};function J(B,C={}){const G=[];if(B instanceof URL)throw new TypeError("Watch URLs are not supported yet");else if(Buffer.isBuffer(B))B=B.toString();else if(typeof B!=="string")throw new TypeError("Expected path to be a string or Buffer");let D=null;if(typeof C==="string")C={encoding:C};return S.watch(B,C||{},(z,A)=>{if(G.push({eventType:z,filename:A}),D){const H=D;D=null,H()}}),{async*[Symbol.asyncIterator](){let z=!1;while(!z){while(G.length){let A=G.shift();if(A.eventType==="close"){z=!0;break}if(A.eventType==="error")throw z=!0,A.filename;yield A}await new Promise((A)=>D=A)}}}}var S=Bun.fs(),I="::bunternal::",q={[I]:(B)=>{var C={[I]:function(G,D,z){var A;try{A=B.apply(S,z),z=void 0}catch(H){z=void 0,D(H);return}G(A)}}[I];return async function(...G){return await new Promise((D,z)=>{process.nextTick(C,D,z,G)})}}}[I],K=q(S.accessSync),L=q(S.appendFileSync),M=q(S.closeSync),N=q(S.copyFileSync),O=q(S.existsSync),P=q(S.chownSync),Q=q(S.chmodSync),U=q(S.fchmodSync),V=q(S.fchownSync),X=q(S.fstatSync),Y=q(S.fsyncSync),Z=q(S.ftruncateSync),_=q(S.futimesSync),$=q(S.lchmodSync),T=q(S.lchownSync),W=q(S.linkSync),k=q(S.lstatSync),E=q(S.mkdirSync),x=q(S.mkdtempSync),F=q(S.openSync),R=q(S.readSync),g=q(S.writeSync),h=q(S.readdirSync),j=q(S.readFileSync),w=q(S.writeFileSync),b=q(S.readlinkSync),u=q(S.realpathSync),d=q(S.renameSync),c=q(S.statSync),v=q(S.symlinkSync),a=q(S.truncateSync),y=q(S.unlinkSync),l=q(S.utimesSync),p=q(S.lutimesSync),m=q(S.rmSync),n=q(S.rmdirSync),t=(B,C,G)=>{return new Promise((D,z)=>{try{var A=S.writevSync(B,C,G)}catch(H){z(H);return}D({bytesWritten:A,buffers:C})})},o=(B,C,G)=>{return new Promise((D,z)=>{try{var A=S.readvSync(B,C,G)}catch(H){z(H);return}D({bytesRead:A,buffers:C})})},f={access:K,appendFile:L,close:M,copyFile:N,exists:O,chown:P,chmod:Q,fchmod:U,fchown:V,fstat:X,fsync:Y,ftruncate:Z,futimes:_,lchmod:$,lchown:T,link:W,lstat:k,mkdir:E,mkdtemp:x,open:F,read:R,write:g,readdir:h,readFile:j,writeFile:w,readlink:b,realpath:u,rename:d,stat:c,symlink:v,truncate:a,unlink:y,utimes:l,lutimes:p,rm:m,rmdir:n,watch:J,writev:t,readv:o,constants,[Symbol.for("CommonJS")]:0};export{t as writev,w as writeFile,g as write,J as watch,l as utimes,y as unlink,a as truncate,v as symlink,c as stat,n as rmdir,m as rm,d as rename,u as realpath,o as readv,b as readlink,h as readdir,j as readFile,R as read,F as open,x as mkdtemp,E as mkdir,p as lutimes,k as lstat,W as link,T as lchown,$ as lchmod,_ as futimes,Z as ftruncate,Y as fsync,X as fstat,V as fchown,U as fchmod,O as exists,f as default,N as copyFile,M as close,P as chown,Q as chmod,L as appendFile,K as access};
+var o=(g)=>{return import.meta.require(g)};function H(g,q={}){const C=[];if(g instanceof URL)throw new TypeError("Watch URLs are not supported yet");else if(Buffer.isBuffer(g))g=g.toString();else if(typeof g!=="string")throw new TypeError("Expected path to be a string or Buffer");let B=null;if(typeof q==="string")q={encoding:q};return P.watch(g,q||{},(A,z)=>{if(C.push({eventType:A,filename:z}),B){const D=B;B=null,D()}}),{async*[Symbol.asyncIterator](){let A=!1;while(!A){while(C.length){let z=C.shift();if(z.eventType==="close"){A=!0;break}if(z.eventType==="error")throw A=!0,z.filename;yield z}await new Promise((z)=>B=z)}}}}var P=Bun.fs(),G="::bunternal::",U={[G]:(g)=>{return async function(...q){return await 1,g.apply(P,q)}}}[G],I=U(P.accessSync),J=U(P.appendFileSync),K=U(P.closeSync),L=U(P.copyFileSync),M=U(P.existsSync),N=U(P.chownSync),O=U(P.chmodSync),Q=U(P.fchmodSync),S=U(P.fchownSync),V=U(P.fstatSync),X=U(P.fsyncSync),Y=U(P.ftruncateSync),Z=U(P.futimesSync),_=U(P.lchmodSync),$=U(P.lchownSync),T=U(P.linkSync),W=P.lstat.bind(P),j=U(P.mkdirSync),x=U(P.mkdtempSync),E=U(P.openSync),F=U(P.readSync),w=U(P.writeSync),k=P.readdir.bind(P),R=U(P.readFileSync),h=U(P.writeFileSync),b=U(P.readlinkSync),u=U(P.realpathSync),c=U(P.renameSync),d=P.stat.bind(P),v=U(P.symlinkSync),a=U(P.truncateSync),l=U(P.unlinkSync),y=U(P.utimesSync),p=U(P.lutimesSync),m=U(P.rmSync),n=U(P.rmdirSync),t=(g,q,C)=>{return new Promise((B,A)=>{try{var z=P.writevSync(g,q,C)}catch(D){A(D);return}B({bytesWritten:z,buffers:q})})},r=(g,q,C)=>{return new Promise((B,A)=>{try{var z=P.readvSync(g,q,C)}catch(D){A(D);return}B({bytesRead:z,buffers:q})})},f={access:I,appendFile:J,close:K,copyFile:L,exists:M,chown:N,chmod:O,fchmod:Q,fchown:S,fstat:V,fsync:X,ftruncate:Y,futimes:Z,lchmod:_,lchown:$,link:T,lstat:W,mkdir:j,mkdtemp:x,open:E,read:F,write:w,readdir:k,readFile:R,writeFile:h,readlink:b,realpath:u,rename:c,stat:d,symlink:v,truncate:a,unlink:l,utimes:y,lutimes:p,rm:m,rmdir:n,watch:H,writev:t,readv:r,constants,[Symbol.for("CommonJS")]:0};export{t as writev,h as writeFile,w as write,H as watch,y as utimes,l as unlink,a as truncate,v as symlink,d as stat,n as rmdir,m as rm,c as rename,u as realpath,r as readv,b as readlink,k as readdir,R as readFile,F as read,E as open,x as mkdtemp,j as mkdir,p as lutimes,W as lstat,T as link,$ as lchown,_ as lchmod,Z as futimes,Y as ftruncate,X as fsync,V as fstat,S as fchown,Q as fchmod,M as exists,f as default,L as copyFile,K as close,N as chown,O as chmod,J as appendFile,I as access};