[bun:ffi] Improve `uint64_t` and `int64_t` performance

3 files changed, 212 insertions, 101 deletions

diff --git a/src/javascript/jsc/api/FFI.h b/src/javascript/jsc/api/FFI.h
index 69af59dc6..399448cae 100644
--- a/src/javascript/jsc/api/FFI.h
+++ b/src/javascript/jsc/api/FFI.h
@@ -7,20 +7,15 @@
 // It must be kept in sync with JSCJSValue.h
 // https://github.com/Jarred-Sumner/WebKit/blob/72c2052b781cbfd4af867ae79ac9de460e392fba/Source/JavaScriptCore/runtime/JSCJSValue.h#L455-L458
 #ifdef IS_CALLBACK
-// #define INJECT_BEFORE printf("bun_call %p cachedJSContext %p cachedCallbackFunction %p\n", &bun_call, cachedJSContext, cachedCallbackFunction);
+#define INJECT_BEFORE int c = 500; // This is a callback, so we need to inject code before the call
 #endif
 
 #define IS_BIG_ENDIAN 0
 #define USE_JSVALUE64 1
 #define USE_JSVALUE32_64 0
 
-// #include <stdint.h>
-#ifdef INJECT_BEFORE
-// #include <stdio.h>
-#endif
-// #include <tcclib.h>
 
-// // /* 7.18.1.1  Exact-width integer types */
+// /* 7.18.1.1  Exact-width integer types */
 typedef unsigned char uint8_t;
 typedef signed char int8_t;
 typedef short int16_t;
@@ -29,7 +24,7 @@ typedef int int32_t;
 typedef unsigned int uint32_t;
 typedef long long int64_t;
 typedef unsigned long long uint64_t;
-typedef uint64_t size_t;
+typedef unsigned long long size_t;
 typedef long intptr_t;
 typedef uint64_t uintptr_t;
 typedef _Bool bool;
@@ -38,6 +33,13 @@ typedef _Bool bool;
 #define false 0
 
 
+#ifdef INJECT_BEFORE
+// #include <stdint.h>
+#endif
+// #include <tcclib.h>
+
+
+
 // This value is 2^49, used to encode doubles such that the encoded value will
 // begin with a 15-bit pattern within the range 0x0002..0xFFFC.
 #define DoubleEncodeOffsetBit 49
@@ -49,6 +51,10 @@ typedef _Bool bool;
 #define TagValueTrue             (OtherTag | BoolTag | true)
 #define TagValueUndefined        (OtherTag | UndefinedTag)
 #define TagValueNull             (OtherTag)
+#define NotCellMask  NumberTag | OtherTag
+
+#define MAX_INT32 2147483648
+#define MAX_INT52 9007199254740991
 
 // If all bits in the mask are set, this indicates an integer number,
 // if any but not all are set this value is a double precision number.
@@ -58,8 +64,8 @@ typedef  void* JSCell;
 
 typedef union EncodedJSValue {
   int64_t asInt64;
-#if USE_JSVALUE32_64
-#elif USE_JSVALUE64
+
+#if USE_JSVALUE64
   JSCell *ptr;
 #endif
 
@@ -84,7 +90,13 @@ EncodedJSValue ValueTrue = { TagValueTrue };
 
 typedef void* JSContext;
 
-#define LOAD_ARGUMENTS_FROM_CALL_FRAME EncodedJSValue* args = (EncodedJSValue*)((size_t*)callFrame + Bun_FFI_PointerOffsetToArgumentsList);
+// Bun_FFI_PointerOffsetToArgumentsList is injected into the build 
+// The value is generated in `make sizegen`
+// The value is 6.
+// On ARM64_32, the value is something else but it really doesn't matter for our case
+// However, I don't want this to subtly break amidst future upgrades to JavaScriptCore
+#define LOAD_ARGUMENTS_FROM_CALL_FRAME \
+  int64_t *argsPtr = (int64_t*)((size_t*)callFrame + Bun_FFI_PointerOffsetToArgumentsList)
 
 
 #ifdef IS_CALLBACK
@@ -93,11 +105,20 @@ JSContext cachedJSContext;
 void* cachedCallbackFunction;
 #endif
 
-uint64_t JSVALUE_TO_UINT64(void* globalObject, EncodedJSValue value);
-int64_t JSVALUE_TO_INT64(EncodedJSValue value);
+static bool JSVALUE_IS_CELL(EncodedJSValue val) __attribute__((__always_inline__));
+static bool JSVALUE_IS_INT32(EncodedJSValue val) __attribute__((__always_inline__)); 
+static bool JSVALUE_IS_NUMBER(EncodedJSValue val) __attribute__((__always_inline__));
+
+static uint64_t JSVALUE_TO_UINT64(void* globalObject, EncodedJSValue value) __attribute__((__always_inline__));
+static int64_t  JSVALUE_TO_INT64(EncodedJSValue value) __attribute__((__always_inline__));
+uint64_t JSVALUE_TO_UINT64_SLOW(void* globalObject, EncodedJSValue value);
+int64_t  JSVALUE_TO_INT64_SLOW(EncodedJSValue value);
+
+EncodedJSValue UINT64_TO_JSVALUE_SLOW(void* globalObject, uint64_t val);
+EncodedJSValue INT64_TO_JSVALUE_SLOW(void* globalObject, int64_t val);
+static EncodedJSValue UINT64_TO_JSVALUE(void* globalObject, uint64_t val) __attribute__((__always_inline__));
+static EncodedJSValue INT64_TO_JSVALUE(void* globalObject, int64_t val) __attribute__((__always_inline__));
 
-EncodedJSValue UINT64_TO_JSVALUE(void* globalObject, uint64_t val);
-EncodedJSValue INT64_TO_JSVALUE(void* globalObject, int64_t val);
 
 static EncodedJSValue INT32_TO_JSVALUE(int32_t val) __attribute__((__always_inline__));
 static EncodedJSValue DOUBLE_TO_JSVALUE(double val) __attribute__((__always_inline__));
@@ -111,6 +132,19 @@ static float JSVALUE_TO_FLOAT(EncodedJSValue val) __attribute__((__always_inline
 static double JSVALUE_TO_DOUBLE(EncodedJSValue val) __attribute__((__always_inline__));
 static bool JSVALUE_TO_BOOL(EncodedJSValue val) __attribute__((__always_inline__));
 
+static bool JSVALUE_IS_CELL(EncodedJSValue val) {
+  return !(val.asInt64 & NotCellMask);
+}
+
+static bool JSVALUE_IS_INT32(EncodedJSValue val) {
+  return (val.asInt64 & NumberTag) == NumberTag;
+}
+
+static bool JSVALUE_IS_NUMBER(EncodedJSValue val) {
+  return val.asInt64 & NumberTag;
+}
+
+
 static void* JSVALUE_TO_PTR(EncodedJSValue val) {
   // must be a double
   return (void*)(val.asInt64 - DoubleEncodeOffset);
@@ -164,9 +198,57 @@ static bool JSVALUE_TO_BOOL(EncodedJSValue val) {
   return val.asInt64 == TagValueTrue;
 }
 
-#define arg(i) ((EncodedJSValue*)args)[i]
+
+static uint64_t JSVALUE_TO_UINT64(void* globalObject, EncodedJSValue value) {
+  if (JSVALUE_IS_INT32(value)) {
+    return (uint64_t)JSVALUE_TO_INT32(value);
+  }
+
+  if (JSVALUE_IS_NUMBER(value)) {
+    return (uint64_t)JSVALUE_TO_DOUBLE(value);
+  }
+
+  return JSVALUE_TO_UINT64_SLOW(globalObject, value);
+}
+static int64_t JSVALUE_TO_INT64(EncodedJSValue value) {
+  if (JSVALUE_IS_INT32(value)) {
+    return (int64_t)JSVALUE_TO_INT32(value);
+  }
+
+  if (JSVALUE_IS_NUMBER(value)) {
+    return (int64_t)JSVALUE_TO_DOUBLE(value);
+  }
+
+  return JSVALUE_TO_INT64_SLOW(value);
+}
+
+static EncodedJSValue UINT64_TO_JSVALUE(void* globalObject, uint64_t val) {
+  if (val < MAX_INT32) {
+    return INT32_TO_JSVALUE((int32_t)val);
+  }
+
+  if (val < MAX_INT52) {
+    return DOUBLE_TO_JSVALUE((double)val);
+  }
+
+  return UINT64_TO_JSVALUE_SLOW(globalObject, val);
+}
+
+static EncodedJSValue INT64_TO_JSVALUE(void* globalObject, int64_t val) {
+  if (val >= -MAX_INT32 && val <= MAX_INT32) {
+    return INT32_TO_JSVALUE((int32_t)val);
+  }
+
+  if (val >= -MAX_INT52 && val <= MAX_INT52) {
+    return DOUBLE_TO_JSVALUE((double)val);
+  }
+
+  return INT64_TO_JSVALUE_SLOW(globalObject, val);
+}
+
 #ifndef IS_CALLBACK
 void* JSFunctionCall(void* globalObject, void* callFrame);
+
 #endif
 
 
diff --git a/src/javascript/jsc/api/ffi.zig b/src/javascript/jsc/api/ffi.zig
index 702d0d6d6..ce162f890 100644
--- a/src/javascript/jsc/api/ffi.zig
+++ b/src/javascript/jsc/api/ffi.zig
@@ -79,28 +79,6 @@ const ComptimeStringMap = @import("../../../comptime_string_map.zig").ComptimeSt
 
 const TCC = @import("../../../../tcc.zig");
 
-/// This is the entry point for generated FFI callback functions
-/// We want to avoid potentially causing LLVM to not inline our regular calls to JSC.C.JSObjectCallAsFunction
-/// to do that, we use a different pointer for the callback function
-/// which is this noinline wrapper
-noinline fn bun_call(
-    ctx: JSC.C.JSContextRef,
-    function: JSC.C.JSObjectRef,
-    count: usize,
-    argv: [*c]const JSC.C.JSValueRef,
-) callconv(.C) JSC.C.JSObjectRef {
-    var exception = [1]JSC.C.JSValueRef{null};
-    Output.debug("[bun_call] {d} args\n", .{count});
-    return JSC.C.JSObjectCallAsFunction(ctx, function, JSC.JSValue.jsUndefined().asObjectRef(), count, argv, &exception);
-}
-
-comptime {
-    if (!JSC.is_bindgen) {
-        _ = bun_call;
-        @export(bun_call, .{ .name = "bun_call" });
-    }
-}
-
 pub const FFI = struct {
     dylib: std.DynLib,
     functions: std.StringArrayHashMapUnmanaged(Function) = .{},
@@ -596,6 +574,7 @@ pub const FFI = struct {
             JSVALUE_TO_UINT64: fn (JSValue0: JSC.JSValue) callconv(.C) u64,
             INT64_TO_JSVALUE: fn (arg0: [*c]JSC.JSGlobalObject, arg1: i64) callconv(.C) JSC.JSValue,
             UINT64_TO_JSVALUE: fn (arg0: [*c]JSC.JSGlobalObject, arg1: u64) callconv(.C) JSC.JSValue,
+            bun_call: *const @TypeOf(JSC.C.JSObjectCallAsFunction),
         };
         const headers = @import("../bindings/headers.zig");
 
@@ -604,9 +583,10 @@ pub const FFI = struct {
             .JSVALUE_TO_UINT64 = headers.JSC__JSValue__toUInt64NoTruncate,
             .INT64_TO_JSVALUE = headers.JSC__JSValue__fromInt64NoTruncate,
             .UINT64_TO_JSVALUE = headers.JSC__JSValue__fromUInt64NoTruncate,
+            .bun_call = &JSC.C.JSObjectCallAsFunction,
         };
 
-        const tcc_options = "-std=c11 -nostdlib -Wl,--export-all-symbols";
+        const tcc_options = "-std=c11 -nostdlib -Wl,--export-all-symbols" ++ if (Environment.isDebug) " -g" else "";
 
         pub fn compile(
             this: *Function,
@@ -663,31 +643,7 @@ pub const FFI = struct {
             }
             CompilerRT.inject(state);
             _ = TCC.tcc_add_symbol(state, this.base_name, this.symbol_from_dynamic_library.?);
-            _ = TCC.tcc_add_symbol(
-                state,
-                "JSVALUE_TO_INT64",
-                workaround.JSVALUE_TO_INT64,
-            );
-            _ = TCC.tcc_add_symbol(
-                state,
-                "JSVALUE_TO_UINT64",
-                workaround.JSVALUE_TO_UINT64,
-            );
-            std.mem.doNotOptimizeAway(headers.JSC__JSValue__toUInt64NoTruncate);
-            std.mem.doNotOptimizeAway(headers.JSC__JSValue__toInt64);
-            std.mem.doNotOptimizeAway(headers.JSC__JSValue__fromInt64NoTruncate);
-            std.mem.doNotOptimizeAway(headers.JSC__JSValue__fromUInt64NoTruncate);
 
-            _ = TCC.tcc_add_symbol(
-                state,
-                "INT64_TO_JSVALUE",
-                workaround.INT64_TO_JSVALUE,
-            );
-            _ = TCC.tcc_add_symbol(
-                state,
-                "UINT64_TO_JSVALUE",
-                workaround.UINT64_TO_JSVALUE,
-            );
             if (this.step == .failed) {
                 return;
             }
@@ -731,7 +687,6 @@ pub const FFI = struct {
             };
             return;
         }
-
         const CompilerRT = struct {
             noinline fn memset(
                 dest: [*]u8,
@@ -752,6 +707,31 @@ pub const FFI = struct {
             pub fn inject(state: *TCC.TCCState) void {
                 _ = TCC.tcc_add_symbol(state, "memset", &memset);
                 _ = TCC.tcc_add_symbol(state, "memcpy", &memcpy);
+
+                _ = TCC.tcc_add_symbol(
+                    state,
+                    "JSVALUE_TO_INT64_SLOW",
+                    workaround.JSVALUE_TO_INT64,
+                );
+                _ = TCC.tcc_add_symbol(
+                    state,
+                    "JSVALUE_TO_UINT64_SLOW",
+                    workaround.JSVALUE_TO_UINT64,
+                );
+                std.mem.doNotOptimizeAway(headers.JSC__JSValue__toUInt64NoTruncate);
+                std.mem.doNotOptimizeAway(headers.JSC__JSValue__toInt64);
+                std.mem.doNotOptimizeAway(headers.JSC__JSValue__fromInt64NoTruncate);
+                std.mem.doNotOptimizeAway(headers.JSC__JSValue__fromUInt64NoTruncate);
+                _ = TCC.tcc_add_symbol(
+                    state,
+                    "INT64_TO_JSVALUE_SLOW",
+                    workaround.INT64_TO_JSVALUE,
+                );
+                _ = TCC.tcc_add_symbol(
+                    state,
+                    "UINT64_TO_JSVALUE_SLOW",
+                    workaround.UINT64_TO_JSVALUE,
+                );
             }
         };
 
@@ -797,12 +777,12 @@ pub const FFI = struct {
 
                 return;
             }
-            Output.debug("here", .{});
+            CompilerRT.inject(state);
 
-            _ = TCC.tcc_add_symbol(state, "bun_call", JSC.C.JSObjectCallAsFunction);
+            Output.debug("here", .{});
+            _ = TCC.tcc_add_symbol(state, "bun_call", workaround.bun_call.*);
             _ = TCC.tcc_add_symbol(state, "cachedJSContext", js_context);
             _ = TCC.tcc_add_symbol(state, "cachedCallbackFunction", js_function);
-            CompilerRT.inject(state);
 
             var relocation_size = TCC.tcc_relocate(state, null);
             if (relocation_size == 0) return;
@@ -826,14 +806,11 @@ pub const FFI = struct {
 
                 return;
             };
-            Output.debug("symbol: {*}", .{symbol});
-            Output.debug("bun_call: {*}", .{&bun_call});
-            Output.debug("js_function: {*}", .{js_function});
 
             this.step = .{
                 .compiled = .{
                     .ptr = symbol,
-                    .buf = &[_]u8{},
+                    .buf = bytes,
                     .js_function = js_function,
                     .js_context = js_context,
                 },
@@ -884,18 +861,69 @@ pub const FFI = struct {
                 try arg.typename(writer);
                 try writer.print(" arg{d}", .{i});
             }
-            try writer.writeAll(");\n\n");
-
-            // -- Generate JavaScriptCore's C wrapper function
             try writer.writeAll(
+                \\);
+                \\
+                \\
                 \\/* ---- Your Wrapper Function ---- */
                 \\void* JSFunctionCall(void* globalObject, void* callFrame) {
-                \\#ifdef HAS_ARGUMENTS
-                \\  LOAD_ARGUMENTS_FROM_CALL_FRAME;
-                \\#endif
                 \\
             );
 
+            if (this.arg_types.items.len > 0) {
+                try writer.writeAll(
+                    \\  LOAD_ARGUMENTS_FROM_CALL_FRAME;
+                    \\
+                );
+                for (this.arg_types.items) |arg, i| {
+                    if (arg.needsACastInC()) {
+                        if (i < this.arg_types.items.len - 1) {
+                            try writer.print(
+                                \\  EncodedJSValue arg{d};
+                                \\  arg{d}.asInt64 = *argsPtr++;
+                                \\
+                            ,
+                                .{
+                                    i,
+                                    i,
+                                },
+                            );
+                        } else {
+                            try writer.print(
+                                \\  EncodedJSValue arg{d};
+                                \\  arg{d}.asInt64 = *argsPtr;
+                                \\
+                            ,
+                                .{
+                                    i,
+                                    i,
+                                },
+                            );
+                        }
+                    } else {
+                        if (i < this.arg_types.items.len - 1) {
+                            try writer.print(
+                                \\  int64_t arg{d} = *argsPtr++;
+                                \\
+                            ,
+                                .{
+                                    i,
+                                },
+                            );
+                        } else {
+                            try writer.print(
+                                \\  int64_t arg{d} = *argsPtr;
+                                \\
+                            ,
+                                .{
+                                    i,
+                                },
+                            );
+                        }
+                    }
+                }
+            }
+
             // try writer.writeAll(
             //     "(JSContext ctx, void* function, void* thisObject, size_t argumentCount, const EncodedJSValue arguments[], void* exception);\n\n",
             // );
@@ -909,7 +937,7 @@ pub const FFI = struct {
             }
             try writer.print("{s}(", .{std.mem.span(this.base_name)});
             first = true;
-            arg_buf[0..4].* = "arg(".*;
+            arg_buf[0..3].* = "arg".*;
             for (this.arg_types.items) |arg, i| {
                 if (!first) {
                     try writer.writeAll(", ");
@@ -917,9 +945,13 @@ pub const FFI = struct {
                 first = false;
 
                 try writer.writeAll("    ");
-                _ = std.fmt.bufPrintIntToSlice(arg_buf["arg(".len..], i, 10, .lower, .{});
-                arg_buf["arg(N".len] = ')';
-                try writer.print("{}", .{arg.toC(arg_buf[0..6])});
+                const lengthBuf = std.fmt.bufPrintIntToSlice(arg_buf["arg".len..], i, 10, .lower, .{});
+                const argName = arg_buf[0 .. 3 + lengthBuf.len];
+                if (arg.needsACastInC()) {
+                    try writer.print("{}", .{arg.toC(argName)});
+                } else {
+                    try writer.writeAll(argName);
+                }
             }
             try writer.writeAll(");\n");
 
@@ -982,6 +1014,7 @@ pub const FFI = struct {
             }
             try writer.writeAll(");\n\n");
 
+            first = true;
             try this.return_type.typename(writer);
 
             try writer.writeAll(" my_callback_function");
@@ -1010,7 +1043,6 @@ pub const FFI = struct {
                 var arg_buf: [512]u8 = undefined;
                 arg_buf[0.."arg".len].* = "arg".*;
                 for (this.arg_types.items) |arg, i| {
-                    try arg.typename(writer);
                     const printed = std.fmt.bufPrintIntToSlice(arg_buf["arg".len..], i, 10, .lower, .{});
                     const arg_name = arg_buf[0 .. "arg".len + printed.len];
                     try writer.print("    {}", .{arg.toJS(arg_name)});
@@ -1025,7 +1057,7 @@ pub const FFI = struct {
 
             try writer.writeAll("  ");
             if (!(this.return_type == .void)) {
-                try writer.writeAll("  EncodedJSValue return_value = {");
+                try writer.writeAll("EncodedJSValue return_value = {");
             }
             // JSC.C.JSObjectCallAsFunction(
             //     ctx,
@@ -1037,9 +1069,9 @@ pub const FFI = struct {
             // );
             try writer.writeAll("bun_call(cachedJSContext, cachedCallbackFunction, (void*)0, ");
             if (this.arg_types.items.len > 0) {
-                try writer.print("{d}, arguments, 0)", .{this.arg_types.items.len});
+                try writer.print("{d}, &arguments[0], (void*)0)", .{this.arg_types.items.len});
             } else {
-                try writer.writeAll("0, arguments, (void*)0)");
+                try writer.writeAll("0, &arguments[0], (void*)0)");
             }
 
             if (this.return_type != .void) {
@@ -1076,6 +1108,14 @@ pub const FFI = struct {
 
         cstring = 14,
 
+        /// Types that we can directly pass through as an `int64_t`
+        pub fn needsACastInC(this: ABIType) bool {
+            return switch (this) {
+                .char, .int8_t, .uint8_t, .int16_t, .uint16_t, .int32_t, .uint32_t => false,
+                else => true,
+            };
+        }
+
         const map = .{
             .{ "bool", ABIType.bool },
             .{ "c_int", ABIType.int32_t },
diff --git a/src/javascript/jsc/ffi.exports.js b/src/javascript/jsc/ffi.exports.js
index 8f6c47e41..949226436 100644
--- a/src/javascript/jsc/ffi.exports.js
+++ b/src/javascript/jsc/ffi.exports.js
@@ -72,7 +72,7 @@ ffiWrappers[FFIType.uint32_t] = function uint32(val) {
 };
 ffiWrappers[FFIType.int64_t] = function int64(val) {
   if (typeof val === "bigint") {
-    if (val < Number.MAX_VALUE) {
+    if (val < BigInt(Number.MAX_VALUE)) {
       return Number(val).valueOf();
     }
   }
@@ -86,7 +86,7 @@ ffiWrappers[FFIType.int64_t] = function int64(val) {
 
 ffiWrappers[FFIType.uint64_t] = function int64(val) {
   if (typeof val === "bigint") {
-    if (val < Number.MAX_VALUE && val > 0) {
+    if (val < BigInt(Number.MAX_VALUE) && val > 0) {
       return Number(val).valueOf();
     }
   }
@@ -100,7 +100,7 @@ ffiWrappers[FFIType.uint64_t] = function int64(val) {
 
 ffiWrappers[FFIType.uint16_t] = function uint64(val) {
   if (typeof val === "bigint") {
-    if (val < Number.MAX_VALUE) {
+    if (val < BigInt(Number.MAX_VALUE)) {
       return Math.abs(Number(val).valueOf());
     }
   }
@@ -238,17 +238,6 @@ export function dlopen(path, options) {
   return result;
 }
 
-export function callback(options) {
-  const result = nativeCallback(options);
-
-  if (options.args || options.return_type) {
-    return FFIBuilder(
-      options.args ?? [],
-      options.return_type ?? FFIType.void,
-      result,
-      "callback"
-    );
-  }
-
-  return result;
+export function callback(options, cb) {
+  return nativeCallback(options, cb);
 }