Add support for INotify-based filesystem watcher

6 files changed, 287 insertions, 93 deletions

diff --git a/Makefile b/Makefile
index 6a7497dc9..5a229cc52 100644
--- a/Makefile
+++ b/Makefile
@@ -121,6 +121,9 @@ release-mac-push: write-package-json-version
 dev-obj:
 	zig build obj
 
+dev-obj-linux:
+	zig build obj -Dtarget=x86_64-linux-gnu
+
 dev: mkdir-dev dev-obj bun-link-lld-debug
 
 mkdir-dev:
@@ -222,6 +225,7 @@ ICU_FLAGS :=
 
 ifeq ($(OS_NAME),linux)
 	JSC_BUILD_STEPS += jsc-build-linux jsc-copy-headers
+	ICU_FLAGS += -licuuc -licudata -licui18n
 endif
 ifeq ($(OS_NAME),darwin)
 ICU_FLAGS += -l icucore \
diff --git a/README.md b/README.md
index 549ebe609..6fa37cc67 100644
--- a/README.md
+++ b/README.md
@@ -574,6 +574,8 @@ For compatibiltiy reasons, these NPM packages are embedded into Bun's binary and
 
 Estimated: 30-90 minutes :(
 
+## macOS
+
 Compile Zig:
 
 ```bash
@@ -590,9 +592,6 @@ In `bun`:
 ```bash
 git submodule update --init --recursive --progress --depth=1
 make vendor
-zig build headers
-make jsc-bindings-mac
-zig build -Drelease-fast
 ```
 
 Note that `brew install zig` won't work. Bun uses a build of Zig with a couple patches.
@@ -601,27 +600,7 @@ Additionally, you'll need `cmake`, `npm` and `esbuild` installed globally.
 
 ## Linux
 
-You will need:
-
-- `clang-12` (clang-13 after next Zig upgrade)
-- ruby (JavaScriptCore dependency)
-
-On Ubuntu:
-
-```
-sudo apt install libc++-12-dev libc++1-12 libc++abi1-12 build-essential
-```
-
-If you **do not** have this setup correctly, you will see an error that looks something like this after compiling JavaScriptCore:
-
-```c
-src/javascript/jsc/WebKit/WebKitBuild/Release/WTF/Headers/wtf/FastMalloc.h:23:10: fatal error: 'new' file not found
-#include <new>
-         ^~~~~
-1 error generated.
+```bash
+git submodule update --init --recursive --progress --depth=1
+make vendor
 ```
-
-
-When trying to run `make jsc` (run during `make vendor`), if you get an error complaining about Ruby missing but you have Ruby installed, it might mean your Ruby installation is corrupt.
-
-Verify that `ruby -v` runs without error.
\ No newline at end of file
diff --git a/build.zig b/build.zig
index 3c3dfabaa..4668aef04 100644
--- a/build.zig
+++ b/build.zig
@@ -261,6 +261,10 @@ pub fn build(b: *std.build.Builder) !void {
                 step.linkSystemLibrary("icucore");
                 step.addLibPath(homebrew_prefix ++ "opt/icu4c/lib");
                 step.addIncludeDir(homebrew_prefix ++ "opt/icu4c/include");
+            } else {
+                step.linkSystemLibrary("icuuc");
+                step.linkSystemLibrary("icudata");
+                step.linkSystemLibrary("icui18n");
             }
 
             for (bindings_files.items) |binding| {
@@ -283,6 +287,8 @@ pub fn build(b: *std.build.Builder) !void {
         obj.setOutputDir(output_dir);
         obj.setBuildMode(mode);
         obj.linkLibC();
+        obj.linkLibCpp();
+
         obj.setTarget(target);
     } else {
         b.default_step.dependOn(&exe.step);
diff --git a/src/env.zig b/src/env.zig
index 8bb47b185..9b25cf523 100644
--- a/src/env.zig
+++ b/src/env.zig
@@ -20,5 +20,5 @@ pub const isWindows = std.Target.current.os.tag == .windows;
 pub const isDebug = std.builtin.Mode.Debug == std.builtin.mode;
 pub const isRelease = std.builtin.Mode.Debug != std.builtin.mode and !isTest;
 pub const isTest = std.builtin.is_test;
-
+pub const isLinux = std.Target.current.os.tag == .linux;
 pub const isAarch64 = std.Target.current.cpu.arch == .aarch64;
diff --git a/src/runtime.version b/src/runtime.version
index 2dcc8c529..57659652b 100644
--- a/src/runtime.version
+++ b/src/runtime.version
@@ -1 +1 @@
-4d09f9efba49d5ac
\ No newline at end of file
+35057197d4ad54bc
\ No newline at end of file
diff --git a/src/watcher.zig b/src/watcher.zig
index 7805222f2..63fdb9007 100644
--- a/src/watcher.zig
+++ b/src/watcher.zig
@@ -6,8 +6,6 @@ const IndexType = @import("./allocators.zig").IndexType;
 
 const os = std.os;
 
-const KEvent = std.os.Kevent;
-
 const Mutex = @import("./lock.zig").Lock;
 const WatchItemIndex = u16;
 const NoWatchItem: WatchItemIndex = std.math.maxInt(WatchItemIndex);
@@ -15,11 +13,161 @@ const PackageJSON = @import("./resolver/package_json.zig").PackageJSON;
 
 const WATCHER_MAX_LIST = 8096;
 
+pub const INotify = struct {
+    pub const IN_CLOEXEC = std.os.O_CLOEXEC;
+    pub const IN_NONBLOCK = std.os.O_NONBLOCK;
+
+    pub const IN_ACCESS = 0x00000001;
+    pub const IN_MODIFY = 0x00000002;
+    pub const IN_ATTRIB = 0x00000004;
+    pub const IN_CLOSE_WRITE = 0x00000008;
+    pub const IN_CLOSE_NOWRITE = 0x00000010;
+    pub const IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE;
+    pub const IN_OPEN = 0x00000020;
+    pub const IN_MOVED_FROM = 0x00000040;
+    pub const IN_MOVED_TO = 0x00000080;
+    pub const IN_MOVE = IN_MOVED_FROM | IN_MOVED_TO;
+    pub const IN_CREATE = 0x00000100;
+    pub const IN_DELETE = 0x00000200;
+    pub const IN_DELETE_SELF = 0x00000400;
+    pub const IN_MOVE_SELF = 0x00000800;
+    pub const IN_ALL_EVENTS = 0x00000fff;
+
+    pub const IN_UNMOUNT = 0x00002000;
+    pub const IN_Q_OVERFLOW = 0x00004000;
+    pub const IN_IGNORED = 0x00008000;
+
+    pub const IN_ONLYDIR = 0x01000000;
+    pub const IN_DONT_FOLLOW = 0x02000000;
+    pub const IN_EXCL_UNLINK = 0x04000000;
+    pub const IN_MASK_ADD = 0x20000000;
+
+    pub const IN_ISDIR = 0x40000000;
+    pub const IN_ONESHOT = 0x80000000;
+
+    pub const EventListIndex = c_int;
+
+    pub const INotifyEvent = extern struct {
+        watch_descriptor: c_int,
+        mask: u32,
+        cookie: u32,
+        name_len: u32,
+    };
+    pub var inotify_fd: EventListIndex = 0;
+    pub var loaded_inotify = false;
+
+    const EventListBuffer = [@sizeOf([128]INotifyEvent) + (128 * std.fs.MAX_PATH_BYTES)]u8;
+    var eventlist: EventListBuffer = undefined;
+    var eventlist_ptrs: [128]*const INotifyEvent = undefined;
+
+    const add_mask = IN_EXCL_UNLINK | IN_MOVE_SELF | IN_CREATE | IN_DELETE | IN_DELETE_SELF;
+
+    pub fn watchPath(pathname: [*:0]const u8) !EventListIndex {
+        std.debug.assert(loaded_inotify);
+
+        return std.os.inotify_add_watchZ(inotify_fd, pathname, add_mask);
+    }
+
+    pub fn unwatch(wd: EventListIndex) void {
+        std.debug.assert(loaded_inotify);
+
+        std.os.inotify_rm_watch(inotify_fd, wd);
+    }
+
+    pub fn init() !void {
+        std.debug.assert(!loaded_inotify);
+        loaded_inotify = true;
+
+        inotify_fd = try std.os.inotify_init1(IN_CLOEXEC);
+    }
+
+    pub fn read() ![]*const INotifyEvent {
+        std.debug.assert(loaded_inotify);
+
+        const rc = std.os.system.read(
+            inotify_fd,
+            @ptrCast([*]u8, @alignCast(@alignOf([*]u8), &eventlist)),
+            @sizeOf(EventListBuffer),
+        );
+
+        switch (std.os.errno(rc)) {
+            .SUCCESS => {
+                const len = @intCast(usize, rc);
+
+                if (len == 0) return &[_]*INotifyEvent{};
+
+                var count: u32 = 0;
+                var i: u32 = 0;
+                while (i < len) : (i += @sizeOf(INotifyEvent)) {
+                    const event = @ptrCast(*const INotifyEvent, @alignCast(@alignOf(*const INotifyEvent), eventlist[i..][0..@sizeOf(INotifyEvent)]));
+                    if (event.name_len > 0) {
+                        i += event.name_len;
+                    }
+
+                    eventlist_ptrs[count] = event;
+                    count += 1;
+                }
+
+                return eventlist_ptrs[0..count];
+            },
+            .INVAL => return error.ShortRead,
+            .BADF => return error.INotifyFailedToStart,
+
+            else => unreachable,
+        }
+
+        unreachable;
+    }
+
+    pub fn stop() void {
+        if (inotify_fd != 0) {
+            std.os.close(inotify_fd);
+            inotify_fd = 0;
+        }
+    }
+};
+
+const DarwinWatcher = struct {
+    pub const EventListIndex = u32;
+
+    const KEvent = std.os.Kevent;
+    // Internal
+    pub var changelist: [128]KEvent = undefined;
+
+    // Everything being watched
+    pub var eventlist: [WATCHER_MAX_LIST]KEvent = undefined;
+    pub var eventlist_index: EventListIndex = 0;
+
+    var fd: EventListIndex = 0;
+
+    pub fn init() !void {
+        std.debug.assert(fd == 0);
+
+        fd = try std.os.kqueue();
+        if (fd == 0) return error.KQueueError;
+    }
+
+    pub fn stop() void {
+        if (fd != 0) {
+            std.os.close(fd);
+        }
+
+        fd = 0;
+    }
+};
+
+const PlatformWatcher = if (Environment.isMac)
+    DarwinWatcher
+else if (Environment.isLinux)
+    INotify
+else
+    void;
+
 pub const WatchItem = struct {
     file_path: string,
     // filepath hash for quick comparison
     hash: u32,
-    eventlist_index: u32,
+    eventlist_index: PlatformWatcher.EventListIndex,
     loader: options.Loader,
     fd: StoredFileDescriptorType,
     count: u32,
@@ -34,12 +182,34 @@ pub const WatchEvent = struct {
     index: WatchItemIndex,
     op: Op,
 
-    pub fn fromKEvent(this: *WatchEvent, kevent: *const KEvent) void {
-        this.op.delete = (kevent.fflags & std.os.NOTE_DELETE) > 0;
-        this.op.metadata = (kevent.fflags & std.os.NOTE_ATTRIB) > 0;
-        this.op.rename = (kevent.fflags & std.os.NOTE_RENAME) > 0;
-        this.op.write = (kevent.fflags & std.os.NOTE_WRITE) > 0;
-        this.index = @truncate(WatchItemIndex, kevent.udata);
+    const KEvent = std.os.Kevent;
+
+    pub fn fromKEvent(this: *WatchEvent, kevent: KEvent) void {
+        this.* =
+            WatchEvent{
+            .op = Op{
+                .delete = (kevent.fflags & std.os.NOTE_DELETE) > 0,
+                .metadata = (kevent.fflags & std.os.NOTE_ATTRIB) > 0,
+                .rename = (kevent.fflags & std.os.NOTE_RENAME) > 0,
+                .write = (kevent.fflags & std.os.NOTE_WRITE) > 0,
+            },
+            .index = @truncate(WatchItemIndex, kevent.udata),
+        };
+    }
+
+    pub fn fromINotify(this: *WatchEvent, event: INotify.INotifyEvent, index: WatchItemIndex) void {
+        this.* = WatchEvent{
+            .op = Op{
+                .delete = (event.mask & INotify.IN_DELETE_SELF) > 0,
+                // only applies to directories
+                .metadata = (event.mask & INotify.IN_CREATE) > 0 or
+                    (event.mask & INotify.IN_DELETE) > 0 or
+                    (event.mask & INotify.IN_MOVE) > 0,
+                .rename = (event.mask & INotify.IN_MOVE_SELF) > 0,
+                .write = (event.mask & INotify.IN_MODIFY) > 0,
+            },
+            .index = index,
+        };
     }
 
     pub const Op = packed struct {
@@ -52,20 +222,6 @@ pub const WatchEvent = struct {
 
 pub const Watchlist = std.MultiArrayList(WatchItem);
 
-const DarwinWatcher = struct {
-
-    // Internal
-    changelist: [128]KEvent = undefined,
-
-    // Everything being watched
-    eventlist: [WATCHER_MAX_LIST]KEvent = undefined,
-};
-
-const PlatformWatcher = if (Environment.isMac)
-    DarwinWatcher
-else
-    void;
-
 // This implementation only works on macOS, for now.
 // The Internet seems to suggest basically always using FSEvents instead of kqueue
 // It seems like the main concern is max open file descriptors
@@ -78,8 +234,6 @@ pub fn NewWatcher(comptime ContextType: type) type {
         watched_count: usize = 0,
         mutex: Mutex,
 
-        eventlist_used: usize = 0,
-
         platform: PlatformWatcher = PlatformWatcher{},
 
         // User-facing
@@ -117,21 +271,8 @@ pub fn NewWatcher(comptime ContextType: type) type {
             return watcher;
         }
 
-        pub fn getQueue(this: *Watcher) !StoredFileDescriptorType {
-            if (this.fd == 0) {
-                if (Environment.isMac) {
-                    this.fd = try os.kqueue();
-                    if (this.fd == 0) {
-                        return error.WatcherFailed;
-                    }
-                }
-            }
-
-            return this.fd;
-        }
-
         pub fn start(this: *Watcher) !void {
-            _ = try this.getQueue();
+            try PlatformWatcher.init();
             std.debug.assert(this.watchloop_handle == null);
             var thread = try std.Thread.spawn(.{}, Watcher.watchLoop, .{this});
             thread.setName("File Watcher") catch {};
@@ -152,8 +293,7 @@ pub fn NewWatcher(comptime ContextType: type) type {
                 Output.prettyErrorln("<r>Watcher crashed: <red><b>{s}<r>", .{@errorName(err)});
 
                 this.watchloop_handle = null;
-                std.os.close(this.fd);
-                this.fd = 0;
+                PlatformWatcher.stop();
                 return;
             };
         }
@@ -193,13 +333,20 @@ pub fn NewWatcher(comptime ContextType: type) type {
 
             var slice = this.watchlist.slice();
             var fds = slice.items(.fd);
+            var event_list_ids = slice.items(.eventlist_index);
             var last_item = NoWatchItem;
 
             for (evict_list[0..evict_list_i]) |item, i| {
                 // catch duplicates, since the list is sorted, duplicates will appear right after each other
                 if (item == last_item) continue;
+
                 // close the file descriptors here. this should automatically remove it from being watched too.
                 std.os.close(fds[item]);
+
+                if (Environment.isLinux) {
+                    INotify.unwatch(event_list_ids[item]);
+                }
+
                 last_item = item;
             }
 
@@ -216,7 +363,8 @@ pub fn NewWatcher(comptime ContextType: type) type {
             const time = std.time;
 
             if (Environment.isMac) {
-                std.debug.assert(this.fd > 0);
+                std.debug.assert(DarwinWatcher.fd > 0);
+                const KEvent = std.os.KEvent;
 
                 var changelist_array: [1]KEvent = std.mem.zeroes([1]KEvent);
                 var changelist = &changelist_array;
@@ -224,7 +372,7 @@ pub fn NewWatcher(comptime ContextType: type) type {
                     defer Output.flush();
 
                     _ = std.os.system.kevent(
-                        try this.getQueue(),
+                        DarwinWatcher.fd,
                         @as([*]KEvent, changelist),
                         0,
                         @as([*]KEvent, changelist),
@@ -235,11 +383,44 @@ pub fn NewWatcher(comptime ContextType: type) type {
 
                     var watchevents = this.watch_events[0..1];
                     for (changelist) |event, i| {
-                        watchevents[i].fromKEvent(&event);
+                        watchevents[i].fromKEvent(event);
                     }
 
                     this.ctx.onFileUpdate(watchevents, this.watchlist);
                 }
+            } else if (Environment.isLinux) {
+                while (true) {
+                    defer Output.flush();
+
+                    var events = try INotify.read();
+                    // TODO: is this thread safe?
+                    const eventlist_index = this.watchlist.items(.eventlist_index);
+                    var remaining_events = events.len;
+
+                    while (remaining_events > 0) {
+                        const slice = events[0..std.math.min(remaining_events, this.watch_events.len)];
+                        var watchevents = this.watch_events[0..slice.len];
+                        var watch_event_id: u32 = 0;
+                        for (slice) |event| {
+                            watchevents[watch_event_id].fromINotify(
+                                event.*,
+                                @intCast(
+                                    WatchItemIndex,
+                                    std.mem.indexOfScalar(
+                                        INotify.EventListIndex,
+                                        eventlist_index,
+                                        event.watch_descriptor,
+                                    ) orelse continue,
+                                ),
+                            );
+
+                            watch_event_id += 1;
+                        }
+
+                        this.ctx.onFileUpdate(watchevents[0..watch_event_id], this.watchlist);
+                        remaining_events -= slice.len;
+                    }
+                }
             }
         }
 
@@ -279,10 +460,17 @@ pub fn NewWatcher(comptime ContextType: type) type {
             package_json: ?*PackageJSON,
             comptime copy_file_path: bool,
         ) !void {
-            const index = this.eventlist_used;
+            var index: PlatformWatcher.EventListIndex = undefined;
             const watchlist_id = this.watchlist.len;
 
+            const file_path_: string = if (comptime copy_file_path)
+                std.mem.span(try this.allocator.dupeZ(u8, file_path))
+            else
+                file_path;
+
             if (Environment.isMac) {
+                const KEvent = std.os.Kevent;
+                index = DarwinWatcher.eventlist_index;
 
                 // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/kqueue.2.html
                 var event = std.mem.zeroes(KEvent);
@@ -291,43 +479,43 @@ pub fn NewWatcher(comptime ContextType: type) type {
                 // we want to know about the vnode
                 event.filter = std.os.EVFILT_VNODE;
 
-                // monitor:
-                // - Write
-                // - Rename
-
-                // we should monitor:
-                // - Delete
                 event.fflags = std.os.NOTE_WRITE | std.os.NOTE_RENAME | std.os.NOTE_DELETE;
 
                 // id
                 event.ident = @intCast(usize, fd);
 
-                this.eventlist_used += 1;
+                DarwinWatcher.eventlist_index += 1;
 
                 // Store the hash for fast filtering later
                 event.udata = @intCast(usize, watchlist_id);
-                this.platform.eventlist[index] = event;
+                DarwinWatcher.eventlist[index] = event;
 
                 // This took a lot of work to figure out the right permutation
                 // Basically:
                 // - We register the event here.
                 // our while(true) loop above receives notification of changes to any of the events created here.
                 _ = std.os.system.kevent(
-                    try this.getQueue(),
-                    this.platform.eventlist[index .. index + 1].ptr,
+                    DarwinWatcher.fd,
+                    DarwinWatcher.eventlist[index .. index + 1].ptr,
                     1,
-                    this.platform.eventlist[index .. index + 1].ptr,
+                    DarwinWatcher.eventlist[index .. index + 1].ptr,
                     0,
                     null,
                 );
+            } else if (Environment.isLinux) {
+                var sentineled = file_path_;
+                var file_path_to_use_ptr: [*c]u8 = @intToPtr([*c]u8, @ptrToInt(file_path_.ptr));
+                var file_path_to_use: [:0]u8 = file_path_to_use_ptr[0..sentineled.len :0];
+
+                index = try INotify.watchPath(file_path_to_use);
             }
 
             this.watchlist.appendAssumeCapacity(.{
-                .file_path = if (copy_file_path) try this.allocator.dupe(u8, file_path) else file_path,
+                .file_path = std.mem.span(file_path_),
                 .fd = fd,
                 .hash = hash,
                 .count = 0,
-                .eventlist_index = @truncate(u32, index),
+                .eventlist_index = index,
                 .loader = loader,
                 .parent_hash = parent_hash,
                 .package_json = package_json,
@@ -350,10 +538,20 @@ pub fn NewWatcher(comptime ContextType: type) type {
             };
 
             const parent_hash = Watcher.getHash(Fs.PathName.init(file_path).dirWithTrailingSlash());
-            const index = this.eventlist_used;
+            var index: PlatformWatcher.EventListIndex = undefined;
+            const file_path_ptr = @intToPtr([*]const u8, @ptrToInt(file_path.ptr));
+            const file_path_len = file_path.len;
+
+            const file_path_: string = if (comptime copy_file_path)
+                std.mem.span(try this.allocator.dupeZ(u8, file_path))
+            else
+                file_path;
+
             const watchlist_id = this.watchlist.len;
 
             if (Environment.isMac) {
+                index = DarwinWatcher.eventlist_index;
+                const KEvent = std.os.Kevent;
 
                 // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/kqueue.2.html
                 var event = std.mem.zeroes(KEvent);
@@ -374,28 +572,35 @@ pub fn NewWatcher(comptime ContextType: type) type {
                 this.eventlist_used += 1;
                 // Store the hash for fast filtering later
                 event.udata = @intCast(usize, watchlist_id);
-                this.platform.eventlist[index] = event;
+                DarwinWatcher.eventlist[index] = event;
 
                 // This took a lot of work to figure out the right permutation
                 // Basically:
                 // - We register the event here.
                 // our while(true) loop above receives notification of changes to any of the events created here.
                 _ = std.os.system.kevent(
-                    try this.getQueue(),
-                    this.platform.eventlist[index .. index + 1].ptr,
+                    DarwinWatcher.fd,
+                    DarwinWatcher.eventlist[index .. index + 1].ptr,
                     1,
-                    this.platform.eventlist[index .. index + 1].ptr,
+                    DarwinWatcher.eventlist[index .. index + 1].ptr,
                     0,
                     null,
                 );
+            } else if (Environment.isLinux) {
+                // This works around a Zig compiler bug when casting a slice from a string to a sentineled string.
+                var sentineled = file_path_;
+                var file_path_to_use_ptr: [*c]u8 = @intToPtr([*c]u8, @ptrToInt(file_path_.ptr));
+                var file_path_to_use: [:0]u8 = file_path_to_use_ptr[0..sentineled.len :0];
+
+                index = try INotify.watchPath(file_path_to_use);
             }
 
             this.watchlist.appendAssumeCapacity(.{
-                .file_path = if (copy_file_path) try this.allocator.dupe(u8, file_path) else file_path,
+                .file_path = file_path_,
                 .fd = fd,
                 .hash = hash,
                 .count = 0,
-                .eventlist_index = @truncate(u32, index),
+                .eventlist_index = index,
                 .loader = options.Loader.file,
                 .parent_hash = parent_hash,
                 .kind = .directory,