diff options
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | src/analytics/analytics.zig | 643 | ||||
| -rw-r--r-- | src/analytics/analytics_thread.zig | 341 | ||||
| -rw-r--r-- | src/analytics/schema.peechy | 49 | ||||
| -rw-r--r-- | src/bundler.zig | 16 | ||||
| -rw-r--r-- | src/env.zig | 2 | ||||
| -rw-r--r-- | src/feature_flags.zig | 2 | ||||
| -rw-r--r-- | src/http.zig | 13 |
8 files changed, 1070 insertions, 1 deletions
@@ -68,7 +68,7 @@ endif bun: vendor build-obj bun-link-lld-release -vendor-without-check: api node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp +vendor-without-check: api analytics node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp vendor: require init-submodules vendor-without-check @@ -380,3 +380,6 @@ sizegen: picohttp: $(CC) -O3 -g -fPIE -c src/deps/picohttpparser.c -Isrc/deps -o src/deps/picohttpparser.o; cd ../../ + +analytics: + ./node_modules/.bin/peechy --schema src/analytics/schema.peechy --go src/analytics/analytics.go --zig src/analytics/analytics.zig diff --git a/src/analytics/analytics.zig b/src/analytics/analytics.zig new file mode 100644 index 000000000..042f9040d --- /dev/null +++ b/src/analytics/analytics.zig @@ -0,0 +1,643 @@ +const std = @import("std"); + +pub const Reader = struct { + const Self = @This(); + pub const ReadError = error{EOF}; + + buf: []u8, + remain: []u8, + allocator: *std.mem.Allocator, + + pub fn init(buf: []u8, allocator: *std.mem.Allocator) Reader { + return Reader{ + .buf = buf, + .remain = buf, + .allocator = allocator, + }; + } + + pub fn read(this: *Self, count: usize) ![]u8 { + const read_count = std.math.min(count, this.remain.len); + if (read_count < count) { + return error.EOF; + } + + var slice = this.remain[0..read_count]; + + this.remain = this.remain[read_count..]; + + return slice; + } + + pub fn readAs(this: *Self, comptime T: type) !T { + if (!std.meta.trait.hasUniqueRepresentation(T)) { + @compileError(@typeName(T) ++ " must have unique representation."); + } + + return std.mem.bytesAsValue(T, try this.read(@sizeOf(T))); + } + + pub fn readByte(this: *Self) !u8 { + return (try this.read(1))[0]; + } + + pub fn readEnum(this: *Self, comptime Enum: type) !Enum { + const E = error{ + /// An integer was read, but it did not match any of the tags in the supplied enum. + InvalidValue, + }; + const type_info = @typeInfo(Enum).Enum; + const tag = try this.readInt(type_info.tag_type); + + inline for (std.meta.fields(Enum)) |field| { + if (tag == field.value) { + return @field(Enum, field.name); + } + } + + return E.InvalidValue; + } + + pub fn readArray(this: *Self, comptime T: type) ![]const T { + const length = try this.readInt(u32); + if (length == 0) { + return &([_]T{}); + } + + switch (T) { + u8 => { + return try this.read(length); + }, + u16, u32, i8, i16, i32 => { + return std.mem.readIntSliceNative(T, this.read(length * @sizeOf(T))); + }, + []const u8 => { + var i: u32 = 0; + var array = try this.allocator.alloc([]const u8, length); + while (i < length) : (i += 1) { + array[i] = try this.readArray(u8); + } + return array; + }, + else => { + switch (@typeInfo(T)) { + .Struct => |Struct| { + switch (Struct.layout) { + .Packed => { + const sizeof = @sizeOf(T); + var slice = try this.read(sizeof * length); + return std.mem.bytesAsSlice(T, slice); + }, + else => {}, + } + }, + .Enum => |type_info| { + const enum_values = try this.read(length * @sizeOf(type_info.tag_type)); + return @ptrCast([*]T, enum_values.ptr)[0..length]; + }, + else => {}, + } + + var i: u32 = 0; + var array = try this.allocator.alloc(T, length); + while (i < length) : (i += 1) { + array[i] = try this.readValue(T); + } + + return array; + }, + } + } + + pub fn readByteArray(this: *Self) ![]u8 { + const length = try this.readInt(u32); + if (length == 0) { + return &([_]u8{}); + } + + return try this.read(@intCast(usize, length)); + } + + pub fn readInt(this: *Self, comptime T: type) !T { + var slice = try this.read(@sizeOf(T)); + + return std.mem.readIntSliceNative(T, slice); + } + + pub fn readBool(this: *Self) !bool { + return (try this.readByte()) > 0; + } + + pub fn readValue(this: *Self, comptime T: type) !T { + switch (T) { + bool => { + return try this.readBool(); + }, + u8 => { + return try this.readByte(); + }, + []const u8 => { + return try this.readArray(u8); + }, + + []const []const u8 => { + return try this.readArray([]const u8); + }, + []u8 => { + return try this.readArray([]u8); + }, + u16, u32, i8, i16, i32 => { + return std.mem.readIntSliceNative(T, try this.read(@sizeOf(T))); + }, + else => { + switch (@typeInfo(T)) { + .Struct => |Struct| { + switch (Struct.layout) { + .Packed => { + const sizeof = @sizeOf(T); + var slice = try this.read(sizeof); + return @ptrCast(*T, slice[0..sizeof]).*; + }, + else => {}, + } + }, + .Enum => |type_info| { + return try this.readEnum(T); + }, + else => {}, + } + + return try T.decode(this); + }, + } + + @compileError("Invalid type passed to readValue"); + } +}; + +pub fn Writer(comptime WritableStream: type) type { + return struct { + const Self = @This(); + writable: WritableStream, + + pub fn init(writable: WritableStream) Self { + return Self{ .writable = writable }; + } + + pub fn write(this: *Self, bytes: anytype) !void { + _ = try this.writable.write(bytes); + } + + pub fn writeByte(this: *Self, byte: u8) !void { + _ = try this.writable.write(&[1]u8{byte}); + } + + pub fn writeInt(this: *Self, int: anytype) !void { + try this.write(std.mem.asBytes(&int)); + } + + pub fn writeFieldID(this: *Self, comptime id: comptime_int) !void { + try this.writeByte(id); + } + + pub fn writeEnum(this: *Self, val: anytype) !void { + try this.writeInt(@enumToInt(val)); + } + + pub fn writeValue(this: *Self, slice: anytype) !void { + switch (@TypeOf(slice)) { + []u16, + []u32, + []i16, + []i32, + []i8, + []const u16, + []const u32, + []const i16, + []const i32, + []const i8, + => { + try this.writeArray(@TypeOf(slice), slice); + }, + + []u8, []const u8 => { + try this.writeArray(u8, slice); + }, + + u8 => { + try this.write(slice); + }, + u16, u32, i16, i32, i8 => { + try this.write(std.mem.asBytes(slice)); + }, + + else => { + try slice.encode(this); + }, + } + } + + pub fn writeArray(this: *Self, comptime T: type, slice: anytype) !void { + try this.writeInt(@truncate(u32, slice.len)); + + switch (T) { + u8 => { + try this.write(slice); + }, + u16, u32, i16, i32, i8 => { + try this.write(std.mem.asBytes(slice)); + }, + []u8, + []u16, + []u32, + []i16, + []i32, + []i8, + []const u8, + []const u16, + []const u32, + []const i16, + []const i32, + []const i8, + => { + for (slice) |num_slice| { + try this.writeArray(std.meta.Child(@TypeOf(num_slice)), num_slice); + } + }, + else => { + for (slice) |val| { + try val.encode(this); + } + }, + } + } + + pub fn endMessage(this: *Self) !void { + try this.writeByte(0); + } + }; +} + +pub const ByteWriter = Writer(*std.io.FixedBufferStream([]u8)); +pub const FileWriter = Writer(std.fs.File); + +pub const Analytics = struct { + pub const OperatingSystem = enum(u8) { + _none, + /// linux + linux, + + /// macos + macos, + + /// windows + windows, + + /// wsl + wsl, + + _, + + pub fn jsonStringify(self: *const @This(), opts: anytype, o: anytype) !void { + return try std.json.stringify(@tagName(self), opts, o); + } + }; + + pub const Architecture = enum(u8) { + _none, + /// x64 + x64, + + /// arm + arm, + + _, + + pub fn jsonStringify(self: *const @This(), opts: anytype, o: anytype) !void { + return try std.json.stringify(@tagName(self), opts, o); + } + }; + + pub const Platform = struct { + /// os + os: OperatingSystem, + + /// arch + arch: Architecture, + + /// version + version: []const u8, + + pub fn decode(reader: anytype) anyerror!Platform { + var this = std.mem.zeroes(Platform); + + this.os = try reader.readValue(OperatingSystem); + this.arch = try reader.readValue(Architecture); + this.version = try reader.readValue([]const u8); + return this; + } + + pub fn encode(this: *const @This(), writer: anytype) anyerror!void { + try writer.writeEnum(this.os); + try writer.writeEnum(this.arch); + try writer.writeValue(this.version); + } + }; + + pub const EventKind = enum(u32) { + _none, + /// bundle_success + bundle_success, + + /// bundle_fail + bundle_fail, + + /// http_start + http_start, + + /// http_build + http_build, + + /// bundle_start + bundle_start, + + _, + + pub fn jsonStringify(self: *const @This(), opts: anytype, o: anytype) !void { + return try std.json.stringify(@tagName(self), opts, o); + } + }; + + pub const Uint64 = packed struct { + /// first + first: u32 = 0, + + /// second + second: u32 = 0, + + pub fn decode(reader: anytype) anyerror!Uint64 { + var this = std.mem.zeroes(Uint64); + + this.first = try reader.readValue(u32); + this.second = try reader.readValue(u32); + return this; + } + + pub fn encode(this: *const @This(), writer: anytype) anyerror!void { + try writer.writeInt(this.first); + try writer.writeInt(this.second); + } + }; + + pub const EventListHeader = struct { + /// machine_id + machine_id: Uint64, + + /// platform + platform: Platform, + + /// build_id + build_id: u32 = 0, + + /// session_length + session_length: u32 = 0, + + pub fn decode(reader: anytype) anyerror!EventListHeader { + var this = std.mem.zeroes(EventListHeader); + + this.machine_id = try reader.readValue(Uint64); + this.platform = try reader.readValue(Platform); + this.build_id = try reader.readValue(u32); + this.session_length = try reader.readValue(u32); + return this; + } + + pub fn encode(this: *const @This(), writer: anytype) anyerror!void { + try writer.writeValue(this.machine_id); + try writer.writeValue(this.platform); + try writer.writeInt(this.build_id); + try writer.writeInt(this.session_length); + } + }; + + pub const EventHeader = struct { + /// timestamp + timestamp: Uint64, + + /// kind + kind: EventKind, + + pub fn decode(reader: anytype) anyerror!EventHeader { + var this = std.mem.zeroes(EventHeader); + + this.timestamp = try reader.readValue(Uint64); + this.kind = try reader.readValue(EventKind); + return this; + } + + pub fn encode(this: *const @This(), writer: anytype) anyerror!void { + try writer.writeValue(this.timestamp); + try writer.writeEnum(this.kind); + } + }; + + pub const EventList = struct { + /// header + header: EventListHeader, + + /// event_count + event_count: u32 = 0, + + pub fn decode(reader: anytype) anyerror!EventList { + var this = std.mem.zeroes(EventList); + + this.header = try reader.readValue(EventListHeader); + this.event_count = try reader.readValue(u32); + return this; + } + + pub fn encode(this: *const @This(), writer: anytype) anyerror!void { + try writer.writeValue(this.header); + try writer.writeInt(this.event_count); + } + }; +}; + +const ExamplePackedStruct = packed struct { + len: u32 = 0, + offset: u32 = 0, + + pub fn encode(this: *const ExamplePackedStruct, writer: anytype) !void { + try writer.write(std.mem.asBytes(this)); + } + + pub fn decode(reader: anytype) !ExamplePackedStruct { + return try reader.readAs(ExamplePackedStruct); + } +}; + +const ExampleStruct = struct { + name: []const u8 = "", + age: u32 = 0, + + pub fn encode(this: *const ExampleStruct, writer: anytype) !void { + try writer.writeArray(u8, this.name); + try writer.writeInt(this.age); + } + + pub fn decode(reader: anytype) !ExampleStruct { + var this = std.mem.zeroes(ExampleStruct); + this.name = try reader.readArray(u8); + this.age = try reader.readInt(u32); + + return this; + } +}; + +const EnumValue = enum(u8) { hey, hi, heyopoo }; + +const ExampleMessage = struct { + examples: ?[]ExampleStruct = &([_]ExampleStruct{}), + pack: ?[]ExamplePackedStruct = &([_]ExamplePackedStruct{}), + hey: ?u8 = 0, + hey16: ?u16 = 0, + hey32: ?u16 = 0, + heyi32: ?i32 = 0, + heyi16: ?i16 = 0, + heyi8: ?i8 = 0, + boolean: ?bool = null, + heyooo: ?EnumValue = null, + + pub fn encode(this: *const ExampleMessage, writer: anytype) !void { + if (this.examples) |examples| { + try writer.writeFieldID(1); + try writer.writeArray(ExampleStruct, examples); + } + + if (this.pack) |pack| { + try writer.writeFieldID(2); + try writer.writeArray(ExamplePackedStruct, pack); + } + + if (this.hey) |hey| { + try writer.writeFieldID(3); + try writer.writeInt(hey); + } + if (this.hey16) |hey16| { + try writer.writeFieldID(4); + try writer.writeInt(hey16); + } + if (this.hey32) |hey32| { + try writer.writeFieldID(5); + try writer.writeInt(hey32); + } + if (this.heyi32) |heyi32| { + try writer.writeFieldID(6); + try writer.writeInt(heyi32); + } + if (this.heyi16) |heyi16| { + try writer.writeFieldID(7); + try writer.writeInt(heyi16); + } + if (this.heyi8) |heyi8| { + try writer.writeFieldID(8); + try writer.writeInt(heyi8); + } + if (this.boolean) |boolean| { + try writer.writeFieldID(9); + try writer.writeInt(boolean); + } + + if (this.heyooo) |heyoo| { + try writer.writeFieldID(10); + try writer.writeEnum(heyoo); + } + + try writer.endMessage(); + } + + pub fn decode(reader: anytype) !ExampleMessage { + var this = std.mem.zeroes(ExampleMessage); + while (true) { + switch (try reader.readByte()) { + 0 => { + return this; + }, + + 1 => { + this.examples = try reader.readArray(std.meta.Child(@TypeOf(this.examples.?))); + }, + 2 => { + this.pack = try reader.readArray(std.meta.Child(@TypeOf(this.pack.?))); + }, + 3 => { + this.hey = try reader.readValue(@TypeOf(this.hey.?)); + }, + 4 => { + this.hey16 = try reader.readValue(@TypeOf(this.hey16.?)); + }, + 5 => { + this.hey32 = try reader.readValue(@TypeOf(this.hey32.?)); + }, + 6 => { + this.heyi32 = try reader.readValue(@TypeOf(this.heyi32.?)); + }, + 7 => { + this.heyi16 = try reader.readValue(@TypeOf(this.heyi16.?)); + }, + 8 => { + this.heyi8 = try reader.readValue(@TypeOf(this.heyi8.?)); + }, + 9 => { + this.boolean = try reader.readValue(@TypeOf(this.boolean.?)); + }, + 10 => { + this.heyooo = try reader.readValue(@TypeOf(this.heyooo.?)); + }, + else => { + return error.InvalidValue; + }, + } + } + + return this; + } +}; + +test "ExampleMessage" { + var base = std.mem.zeroes(ExampleMessage); + base.hey = 1; + var buf: [4096]u8 = undefined; + var writable = std.io.fixedBufferStream(&buf); + var writer = ByteWriter.init(writable); + var examples = [_]ExamplePackedStruct{ + .{ .len = 2, .offset = 5 }, + .{ .len = 0, .offset = 10 }, + }; + + var more_examples = [_]ExampleStruct{ + .{ .name = "bacon", .age = 10 }, + .{ .name = "slime", .age = 300 }, + }; + base.examples = &more_examples; + base.pack = &examples; + base.heyooo = EnumValue.hey; + try base.encode(&writer); + var reader = Reader.init(&buf, std.heap.c_allocator); + var compare = try ExampleMessage.decode(&reader); + try std.testing.expectEqual(base.hey orelse 255, 1); + + const cmp_pack = compare.pack.?; + for (cmp_pack) |item, id| { + try std.testing.expectEqual(item, examples[id]); + } + + const cmp_ex = compare.examples.?; + for (cmp_ex) |item, id| { + try std.testing.expectEqualStrings(item.name, more_examples[id].name); + try std.testing.expectEqual(item.age, more_examples[id].age); + } + + try std.testing.expectEqual(cmp_pack[0].len, examples[0].len); + try std.testing.expectEqual(base.heyooo, compare.heyooo); +} diff --git a/src/analytics/analytics_thread.zig b/src/analytics/analytics_thread.zig new file mode 100644 index 000000000..0b986af18 --- /dev/null +++ b/src/analytics/analytics_thread.zig @@ -0,0 +1,341 @@ +usingnamespace @import("../global.zig"); + +const sync = @import("../sync.zig"); +const std = @import("std"); +const HTTPClient = @import("../http_client.zig"); +const URL = @import("../query_string_map.zig").URL; +const Fs = @import("../fs.zig"); +const Analytics = @import("./analytics.zig").Analytics; +const Writer = @import("./analytics.zig").Writer; +const Headers = @import("../javascript/jsc/webcore/response.zig").Headers; + +pub const EventName = enum(u8) { + bundle_success, + bundle_fail, + bundle_start, + http_start, + http_build, +}; + +const platform_arch = if (Environment.isAarch64) Analytics.Architecture.arm else Analytics.Architecture.x64; + +pub const Event = struct { + timestamp: u64, + data: Data, + + pub fn init(comptime name: EventName) Event { + const millis = std.time.milliTimestamp(); + + const timestamp = if (millis < 0) 0 else @intCast(u64, millis); + + return Event{ .timestamp = timestamp, .data = @unionInit(Data, @tagName(name), void{}) }; + } +}; + +pub const Data = union(EventName) { + bundle_success: void, + bundle_fail: void, + bundle_start: void, + http_start: void, + http_build: void, + + pub fn toKind(this: Data) Analytics.EventKind { + return switch (this) { + .bundle_success => .bundle_success, + .bundle_fail => .bundle_fail, + .bundle_start => .bundle_start, + .http_start => .http_start, + .http_build => .http_build, + }; + } +}; + +const EventQueue = sync.Channel(Event, .Dynamic); +var event_queue: EventQueue = undefined; + +pub const GenerateHeader = struct { + pub fn generate() Analytics.EventListHeader { + if (Environment.isMac) { + return Analytics.EventListHeader{ + .machine_id = GenerateMachineID.forMac() catch Analytics.Uint64{}, + .platform = GeneratePlatform.forMac(), + .build_id = comptime @truncate(u32, Global.build_id), + }; + } + + if (Environment.isLinux) { + return Analytics.EventListHeader{ + .machine_id = GenerateMachineID.forLinux() catch Analytics.Uint64{}, + .platform = GeneratePlatform.forLinux(), + .build_id = comptime @truncate(u32, Global.build_id), + }; + } + + unreachable; + } + + pub const GeneratePlatform = struct { + var osversion_name: [32]u8 = undefined; + pub fn forMac() Analytics.Platform { + std.mem.set(u8, std.mem.span(&osversion_name), 0); + + var platform = Analytics.Platform{ .os = Analytics.OperatingSystem.macos, .version = "", .arch = platform_arch }; + var osversion_name_buf: [2]c_int = undefined; + var osversion_name_ptr = osversion_name.len - 1; + var len = osversion_name.len - 1; + if (std.c.sysctlbyname("kern.osrelease", &osversion_name, &len, null, 0) == -1) return platform; + + platform.version = std.mem.span(std.mem.sliceTo(std.mem.span(&osversion_name), @as(u8, 0))); + return platform; + } + + pub var linux_os_name: std.c.utsname = undefined; + + pub fn forLinux() Analytics.Platform { + linux_os_name = std.mem.zeroes(linux_os_name); + + std.c.uname(&linux_os_name); + + const release = std.mem.span(linux_os_name.release); + const version = std.mem.sliceTo(std.mem.span(linux_os_name.version).ptr, @as(u8, 0)); + // Linux DESKTOP-P4LCIEM 5.10.16.3-microsoft-standard-WSL2 #1 SMP Fri Apr 2 22:23:49 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux + if (std.mem.indexOf(u8, release, "microsoft") != null) { + return Analytics.Platform{ .os = Analytics.OperatingSystem.wsl, .version = version, .arch = platform_arch }; + } + + return Analytics.Platform{ .os = Analytics.OperatingSystem.linux, .version = version, .arch = platform_arch }; + } + }; + + // https://github.com/denisbrodbeck/machineid + pub const GenerateMachineID = struct { + pub fn forMac() !Analytics.Uint64 { + const cmds = [_]string{ + "/usr/sbin/ioreg", + "-rd1", + "-c", + "IOPlatformExpertDevice", + }; + + const result = try std.ChildProcess.exec(.{ + .allocator = default_allocator, + .cwd = Fs.FileSystem.instance.top_level_dir, + .argv = std.mem.span(&cmds), + }); + + var out: []const u8 = result.stdout; + var offset: usize = 0; + offset = std.mem.lastIndexOf(u8, result.stdout, "\"IOPlatformUUID\"") orelse return Analytics.Uint64{}; + out = std.mem.trimLeft(u8, out[offset + "\"IOPlatformUUID\"".len ..], " \n\t="); + if (out.len == 0 or out[0] != '"') return Analytics.Uint64{}; + out = out[1..]; + offset = std.mem.indexOfScalar(u8, out, '"') orelse return Analytics.Uint64{}; + out = out[0..offset]; + + const hash = std.hash.Wyhash.hash(0, std.mem.trim(u8, out, "\n\r ")); + var hash_bytes = std.mem.asBytes(&hash); + return Analytics.Uint64{ + .first = std.mem.readIntNative(u32, hash_bytes[0..4]), + .second = std.mem.readIntNative(u32, hash_bytes[4..8]), + }; + } + + pub var linux_machine_id: [256]u8 = undefined; + + pub fn forLinux() !Analytics.Uint64 { + var file = std.fs.openFileAbsoluteZ("/var/lib/dbus/machine-id", .{ .read = true }) catch |err| brk: { + break :brk try std.fs.openFileAbsoluteZ("/etc/machine-id", .{ .read = true }); + }; + defer file.close(); + var read_count = try file.read(&linux_machine_id); + + const hash = std.hash.Wyhash.hash(0, std.mem.trim(u8, linux_machine_id[0..read_count], "\n\r ")); + var hash_bytes = std.mem.asBytes(&hash); + return Analytics.Uint64{ + .first = std.mem.readIntNative(u32, hash_bytes[0..4]), + .second = std.mem.readIntNative(u32, hash_bytes[4..8]), + }; + } + }; +}; + +pub var has_loaded = false; +pub var disabled = false; +pub fn enqueue(comptime name: EventName) void { + if (disabled) return; + + if (!has_loaded) { + defer has_loaded = true; + event_queue = EventQueue.init(std.heap.c_allocator); + spawn() catch |err| { + if (comptime isDebug) { + Output.prettyErrorln("[Analytics] error spawning thread {s}", .{@errorName(err)}); + Output.flush(); + } + + disabled = true; + return; + }; + } + + _ = event_queue.tryWriteItem(Event.init(name)) catch false; +} + +pub var thread: std.Thread = undefined; + +pub fn spawn() !void { + @setCold(true); + has_loaded = true; + thread = try std.Thread.spawn(.{}, readloop, .{}); +} + +fn readloop() anyerror!void { + defer disabled = true; + Output.Source.configureThread(); + defer Output.flush(); + thread.setName("Analytics") catch {}; + + var event_list = EventList.init(); + // everybody's random should be random + while (true) { + while (event_queue.tryReadItem() catch null) |item| { + event_list.push(item); + } + + if (event_list.events.items.len > 0) { + event_list.flush(); + } + + event_queue.getters.wait(&event_queue.mutex); + } +} + +pub const EventList = struct { + header: Analytics.EventListHeader, + events: std.ArrayList(Event), + client: HTTPClient, + + out_buffer: MutableString, + in_buffer: std.ArrayList(u8), + + var random: std.rand.DefaultPrng = undefined; + + pub fn init() EventList { + random = std.rand.DefaultPrng.init(@intCast(u64, std.time.milliTimestamp())); + return EventList{ + .header = GenerateHeader.generate(), + .events = std.ArrayList(Event).init(default_allocator), + .in_buffer = std.ArrayList(u8).init(default_allocator), + .client = HTTPClient.init( + default_allocator, + .POST, + URL.parse(Environment.analytics_url), + Headers.Entries{}, + "", + ), + .out_buffer = MutableString.init(default_allocator, 0) catch unreachable, + }; + } + + pub fn push(this: *EventList, event: Event) void { + this.events.append(event) catch unreachable; + } + + pub fn flush(this: *EventList) void { + this._flush() catch |err| { + Output.prettyErrorln("[Analytics] Error: {s}", .{@errorName(err)}); + Output.flush(); + }; + } + + pub var is_stuck = false; + fn _flush(this: *EventList) !void { + this.in_buffer.clearRetainingCapacity(); + + const AnalyticsWriter = Writer(*std.ArrayList(u8).Writer); + + var in_buffer = &this.in_buffer; + var buffer_writer = in_buffer.writer(); + var analytics_writer = AnalyticsWriter.init(&buffer_writer); + + const start_time = @import("root").start_time; + const now = std.time.nanoTimestamp(); + + this.header.session_length = @truncate(u32, @intCast(u64, (now - start_time)) / std.time.ns_per_ms); + + var list = Analytics.EventList{ + .header = this.header, + .event_count = @intCast(u32, this.events.items.len), + }; + + try list.encode(&analytics_writer); + + for (this.events.items) |_event| { + const event: Event = _event; + + var time_bytes = std.mem.asBytes(&event.timestamp); + + const analytics_event = Analytics.EventHeader{ + .timestamp = Analytics.Uint64{ + .first = std.mem.readIntNative(u32, time_bytes[0..4]), + .second = std.mem.readIntNative(u32, time_bytes[4..8]), + }, + .kind = event.data.toKind(), + }; + + try analytics_event.encode(&analytics_writer); + } + + const count = this.events.items.len; + + if (comptime FeatureFlags.verbose_analytics) { + Output.prettyErrorln("[Analytics] Sending {d} events", .{count}); + Output.flush(); + } + + this.events.clearRetainingCapacity(); + + var retry_remaining: usize = 10; + retry: while (retry_remaining > 0) { + const response = this.client.send(this.in_buffer.items, &this.out_buffer) catch |err| { + if (FeatureFlags.verbose_analytics) { + Output.prettyErrorln("[Analytics] failed due to error {s} ({d} retries remain)", .{ @errorName(err), retry_remaining }); + } + + retry_remaining -= 1; + @atomicStore(bool, &is_stuck, true, .Release); + const min_delay = (11 - retry_remaining) * std.time.ns_per_s / 2; + Output.flush(); + std.time.sleep(random.random.intRangeAtMost(u64, min_delay, min_delay * 2)); + continue :retry; + }; + + if (response.status_code >= 500 and response.status_code <= 599) { + if (FeatureFlags.verbose_analytics) { + Output.prettyErrorln("[Analytics] failed due to status code {d} ({d} retries remain)", .{ response.status_code, retry_remaining }); + } + + retry_remaining -= 1; + @atomicStore(bool, &is_stuck, true, .Release); + const min_delay = (11 - retry_remaining) * std.time.ns_per_s / 2; + Output.flush(); + std.time.sleep(random.random.intRangeAtMost(u64, min_delay, min_delay * 2)); + continue :retry; + } + + break :retry; + } + + @atomicStore(bool, &is_stuck, retry_remaining == 0, .Release); + + this.in_buffer.clearRetainingCapacity(); + this.out_buffer.reset(); + + if (comptime FeatureFlags.verbose_analytics) { + Output.prettyErrorln("[Analytics] Sent {d} events", .{count}); + Output.flush(); + } + } +}; + +pub var is_ci = false; diff --git a/src/analytics/schema.peechy b/src/analytics/schema.peechy new file mode 100644 index 000000000..ec1233c40 --- /dev/null +++ b/src/analytics/schema.peechy @@ -0,0 +1,49 @@ +package Analytics; + +smol OperatingSystem { + linux = 1; + macos = 2; + windows = 3; + wsl = 4; +} + +smol Architecture { + x64 = 1; + arm = 2; +} + +struct Platform { + OperatingSystem os; + Architecture arch; + string version; +} + +enum EventKind { + bundle_success = 1; + bundle_fail = 2; + http_start = 3; + http_build = 4; + bundle_start = 5; +} + +struct Uint64 { + uint32 first; + uint32 second; +} + +struct EventListHeader { + Uint64 machine_id; + Platform platform; + uint32 build_id; + uint32 session_length; +} + +struct EventHeader { + Uint64 timestamp; + EventKind kind; +} + +struct EventList { + EventListHeader header; + uint32 event_count; +}
\ No newline at end of file diff --git a/src/bundler.zig b/src/bundler.zig index 2ca075760..8409f30e9 100644 --- a/src/bundler.zig +++ b/src/bundler.zig @@ -38,6 +38,7 @@ const Lock = @import("./lock.zig").Lock; const NewBunQueue = @import("./bun_queue.zig").NewBunQueue; const NodeFallbackModules = @import("./node_fallbacks.zig"); const CacheEntry = @import("./cache.zig").FsCacheEntry; +const Analytics = @import("./analytics/analytics_thread.zig"); const Linker = linker.Linker; const Resolver = _resolver.Resolver; @@ -231,8 +232,23 @@ pub const Bundler = struct { try this.env.load(&this.fs.fs, dir, true); } }, + .disable => { + this.env.loadProcess(); + }, else => {}, } + + if (this.env.map.get("DISABLE_BUN_ANALYTICS")) |should_disable| { + if (strings.eqlComptime(should_disable, "1")) { + Analytics.disabled = true; + } + } + + if (this.env.map.get("CI")) |IS_CI| { + if (strings.eqlComptime(IS_CI, "true")) { + Analytics.is_ci = true; + } + } } // This must be run after a framework is configured, if a framework is enabled diff --git a/src/env.zig b/src/env.zig index 9b25cf523..8df580cdc 100644 --- a/src/env.zig +++ b/src/env.zig @@ -22,3 +22,5 @@ pub const isRelease = std.builtin.Mode.Debug != std.builtin.mode and !isTest; pub const isTest = std.builtin.is_test; pub const isLinux = std.Target.current.os.tag == .linux; pub const isAarch64 = std.Target.current.cpu.arch == .aarch64; + +pub const analytics_url = "http://localhost:3008/events"; diff --git a/src/feature_flags.zig b/src/feature_flags.zig index 8087cd6ba..a132d0642 100644 --- a/src/feature_flags.zig +++ b/src/feature_flags.zig @@ -72,3 +72,5 @@ pub const is_macro_enabled = true; pub const force_macro = false; pub const include_filename_in_jsx = false; + +pub const verbose_analytics = true; diff --git a/src/http.zig b/src/http.zig index 7d47b82cd..a9b0c14a3 100644 --- a/src/http.zig +++ b/src/http.zig @@ -21,6 +21,7 @@ const OutputFile = Options.OutputFile; const DotEnv = @import("./env_loader.zig"); const mimalloc = @import("./allocators/mimalloc.zig"); const MacroMap = @import("./resolver/package_json.zig").MacroMap; +const Analytics = @import("./analytics/analytics_thread.zig"); pub fn constStrToU8(s: string) []u8 { return @intToPtr([*]u8, @ptrToInt(s.ptr))[0..s.len]; } @@ -544,6 +545,14 @@ pub const RequestContext = struct { return ctx; } + pub inline fn isBrowserNavigation(req: *RequestContext) bool { + if (req.header("Sec-Fetch-Mode")) |mode| { + return strings.eqlComptime(mode.value, "navigate"); + } + + return false; + } + pub fn sendNotFound(req: *RequestContext) !void { std.debug.assert(!req.has_called_done); @@ -2686,6 +2695,7 @@ pub const Server = struct { server.detectTSConfig(); try server.initWatcher(); did_init = true; + Analytics.enqueue(Analytics.EventName.http_start); server.handleConnection(&conn, comptime features); } @@ -2751,6 +2761,9 @@ pub const Server = struct { var req_ctx = &req_ctx_; req_ctx.timer.reset(); + const is_navigation_request = req_ctx_.isBrowserNavigation(); + defer if (is_navigation_request) Analytics.enqueue(Analytics.EventName.http_build); + if (req_ctx.url.needs_redirect) { req_ctx.handleRedirect(req_ctx.url.path) catch |err| { Output.prettyErrorln("<r>[<red>{s}<r>] - <b>{s}<r>: {s}", .{ @errorName(err), req.method, req.path }); |