diff options
author | 2021-10-14 18:55:41 -0700 | |
---|---|---|
committer | 2021-10-14 18:55:41 -0700 | |
commit | bbc1bcbed125e4aeacac0c374f717f65adb838ea (patch) | |
tree | a3ae72a500afc507231d3f97c7d0762c76614a51 | |
parent | 3ed824fe0fc14d21a5c035d84891b8ecf28e3c44 (diff) | |
download | bun-bbc1bcbed125e4aeacac0c374f717f65adb838ea.tar.gz bun-bbc1bcbed125e4aeacac0c374f717f65adb838ea.tar.zst bun-bbc1bcbed125e4aeacac0c374f717f65adb838ea.zip |
Support local templates
-rw-r--r-- | README.md | 161 | ||||
-rw-r--r-- | examples/README.md | 109 | ||||
-rw-r--r-- | examples/react/package.json | 8 | ||||
-rw-r--r-- | misctools/publish-examples.js | 27 | ||||
-rw-r--r-- | src/cli/create_command.zig | 193 | ||||
-rw-r--r-- | src/copy_file.zig | 50 | ||||
-rw-r--r-- | src/string_immutable.zig | 8 | ||||
-rw-r--r-- | src/walker_skippable.zig | 147 |
8 files changed, 558 insertions, 145 deletions
@@ -31,8 +31,8 @@ npm install -g bun-cli In your project folder root (where `package.json` is): ```bash -npm install -D bun-framework-next -bun bun --use next +bun create next app +cd app bun ``` @@ -77,17 +77,10 @@ If `public/index.html` exists, it becomes the default page instead of a 404 page #### Using Bun with Create React App -To use Bun with `create-react-app`, there are two changes you will need to make in `public/index.html`: - -1. Replace `%PUBLIC_URL%` with `/` -2. Insert `<script type="module" async src="/src/index.js">` just before `</body>` - -These changes are (sadly) necessary until Bun supports parsing & transpiling HTML. - -In your project folder root (where `package.json` is): +Run this: ```bash -bun bun ./src/index.js +bun create react app bun ``` @@ -522,6 +515,152 @@ Is generated like this: The implementation details of this module ID hash will vary between versions of Bun. The important part is the metadata contains the module IDs, the package paths, and the package hashes so it shouldn't really matter in practice if other tooling wants to make use of any of this. +### bun create + +`bun create` is a fast way to create a new project from a template. + +At the time of writing, `bun create react app` runs ~14x faster on my local computer than `yarn create react-app app`. `bun create` currently does no caching (though your npm client does) + +#### Usage + +By default, templates are downloaded from folders inside `examples/` in Bun's GitHub repo. Running `bun create react ./local-path` downloads the `react` folder from `examples/react`. + +Create a new Next.js project: + +```bash +bun create next ./app` +``` + +Create a new React project: + +```bash +bun create react ./app +``` + +To see a list of available templates, run + +```bash +bun create +``` + +##### Local templates + +If you have your own boilerplate you prefer using, copy it into `$HOME/.bun-create/my-boilerplate-name`. + +Before checking Bun's examples folder, `bun create` checks for a local folder matching the input in: + +- `$BUN_CREATE_DIR/` +- `$HOME/.bun-create/` +- `$(pwd)/.bun-create/` + +If a folder exists in any of those folders with the input, bun will use that instead of a remote template. + +This lets you run: + +```bash +bun create my-boilerplate ./app +``` + +Now your new template should appear when you run: + +```bash +bun create +``` + +Warning: unlike with remote templates, **bun will delete the entire destination folder if it already exists.** + +##### Flags + +| Flag | Description | +| ---------------------- | -------------------------------------- | +| --npm | Use `npm` for tasks & install | +| --yarn | Use `yarn` for tasks & install | +| --pnpm | Use `pnpm` for tasks & install | +| --force | Overwrite existing files | +| --no-install | Skip installing `node_modules` & tasks | +| --no-git | Don't initialize a git repository | +| ---------------------- | ----------------------------------- | + +By default, `bun create` will cancel if there are existing files it would overwrite and its a remote template. You can pass `--force` to disable this behavior. + +##### Publishing a new template + +Clone this repository and a new folder in `examples/` with your new template. The `package.json` must have a `name` that starts with `@bun-examples/`. Do not worry about publishing it, that will happen automaticallly after the PR is merged. + +Make sure to include a `.gitignore` that includes `node_modules` so that `node_modules` aren't checked in to git when people download the template. + +##### Testing your new template + +To test your new template, add it as a local template or pass the absolute path. + +```bash +bun create /path/to/my/new/template destination-dir +``` + +Warning: **This will always delete everything in destination-dir**. + +##### Config + +The `bun-create` section of package.json is automatically removed from the `package.json` on disk. This lets you add create-only steps without waiting for an extra package to install. + +There are currently two options: + +- `postinstall` +- `preinstall` + +They can be an array of strings or one string. An array of steps will be executed in order. + +Here is an example: + +```json +{ + "name": "@bun-examples/next", + "version": "0.0.31", + "main": "index.js", + "dependencies": { + "next": "11.1.2", + "react": "^17.0.2", + "react-dom": "^17.0.2", + "react-is": "^17.0.2" + }, + "devDependencies": { + "@types/react": "^17.0.19", + "bun-framework-next": "^0.0.0-21", + "typescript": "^4.3.5" + }, + "bun-create": { + "postinstall": ["bun bun --use next"] + } +} +``` + +By default, all commands run inside the environment exposed by the auto-detected npm client. This incurs a significant performance penalty, something like 150ms spent waiting for the npm client to start on each invocation. + +Any command that starts with `"bun "` will be run without npm, relying on the first `bun` binary in `$PATH`. + +##### How `bun create` works + +When you run `bun create ${template} ${destination}`, here's what happens: + +1. GET `registry.npmjs.org/@bun-examples/${template}/latest` and parse it +2. GET `registry.npmjs.org/@bun-examples/${template}/-/${template}-${latestVersion}.tgz` +3. Decompress & extract `${template}-${latestVersion}.tgz` into `${destination}` + + - If there are files that would overwrite, warn and exit unless `--force` is passed + +4. Parse the `package.json` (again!), update `name` to be `${basename(destination)}`, remove the `bun-create` section from the `package.json` and save updated `package.json` to disk +5. Auto-detect the npm client, preferring `pnpm`, `yarn` (v1), and lastly `npm` +6. Run any tasks defined in `"bun-create": { "preinstall" }` with the npm client +7. Run `${npmClient} install` +8. Run any tasks defined in `"bun-create": { "preinstall" }` with the npm client +9. Run `git init; git add -A .; git commit -am "Initial Commit";`. + + - Rename `gitignore` to `.gitignore`. NPM automatically removes `.gitignore` files from appearing in packages. + +10. Done + +`../misctools/publish-examples.js` publishes all examples to npm. + ### Environment variables - `GOMAXPROCS`: For `bun bun`, this sets the maximum number of threads to use. If you're experiencing an issue with `bun bun`, try setting `GOMAXPROCS=1` to force bun to run single-threaded diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 5b857090a..000000000 --- a/examples/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# bun create - -`bun create` is a fast way to create a new project from a template. At the time of writing, `bun create react app` runs ~14x faster on my local computer than `yarn create react-app app`. `bun create` currently does no caching (though your npm client does) - -## Usage - -Templates are downloaded from folders inside `examples/` in Bun's GitHub repo. Running `bun create react ./local-path` downloads the `react` folder from `examples/react`. - -Create a new Next.js project: - -```bash -bun create next ./app` -``` - -Create a new React project: - -```bash -bun create react ./app -``` - -To see a list of available templates, run - -```bash -bun create -``` - -### Advanced - -| Flag | Description | -| ---------------------- | -------------------------------------- | -| --npm | Use `npm` for tasks & install | -| --yarn | Use `yarn` for tasks & install | -| --pnpm | Use `pnpm` for tasks & install | -| --force | Overwrite existing files | -| --no-install | Skip installing `node_modules` & tasks | -| --no-git | Don't initialize a git repository | -| ---------------------- | ----------------------------------- | - -By default, `bun create` will cancel if there are existing files it would overwrite. You can pass `--force` to disable this behavior. - -## Adding a new template - -Clone this repository and a new folder in `examples/` with your new template. The `package.json` must have a `name` that starts with `@bun-examples/`. Do not worry about publishing it, that will happen automaticallly after the PR is merged. - -Make sure to include a `.gitignore` that includes `node_modules` so that `node_modules` aren't checked in to git when people download the template. - -#### Testing your new template - -### Config - -The `bun-create` section of package.json is automatically removed from the `package.json` on disk. This lets you add create-only steps without waiting for an extra package to install. - -There are currently two options: - -- `postinstall` -- `preinstall` - -They can be an array of strings or one string. An array of steps will be executed in order. - -Here is an example: - -```json -{ - "name": "@bun-examples/next", - "version": "0.0.31", - "main": "index.js", - "dependencies": { - "next": "11.1.2", - "react": "^17.0.2", - "react-dom": "^17.0.2", - "react-is": "^17.0.2" - }, - "devDependencies": { - "@types/react": "^17.0.19", - "bun-framework-next": "^0.0.0-21", - "typescript": "^4.3.5" - }, - "bun-create": { - "postinstall": ["bun bun --use next"] - } -} -``` - -By default, all commands run inside the environment exposed by the auto-detected npm client. This incurs a significant performance penalty, something like 150ms spent waiting for the npm client to start on each invocation. - -Any command that starts with `"bun "` will be run without npm, relying on the first `bun` binary in `$PATH`. - -## How `bun create` works - -When you run `bun create ${template} ${destination}`, here's what happens: - -1. GET `registry.npmjs.org/@bun-examples/${template}/latest` and parse it -2. GET `registry.npmjs.org/@bun-examples/${template}/-/${template}-${latestVersion}.tgz` -3. Decompress & extract `${template}-${latestVersion}.tgz` into `${destination}` - - - If there are files that would overwrite, warn and exit unless `--force` is passed - -4. Parse the `package.json` (again!), update `name` to be `${basename(destination)}`, remove the `bun-create` section from the `package.json` and save updated `package.json` to disk -5. Auto-detect the npm client, preferring `pnpm`, `yarn` (v1), and lastly `npm` -6. Run any tasks defined in `"bun-create": { "preinstall" }` with the npm client -7. Run `${npmClient} install` -8. Run any tasks defined in `"bun-create": { "preinstall" }` with the npm client -9. Run `git init; git add -A .; git commit -am "Initial Commit";`. - - - Rename `gitignore` to `.gitignore`. NPM automatically removes `.gitignore` files from appearing in packages. - -10. Done - -`../misctools/publish-examples.js` publishes all examples to npm. diff --git a/examples/react/package.json b/examples/react/package.json index fcf44635f..429a0b5aa 100644 --- a/examples/react/package.json +++ b/examples/react/package.json @@ -34,6 +34,12 @@ ] }, "devDependencies": { - "typescript": "^4.4.2" + "typescript": "^4.4.2", + "react-refresh": "^0.10.0" + }, + "bun-create": { + "postinstall": [ + "bun bun ./src/index.jsx" + ] } } diff --git a/misctools/publish-examples.js b/misctools/publish-examples.js index f29be6035..b774e234f 100644 --- a/misctools/publish-examples.js +++ b/misctools/publish-examples.js @@ -56,18 +56,31 @@ for (let folder of examplesFolderEntries) { } var retryCount = 5; + // Never commit lockfiles + try { + fs.rmSync(path.join(absolute, "package-lock.json")); + } catch (exception) {} + + try { + fs.rmSync(path.join(absolute, "yarn.lock")); + } catch (exception) {} + + try { + fs.rmSync(path.join(absolute, "pnpm-lock.yaml")); + } catch (exception) {} + + try { + fs.copyFileSync( + path.join(absolute, ".gitignore"), + path.join(absolute, "gitignore") + ); + } catch (exception) {} + restart: while (retryCount-- > 0) { packageJSON.version = version; if ("private" in packageJSON) delete packageJSON.private; if ("license" in packageJSON) delete packageJSON.license; - try { - fs.copyFileSync( - path.join(absolute, ".gitignore"), - path.join(absolute, "gitignore") - ); - } catch (exception) {} - fs.writeFileSync( path.join(absolute, "package.json"), JSON.stringify(packageJSON, null, 2) diff --git a/src/cli/create_command.zig b/src/cli/create_command.zig index d501e0473..d2a780430 100644 --- a/src/cli/create_command.zig +++ b/src/cli/create_command.zig @@ -31,7 +31,16 @@ const NPMClient = @import("../which_npm_client.zig").NPMClient; const which = @import("../which.zig").which; const clap = @import("clap"); +const CopyFile = @import("../copy_file.zig"); var bun_path_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; + +const skip_dirs = &[_]string{ "node_modules", ".git" }; +const skip_files = &[_]string{ + "package-lock.json", + "yarn.lock", + "pnpm-lock.yaml", +}; + var bun_path: ?[:0]const u8 = null; fn execTask(allocator: *std.mem.Allocator, task_: string, cwd: string, PATH: string, npm_client: NPMClient) void { const task = std.mem.trim(u8, task_, " \n\r\t"); @@ -156,6 +165,8 @@ const CreateOptions = struct { } }; +const BUN_CREATE_DIR = ".bun-create"; +var home_dir_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; pub const CreateCommand = struct { var client: HTTPClient = undefined; var extracting_name_buf: [1024]u8 = undefined; @@ -172,7 +183,44 @@ pub const CreateCommand = struct { env_loader.loadProcess(); - const template = positionals[0]; + const template = brk: { + var positional = positionals[0]; + + if (!std.fs.path.isAbsolute(positional)) { + outer: { + if (env_loader.map.get("BUN_CREATE_DIR")) |home_dir| { + var parts = [_]string{ home_dir, positional }; + var outdir_path = filesystem.absBuf(&parts, &home_dir_buf); + home_dir_buf[outdir_path.len] = 0; + var outdir_path_ = home_dir_buf[0..outdir_path.len :0]; + std.fs.accessAbsoluteZ(outdir_path_, .{}) catch break :outer; + break :brk outdir_path; + } + } + + outer: { + var parts = [_]string{ filesystem.top_level_dir, BUN_CREATE_DIR, positional }; + var outdir_path = filesystem.absBuf(&parts, &home_dir_buf); + home_dir_buf[outdir_path.len] = 0; + var outdir_path_ = home_dir_buf[0..outdir_path.len :0]; + std.fs.accessAbsoluteZ(outdir_path_, .{}) catch break :outer; + break :brk outdir_path; + } + + outer: { + if (env_loader.map.get("HOME")) |home_dir| { + var parts = [_]string{ home_dir, BUN_CREATE_DIR, positional }; + var outdir_path = filesystem.absBuf(&parts, &home_dir_buf); + home_dir_buf[outdir_path.len] = 0; + var outdir_path_ = home_dir_buf[0..outdir_path.len :0]; + std.fs.accessAbsoluteZ(outdir_path_, .{}) catch break :outer; + break :brk outdir_path; + } + } + } + + break :brk positional; + }; const dirname = positionals[1]; var filename_writer = filesystem.dirname_store; const destination = try filesystem.dirname_store.append([]const u8, resolve_path.joinAbs(filesystem.top_level_dir, .auto, dirname)); @@ -196,7 +244,9 @@ pub const CreateCommand = struct { var package_json_contents: MutableString = undefined; var package_json_file: std.fs.File = undefined; - if (!std.fs.path.isAbsolute(template)) { + const is_remote_template = !std.fs.path.isAbsolute(template); + + if (is_remote_template) { var tarball_bytes: MutableString = try Example.fetch(ctx, template, &progress, &node); node.end(); @@ -293,7 +343,8 @@ pub const CreateCommand = struct { package_json_contents = plucker.contents; package_json_file = std.fs.File{ .handle = plucker.fd }; } else { - const template_dir = std.fs.openDirAbsolute(template, .{ .iterate = true }) catch |err| { + var template_parts = [_]string{template}; + const template_dir = std.fs.openDirAbsolute(filesystem.abs(&template_parts), .{ .iterate = true }) catch |err| { node.end(); progress.root.end(); progress.refresh(); @@ -314,19 +365,33 @@ pub const CreateCommand = struct { std.os.exit(1); }; - var walker = try template_dir.walk(ctx.allocator); + const Walker = @import("../walker_skippable.zig"); + var walker = try Walker.walk(template_dir, ctx.allocator, skip_files, skip_dirs); defer walker.deinit(); + while (try walker.next()) |entry| { // TODO: make this not walk these folders entirely // rather than checking each file path..... - if (entry.kind != .File or - std.mem.indexOf(u8, entry.path, "node_modules") != null or - std.mem.indexOf(u8, entry.path, ".git") != null) continue; - - entry.dir.copyFile(entry.basename, destination_dir, entry.path, .{}) catch { + if (entry.kind != .File) continue; + var outfile = destination_dir.createFile(entry.path, .{}) catch brk: { if (std.fs.path.dirname(entry.path)) |entry_dirname| { destination_dir.makePath(entry_dirname) catch {}; } + break :brk destination_dir.createFile(entry.path, .{}) catch |err| { + node.end(); + progress.root.end(); + progress.refresh(); + + Output.prettyErrorln("<r><red>{s}<r>: copying file {s}", .{ @errorName(err), entry.path }); + Output.flush(); + std.os.exit(1); + }; + }; + defer outfile.close(); + + var infile = try entry.dir.openFile(entry.basename, .{ .read = true }); + defer infile.close(); + CopyFile.copy(infile.handle, outfile.handle) catch { entry.dir.copyFile(entry.basename, destination_dir, entry.path, .{}) catch |err| { node.end(); progress.root.end(); @@ -337,6 +402,8 @@ pub const CreateCommand = struct { std.os.exit(1); }; }; + var stat = outfile.stat() catch continue; + _ = C.fchmod(outfile.handle, stat.mode); } package_json_file = destination_dir.openFile("package.json", .{ .read = true, .write = true }) catch |err| { @@ -368,7 +435,9 @@ pub const CreateCommand = struct { std.os.exit(1); } package_json_contents = try MutableString.init(ctx.allocator, stat.size); - package_json_contents.inflate(package_json_file.readAll(package_json_contents.list.items) catch |err| { + package_json_contents.list.expandToCapacity(); + + _ = package_json_file.preadAll(package_json_contents.list.items, 0) catch |err| { node.end(); progress.root.end(); progress.refresh(); @@ -376,7 +445,12 @@ pub const CreateCommand = struct { Output.prettyErrorln("Error reading package.json: <r><red>{s}", .{@errorName(err)}); Output.flush(); std.os.exit(1); - }) catch unreachable; + }; + // The printer doesn't truncate, so we must do so manually + std.os.ftruncate(package_json_file.handle, 0) catch {}; + + js_ast.Expr.Data.Store.create(default_allocator); + js_ast.Stmt.Data.Store.create(default_allocator); } var source = logger.Source.initPathString("package.json", package_json_contents.list.items); @@ -679,6 +753,7 @@ pub const Example = struct { name: string, version: string, description: string, + local: bool = false, var client: HTTPClient = undefined; const examples_url: string = "https://registry.npmjs.org/bun-examples-all/latest"; @@ -848,7 +923,7 @@ pub const Example = struct { return thread.buffer; } - pub fn fetchAll(ctx: Command.Context) ![]const Example { + pub fn fetchAll(ctx: Command.Context) ![]Example { url = URL.parse(examples_url); client = HTTPClient.init(ctx.allocator, .GET, url, .{}, ""); client.timeout = timeout; @@ -934,19 +1009,103 @@ pub const Example = struct { pub const CreateListExamplesCommand = struct { pub fn exec(ctx: Command.Context) !void { + var filesystem = try fs.FileSystem.init1(ctx.allocator, null); + var env_loader: DotEnv.Loader = brk: { + var map = try ctx.allocator.create(DotEnv.Map); + map.* = DotEnv.Map.init(ctx.allocator); + + break :brk DotEnv.Loader.init(map, ctx.allocator); + }; + + env_loader.loadProcess(); + const time = std.time.nanoTimestamp(); - const examples = try Example.fetchAll(ctx); + const remote_examples = try Example.fetchAll(ctx); + + var examples = std.ArrayList(Example).fromOwnedSlice(ctx.allocator, remote_examples); + { + var folders = [3]std.fs.Dir{ std.fs.Dir{ .fd = 0 }, std.fs.Dir{ .fd = 0 }, std.fs.Dir{ .fd = 0 } }; + if (env_loader.map.get("BUN_CREATE_DIR")) |home_dir| { + var parts = [_]string{home_dir}; + var outdir_path = filesystem.absBuf(&parts, &home_dir_buf); + folders[0] = std.fs.openDirAbsolute(outdir_path, .{ .iterate = true }) catch std.fs.Dir{ .fd = 0 }; + } + + { + var parts = [_]string{ filesystem.top_level_dir, BUN_CREATE_DIR }; + var outdir_path = filesystem.absBuf(&parts, &home_dir_buf); + folders[1] = std.fs.openDirAbsolute(outdir_path, .{ .iterate = true }) catch std.fs.Dir{ .fd = 0 }; + } + + if (env_loader.map.get("HOME")) |home_dir| { + var parts = [_]string{ home_dir, BUN_CREATE_DIR }; + var outdir_path = filesystem.absBuf(&parts, &home_dir_buf); + folders[2] = std.fs.openDirAbsolute(outdir_path, .{ .iterate = true }) catch std.fs.Dir{ .fd = 0 }; + } + + // subfolders with package.json + for (folders) |folder_| { + if (folder_.fd != 0) { + const folder: std.fs.Dir = folder_; + var iter = folder.iterate(); + + loop: while (iter.next() catch null) |entry_| { + const entry: std.fs.Dir.Entry = entry_; + + switch (entry.kind) { + .Directory => { + inline for (skip_dirs) |skip_dir| { + if (strings.eqlComptime(entry.name, skip_dir)) { + continue :loop; + } + } + + std.mem.copy(u8, &home_dir_buf, entry.name); + home_dir_buf[entry.name.len] = std.fs.path.sep; + std.mem.copy(u8, home_dir_buf[entry.name.len + 1 ..], "package.json"); + home_dir_buf[entry.name.len + 1 + "package.json".len] = 0; + + var path: [:0]u8 = home_dir_buf[0 .. entry.name.len + 1 + "package.json".len :0]; + + folder.accessZ(path, .{ + .read = true, + }) catch continue :loop; + + try examples.append( + Example{ + .name = try filesystem.filename_store.append(@TypeOf(entry.name), entry.name), + .version = "", + .local = true, + .description = "", + }, + ); + continue :loop; + }, + else => continue, + } + } + } + } + } Output.printStartEnd(time, std.time.nanoTimestamp()); Output.prettyln(" <d>Fetched examples<r>", .{}); - Output.prettyln("Welcome to Bun! Create a new project by pasting any of the following:\n\n", .{}); Output.flush(); - Example.print(examples); + Example.print(examples.items); - _ = try CreateOptions.parse(ctx.allocator, true); + if (env_loader.map.get("HOME")) |homedir| { + Output.prettyln( + "<d>This command is completely optional. To add a new local template, create a folder in {s}/.bun-create/. To publish a new template, git clone https://github.com/jarred-sumner/bun, add a new folder to the \"examples\" folder, and submit a PR.<r>", + .{homedir}, + ); + } else { + Output.prettyln( + "<d>This command is completely optional. To add a new local template, create a folder in $HOME/.bun-create/. To publish a new template, git clone https://github.com/jarred-sumner/bun, add a new folder to the \"examples\" folder, and submit a PR.<r>", + .{}, + ); + } - Output.pretty("<d>To add a new template, git clone https://github.com/jarred-sumner/bun, add a new folder to the \"examples\" folder, and submit a PR.<r>", .{}); Output.flush(); } }; diff --git a/src/copy_file.zig b/src/copy_file.zig new file mode 100644 index 000000000..57738363f --- /dev/null +++ b/src/copy_file.zig @@ -0,0 +1,50 @@ +const std = @import("std"); +const os = std.os; +const math = std.math; + +const CopyFileError = error{SystemResources} || os.CopyFileRangeError || os.SendFileError; + +// Transfer all the data between two file descriptors in the most efficient way. +// The copy starts at offset 0, the initial offsets are preserved. +// No metadata is transferred over. +pub fn copy(fd_in: os.fd_t, fd_out: os.fd_t) CopyFileError!void { + if (comptime std.Target.current.isDarwin()) { + const rc = os.system.fcopyfile(fd_in, fd_out, null, os.system.COPYFILE_DATA); + switch (os.errno(rc)) { + .SUCCESS => return, + .INVAL => unreachable, + .NOMEM => return error.SystemResources, + // The source file is not a directory, symbolic link, or regular file. + // Try with the fallback path before giving up. + .OPNOTSUPP => {}, + else => |err| return os.unexpectedErrno(err), + } + } + + if (std.Target.current.os.tag == .linux) { + // Try copy_file_range first as that works at the FS level and is the + // most efficient method (if available). + var offset: u64 = 0; + cfr_loop: while (true) { + // The kernel checks the u64 value `offset+count` for overflow, use + // a 32 bit value so that the syscall won't return EINVAL except for + // impossibly large files (> 2^64-1 - 2^32-1). + const amt = try os.copy_file_range(fd_in, offset, fd_out, offset, math.maxInt(u32), 0); + // Terminate when no data was copied + if (amt == 0) break :cfr_loop; + offset += amt; + } + return; + } + + // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the + // fallback code will copy the contents chunk by chunk. + const empty_iovec = [0]os.iovec_const{}; + var offset: u64 = 0; + sendfile_loop: while (true) { + const amt = try os.sendfile(fd_out, fd_in, offset, 0, &empty_iovec, &empty_iovec, 0); + // Terminate when no data was copied + if (amt == 0) break :sendfile_loop; + offset += amt; + } +} diff --git a/src/string_immutable.zig b/src/string_immutable.zig index bbdea59cf..c9168d336 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -388,6 +388,14 @@ inline fn eqlComptimeCheckLen(self: string, comptime alt: anytype, comptime chec const second = comptime std.mem.readIntNative(u64, alt[8..16]); return ((comptime !check_len) or self.len == alt.len) and first == std.mem.readIntNative(u64, self[0..8]) and second == std.mem.readIntNative(u64, self[8..16]); }, + 17 => { + const first = comptime std.mem.readIntNative(u64, alt[0..8]); + const second = comptime std.mem.readIntNative(u64, alt[8..16]); + return ((comptime !check_len) or self.len == alt.len) and + first == std.mem.readIntNative(u64, self[0..8]) and second == + std.mem.readIntNative(u64, self[8..16]) and + alt[16] == self[16]; + }, 23 => { const first = comptime std.mem.readIntNative(u64, alt[0..8]); const second = comptime std.mem.readIntNative(u64, alt[8..15]); diff --git a/src/walker_skippable.zig b/src/walker_skippable.zig new file mode 100644 index 000000000..809901bc0 --- /dev/null +++ b/src/walker_skippable.zig @@ -0,0 +1,147 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Walker = @This(); +const path = std.fs.path; + +stack: std.ArrayList(StackItem), +name_buffer: std.ArrayList(u8), +skip_filenames: []const u64 = &[_]u64{}, +skip_dirnames: []const u64 = &[_]u64{}, +skip_all: []const u64 = &[_]u64{}, +seed: u64 = 0, + +const Dir = std.fs.Dir; + +pub const WalkerEntry = struct { + /// The containing directory. This can be used to operate directly on `basename` + /// rather than `path`, avoiding `error.NameTooLong` for deeply nested paths. + /// The directory remains open until `next` or `deinit` is called. + dir: Dir, + basename: []const u8, + path: []const u8, + kind: Dir.Entry.Kind, +}; + +const StackItem = struct { + iter: Dir.Iterator, + dirname_len: usize, +}; + +/// After each call to this function, and on deinit(), the memory returned +/// from this function becomes invalid. A copy must be made in order to keep +/// a reference to the path. +pub fn next(self: *Walker) !?WalkerEntry { + while (self.stack.items.len != 0) { + // `top` becomes invalid after appending to `self.stack` + var top = &self.stack.items[self.stack.items.len - 1]; + var dirname_len = top.dirname_len; + if (try top.iter.next()) |base| { + switch (base.kind) { + .Directory => { + if (std.mem.indexOfScalar(u64, self.skip_dirnames, std.hash.Wyhash.hash(self.seed, base.name)) != null) continue; + }, + .File => { + if (std.mem.indexOfScalar(u64, self.skip_filenames, std.hash.Wyhash.hash(self.seed, base.name)) != null) continue; + }, + + // we don't know what it is for a symlink + .SymLink => { + if (std.mem.indexOfScalar(u64, self.skip_all, std.hash.Wyhash.hash(self.seed, base.name)) != null) continue; + }, + + else => {}, + } + + self.name_buffer.shrinkRetainingCapacity(dirname_len); + if (self.name_buffer.items.len != 0) { + try self.name_buffer.append(path.sep); + dirname_len += 1; + } + try self.name_buffer.appendSlice(base.name); + if (base.kind == .Directory) { + var new_dir = top.iter.dir.openDir(base.name, .{ .iterate = true }) catch |err| switch (err) { + error.NameTooLong => unreachable, // no path sep in base.name + else => |e| return e, + }; + { + errdefer new_dir.close(); + try self.stack.append(StackItem{ + .iter = new_dir.iterate(), + .dirname_len = self.name_buffer.items.len, + }); + top = &self.stack.items[self.stack.items.len - 1]; + } + } + return WalkerEntry{ + .dir = top.iter.dir, + .basename = self.name_buffer.items[dirname_len..], + .path = self.name_buffer.items, + .kind = base.kind, + }; + } else { + var item = self.stack.pop(); + if (self.stack.items.len != 0) { + item.iter.dir.close(); + } + } + } + return null; +} + +pub fn deinit(self: *Walker) void { + while (self.stack.popOrNull()) |*item| { + if (self.stack.items.len != 0) { + item.iter.dir.close(); + } + } + self.stack.deinit(); + self.name_buffer.allocator.free(self.skip_all); + self.name_buffer.deinit(); +} + +/// Recursively iterates over a directory. +/// `self` must have been opened with `OpenDirOptions{.iterate = true}`. +/// Must call `Walker.deinit` when done. +/// The order of returned file system entries is undefined. +/// `self` will not be closed after walking it. +pub fn walk( + self: Dir, + allocator: *Allocator, + skip_filenames: []const []const u8, + skip_dirnames: []const []const u8, +) !Walker { + var name_buffer = std.ArrayList(u8).init(allocator); + errdefer name_buffer.deinit(); + + var stack = std.ArrayList(Walker.StackItem).init(allocator); + errdefer stack.deinit(); + + var skip_names = try allocator.alloc(u64, skip_filenames.len + skip_dirnames.len); + const seed = skip_filenames.len + skip_dirnames.len; + var skip_name_i: usize = 0; + + for (skip_filenames) |name| { + skip_names[skip_name_i] = std.hash.Wyhash.hash(seed, name); + skip_name_i += 1; + } + var skip_filenames_ = skip_names[0..skip_name_i]; + var skip_dirnames_ = skip_names[skip_name_i..]; + + for (skip_dirnames) |name, i| { + skip_dirnames_[i] = std.hash.Wyhash.hash(seed, name); + } + + try stack.append(Walker.StackItem{ + .iter = self.iterate(), + .dirname_len = 0, + }); + + return Walker{ + .stack = stack, + .name_buffer = name_buffer, + .skip_all = skip_names, + .seed = seed, + .skip_filenames = skip_filenames_, + .skip_dirnames = skip_dirnames_, + }; +} |