diff options
author | 2024-06-07 04:17:51 -0400 | |
---|---|---|
committer | 2024-06-07 10:17:51 +0200 | |
commit | 025f184c0ebda4d33a3920ff34e7011655adb0a7 (patch) | |
tree | fe46b772bfa3545b57938ef07c6822abf34c9e50 /packages/integrations/cloudflare/src | |
parent | 47a841901b74e401d05c042964865c1646b11c24 (diff) | |
download | astro-025f184c0ebda4d33a3920ff34e7011655adb0a7.tar.gz astro-025f184c0ebda4d33a3920ff34e7011655adb0a7.tar.zst astro-025f184c0ebda4d33a3920ff34e7011655adb0a7.zip |
feat(cloudflare): support `txt` and `bin`(#251)
Diffstat (limited to 'packages/integrations/cloudflare/src')
-rw-r--r-- | packages/integrations/cloudflare/src/index.ts | 19 | ||||
-rw-r--r-- | packages/integrations/cloudflare/src/utils/cloudflare-module-loader.ts (renamed from packages/integrations/cloudflare/src/utils/wasm-module-loader.ts) | 111 |
2 files changed, 84 insertions, 46 deletions
diff --git a/packages/integrations/cloudflare/src/index.ts b/packages/integrations/cloudflare/src/index.ts index 3f55f693c..a0f3d8ba1 100644 --- a/packages/integrations/cloudflare/src/index.ts +++ b/packages/integrations/cloudflare/src/index.ts @@ -1,7 +1,6 @@ import type { AstroConfig, AstroIntegration, RouteData } from 'astro'; import type { OutputChunk, ProgramNode } from 'rollup'; import type { PluginOption } from 'vite'; -import type { CloudflareModulePluginExtra } from './utils/wasm-module-loader.js'; import { createReadStream } from 'node:fs'; import { appendFile, rename, stat, unlink } from 'node:fs/promises'; @@ -16,11 +15,14 @@ import { AstroError } from 'astro/errors'; import { walk } from 'estree-walker'; import MagicString from 'magic-string'; import { getPlatformProxy } from 'wrangler'; +import { + type CloudflareModulePluginExtra, + cloudflareModuleLoader, +} from './utils/cloudflare-module-loader.js'; import { createRoutesFile, getParts } from './utils/generate-routes-json.js'; import { setImageConfig } from './utils/image-config.js'; import { mutateDynamicPageImportsInPlace, mutatePageMapInPlace } from './utils/index.js'; import { NonServerChunkDetector } from './utils/non-server-chunk-detector.js'; -import { cloudflareModuleLoader } from './utils/wasm-module-loader.js'; export type { Runtime } from './entrypoints/server.js'; @@ -64,10 +66,17 @@ export type Options = { /** Configuration persistence settings. Default '.wrangler/state/v3' */ persist?: boolean | { path: string }; }; + /** - * Allow bundling cloudflare worker specific file types - * https://developers.cloudflare.com/workers/wrangler/bundling/ + * Allow bundling cloudflare worker specific file types as importable modules. Defaults to true. + * When enabled, allows imports of '.wasm', '.bin', and '.txt' file types + * + * See https://developers.cloudflare.com/pages/functions/module-support/ + * for reference on how these file types are exported */ + cloudflareModules?: boolean; + + /** @deprecated - use `cloudflareModules`, which defaults to true. You can set `cloudflareModuleLoading: false` to disable */ wasmModuleImports?: boolean; }; @@ -75,7 +84,7 @@ export default function createIntegration(args?: Options): AstroIntegration { let _config: AstroConfig; const cloudflareModulePlugin: PluginOption & CloudflareModulePluginExtra = cloudflareModuleLoader( - args?.wasmModuleImports ?? false + args?.cloudflareModules ?? args?.wasmModuleImports ?? true ); // Initialize the unused chunk analyzer as a shared state between hooks. diff --git a/packages/integrations/cloudflare/src/utils/wasm-module-loader.ts b/packages/integrations/cloudflare/src/utils/cloudflare-module-loader.ts index fdc3415f4..33eb40d74 100644 --- a/packages/integrations/cloudflare/src/utils/wasm-module-loader.ts +++ b/packages/integrations/cloudflare/src/utils/cloudflare-module-loader.ts @@ -8,25 +8,41 @@ import type { PluginOption } from 'vite'; export interface CloudflareModulePluginExtra { afterBuildCompleted(config: AstroConfig): Promise<void>; } + +export type ModuleType = 'CompiledWasm' | 'Text' | 'Data'; + /** - * Enables support for wasm modules within cloudflare pages functions + * Enables support for various non-standard extensions in module imports that cloudflare workers supports. + * + * See https://developers.cloudflare.com/pages/functions/module-support/ for reference * - * Loads '*.wasm?module' and `*.wasm` imports as WebAssembly modules, which is the only way to load WASM in cloudflare workers. - * Current proposal for WASM modules: https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration - * Cloudflare worker WASM from javascript support: https://developers.cloudflare.com/workers/runtime-apis/webassembly/javascript/ - * @param enabled - if true, load '.wasm' imports as Uint8Arrays, otherwise will throw errors when encountered to clarify that it must be enabled - * @returns Vite plugin to load WASM tagged with '?module' as a WASM modules + * This adds supports for imports in the following formats: + * - .wasm + * - .wasm?module + * - .bin + * - .txt + * + * @param enabled - if true, will load all cloudflare pages supported types + * @returns Vite plugin with additional extension method to hook into astro build */ export function cloudflareModuleLoader( enabled: boolean ): PluginOption & CloudflareModulePluginExtra { - const enabledAdapters = cloudflareImportAdapters.filter((x) => enabled); + /** + * It's likely that eventually cloudflare will add support for custom extensions, like they do in vanilla cloudflare workers, + * by adding rules to your wrangler.tome + * https://developers.cloudflare.com/workers/wrangler/bundling/ + */ + const adaptersByExtension: Record<string, ModuleType> = enabled ? { ...defaultAdapters } : {}; + + const extensions = Object.keys(adaptersByExtension); + let isDev = false; const MAGIC_STRING = '__CLOUDFLARE_ASSET__'; const replacements: Replacement[] = []; return { - name: 'vite:wasm-module-loader', + name: 'vite:cf-module-loader', enforce: 'pre', configResolved(config) { isDev = config.command === 'serve'; @@ -34,12 +50,12 @@ export function cloudflareModuleLoader( config(_, __) { // let vite know that file format and the magic import string is intentional, and will be handled in this plugin return { - assetsInclude: enabledAdapters.map((x) => `**/*.${x.qualifiedExtension}`), + assetsInclude: extensions.map((x) => `**/*${x}`), build: { rollupOptions: { // mark the wasm files as external so that they are not bundled and instead are loaded from the files - external: enabledAdapters.map( - (x) => new RegExp(`^${MAGIC_STRING}.+\\.${x.extension}.mjs$`, 'i') + external: extensions.map( + (x) => new RegExp(`^${MAGIC_STRING}.+${escapeRegExp(x)}.mjs$`, 'i') ), }, }, @@ -47,22 +63,27 @@ export function cloudflareModuleLoader( }, async load(id, _) { - const importAdapter = cloudflareImportAdapters.find((x) => id.endsWith(x.qualifiedExtension)); - if (!importAdapter) { + const maybeExtension = extensions.find((x) => id.endsWith(x)); + const moduleType: ModuleType | undefined = + (maybeExtension && adaptersByExtension[maybeExtension]) || undefined; + if (!moduleType || !maybeExtension) { return; } if (!enabled) { throw new Error( - `Cloudflare module loading is experimental. The ${importAdapter.qualifiedExtension} module cannot be loaded unless you add \`wasmModuleImports: true\` to your astro config.` + `Cloudflare module loading is experimental. The ${maybeExtension} module cannot be loaded unless you add \`wasmModuleImports: true\` to your astro config.` ); } - const filePath = id.replace(/\?module$/, ''); + const moduleLoader = renderers[moduleType]; + + const filePath = id.replace(/\?\w+$/, ''); + const extension = maybeExtension.replace(/\?\w+$/, ''); const data = await fs.readFile(filePath); const base64 = data.toString('base64'); - const inlineModule = importAdapter.asNodeModule(data); + const inlineModule = moduleLoader(data); if (isDev) { // no need to wire up the assets in dev mode, just rewrite @@ -72,9 +93,7 @@ export function cloudflareModuleLoader( const hash = hashString(base64); // emit the wasm binary as an asset file, to be picked up later by the esbuild bundle for the worker. // give it a shared deterministic name to make things easy for esbuild to switch on later - const assetName = `${path.basename(filePath).split('.')[0]}.${hash}.${ - importAdapter.extension - }`; + const assetName = `${path.basename(filePath).split('.')[0]}.${hash}${extension}`; this.emitFile({ type: 'asset', // emit the data explicitly as an esset with `fileName` rather than `name` so that @@ -91,7 +110,7 @@ export function cloudflareModuleLoader( code: inlineModule, }); - return `import module from "${MAGIC_STRING}${chunkId}.${importAdapter.extension}.mjs";export default module;`; + return `import module from "${MAGIC_STRING}${chunkId}${extension}.mjs";export default module;`; }, // output original wasm file relative to the chunk now that chunking has been achieved @@ -103,10 +122,11 @@ export function cloudflareModuleLoader( // SSR will need the .mjs suffix removed from the import before this works in cloudflare, but this is done as a final step // so as to support prerendering from nodejs runtime let replaced = code; - for (const loader of enabledAdapters) { + for (const ext of extensions) { + const extension = ext.replace(/\?\w+$/, ''); + // chunk id can be many things, (alpha numeric, dollars, or underscores, maybe more) replaced = replaced.replaceAll( - // chunk id can be many things, (alpha numeric, dollars, or underscores, maybe more) - new RegExp(`${MAGIC_STRING}([^\\s]+?)\\.${loader.extension}\\.mjs`, 'g'), + new RegExp(`${MAGIC_STRING}([^\\s]+?)${escapeRegExp(extension)}\\.mjs`, 'g'), (s, assetId) => { const fileName = this.getFileName(assetId); const relativePath = path @@ -123,9 +143,6 @@ export function cloudflareModuleLoader( } ); } - if (replaced.includes(MAGIC_STRING)) { - console.error('failed to replace', replaced); - } return { code: replaced }; }, @@ -156,7 +173,9 @@ export function cloudflareModuleLoader( const baseDir = url.fileURLToPath(config.outDir); const replacementsByFileName = new Map<string, Replacement[]>(); for (const replacement of replacements) { - if (!replacement.fileName) continue; + if (!replacement.fileName) { + continue; + } const repls = replacementsByFileName.get(replacement.fileName) || []; if (!repls.length) { replacementsByFileName.set(replacement.fileName, repls); @@ -168,7 +187,7 @@ export function cloudflareModuleLoader( const contents = await fs.readFile(filepath, 'utf-8'); let updated = contents; for (const replacement of repls) { - updated = contents.replaceAll(replacement.nodejsImport, replacement.cloudflareImport); + updated = updated.replaceAll(replacement.nodejsImport, replacement.cloudflareImport); } await fs.writeFile(filepath, updated, 'utf-8'); } @@ -176,8 +195,6 @@ export function cloudflareModuleLoader( }; } -export type ImportType = 'wasm'; - interface Replacement { fileName?: string; chunkName: string; @@ -187,22 +204,30 @@ interface Replacement { nodejsImport: string; } -interface ModuleImportAdapter { - extension: ImportType; - qualifiedExtension: string; - asNodeModule(fileContents: Buffer): string; -} - -const wasmImportAdapter: ModuleImportAdapter = { - extension: 'wasm', - qualifiedExtension: 'wasm?module', - asNodeModule(fileContents: Buffer) { +const renderers: Record<ModuleType, (fileContents: Buffer) => string> = { + CompiledWasm(fileContents: Buffer) { const base64 = fileContents.toString('base64'); return `const wasmModule = new WebAssembly.Module(Uint8Array.from(atob("${base64}"), c => c.charCodeAt(0)));export default wasmModule;`; }, + Data(fileContents: Buffer) { + const base64 = fileContents.toString('base64'); + return `const binModule = Uint8Array.from(atob("${base64}"), c => c.charCodeAt(0)).buffer;export default binModule;`; + }, + Text(fileContents: Buffer) { + const escaped = JSON.stringify(fileContents.toString('utf-8')); + return `const stringModule = ${escaped};export default stringModule;`; + }, }; -const cloudflareImportAdapters = [wasmImportAdapter]; +const defaultAdapters: Record<string, ModuleType> = { + // Loads '*.wasm?module' imports as WebAssembly modules, which is the only way to load WASM in cloudflare workers. + // Current proposal for WASM modules: https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration + '.wasm?module': 'CompiledWasm', + // treats the module as a WASM module + '.wasm': 'CompiledWasm', + '.bin': 'Data', + '.txt': 'Text', +}; /** * Returns a deterministic 32 bit hash code from a string @@ -216,3 +241,7 @@ function hashString(str: string): string { } return new Uint32Array([hash])[0].toString(36); } + +function escapeRegExp(string: string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string +} |