diff options
| -rw-r--r-- | README.md | 118 | ||||
| -rw-r--r-- | src/defines-table.zig | 798 | ||||
| -rw-r--r-- | src/defines.zig | 12 | ||||
| -rw-r--r-- | src/js_lexer.zig | 2202 | ||||
| -rw-r--r-- | src/js_parser.zig | 58 | ||||
| -rw-r--r-- | src/json_parser.zig | 212 | ||||
| -rw-r--r-- | src/logger.zig | 18 |
7 files changed, 2360 insertions, 1058 deletions
@@ -67,31 +67,115 @@ If bundler means "turn my development code into something a browser can run", | Feature | esbuild | esdev | | ------------------------------------ | ------- | ----- | -| React Fast Refresh | ❌[1] | ⌛ | -| Hot Module Reloading | ❌[1] | ⌛ | -| Minification | ✅ | ❌ | | JSX (transform) | ✅ | ⌛ | | TypeScript (transform) | ✅ | ⌛ | +| React Fast Refresh | ❌ | ⌛ | +| Hot Module Reloading | ❌ | ⌛ | +| Minification | ✅ | ❌ | | Tree Shaking | ✅ | ⌛ | | Incremental builds | ✅ | ⌛ | | CSS | ✅ | 🗓️ | +| Expose CSS dependencies per file | ✅ | 🗓️ | | CommonJS, IIFE, UMD outputs | ✅ | ❌ | | Node.js build target | ✅ | ❌ | +| Code Splitting | ✅ | ⌛ | | Browser build target | ✅ | ⌛ | -| Support older browsers | ✅ | ❌[2] | -| Plugins | ✅ | ⌛[3] | -| AST Plugins | ❌ | ❌[4] | -| Filesystem Cache API (for plugins) | ❓ | 🗓️[4] | +| Bundling for production | ✅ | ❌ | +| Support older browsers | ✅ | ❌ | +| Plugins | ✅ | 🗓️ | +| AST Plugins | ❌ | ❌ | +| Filesystem Cache API (for plugins) | ❓ | 🗓️ | | Transform to ESM with `bundle` false | ❓ | ⌛ | Key: -|Tag | Meaning -|----|----------------------------------------------| -| ✅ | Compatible | -| ❌ | Not supported, and no plans to change that | -| ⌛ | In-progress | -| 🗓️ | Planned but work has not started | -| ❓ | Unknown | - -Notes: -[1]: https://esbuild.github.io/faq/#upcoming-roadmap + +| Tag | Meaning | +| --- | ------------------------------------------ | +| ✅ | Compatible | +| ❌ | Not supported, and no plans to change that | +| ⌛ | In-progress | +| 🗓️ | Planned but work has not started | +| ❓ | Unknown | + +#### Notes + +##### Hot Module Reloading & React Fast Refresh + +esdev exposes a runtime API to support Hot Module Reloading and React Fast Refresh. React Fast Refresh depends on Hot Module Reloading to work, but you can turn either of them off. esdev itself doesn't serve bundled files, it's up to the development server to provide that. + +##### Code Splitting + +esdev supports code splitting the way browsers do natively: through ES Modules. This works great for local development files. It doesn't work great for node_modules or for production due to the sheer number of network requests. There are plans to make this better, stay tuned. + +##### Support older browsers + +To simplify the parser, esdev doesn't support lowering features to non-current browsers. This means if you run a development build with esdev with, for example, optional chaining, it won't work in Internet Explorer 11. If you want to support older browsers, use a different tool. + +#### Implementation Notes + +##### HMR & Fast Refresh implementation + +This section only applies when Hot Module Reloading is enabled. When it's off, none of this part runs. React Fast Refresh depends on Hot Module Reloading. + +###### What is hot module reloading? + +HMR: "hot module reloading" + +A lot of developers know what it does -- but what actually is it and how does it work? Essentially, it means when a source file changes, automatically reload the code without reloading the web page. + +A big caveat here is JavaScript VMs don't expose an API to "unload" parts of the JavaScript context. In all HMR implementations, What really happens is this: + +1. Load a new copy of the code that changed +2. Update references to the old code to point to the new code +3. Handle errors + +The old code still lives there, in your browser's JavaScript VM until the page is refreshed. If any past references are kept (side effects!), undefined behavior happens. That's why, historically (by web standards), HMR has a reputation for being buggy. + +Loading code is easy. The hard parts are updating references and handling errors. + +There are two ways to update references: + +- Update all module imports +- Update the exports + +Either approach works. + +###### How it's implemented in esdev + +At build time, esdev replaces all import URLs with import manifests that wrap the real module. + +In the simple case, that looks like this: + +```ts +import { Button as _Button } from "http://localhost:3000/src/components/button.KXk23UX3.js"; + +export let Button = _Button; + +import.meta.onUpdate(import.meta.url, (exports) => { + if ("Button" in exports) { + Button = exports["Button"]; + } +}); +``` + +Then, lets say you updated `button.tsx` from this: + +```tsx +export const Button = ({ children }) => ( + <div className="Button">{children}</div> +); +``` + +To this: + +```tsx +export const Button = ({ children }) => ( + <div className="Button"> + <div className="Button-label">{children}</div> + </div> +); +``` + +This triggers the HMR client in esdev to: + +1. import `/src/components/button.js` once again diff --git a/src/defines-table.zig b/src/defines-table.zig new file mode 100644 index 000000000..3e3a720a3 --- /dev/null +++ b/src/defines-table.zig @@ -0,0 +1,798 @@ +usingnamespace @import("strings.zig"); + +// If something is in this list, then a direct identifier expression or property +// access chain matching this will be assumed to have no side effects and will +// be removed. +// +// This also means code is allowed to be reordered past things in this list. For +// example, if "console.log" is in this list, permitting reordering allows for +// "if (a) console.log(b); else console.log(c)" to be reordered and transformed +// into "console.log(a ? b : c)". Notice that "a" and "console.log" are in a +// different order, which can only happen if evaluating the "console.log" +// property access can be assumed to not change the value of "a". +// +// Note that membership in this list says nothing about whether calling any of +// these functions has any side effects. It only says something about +// referencing these function without calling them. +pub const GlobalDefinesKey = [][]string{ + // These global identifiers should exist in all JavaScript environments. This + // deliberately omits "NaN", "Infinity", and "undefined" because these are + // treated as automatically-inlined constants instead of identifiers. + []string{"Array"}, + []string{"Boolean"}, + []string{"Function"}, + []string{"Math"}, + []string{"Number"}, + []string{"Object"}, + []string{"RegExp"}, + []string{"String"}, + + // Object: Static methods + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object#Static_methods + []string{ "Object", "assign" }, + []string{ "Object", "create" }, + []string{ "Object", "defineProperties" }, + []string{ "Object", "defineProperty" }, + []string{ "Object", "entries" }, + []string{ "Object", "freeze" }, + []string{ "Object", "fromEntries" }, + []string{ "Object", "getOwnPropertyDescriptor" }, + []string{ "Object", "getOwnPropertyDescriptors" }, + []string{ "Object", "getOwnPropertyNames" }, + []string{ "Object", "getOwnPropertySymbols" }, + []string{ "Object", "getPrototypeOf" }, + []string{ "Object", "is" }, + []string{ "Object", "isExtensible" }, + []string{ "Object", "isFrozen" }, + []string{ "Object", "isSealed" }, + []string{ "Object", "keys" }, + []string{ "Object", "preventExtensions" }, + []string{ "Object", "seal" }, + []string{ "Object", "setPrototypeOf" }, + []string{ "Object", "values" }, + + // Object: Instance methods + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object#Instance_methods + []string{ "Object", "prototype", "__defineGetter__" }, + []string{ "Object", "prototype", "__defineSetter__" }, + []string{ "Object", "prototype", "__lookupGetter__" }, + []string{ "Object", "prototype", "__lookupSetter__" }, + []string{ "Object", "prototype", "hasOwnProperty" }, + []string{ "Object", "prototype", "isPrototypeOf" }, + []string{ "Object", "prototype", "propertyIsEnumerable" }, + []string{ "Object", "prototype", "toLocaleString" }, + []string{ "Object", "prototype", "toString" }, + []string{ "Object", "prototype", "unwatch" }, + []string{ "Object", "prototype", "valueOf" }, + []string{ "Object", "prototype", "watch" }, + + // Math: Static properties + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math#Static_properties + []string{ "Math", "E" }, + []string{ "Math", "LN10" }, + []string{ "Math", "LN2" }, + []string{ "Math", "LOG10E" }, + []string{ "Math", "LOG2E" }, + []string{ "Math", "PI" }, + []string{ "Math", "SQRT1_2" }, + []string{ "Math", "SQRT2" }, + + // Math: Static methods + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math#Static_methods + []string{ "Math", "abs" }, + []string{ "Math", "acos" }, + []string{ "Math", "acosh" }, + []string{ "Math", "asin" }, + []string{ "Math", "asinh" }, + []string{ "Math", "atan" }, + []string{ "Math", "atan2" }, + []string{ "Math", "atanh" }, + []string{ "Math", "cbrt" }, + []string{ "Math", "ceil" }, + []string{ "Math", "clz32" }, + []string{ "Math", "cos" }, + []string{ "Math", "cosh" }, + []string{ "Math", "exp" }, + []string{ "Math", "expm1" }, + []string{ "Math", "floor" }, + []string{ "Math", "fround" }, + []string{ "Math", "hypot" }, + []string{ "Math", "imul" }, + []string{ "Math", "log" }, + []string{ "Math", "log10" }, + []string{ "Math", "log1p" }, + []string{ "Math", "log2" }, + []string{ "Math", "max" }, + []string{ "Math", "min" }, + []string{ "Math", "pow" }, + []string{ "Math", "random" }, + []string{ "Math", "round" }, + []string{ "Math", "sign" }, + []string{ "Math", "sin" }, + []string{ "Math", "sinh" }, + []string{ "Math", "sqrt" }, + []string{ "Math", "tan" }, + []string{ "Math", "tanh" }, + []string{ "Math", "trunc" }, + + // Other globals present in both the browser and node (except "eval" because + // it has special behavior) + []string{"AbortController"}, + []string{"AbortSignal"}, + []string{"AggregateError"}, + []string{"ArrayBuffer"}, + []string{"BigInt"}, + []string{"DataView"}, + []string{"Date"}, + []string{"Error"}, + []string{"EvalError"}, + []string{"Event"}, + []string{"EventTarget"}, + []string{"Float32Array"}, + []string{"Float64Array"}, + []string{"Int16Array"}, + []string{"Int32Array"}, + []string{"Int8Array"}, + []string{"Intl"}, + []string{"JSON"}, + []string{"Map"}, + []string{"MessageChannel"}, + []string{"MessageEvent"}, + []string{"MessagePort"}, + []string{"Promise"}, + []string{"Proxy"}, + []string{"RangeError"}, + []string{"ReferenceError"}, + []string{"Reflect"}, + []string{"Set"}, + []string{"Symbol"}, + []string{"SyntaxError"}, + []string{"TextDecoder"}, + []string{"TextEncoder"}, + []string{"TypeError"}, + []string{"URIError"}, + []string{"URL"}, + []string{"URLSearchParams"}, + []string{"Uint16Array"}, + []string{"Uint32Array"}, + []string{"Uint8Array"}, + []string{"Uint8ClampedArray"}, + []string{"WeakMap"}, + []string{"WeakSet"}, + []string{"WebAssembly"}, + []string{"clearInterval"}, + []string{"clearTimeout"}, + []string{"console"}, + []string{"decodeURI"}, + []string{"decodeURIComponent"}, + []string{"encodeURI"}, + []string{"encodeURIComponent"}, + []string{"escape"}, + []string{"globalThis"}, + []string{"isFinite"}, + []string{"isNaN"}, + []string{"parseFloat"}, + []string{"parseInt"}, + []string{"queueMicrotask"}, + []string{"setInterval"}, + []string{"setTimeout"}, + []string{"unescape"}, + + // Console method references are assumed to have no side effects + // https://developer.mozilla.org/en-US/docs/Web/API/console + []string{ "console", "assert" }, + []string{ "console", "clear" }, + []string{ "console", "count" }, + []string{ "console", "countReset" }, + []string{ "console", "debug" }, + []string{ "console", "dir" }, + []string{ "console", "dirxml" }, + []string{ "console", "error" }, + []string{ "console", "group" }, + []string{ "console", "groupCollapsed" }, + []string{ "console", "groupEnd" }, + []string{ "console", "info" }, + []string{ "console", "log" }, + []string{ "console", "table" }, + []string{ "console", "time" }, + []string{ "console", "timeEnd" }, + []string{ "console", "timeLog" }, + []string{ "console", "trace" }, + []string{ "console", "warn" }, + + // CSSOM APIs + []string{"CSSAnimation"}, + []string{"CSSFontFaceRule"}, + []string{"CSSImportRule"}, + []string{"CSSKeyframeRule"}, + []string{"CSSKeyframesRule"}, + []string{"CSSMediaRule"}, + []string{"CSSNamespaceRule"}, + []string{"CSSPageRule"}, + []string{"CSSRule"}, + []string{"CSSRuleList"}, + []string{"CSSStyleDeclaration"}, + []string{"CSSStyleRule"}, + []string{"CSSStyleSheet"}, + []string{"CSSSupportsRule"}, + []string{"CSSTransition"}, + + // SVG DOM + []string{"SVGAElement"}, + []string{"SVGAngle"}, + []string{"SVGAnimateElement"}, + []string{"SVGAnimateMotionElement"}, + []string{"SVGAnimateTransformElement"}, + []string{"SVGAnimatedAngle"}, + []string{"SVGAnimatedBoolean"}, + []string{"SVGAnimatedEnumeration"}, + []string{"SVGAnimatedInteger"}, + []string{"SVGAnimatedLength"}, + []string{"SVGAnimatedLengthList"}, + []string{"SVGAnimatedNumber"}, + []string{"SVGAnimatedNumberList"}, + []string{"SVGAnimatedPreserveAspectRatio"}, + []string{"SVGAnimatedRect"}, + []string{"SVGAnimatedString"}, + []string{"SVGAnimatedTransformList"}, + []string{"SVGAnimationElement"}, + []string{"SVGCircleElement"}, + []string{"SVGClipPathElement"}, + []string{"SVGComponentTransferFunctionElement"}, + []string{"SVGDefsElement"}, + []string{"SVGDescElement"}, + []string{"SVGElement"}, + []string{"SVGEllipseElement"}, + []string{"SVGFEBlendElement"}, + []string{"SVGFEColorMatrixElement"}, + []string{"SVGFEComponentTransferElement"}, + []string{"SVGFECompositeElement"}, + []string{"SVGFEConvolveMatrixElement"}, + []string{"SVGFEDiffuseLightingElement"}, + []string{"SVGFEDisplacementMapElement"}, + []string{"SVGFEDistantLightElement"}, + []string{"SVGFEDropShadowElement"}, + []string{"SVGFEFloodElement"}, + []string{"SVGFEFuncAElement"}, + []string{"SVGFEFuncBElement"}, + []string{"SVGFEFuncGElement"}, + []string{"SVGFEFuncRElement"}, + []string{"SVGFEGaussianBlurElement"}, + []string{"SVGFEImageElement"}, + []string{"SVGFEMergeElement"}, + []string{"SVGFEMergeNodeElement"}, + []string{"SVGFEMorphologyElement"}, + []string{"SVGFEOffsetElement"}, + []string{"SVGFEPointLightElement"}, + []string{"SVGFESpecularLightingElement"}, + []string{"SVGFESpotLightElement"}, + []string{"SVGFETileElement"}, + []string{"SVGFETurbulenceElement"}, + []string{"SVGFilterElement"}, + []string{"SVGForeignObjectElement"}, + []string{"SVGGElement"}, + []string{"SVGGeometryElement"}, + []string{"SVGGradientElement"}, + []string{"SVGGraphicsElement"}, + []string{"SVGImageElement"}, + []string{"SVGLength"}, + []string{"SVGLengthList"}, + []string{"SVGLineElement"}, + []string{"SVGLinearGradientElement"}, + []string{"SVGMPathElement"}, + []string{"SVGMarkerElement"}, + []string{"SVGMaskElement"}, + []string{"SVGMatrix"}, + []string{"SVGMetadataElement"}, + []string{"SVGNumber"}, + []string{"SVGNumberList"}, + []string{"SVGPathElement"}, + []string{"SVGPatternElement"}, + []string{"SVGPoint"}, + []string{"SVGPointList"}, + []string{"SVGPolygonElement"}, + []string{"SVGPolylineElement"}, + []string{"SVGPreserveAspectRatio"}, + []string{"SVGRadialGradientElement"}, + []string{"SVGRect"}, + []string{"SVGRectElement"}, + []string{"SVGSVGElement"}, + []string{"SVGScriptElement"}, + []string{"SVGSetElement"}, + []string{"SVGStopElement"}, + []string{"SVGStringList"}, + []string{"SVGStyleElement"}, + []string{"SVGSwitchElement"}, + []string{"SVGSymbolElement"}, + []string{"SVGTSpanElement"}, + []string{"SVGTextContentElement"}, + []string{"SVGTextElement"}, + []string{"SVGTextPathElement"}, + []string{"SVGTextPositioningElement"}, + []string{"SVGTitleElement"}, + []string{"SVGTransform"}, + []string{"SVGTransformList"}, + []string{"SVGUnitTypes"}, + []string{"SVGUseElement"}, + []string{"SVGViewElement"}, + + // Other browser APIs + // + // This list contains all globals present in modern versions of Chrome, Safari, + // and Firefox except for the following properties, since they have a side effect + // of triggering layout (https://gist.github.com/paulirish/5d52fb081b3570c81e3a): + // + // - scrollX + // - scrollY + // - innerWidth + // - innerHeight + // - pageXOffset + // - pageYOffset + // + // The following globals have also been removed since they sometimes throw an + // exception when accessed, which is a side effect (for more information see + // https://stackoverflow.com/a/33047477): + // + // - localStorage + // - sessionStorage + // + []string{"AnalyserNode"}, + []string{"Animation"}, + []string{"AnimationEffect"}, + []string{"AnimationEvent"}, + []string{"AnimationPlaybackEvent"}, + []string{"AnimationTimeline"}, + []string{"Attr"}, + []string{"Audio"}, + []string{"AudioBuffer"}, + []string{"AudioBufferSourceNode"}, + []string{"AudioDestinationNode"}, + []string{"AudioListener"}, + []string{"AudioNode"}, + []string{"AudioParam"}, + []string{"AudioProcessingEvent"}, + []string{"AudioScheduledSourceNode"}, + []string{"BarProp"}, + []string{"BeforeUnloadEvent"}, + []string{"BiquadFilterNode"}, + []string{"Blob"}, + []string{"BlobEvent"}, + []string{"ByteLengthQueuingStrategy"}, + []string{"CDATASection"}, + []string{"CSS"}, + []string{"CanvasGradient"}, + []string{"CanvasPattern"}, + []string{"CanvasRenderingContext2D"}, + []string{"ChannelMergerNode"}, + []string{"ChannelSplitterNode"}, + []string{"CharacterData"}, + []string{"ClipboardEvent"}, + []string{"CloseEvent"}, + []string{"Comment"}, + []string{"CompositionEvent"}, + []string{"ConvolverNode"}, + []string{"CountQueuingStrategy"}, + []string{"Crypto"}, + []string{"CustomElementRegistry"}, + []string{"CustomEvent"}, + []string{"DOMException"}, + []string{"DOMImplementation"}, + []string{"DOMMatrix"}, + []string{"DOMMatrixReadOnly"}, + []string{"DOMParser"}, + []string{"DOMPoint"}, + []string{"DOMPointReadOnly"}, + []string{"DOMQuad"}, + []string{"DOMRect"}, + []string{"DOMRectList"}, + []string{"DOMRectReadOnly"}, + []string{"DOMStringList"}, + []string{"DOMStringMap"}, + []string{"DOMTokenList"}, + []string{"DataTransfer"}, + []string{"DataTransferItem"}, + []string{"DataTransferItemList"}, + []string{"DelayNode"}, + []string{"Document"}, + []string{"DocumentFragment"}, + []string{"DocumentTimeline"}, + []string{"DocumentType"}, + []string{"DragEvent"}, + []string{"DynamicsCompressorNode"}, + []string{"Element"}, + []string{"ErrorEvent"}, + []string{"EventSource"}, + []string{"File"}, + []string{"FileList"}, + []string{"FileReader"}, + []string{"FocusEvent"}, + []string{"FontFace"}, + []string{"FormData"}, + []string{"GainNode"}, + []string{"Gamepad"}, + []string{"GamepadButton"}, + []string{"GamepadEvent"}, + []string{"Geolocation"}, + []string{"GeolocationPositionError"}, + []string{"HTMLAllCollection"}, + []string{"HTMLAnchorElement"}, + []string{"HTMLAreaElement"}, + []string{"HTMLAudioElement"}, + []string{"HTMLBRElement"}, + []string{"HTMLBaseElement"}, + []string{"HTMLBodyElement"}, + []string{"HTMLButtonElement"}, + []string{"HTMLCanvasElement"}, + []string{"HTMLCollection"}, + []string{"HTMLDListElement"}, + []string{"HTMLDataElement"}, + []string{"HTMLDataListElement"}, + []string{"HTMLDetailsElement"}, + []string{"HTMLDirectoryElement"}, + []string{"HTMLDivElement"}, + []string{"HTMLDocument"}, + []string{"HTMLElement"}, + []string{"HTMLEmbedElement"}, + []string{"HTMLFieldSetElement"}, + []string{"HTMLFontElement"}, + []string{"HTMLFormControlsCollection"}, + []string{"HTMLFormElement"}, + []string{"HTMLFrameElement"}, + []string{"HTMLFrameSetElement"}, + []string{"HTMLHRElement"}, + []string{"HTMLHeadElement"}, + []string{"HTMLHeadingElement"}, + []string{"HTMLHtmlElement"}, + []string{"HTMLIFrameElement"}, + []string{"HTMLImageElement"}, + []string{"HTMLInputElement"}, + []string{"HTMLLIElement"}, + []string{"HTMLLabelElement"}, + []string{"HTMLLegendElement"}, + []string{"HTMLLinkElement"}, + []string{"HTMLMapElement"}, + []string{"HTMLMarqueeElement"}, + []string{"HTMLMediaElement"}, + []string{"HTMLMenuElement"}, + []string{"HTMLMetaElement"}, + []string{"HTMLMeterElement"}, + []string{"HTMLModElement"}, + []string{"HTMLOListElement"}, + []string{"HTMLObjectElement"}, + []string{"HTMLOptGroupElement"}, + []string{"HTMLOptionElement"}, + []string{"HTMLOptionsCollection"}, + []string{"HTMLOutputElement"}, + []string{"HTMLParagraphElement"}, + []string{"HTMLParamElement"}, + []string{"HTMLPictureElement"}, + []string{"HTMLPreElement"}, + []string{"HTMLProgressElement"}, + []string{"HTMLQuoteElement"}, + []string{"HTMLScriptElement"}, + []string{"HTMLSelectElement"}, + []string{"HTMLSlotElement"}, + []string{"HTMLSourceElement"}, + []string{"HTMLSpanElement"}, + []string{"HTMLStyleElement"}, + []string{"HTMLTableCaptionElement"}, + []string{"HTMLTableCellElement"}, + []string{"HTMLTableColElement"}, + []string{"HTMLTableElement"}, + []string{"HTMLTableRowElement"}, + []string{"HTMLTableSectionElement"}, + []string{"HTMLTemplateElement"}, + []string{"HTMLTextAreaElement"}, + []string{"HTMLTimeElement"}, + []string{"HTMLTitleElement"}, + []string{"HTMLTrackElement"}, + []string{"HTMLUListElement"}, + []string{"HTMLUnknownElement"}, + []string{"HTMLVideoElement"}, + []string{"HashChangeEvent"}, + []string{"Headers"}, + []string{"History"}, + []string{"IDBCursor"}, + []string{"IDBCursorWithValue"}, + []string{"IDBDatabase"}, + []string{"IDBFactory"}, + []string{"IDBIndex"}, + []string{"IDBKeyRange"}, + []string{"IDBObjectStore"}, + []string{"IDBOpenDBRequest"}, + []string{"IDBRequest"}, + []string{"IDBTransaction"}, + []string{"IDBVersionChangeEvent"}, + []string{"Image"}, + []string{"ImageData"}, + []string{"InputEvent"}, + []string{"IntersectionObserver"}, + []string{"IntersectionObserverEntry"}, + []string{"KeyboardEvent"}, + []string{"KeyframeEffect"}, + []string{"Location"}, + []string{"MediaCapabilities"}, + []string{"MediaElementAudioSourceNode"}, + []string{"MediaEncryptedEvent"}, + []string{"MediaError"}, + []string{"MediaList"}, + []string{"MediaQueryList"}, + []string{"MediaQueryListEvent"}, + []string{"MediaRecorder"}, + []string{"MediaSource"}, + []string{"MediaStream"}, + []string{"MediaStreamAudioDestinationNode"}, + []string{"MediaStreamAudioSourceNode"}, + []string{"MediaStreamTrack"}, + []string{"MediaStreamTrackEvent"}, + []string{"MimeType"}, + []string{"MimeTypeArray"}, + []string{"MouseEvent"}, + []string{"MutationEvent"}, + []string{"MutationObserver"}, + []string{"MutationRecord"}, + []string{"NamedNodeMap"}, + []string{"Navigator"}, + []string{"Node"}, + []string{"NodeFilter"}, + []string{"NodeIterator"}, + []string{"NodeList"}, + []string{"Notification"}, + []string{"OfflineAudioCompletionEvent"}, + []string{"Option"}, + []string{"OscillatorNode"}, + []string{"PageTransitionEvent"}, + []string{"Path2D"}, + []string{"Performance"}, + []string{"PerformanceEntry"}, + []string{"PerformanceMark"}, + []string{"PerformanceMeasure"}, + []string{"PerformanceNavigation"}, + []string{"PerformanceObserver"}, + []string{"PerformanceObserverEntryList"}, + []string{"PerformanceResourceTiming"}, + []string{"PerformanceTiming"}, + []string{"PeriodicWave"}, + []string{"Plugin"}, + []string{"PluginArray"}, + []string{"PointerEvent"}, + []string{"PopStateEvent"}, + []string{"ProcessingInstruction"}, + []string{"ProgressEvent"}, + []string{"PromiseRejectionEvent"}, + []string{"RTCCertificate"}, + []string{"RTCDTMFSender"}, + []string{"RTCDTMFToneChangeEvent"}, + []string{"RTCDataChannel"}, + []string{"RTCDataChannelEvent"}, + []string{"RTCIceCandidate"}, + []string{"RTCPeerConnection"}, + []string{"RTCPeerConnectionIceEvent"}, + []string{"RTCRtpReceiver"}, + []string{"RTCRtpSender"}, + []string{"RTCRtpTransceiver"}, + []string{"RTCSessionDescription"}, + []string{"RTCStatsReport"}, + []string{"RTCTrackEvent"}, + []string{"RadioNodeList"}, + []string{"Range"}, + []string{"ReadableStream"}, + []string{"Request"}, + []string{"ResizeObserver"}, + []string{"ResizeObserverEntry"}, + []string{"Response"}, + []string{"Screen"}, + []string{"ScriptProcessorNode"}, + []string{"SecurityPolicyViolationEvent"}, + []string{"Selection"}, + []string{"ShadowRoot"}, + []string{"SourceBuffer"}, + []string{"SourceBufferList"}, + []string{"SpeechSynthesisEvent"}, + []string{"SpeechSynthesisUtterance"}, + []string{"StaticRange"}, + []string{"Storage"}, + []string{"StorageEvent"}, + []string{"StyleSheet"}, + []string{"StyleSheetList"}, + []string{"Text"}, + []string{"TextMetrics"}, + []string{"TextTrack"}, + []string{"TextTrackCue"}, + []string{"TextTrackCueList"}, + []string{"TextTrackList"}, + []string{"TimeRanges"}, + []string{"TrackEvent"}, + []string{"TransitionEvent"}, + []string{"TreeWalker"}, + []string{"UIEvent"}, + []string{"VTTCue"}, + []string{"ValidityState"}, + []string{"VisualViewport"}, + []string{"WaveShaperNode"}, + []string{"WebGLActiveInfo"}, + []string{"WebGLBuffer"}, + []string{"WebGLContextEvent"}, + []string{"WebGLFramebuffer"}, + []string{"WebGLProgram"}, + []string{"WebGLQuery"}, + []string{"WebGLRenderbuffer"}, + []string{"WebGLRenderingContext"}, + []string{"WebGLSampler"}, + []string{"WebGLShader"}, + []string{"WebGLShaderPrecisionFormat"}, + []string{"WebGLSync"}, + []string{"WebGLTexture"}, + []string{"WebGLUniformLocation"}, + []string{"WebKitCSSMatrix"}, + []string{"WebSocket"}, + []string{"WheelEvent"}, + []string{"Window"}, + []string{"Worker"}, + []string{"XMLDocument"}, + []string{"XMLHttpRequest"}, + []string{"XMLHttpRequestEventTarget"}, + []string{"XMLHttpRequestUpload"}, + []string{"XMLSerializer"}, + []string{"XPathEvaluator"}, + []string{"XPathExpression"}, + []string{"XPathResult"}, + []string{"XSLTProcessor"}, + []string{"alert"}, + []string{"atob"}, + []string{"blur"}, + []string{"btoa"}, + []string{"cancelAnimationFrame"}, + []string{"captureEvents"}, + []string{"close"}, + []string{"closed"}, + []string{"confirm"}, + []string{"customElements"}, + []string{"devicePixelRatio"}, + []string{"document"}, + []string{"event"}, + []string{"fetch"}, + []string{"find"}, + []string{"focus"}, + []string{"frameElement"}, + []string{"frames"}, + []string{"getComputedStyle"}, + []string{"getSelection"}, + []string{"history"}, + []string{"indexedDB"}, + []string{"isSecureContext"}, + []string{"length"}, + []string{"location"}, + []string{"locationbar"}, + []string{"matchMedia"}, + []string{"menubar"}, + []string{"moveBy"}, + []string{"moveTo"}, + []string{"name"}, + []string{"navigator"}, + []string{"onabort"}, + []string{"onafterprint"}, + []string{"onanimationend"}, + []string{"onanimationiteration"}, + []string{"onanimationstart"}, + []string{"onbeforeprint"}, + []string{"onbeforeunload"}, + []string{"onblur"}, + []string{"oncanplay"}, + []string{"oncanplaythrough"}, + []string{"onchange"}, + []string{"onclick"}, + []string{"oncontextmenu"}, + []string{"oncuechange"}, + []string{"ondblclick"}, + []string{"ondrag"}, + []string{"ondragend"}, + []string{"ondragenter"}, + []string{"ondragleave"}, + []string{"ondragover"}, + []string{"ondragstart"}, + []string{"ondrop"}, + []string{"ondurationchange"}, + []string{"onemptied"}, + []string{"onended"}, + []string{"onerror"}, + []string{"onfocus"}, + []string{"ongotpointercapture"}, + []string{"onhashchange"}, + []string{"oninput"}, + []string{"oninvalid"}, + []string{"onkeydown"}, + []string{"onkeypress"}, + []string{"onkeyup"}, + []string{"onlanguagechange"}, + []string{"onload"}, + []string{"onloadeddata"}, + []string{"onloadedmetadata"}, + []string{"onloadstart"}, + []string{"onlostpointercapture"}, + []string{"onmessage"}, + []string{"onmousedown"}, + []string{"onmouseenter"}, + []string{"onmouseleave"}, + []string{"onmousemove"}, + []string{"onmouseout"}, + []string{"onmouseover"}, + []string{"onmouseup"}, + []string{"onoffline"}, + []string{"ononline"}, + []string{"onpagehide"}, + []string{"onpageshow"}, + []string{"onpause"}, + []string{"onplay"}, + []string{"onplaying"}, + []string{"onpointercancel"}, + []string{"onpointerdown"}, + []string{"onpointerenter"}, + []string{"onpointerleave"}, + []string{"onpointermove"}, + []string{"onpointerout"}, + []string{"onpointerover"}, + []string{"onpointerup"}, + []string{"onpopstate"}, + []string{"onprogress"}, + []string{"onratechange"}, + []string{"onrejectionhandled"}, + []string{"onreset"}, + []string{"onresize"}, + []string{"onscroll"}, + []string{"onseeked"}, + []string{"onseeking"}, + []string{"onselect"}, + []string{"onstalled"}, + []string{"onstorage"}, + []string{"onsubmit"}, + []string{"onsuspend"}, + []string{"ontimeupdate"}, + []string{"ontoggle"}, + []string{"ontransitioncancel"}, + []string{"ontransitionend"}, + []string{"ontransitionrun"}, + []string{"ontransitionstart"}, + []string{"onunhandledrejection"}, + []string{"onunload"}, + []string{"onvolumechange"}, + []string{"onwaiting"}, + []string{"onwebkitanimationend"}, + []string{"onwebkitanimationiteration"}, + []string{"onwebkitanimationstart"}, + []string{"onwebkittransitionend"}, + []string{"onwheel"}, + []string{"open"}, + []string{"opener"}, + []string{"origin"}, + []string{"outerHeight"}, + []string{"outerWidth"}, + []string{"parent"}, + []string{"performance"}, + []string{"personalbar"}, + []string{"postMessage"}, + []string{"print"}, + []string{"prompt"}, + []string{"releaseEvents"}, + []string{"requestAnimationFrame"}, + []string{"resizeBy"}, + []string{"resizeTo"}, + []string{"screen"}, + []string{"screenLeft"}, + []string{"screenTop"}, + []string{"screenX"}, + []string{"screenY"}, + []string{"scroll"}, + []string{"scrollBy"}, + []string{"scrollTo"}, + []string{"scrollbars"}, + []string{"self"}, + []string{"speechSynthesis"}, + []string{"status"}, + []string{"statusbar"}, + []string{"stop"}, + []string{"toolbar"}, + []string{"top"}, + []string{"webkitURL"}, + []string{"window"}, +}; diff --git a/src/defines.zig b/src/defines.zig new file mode 100644 index 000000000..6021df57f --- /dev/null +++ b/src/defines.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const js_ast = @import("./js_ast.zig"); + +const GlobalDefinesKey = @import("./defines-table.zig").GlobalDefinesKey; + +pub const defaultIdentifierDefines = comptime {}; + +pub const IdentifierDefine = struct {}; + +pub const DotDefine = struct {}; + +pub const Defines = struct {}; diff --git a/src/js_lexer.zig b/src/js_lexer.zig index a16c72c19..fa95341a5 100644 --- a/src/js_lexer.zig +++ b/src/js_lexer.zig @@ -22,1123 +22,1264 @@ pub const StrictModeReservedWords = tables.StrictModeReservedWords; pub const PropertyModifierKeyword = tables.PropertyModifierKeyword; pub const TypescriptStmtKeyword = tables.TypescriptStmtKeyword; -// TODO: JSON -const IS_JSON_FILE = false; - -pub const Lexer = struct { - // pub const Error = error{ - // UnexpectedToken, - // EndOfFile, - // }; - - // err: ?Lexer.Error, - log: logger.Log, - source: logger.Source, - current: usize = 0, - start: usize = 0, - end: usize = 0, - approximate_newline_count: i32 = 0, - legacy_octal_loc: logger.Loc = logger.Loc.Empty, - previous_backslash_quote_in_jsx: logger.Range = logger.Range.None, - token: T = T.t_end_of_file, - has_newline_before: bool = false, - has_pure_comment_before: bool = false, - preserve_all_comments_before: bool = false, - is_legacy_octal_literal: bool = false, - comments_to_preserve_before: ?[]js_ast.G.Comment = null, - all_original_comments: ?[]js_ast.G.Comment = null, - code_point: CodePoint = -1, - string_literal: JavascriptString, - identifier: []const u8 = "", - jsx_factory_pragma_comment: ?js_ast.Span = null, - jsx_fragment_pragma_comment: ?js_ast.Span = null, - source_mapping_url: ?js_ast.Span = null, - number: f64 = 0.0, - rescan_close_brace_as_template_token: bool = false, - for_global_name: bool = false, - prev_error_loc: logger.Loc = logger.Loc.Empty, - allocator: *std.mem.Allocator, - - pub fn loc(self: *Lexer) logger.Loc { - return logger.usize2Loc(self.start); - } +pub const JSONOptions = struct { + allow_comments: bool = false, + allow_trailing_commas: bool = false, +}; - fn nextCodepointSlice(it: *Lexer) callconv(.Inline) ?[]const u8 { - if (it.current >= it.source.contents.len) { - return null; +pub fn NewLexerType(comptime jsonOptions: ?JSONOptions) type { + return struct { + // pub const Error = error{ + // UnexpectedToken, + // EndOfFile, + // }; + + // err: ?@This().Error, + log: logger.Log, + source: logger.Source, + current: usize = 0, + start: usize = 0, + end: usize = 0, + approximate_newline_count: i32 = 0, + legacy_octal_loc: logger.Loc = logger.Loc.Empty, + previous_backslash_quote_in_jsx: logger.Range = logger.Range.None, + token: T = T.t_end_of_file, + has_newline_before: bool = false, + has_pure_comment_before: bool = false, + preserve_all_comments_before: bool = false, + is_legacy_octal_literal: bool = false, + comments_to_preserve_before: std.ArrayList(js_ast.G.Comment), + all_original_comments: ?[]js_ast.G.Comment = null, + code_point: CodePoint = -1, + string_literal: JavascriptString, + identifier: []const u8 = "", + jsx_factory_pragma_comment: ?js_ast.Span = null, + jsx_fragment_pragma_comment: ?js_ast.Span = null, + source_mapping_url: ?js_ast.Span = null, + number: f64 = 0.0, + rescan_close_brace_as_template_token: bool = false, + for_global_name: bool = false, + prev_error_loc: logger.Loc = logger.Loc.Empty, + allocator: *std.mem.Allocator, + + pub fn loc(self: *@This()) logger.Loc { + return logger.usize2Loc(self.start); } - const cp_len = unicode.utf8ByteSequenceLength(it.source.contents[it.current]) catch unreachable; - it.end = it.current; - it.current += cp_len; - - return it.source.contents[it.current - cp_len .. it.current]; - } - - pub fn syntaxError(self: *Lexer) void { - self.addError(self.start, "Syntax Error!!", .{}, true); - } + fn nextCodepointSlice(it: *@This()) callconv(.Inline) ?[]const u8 { + if (it.current >= it.source.contents.len) { + return null; + } - pub fn addDefaultError(self: *Lexer, msg: []const u8) void { - self.addError(self.start, "{s}", .{msg}, true); - } + const cp_len = unicode.utf8ByteSequenceLength(it.source.contents[it.current]) catch unreachable; + it.end = it.current; + it.current += cp_len; - pub fn addError(self: *Lexer, _loc: usize, comptime format: []const u8, args: anytype, panic: bool) void { - var __loc = logger.usize2Loc(_loc); - if (__loc.eql(self.prev_error_loc)) { - return; + return it.source.contents[it.current - cp_len .. it.current]; } - const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; - self.log.addError(self.source, __loc, errorMessage) catch unreachable; - self.prev_error_loc = __loc; - var msg = self.log.msgs.items[self.log.msgs.items.len - 1]; - msg.formatNoWriter(std.debug.panic); - } + pub fn syntaxError(self: *@This()) void { + self.addError(self.start, "Syntax Error!!", .{}, true); + } - pub fn addRangeError(self: *Lexer, r: logger.Range, comptime format: []const u8, args: anytype, panic: bool) void { - if (self.prev_error_loc.eql(r.loc)) { - return; + pub fn addDefaultError(self: *@This(), msg: []const u8) void { + self.addError(self.start, "{s}", .{msg}, true); } - const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; - var msg = self.log.addRangeError(self.source, r, errorMessage); - self.prev_error_loc = r.loc; + pub fn addError(self: *@This(), _loc: usize, comptime format: []const u8, args: anytype, panic: bool) void { + var __loc = logger.usize2Loc(_loc); + if (__loc.eql(self.prev_error_loc)) { + return; + } - if (panic) { - self.doPanic(errorMessage); + const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; + self.log.addError(self.source, __loc, errorMessage) catch unreachable; + self.prev_error_loc = __loc; + var msg = self.log.msgs.items[self.log.msgs.items.len - 1]; + msg.formatNoWriter(std.debug.panic); } - } - fn doPanic(self: *Lexer, content: []const u8) void { - std.debug.panic("{s}", .{content}); - } + pub fn addRangeError(self: *@This(), r: logger.Range, comptime format: []const u8, args: anytype, panic: bool) void { + if (self.prev_error_loc.eql(r.loc)) { + return; + } - pub fn codePointEql(self: *Lexer, a: u8) bool { - return @intCast(CodePoint, a) == self.code_point; - } + const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; + var msg = self.log.addRangeError(self.source, r, errorMessage); + self.prev_error_loc = r.loc; - fn nextCodepoint(it: *Lexer) callconv(.Inline) CodePoint { - const slice = it.nextCodepointSlice() orelse return @as(CodePoint, -1); + if (panic) { + self.doPanic(errorMessage); + } + } - switch (slice.len) { - 1 => return @as(CodePoint, slice[0]), - 2 => return @as(CodePoint, unicode.utf8Decode2(slice) catch unreachable), - 3 => return @as(CodePoint, unicode.utf8Decode3(slice) catch unreachable), - 4 => return @as(CodePoint, unicode.utf8Decode4(slice) catch unreachable), - else => unreachable, + fn doPanic(self: *@This(), content: []const u8) void { + std.debug.panic("{s}", .{content}); } - } - /// Look ahead at the next n codepoints without advancing the iterator. - /// If fewer than n codepoints are available, then return the remainder of the string. - fn peek(it: *Lexer, n: usize) []const u8 { - const original_i = it.current; - defer it.current = original_i; - - var end_ix = original_i; - var found: usize = 0; - while (found < n) : (found += 1) { - const next_codepoint = it.nextCodepointSlice() orelse return it.source.contents[original_i..]; - end_ix += next_codepoint.len; + pub fn codePointEql(self: *@This(), a: u8) bool { + return @intCast(CodePoint, a) == self.code_point; } - return it.source.contents[original_i..end_ix]; - } + fn nextCodepoint(it: *@This()) callconv(.Inline) CodePoint { + const slice = it.nextCodepointSlice() orelse return @as(CodePoint, -1); - pub fn isIdentifierOrKeyword(lexer: Lexer) bool { - return @enumToInt(lexer.token) >= @enumToInt(T.t_identifier); - } + switch (slice.len) { + 1 => return @as(CodePoint, slice[0]), + 2 => return @as(CodePoint, unicode.utf8Decode2(slice) catch unreachable), + 3 => return @as(CodePoint, unicode.utf8Decode3(slice) catch unreachable), + 4 => return @as(CodePoint, unicode.utf8Decode4(slice) catch unreachable), + else => unreachable, + } + } - fn parseStringLiteral(lexer: *Lexer) void { - var quote: CodePoint = lexer.code_point; - var needs_slow_path = false; - var suffixLen: usize = 1; - - if (quote != '`') { - lexer.token = T.t_string_literal; - } else if (lexer.rescan_close_brace_as_template_token) { - lexer.token = T.t_template_tail; - } else { - lexer.token = T.t_no_substitution_template_literal; + /// Look ahead at the next n codepoints without advancing the iterator. + /// If fewer than n codepoints are available, then return the remainder of the string. + fn peek(it: *@This(), n: usize) []const u8 { + const original_i = it.current; + defer it.current = original_i; + + var end_ix = original_i; + var found: usize = 0; + while (found < n) : (found += 1) { + const next_codepoint = it.nextCodepointSlice() orelse return it.source.contents[original_i..]; + end_ix += next_codepoint.len; + } + + return it.source.contents[original_i..end_ix]; } - lexer.step(); - stringLiteral: while (true) { - switch (lexer.code_point) { - '\\' => { - needs_slow_path = true; - lexer.step(); + pub fn isIdentifierOrKeyword(lexer: @This()) bool { + return @enumToInt(lexer.token) >= @enumToInt(T.t_identifier); + } - // Handle Windows CRLF - if (lexer.code_point == '\r' and !IS_JSON_FILE) { + fn parseStringLiteral(lexer: *@This()) void { + var quote: CodePoint = lexer.code_point; + var needs_slow_path = false; + var suffixLen: usize = 1; + + if (quote != '`') { + lexer.token = T.t_string_literal; + } else if (lexer.rescan_close_brace_as_template_token) { + lexer.token = T.t_template_tail; + } else { + lexer.token = T.t_no_substitution_template_literal; + } + lexer.step(); + + stringLiteral: while (true) { + switch (lexer.code_point) { + '\\' => { + needs_slow_path = true; lexer.step(); - if (lexer.code_point == '\n') { + + // Handle Windows CRLF + if (lexer.code_point == '\r' and jsonOptions != null) { lexer.step(); + if (lexer.code_point == '\n') { + lexer.step(); + } + continue :stringLiteral; } - continue :stringLiteral; - } - }, - // This indicates the end of the file - - -1 => { - lexer.addDefaultError("Unterminated string literal"); - }, + }, + // This indicates the end of the file - '\r' => { - if (quote != '`') { + -1 => { lexer.addDefaultError("Unterminated string literal"); - } + }, - // Template literals require newline normalization - needs_slow_path = true; - }, + '\r' => { + if (quote != '`') { + lexer.addDefaultError("Unterminated string literal"); + } - '\n' => { - if (quote != '`') { - lexer.addDefaultError("Unterminated string literal"); - } - }, + // Template literals require newline normalization + needs_slow_path = true; + }, - '$' => { - if (quote == '`') { - lexer.step(); - if (lexer.code_point == '{') { - suffixLen = 2; + '\n' => { + if (quote != '`') { + lexer.addDefaultError("Unterminated string literal"); + } + }, + + '$' => { + if (quote == '`') { lexer.step(); - if (lexer.rescan_close_brace_as_template_token) { - lexer.token = T.t_template_middle; - } else { - lexer.token = T.t_template_head; + if (lexer.code_point == '{') { + suffixLen = 2; + lexer.step(); + if (lexer.rescan_close_brace_as_template_token) { + lexer.token = T.t_template_middle; + } else { + lexer.token = T.t_template_head; + } + break :stringLiteral; } - break :stringLiteral; + continue :stringLiteral; } - continue :stringLiteral; - } - }, + }, - else => { - if (quote == lexer.code_point) { - lexer.step(); - break :stringLiteral; - } - // Non-ASCII strings need the slow path - if (lexer.code_point >= 0x80) { - needs_slow_path = true; - } else if (IS_JSON_FILE and lexer.code_point < 0x20) { - lexer.syntaxError(); - } - }, + else => { + if (quote == lexer.code_point) { + lexer.step(); + break :stringLiteral; + } + // Non-ASCII strings need the slow path + if (lexer.code_point >= 0x80) { + needs_slow_path = true; + } else if (jsonOptions != null and lexer.code_point < 0x20) { + lexer.syntaxError(); + } + }, + } + lexer.step(); } - lexer.step(); - } - const text = lexer.source.contents[lexer.start + 1 .. lexer.end - suffixLen]; - // TODO: actually implement proper utf16 - lexer.string_literal = lexer.allocator.alloc(u16, text.len) catch unreachable; - var i: usize = 0; - for (text) |byte| { - lexer.string_literal[i] = byte; - i += 1; - } - // for (text) - // // if (needs_slow_path) { - // // // Slow path + const text = lexer.source.contents[lexer.start + 1 .. lexer.end - suffixLen]; + if (needs_slow_path) { + lexer.string_literal = lexer.stringToUTF16(text); + } else { + lexer.string_literal = lexer.allocator.alloc(u16, text.len) catch unreachable; + var i: usize = 0; + for (text) |byte| { + lexer.string_literal[i] = byte; + i += 1; + } + } - // // // lexer.string_literal = lexer.(lexer.start + 1, text); - // // } else { - // // // Fast path + if (quote == '\'' and jsonOptions != null) { + lexer.addRangeError(lexer.range(), "JSON strings must use double quotes", .{}, true); + } + // for (text) + // // if (needs_slow_path) { + // // // Slow path - // // } - } + // // // lexer.string_literal = lexer.(lexer.start + 1, text); + // // } else { + // // // Fast path - fn step(lexer: *Lexer) void { - lexer.code_point = lexer.nextCodepoint(); - - // Track the approximate number of newlines in the file so we can preallocate - // the line offset table in the printer for source maps. The line offset table - // is the #1 highest allocation in the heap profile, so this is worth doing. - // This count is approximate because it handles "\n" and "\r\n" (the common - // cases) but not "\r" or "\u2028" or "\u2029". Getting this wrong is harmless - // because it's only a preallocation. The array will just grow if it's too small. - if (lexer.code_point == '\n') { - lexer.approximate_newline_count += 1; + // // } } - } - pub fn expect(self: *Lexer, comptime token: T) void { - if (self.token != token) { - self.expected(token); + fn step(lexer: *@This()) void { + lexer.code_point = lexer.nextCodepoint(); + + // Track the approximate number of newlines in the file so we can preallocate + // the line offset table in the printer for source maps. The line offset table + // is the #1 highest allocation in the heap profile, so this is worth doing. + // This count is approximate because it handles "\n" and "\r\n" (the common + // cases) but not "\r" or "\u2028" or "\u2029". Getting this wrong is harmless + // because it's only a preallocation. The array will just grow if it's too small. + if (lexer.code_point == '\n') { + lexer.approximate_newline_count += 1; + } } - self.next(); - } + pub fn expect(self: *@This(), comptime token: T) void { + if (self.token != token) { + self.expected(token); + } - pub fn expectOrInsertSemicolon(lexer: *Lexer) void { - if (lexer.token == T.t_semicolon or (!lexer.has_newline_before and - lexer.token != T.t_close_brace and lexer.token != T.t_end_of_file)) - { - lexer.expect(T.t_semicolon); + self.next(); } - } - pub fn addUnsupportedSyntaxError(self: *Lexer, msg: []const u8) void { - self.addError(self.end, "Unsupported syntax: {s}", .{msg}, true); - } + pub fn expectOrInsertSemicolon(lexer: *@This()) void { + if (lexer.token == T.t_semicolon or (!lexer.has_newline_before and + lexer.token != T.t_close_brace and lexer.token != T.t_end_of_file)) + { + lexer.expect(T.t_semicolon); + } + } - pub fn scanIdentifierWithEscapes(self: *Lexer) void { - self.addUnsupportedSyntaxError("escape sequence"); - return; - } + pub fn addUnsupportedSyntaxError(self: *@This(), msg: []const u8) void { + self.addError(self.end, "Unsupported syntax: {s}", .{msg}, true); + } + + pub fn scanIdentifierWithEscapes(self: *@This()) void { + self.addUnsupportedSyntaxError("escape sequence"); + return; + } - pub fn debugInfo(self: *Lexer) void { - if (self.log.errors > 0) { - const stderr = std.io.getStdErr().writer(); - self.log.print(stderr) catch unreachable; - } else { - if (self.token == T.t_identifier or self.token == T.t_string_literal) { - std.debug.print(" {s} ", .{self.raw()}); + pub fn debugInfo(self: *@This()) void { + if (self.log.errors > 0) { + const stderr = std.io.getStdErr().writer(); + self.log.print(stderr) catch unreachable; } else { - std.debug.print(" <{s}> ", .{tokenToString.get(self.token)}); + if (self.token == T.t_identifier or self.token == T.t_string_literal) { + std.debug.print(" {s} ", .{self.raw()}); + } else { + std.debug.print(" <{s}> ", .{tokenToString.get(self.token)}); + } } } - } - pub fn expectContextualKeyword(self: *Lexer, comptime keyword: string) void { - if (!self.isContextualKeyword(keyword)) { - self.addError(self.start, "\"{s}\"", .{keyword}, true); + pub fn expectContextualKeyword(self: *@This(), comptime keyword: string) void { + if (!self.isContextualKeyword(keyword)) { + self.addError(self.start, "\"{s}\"", .{keyword}, true); + } + self.next(); } - self.next(); - } - - pub fn next(lexer: *Lexer) void { - lexer.has_newline_before = lexer.end == 0; - lex: while (lexer.log.errors == 0) { - lexer.start = lexer.end; - lexer.token = T.t_end_of_file; + pub fn next(lexer: *@This()) void { + lexer.has_newline_before = lexer.end == 0; - switch (lexer.code_point) { - -1 => { - lexer.token = T.t_end_of_file; - break :lex; - }, + lex: while (lexer.log.errors == 0) { + lexer.start = lexer.end; + lexer.token = T.t_end_of_file; - '#' => { - if (lexer.start == 0 and lexer.source.contents[1] == '!') { - lexer.addUnsupportedSyntaxError("#!hashbang is not supported yet."); - return; - } + switch (lexer.code_point) { + -1 => { + lexer.token = T.t_end_of_file; + break :lex; + }, - lexer.step(); - if (!isIdentifierStart(lexer.code_point)) { - lexer.syntaxError(); - } - lexer.step(); + '#' => { + if (lexer.start == 0 and lexer.source.contents[1] == '!') { + lexer.addUnsupportedSyntaxError("#!hashbang is not supported yet."); + return; + } - if (isIdentifierStart(lexer.code_point)) { lexer.step(); - while (isIdentifierContinue(lexer.code_point)) { - lexer.step(); - } - if (lexer.code_point == '\\') { - lexer.scanIdentifierWithEscapes(); - lexer.token = T.t_private_identifier; - // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); - } else { - lexer.token = T.t_private_identifier; - lexer.identifier = lexer.raw(); + if (!isIdentifierStart(lexer.code_point)) { + lexer.syntaxError(); } - break; - } - }, - '\r', '\n', 0x2028, 0x2029 => { - lexer.step(); - lexer.has_newline_before = true; - continue; - }, - - '\t', ' ' => { - lexer.step(); - continue; - }, - - '(' => { - lexer.step(); - lexer.token = T.t_open_paren; - }, - ')' => { - lexer.step(); - lexer.token = T.t_close_paren; - }, - '[' => { - lexer.step(); - lexer.token = T.t_open_bracket; - }, - ']' => { - lexer.step(); - lexer.token = T.t_close_bracket; - }, - '{' => { - lexer.step(); - lexer.token = T.t_open_brace; - }, - '}' => { - lexer.step(); - lexer.token = T.t_close_brace; - }, - ',' => { - lexer.step(); - lexer.token = T.t_comma; - }, - ':' => { - lexer.step(); - lexer.token = T.t_colon; - }, - ';' => { - lexer.step(); - lexer.token = T.t_semicolon; - }, - '@' => { - lexer.step(); - lexer.token = T.t_at; - }, - '~' => { - lexer.step(); - lexer.token = T.t_tilde; - }, + lexer.step(); - '?' => { - // '?' or '?.' or '??' or '??=' - lexer.step(); - switch (lexer.code_point) { - '?' => { + if (isIdentifierStart(lexer.code_point)) { lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_question_question_equals; - }, - else => { - lexer.token = T.t_question_question; - }, + while (isIdentifierContinue(lexer.code_point)) { + lexer.step(); } - }, - - '.' => { - lexer.token = T.t_question; - const current = lexer.current; - const contents = lexer.source.contents; - - // Lookahead to disambiguate with 'a?.1:b' - if (current < contents.len) { - const c = contents[current]; - if (c < '0' or c > '9') { - lexer.step(); - lexer.token = T.t_question_dot; - } + if (lexer.code_point == '\\') { + lexer.scanIdentifierWithEscapes(); + lexer.token = T.t_private_identifier; + // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); + } else { + lexer.token = T.t_private_identifier; + lexer.identifier = lexer.raw(); } - }, - else => { - lexer.token = T.t_question; - }, - } - }, - - '%' => { - // '%' or '%=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_percent_equals; - }, + break; + } + }, + '\r', '\n', 0x2028, 0x2029 => { + lexer.step(); + lexer.has_newline_before = true; + continue; + }, + '\t', ' ' => { + lexer.step(); + continue; + }, + '(' => { + lexer.step(); + lexer.token = T.t_open_paren; + }, + ')' => { + lexer.step(); + lexer.token = T.t_close_paren; + }, + '[' => { + lexer.step(); + lexer.token = T.t_open_bracket; + }, + ']' => { + lexer.step(); + lexer.token = T.t_close_bracket; + }, + '{' => { + lexer.step(); + lexer.token = T.t_open_brace; + }, + '}' => { + lexer.step(); + lexer.token = T.t_close_brace; + }, + ',' => { + lexer.step(); + lexer.token = T.t_comma; + }, + ':' => { + lexer.step(); + lexer.token = T.t_colon; + }, + ';' => { + lexer.step(); + lexer.token = T.t_semicolon; + }, + '@' => { + lexer.step(); + lexer.token = T.t_at; + }, + '~' => { + lexer.step(); + lexer.token = T.t_tilde; + }, + '?' => { + // '?' or '?.' or '??' or '??=' + lexer.step(); + switch (lexer.code_point) { + '?' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_question_question_equals; + }, + else => { + lexer.token = T.t_question_question; + }, + } + }, - else => { - lexer.token = T.t_percent; - }, - } - }, + '.' => { + lexer.token = T.t_question; + const current = lexer.current; + const contents = lexer.source.contents; + + // Lookahead to disambiguate with 'a?.1:b' + if (current < contents.len) { + const c = contents[current]; + if (c < '0' or c > '9') { + lexer.step(); + lexer.token = T.t_question_dot; + } + } + }, + else => { + lexer.token = T.t_question; + }, + } + }, + '%' => { + // '%' or '%=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_percent_equals; + }, - '&' => { - // '&' or '&=' or '&&' or '&&=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_ampersand_equals; - }, + else => { + lexer.token = T.t_percent; + }, + } + }, - '&' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_ampersand_ampersand_equals; - }, + '&' => { + // '&' or '&=' or '&&' or '&&=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_ampersand_equals; + }, - else => { - lexer.token = T.t_ampersand_ampersand; - }, - } - }, - else => { - lexer.token = T.t_ampersand; - }, - } - }, + '&' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_ampersand_ampersand_equals; + }, + + else => { + lexer.token = T.t_ampersand_ampersand; + }, + } + }, + else => { + lexer.token = T.t_ampersand; + }, + } + }, - '|' => { + '|' => { - // '|' or '|=' or '||' or '||=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_bar_equals; - }, - '|' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_bar_bar_equals; - }, + // '|' or '|=' or '||' or '||=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_bar_equals; + }, + '|' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_bar_bar_equals; + }, + + else => { + lexer.token = T.t_bar_bar; + }, + } + }, + else => { + lexer.token = T.t_bar; + }, + } + }, - else => { - lexer.token = T.t_bar_bar; - }, - } - }, - else => { - lexer.token = T.t_bar; - }, - } - }, + '^' => { + // '^' or '^=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_caret_equals; + }, - '^' => { - // '^' or '^=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_caret_equals; - }, + else => { + lexer.token = T.t_caret; + }, + } + }, - else => { - lexer.token = T.t_caret; - }, - } - }, + '+' => { + // '+' or '+=' or '++' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_plus_equals; + }, - '+' => { - // '+' or '+=' or '++' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_plus_equals; - }, + '+' => { + lexer.step(); + lexer.token = T.t_plus_plus; + }, - '+' => { - lexer.step(); - lexer.token = T.t_plus_plus; - }, + else => { + lexer.token = T.t_plus; + }, + } + }, - else => { - lexer.token = T.t_plus; - }, - } - }, + '-' => { + // '+' or '+=' or '++' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_minus_equals; + }, - '=' => { - // '=' or '=>' or '==' or '===' - lexer.step(); - switch (lexer.code_point) { - '>' => { - lexer.step(); - lexer.token = T.t_equals_greater_than; - }, + '-' => { + lexer.step(); - '=' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { + if (lexer.code_point == '>' and lexer.has_newline_before) { lexer.step(); - lexer.token = T.t_equals_equals_equals; - }, - - else => { - lexer.token = T.t_equals_equals; - }, - } - }, + lexer.log.addRangeWarning(lexer.source, lexer.range(), "Treating \"-->\" as the start of a legacy HTML single-line comment") catch unreachable; + + singleLineHTMLCloseComment: while (true) { + switch (lexer.code_point) { + '\r', '\n', 0x2028, 0x2029 => { + break :singleLineHTMLCloseComment; + }, + -1 => { + break :singleLineHTMLCloseComment; + }, + else => {}, + } + lexer.step(); + } + continue; + } - else => { - lexer.token = T.t_equals; - }, - } - }, + lexer.token = T.t_minus_minus; + }, - '<' => { - // '<' or '<<' or '<=' or '<<=' or '<!--' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_less_than_equals; - }, + else => { + lexer.token = T.t_plus; + }, + } + }, - '<' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_less_than_less_than_equals; - }, + '*' => { + // '*' or '*=' or '**' or '**=' - else => { - lexer.token = T.t_less_than_less_than; - }, - } - }, - // Handle legacy HTML-style comments - '!' => { - if (std.mem.eql(u8, lexer.peek("--".len), "--")) { - lexer.addUnsupportedSyntaxError("Legacy HTML comments not implemented yet!"); - return; - } - - lexer.token = T.t_less_than; - }, + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = .t_asterisk_equals; + }, + '*' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = .t_asterisk_asterisk_equals; + }, + else => { + lexer.token = .t_asterisk_asterisk; + }, + } + }, + else => { + lexer.token = .t_asterisk; + }, + } + }, + '/' => { + // '/' or '/=' or '//' or '/* ... */' + lexer.step(); + // TODO: forGlobalName - else => { - lexer.token = T.t_less_than; - }, - } - }, + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = .t_slash_equals; + }, + '/' => { + lexer.step(); + singleLineComment: while (true) { + lexer.step(); + switch (lexer.code_point) { + '\r', '\n', 0x2028, 0x2029 => { + break :singleLineComment; + }, + -1 => { + break :singleLineComment; + }, + else => {}, + } + } - '>' => { - // '>' or '>>' or '>>>' or '>=' or '>>=' or '>>>=' - lexer.step(); + if (jsonOptions) |json| { + if (!json.allow_comments) { + lexer.addRangeError(lexer.range(), "JSON does not support comments", .{}, true); + return; + } + } + lexer.scanCommentText(); + continue; + }, + '*' => { + lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_greater_than_equals; - }, - '>' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_greater_than_greater_than_equals; - }, - '>' => { - lexer.step(); + multiLineComment: while (true) { switch (lexer.code_point) { - '=' => { + '*' => { + lexer.step(); + if (lexer.code_point == '/') { + lexer.step(); + break :multiLineComment; + } + }, + '\r', '\n', 0x2028, 0x2029 => { lexer.step(); - lexer.token = T.t_greater_than_greater_than_greater_than_equals; + lexer.has_newline_before = true; + }, + -1 => { + lexer.start = lexer.end; + lexer.addError(lexer.start, "Expected \"*/\" to terminate multi-line comment", .{}, true); }, else => { - lexer.token = T.t_greater_than_greater_than_greater_than; + lexer.step(); }, } - }, - else => { - lexer.token = T.t_greater_than_greater_than; - }, - } - }, - else => { - lexer.token = T.t_greater_than; - }, - } - }, + } + if (jsonOptions) |json| { + if (!json.allow_comments) { + lexer.addRangeError(lexer.range(), "JSON does not support comments", .{}, true); + return; + } + } + lexer.scanCommentText(); + continue; + }, + else => { + lexer.token = .t_slash; + }, + } + }, - '!' => { - // '!' or '!=' or '!==' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_exclamation_equals_equals; - }, + '=' => { + // '=' or '=>' or '==' or '===' + lexer.step(); + switch (lexer.code_point) { + '>' => { + lexer.step(); + lexer.token = T.t_equals_greater_than; + }, - else => { - lexer.token = T.t_exclamation_equals; - }, - } - }, - else => { - lexer.token = T.t_exclamation; - }, - } - }, + '=' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_equals_equals_equals; + }, + + else => { + lexer.token = T.t_equals_equals; + }, + } + }, - '\'', '"', '`' => { - lexer.parseStringLiteral(); - }, + else => { + lexer.token = T.t_equals; + }, + } + }, - '_', '$', 'a'...'z', 'A'...'Z' => { - lexer.step(); - while (isIdentifierContinue(lexer.code_point)) { + '<' => { + // '<' or '<<' or '<=' or '<<=' or '<!--' lexer.step(); - } + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_less_than_equals; + }, - if (lexer.code_point == '\\') { - lexer.scanIdentifierWithEscapes(); - } else { - const contents = lexer.raw(); - lexer.identifier = contents; - if (Keywords.get(contents)) |keyword| { - lexer.token = keyword; - } else { - lexer.token = T.t_identifier; + '<' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_less_than_less_than_equals; + }, + + else => { + lexer.token = T.t_less_than_less_than; + }, + } + }, + // Handle legacy HTML-style comments + '!' => { + if (std.mem.eql(u8, lexer.peek("--".len), "--")) { + lexer.addUnsupportedSyntaxError("Legacy HTML comments not implemented yet!"); + return; + } + + lexer.token = T.t_less_than; + }, + + else => { + lexer.token = T.t_less_than; + }, } - } - }, + }, - '\\' => { - // TODO: normal - lexer.scanIdentifierWithEscapes(); - }, + '>' => { + // '>' or '>>' or '>>>' or '>=' or '>>=' or '>>>=' + lexer.step(); - '.', '0'...'9' => { - lexer.parseNumericLiteralOrDot(); - }, + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_greater_than_equals; + }, + '>' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_greater_than_greater_than_equals; + }, + '>' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_greater_than_greater_than_greater_than_equals; + }, + else => { + lexer.token = T.t_greater_than_greater_than_greater_than; + }, + } + }, + else => { + lexer.token = T.t_greater_than_greater_than; + }, + } + }, + else => { + lexer.token = T.t_greater_than; + }, + } + }, - else => { - // Check for unusual whitespace characters - if (isWhitespace(lexer.code_point)) { + '!' => { + // '!' or '!=' or '!==' lexer.step(); - continue; - } + switch (lexer.code_point) { + '=' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_exclamation_equals_equals; + }, + + else => { + lexer.token = T.t_exclamation_equals; + }, + } + }, + else => { + lexer.token = T.t_exclamation; + }, + } + }, - if (isIdentifierStart(lexer.code_point)) { + '\'', '"', '`' => { + lexer.parseStringLiteral(); + }, + + '_', '$', 'a'...'z', 'A'...'Z' => { lexer.step(); while (isIdentifierContinue(lexer.code_point)) { lexer.step(); } - if (lexer.code_point == '\\') { - // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); + if (lexer.code_point == '\\') { + lexer.scanIdentifierWithEscapes(); } else { - lexer.token = T.t_identifier; - lexer.identifier = lexer.raw(); + const contents = lexer.raw(); + lexer.identifier = contents; + if (Keywords.get(contents)) |keyword| { + lexer.token = keyword; + } else { + lexer.token = T.t_identifier; + } } - break; - } - - lexer.end = lexer.current; - lexer.token = T.t_syntax_error; - }, - } + }, - return; - } - } + '\\' => { + // TODO: normal + lexer.scanIdentifierWithEscapes(); + }, - pub fn expected(self: *Lexer, token: T) void { - if (tokenToString.get(token).len > 0) { - self.expectedString(tokenToString.get(token)); - } else { - self.unexpected(); - } - } + '.', '0'...'9' => { + lexer.parseNumericLiteralOrDot(); + }, - pub fn unexpected(lexer: *Lexer) void { - var found: string = undefined; - if (lexer.start == lexer.source.contents.len) { - found = "end of file"; - } else { - found = lexer.raw(); - } + else => { + // Check for unusual whitespace characters + if (isWhitespace(lexer.code_point)) { + lexer.step(); + continue; + } - lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}, true); - } + if (isIdentifierStart(lexer.code_point)) { + lexer.step(); + while (isIdentifierContinue(lexer.code_point)) { + lexer.step(); + } + if (lexer.code_point == '\\') { - pub fn raw(self: *Lexer) []const u8 { - return self.source.contents[self.start..self.end]; - } + // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); + } else { + lexer.token = T.t_identifier; + lexer.identifier = lexer.raw(); + } + break; + } - pub fn isContextualKeyword(self: *Lexer, comptime keyword: string) bool { - return self.token == .t_identifier and strings.eql(self.raw(), keyword); - } + lexer.end = lexer.current; + lexer.token = T.t_syntax_error; + }, + } - pub fn expectedString(self: *Lexer, text: string) void { - var found = text; - if (self.source.contents.len == self.start) { - found = "end of file"; + return; + } } - self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }, true); - } - - pub fn range(self: *Lexer) logger.Range { - return logger.Range{ - .loc = logger.usize2Loc(self.start), - .len = std.math.lossyCast(i32, self.end - self.start), - }; - } - pub fn init(log: logger.Log, source: logger.Source, allocator: *std.mem.Allocator) !Lexer { - var empty_string_literal: JavascriptString = undefined; - var lex = Lexer{ - .log = log, - .source = source, - .string_literal = empty_string_literal, - .prev_error_loc = logger.Loc.Empty, - .allocator = allocator, - }; - lex.step(); - lex.next(); - - return lex; - } + pub fn expected(self: *@This(), token: T) void { + if (tokenToString.get(token).len > 0) { + self.expectedString(tokenToString.get(token)); + } else { + self.unexpected(); + } + } - pub fn scanRegExp(lexer: *Lexer) void { - while (true) { - switch (lexer.code_point) { - '/' => { - lexer.step(); - while (isIdentifierContinue(lexer.code_point)) { - switch (lexer.code_point) { - 'g', 'i', 'm', 's', 'u', 'y' => { - lexer.step(); - }, - else => { - lexer.syntaxError(); - }, - } - } - }, - '[' => { - lexer.step(); - while (lexer.code_point != ']') { - lexer.scanRegExpValidateAndStep(); - } - lexer.step(); - }, - else => { - lexer.scanRegExpValidateAndStep(); - }, + pub fn unexpected(lexer: *@This()) void { + var found: string = undefined; + if (lexer.start == lexer.source.contents.len) { + found = "end of file"; + } else { + found = lexer.raw(); } + + lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}, true); } - } - // TODO: use wtf-8 encoding. - pub fn stringToUTF16(lexer: *Lexer, str: string) JavascriptString { - var buf: JavascriptString = lexer.allocator.alloc(u16, std.mem.len(str)) catch unreachable; - var i: usize = 0; - // theres prob a faster/better way - for (str) |char| { - buf[i] = char; - i += 1; + pub fn raw(self: *@This()) []const u8 { + return self.source.contents[self.start..self.end]; } - return buf; - } + pub fn isContextualKeyword(self: *@This(), comptime keyword: string) bool { + return self.token == .t_identifier and strings.eql(self.raw(), keyword); + } - // TODO: use wtf-8 encoding. - pub fn utf16ToStringWithValidation(lexer: *Lexer, js: JavascriptString) !string { - return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js); - } + pub fn expectedString(self: *@This(), text: string) void { + var found = text; + if (self.source.contents.len == self.start) { + found = "end of file"; + } + self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }, true); + } - // TODO: use wtf-8 encoding. - pub fn utf16ToString(lexer: *Lexer, js: JavascriptString) string { - return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js) catch unreachable; - } + pub fn scanCommentText(lexer: *@This()) void { + var text = lexer.source.contents[lexer.start..lexer.end]; + const has_preserve_annotation = text.len > 2 and text[2] == '!'; + const is_multiline_comment = text[1] == '*'; - pub fn nextInsideJSXElement() void { - std.debug.panic("JSX not implemented yet.", .{}); - } + // Omit the trailing "*/" from the checks below + var endCommentText = text.len; + if (is_multiline_comment) { + endCommentText -= 2; + } - fn scanRegExpValidateAndStep(lexer: *Lexer) void { - if (lexer.code_point == '\\') { - lexer.step(); - } + if (has_preserve_annotation or lexer.preserve_all_comments_before) { + if (is_multiline_comment) { + // text = lexer.removeMultilineCommentIndent(lexer.source.contents[0..lexer.start], text); + } - switch (lexer.code_point) { - '\r', '\n', 0x2028, 0x2029 => { - // Newlines aren't allowed in regular expressions - lexer.syntaxError(); - }, - -1 => { // EOF - lexer.syntaxError(); - }, - else => { - lexer.step(); - }, + lexer.comments_to_preserve_before.append(js_ast.G.Comment{ + .text = text, + .loc = lexer.loc(), + }) catch unreachable; + } } - } - pub fn rescanCloseBraceAsTemplateToken(lexer: *Lexer) void { - if (lexer.token != .t_close_brace) { - lexer.expected(.t_close_brace); + // TODO: implement this + // it's too complicated to handle all the edgecases right now given the state of Zig's standard library + pub fn removeMultilineCommentIndent(lexer: *@This(), _prefix: string, text: string) string { + return text; } - lexer.rescan_close_brace_as_template_token = true; - lexer.code_point = '`'; - lexer.current = lexer.end; - lexer.end -= 1; - lexer.next(); - lexer.rescan_close_brace_as_template_token = false; - } - - pub fn rawTemplateContents(lexer: *Lexer) string { - var text: string = undefined; - - switch (lexer.token) { - .t_no_substitution_template_literal, .t_template_tail => { - text = lexer.source.contents[lexer.start + 1 .. lexer.end - 1]; - }, - .t_template_middle, .t_template_head => { - text = lexer.source.contents[lexer.start + 1 .. lexer.end - 2]; - }, - else => {}, + pub fn range(self: *@This()) logger.Range { + return logger.Range{ + .loc = logger.usize2Loc(self.start), + .len = std.math.lossyCast(i32, self.end - self.start), + }; } - if (strings.indexOfChar(text, '\r') == null) { - return text; + pub fn init(log: logger.Log, source: logger.Source, allocator: *std.mem.Allocator) !@This() { + var empty_string_literal: JavascriptString = undefined; + var lex = @This(){ + .log = log, + .source = source, + .string_literal = empty_string_literal, + .prev_error_loc = logger.Loc.Empty, + .allocator = allocator, + .comments_to_preserve_before = std.ArrayList(js_ast.G.Comment).init(allocator), + }; + lex.step(); + lex.next(); + + return lex; } - // From the specification: - // - // 11.8.6.1 Static Semantics: TV and TRV - // - // TV excludes the code units of LineContinuation while TRV includes - // them. <CR><LF> and <CR> LineTerminatorSequences are normalized to - // <LF> for both TV and TRV. An explicit EscapeSequence is needed to - // include a <CR> or <CR><LF> sequence. - var bytes = MutableString.initCopy(lexer.allocator, text) catch unreachable; - var end: usize = 0; - var i: usize = 0; - var c: u8 = '0'; - while (i < bytes.list.items.len) { - c = bytes.list.items[i]; - i += 1; - - if (c == '\r') { - // Convert '\r\n' into '\n' - if (i < bytes.list.items.len and bytes.list.items[i] == '\n') { - i += 1; + pub fn scanRegExp(lexer: *@This()) void { + while (true) { + switch (lexer.code_point) { + '/' => { + lexer.step(); + while (isIdentifierContinue(lexer.code_point)) { + switch (lexer.code_point) { + 'g', 'i', 'm', 's', 'u', 'y' => { + lexer.step(); + }, + else => { + lexer.syntaxError(); + }, + } + } + }, + '[' => { + lexer.step(); + while (lexer.code_point != ']') { + lexer.scanRegExpValidateAndStep(); + } + lexer.step(); + }, + else => { + lexer.scanRegExpValidateAndStep(); + }, } + } + } - // Convert '\r' into '\n' - c = '\n'; + // TODO: use wtf-8 encoding. + pub fn stringToUTF16(lexer: *@This(), str: string) JavascriptString { + var buf: JavascriptString = lexer.allocator.alloc(u16, std.mem.len(str)) catch unreachable; + var i: usize = 0; + // theres prob a faster/better way + for (str) |char| { + buf[i] = char; + i += 1; } - bytes.list.items[end] = c; - end += 1; + return buf; } - return bytes.toOwnedSliceLength(end + 1); - } - - fn parseNumericLiteralOrDot(lexer: *Lexer) void { - // Number or dot; - var first = lexer.code_point; - lexer.step(); - - // Dot without a digit after it; - if (first == '.' and (lexer.code_point < '0' or lexer.code_point > '9')) { - // "..." - if ((lexer.code_point == '.' and - lexer.current < lexer.source.contents.len) and - lexer.source.contents[lexer.current] == '.') - { - lexer.step(); - lexer.step(); - lexer.token = T.t_dot_dot_dot; - return; - } + // TODO: use wtf-8 encoding. + pub fn utf16ToStringWithValidation(lexer: *@This(), js: JavascriptString) !string { + return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js); + } - // "." - lexer.token = T.t_dot; - return; + // TODO: use wtf-8 encoding. + pub fn utf16ToString(lexer: *@This(), js: JavascriptString) string { + return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js) catch unreachable; } - var underscoreCount: usize = 0; - var lastUnderscoreEnd: usize = 0; - var hasDotOrExponent = first == '.'; - var base: f32 = 0.0; - lexer.is_legacy_octal_literal = false; + pub fn nextInsideJSXElement() void { + std.debug.panic("JSX not implemented yet.", .{}); + } - // Assume this is a number, but potentially change to a bigint later; - lexer.token = T.t_numeric_literal; + fn scanRegExpValidateAndStep(lexer: *@This()) void { + if (lexer.code_point == '\\') { + lexer.step(); + } - // Check for binary, octal, or hexadecimal literal; - if (first == '0') { switch (lexer.code_point) { - 'b', 'B' => { - base = 2; + '\r', '\n', 0x2028, 0x2029 => { + // Newlines aren't allowed in regular expressions + lexer.syntaxError(); }, - - 'o', 'O' => { - base = 8; + -1 => { // EOF + lexer.syntaxError(); }, - - 'x', 'X' => { - base = 16; + else => { + lexer.step(); }, + } + } + + pub fn rescanCloseBraceAsTemplateToken(lexer: *@This()) void { + if (lexer.token != .t_close_brace) { + lexer.expected(.t_close_brace); + } - '0'...'7', '_' => { - base = 8; - lexer.is_legacy_octal_literal = true; + lexer.rescan_close_brace_as_template_token = true; + lexer.code_point = '`'; + lexer.current = lexer.end; + lexer.end -= 1; + lexer.next(); + lexer.rescan_close_brace_as_template_token = false; + } + + pub fn rawTemplateContents(lexer: *@This()) string { + var text: string = undefined; + + switch (lexer.token) { + .t_no_substitution_template_literal, .t_template_tail => { + text = lexer.source.contents[lexer.start + 1 .. lexer.end - 1]; + }, + .t_template_middle, .t_template_head => { + text = lexer.source.contents[lexer.start + 1 .. lexer.end - 2]; }, else => {}, } + + if (strings.indexOfChar(text, '\r') == null) { + return text; + } + + // From the specification: + // + // 11.8.6.1 Static Semantics: TV and TRV + // + // TV excludes the code units of LineContinuation while TRV includes + // them. <CR><LF> and <CR> LineTerminatorSequences are normalized to + // <LF> for both TV and TRV. An explicit EscapeSequence is needed to + // include a <CR> or <CR><LF> sequence. + var bytes = MutableString.initCopy(lexer.allocator, text) catch unreachable; + var end: usize = 0; + var i: usize = 0; + var c: u8 = '0'; + while (i < bytes.list.items.len) { + c = bytes.list.items[i]; + i += 1; + + if (c == '\r') { + // Convert '\r\n' into '\n' + if (i < bytes.list.items.len and bytes.list.items[i] == '\n') { + i += 1; + } + + // Convert '\r' into '\n' + c = '\n'; + } + + bytes.list.items[end] = c; + end += 1; + } + + return bytes.toOwnedSliceLength(end + 1); } - if (base != 0) { - // Integer literal; - var isFirst = true; - var isInvalidLegacyOctalLiteral = false; - lexer.number = 0; - if (!lexer.is_legacy_octal_literal) { - lexer.step(); + fn parseNumericLiteralOrDot(lexer: *@This()) void { + // Number or dot; + var first = lexer.code_point; + lexer.step(); + + // Dot without a digit after it; + if (first == '.' and (lexer.code_point < '0' or lexer.code_point > '9')) { + // "..." + if ((lexer.code_point == '.' and + lexer.current < lexer.source.contents.len) and + lexer.source.contents[lexer.current] == '.') + { + lexer.step(); + lexer.step(); + lexer.token = T.t_dot_dot_dot; + return; + } + + // "." + lexer.token = T.t_dot; + return; } - integerLiteral: while (true) { - switch (lexer.code_point) { - '_' => { - // Cannot have multiple underscores in a row; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.syntaxError(); - } + var underscoreCount: usize = 0; + var lastUnderscoreEnd: usize = 0; + var hasDotOrExponent = first == '.'; + var base: f32 = 0.0; + lexer.is_legacy_octal_literal = false; - // The first digit must exist; - if (isFirst or lexer.is_legacy_octal_literal) { - lexer.syntaxError(); - } + // Assume this is a number, but potentially change to a bigint later; + lexer.token = T.t_numeric_literal; - lastUnderscoreEnd = lexer.end; - underscoreCount += 1; + // Check for binary, octal, or hexadecimal literal; + if (first == '0') { + switch (lexer.code_point) { + 'b', 'B' => { + base = 2; }, - '0', '1' => { - lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + 'o', 'O' => { + base = 8; }, - '2', '3', '4', '5', '6', '7' => { - if (base == 2) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point - '0'); - }, - '8', '9' => { - if (lexer.is_legacy_octal_literal) { - isInvalidLegacyOctalLiteral = true; - } else if (base < 10) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point - '0'); - }, - 'A', 'B', 'C', 'D', 'E', 'F' => { - if (base != 16) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'A'); + 'x', 'X' => { + base = 16; }, - 'a', 'b', 'c', 'd', 'e', 'f' => { - if (base != 16) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'a'); + '0'...'7', '_' => { + base = 8; + lexer.is_legacy_octal_literal = true; }, - else => { - // The first digit must exist; - if (isFirst) { - lexer.syntaxError(); - } + else => {}, + } + } - break :integerLiteral; - }, + if (base != 0) { + // Integer literal; + var isFirst = true; + var isInvalidLegacyOctalLiteral = false; + lexer.number = 0; + if (!lexer.is_legacy_octal_literal) { + lexer.step(); } - lexer.step(); - isFirst = false; - } + integerLiteral: while (true) { + switch (lexer.code_point) { + '_' => { + // Cannot have multiple underscores in a row; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.syntaxError(); + } - var isBigIntegerLiteral = lexer.code_point == 'n' and !hasDotOrExponent; + // The first digit must exist; + if (isFirst or lexer.is_legacy_octal_literal) { + lexer.syntaxError(); + } - // Slow path: do we need to re-scan the input as text? - if (isBigIntegerLiteral or isInvalidLegacyOctalLiteral) { - var text = lexer.raw(); + lastUnderscoreEnd = lexer.end; + underscoreCount += 1; + }, - // Can't use a leading zero for bigint literals; - if (isBigIntegerLiteral and lexer.is_legacy_octal_literal) { - lexer.syntaxError(); - } + '0', '1' => { + lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + }, - // Filter out underscores; - if (underscoreCount > 0) { - var bytes = lexer.allocator.alloc(u8, text.len - underscoreCount) catch unreachable; - var i: usize = 0; - for (text) |char| { - if (char != '_') { - bytes[i] = char; - i += 1; - } - } - } + '2', '3', '4', '5', '6', '7' => { + if (base == 2) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + }, + '8', '9' => { + if (lexer.is_legacy_octal_literal) { + isInvalidLegacyOctalLiteral = true; + } else if (base < 10) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + }, + 'A', 'B', 'C', 'D', 'E', 'F' => { + if (base != 16) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'A'); + }, - // Store bigints as text to avoid precision loss; - if (isBigIntegerLiteral) { - lexer.identifier = text; - } else if (isInvalidLegacyOctalLiteral) { - if (std.fmt.parseFloat(f64, text)) |num| { - lexer.number = num; - } else |err| { - lexer.addError(lexer.start, "Invalid number {s}", .{text}, true); + 'a', 'b', 'c', 'd', 'e', 'f' => { + if (base != 16) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'a'); + }, + else => { + // The first digit must exist; + if (isFirst) { + lexer.syntaxError(); + } + + break :integerLiteral; + }, } + + lexer.step(); + isFirst = false; } - } - } else { - // Floating-point literal; - var isInvalidLegacyOctalLiteral = first == '0' and (lexer.code_point == '8' or lexer.code_point == '9'); - // Initial digits; - while (true) { - if (lexer.code_point < '0' or lexer.code_point > '9') { - if (lexer.code_point != '_') { - break; - } + var isBigIntegerLiteral = lexer.code_point == 'n' and !hasDotOrExponent; - // Cannot have multiple underscores in a row; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.syntaxError(); - } + // Slow path: do we need to re-scan the input as text? + if (isBigIntegerLiteral or isInvalidLegacyOctalLiteral) { + var text = lexer.raw(); - // The specification forbids underscores in this case; - if (isInvalidLegacyOctalLiteral) { + // Can't use a leading zero for bigint literals; + if (isBigIntegerLiteral and lexer.is_legacy_octal_literal) { lexer.syntaxError(); } - lastUnderscoreEnd = lexer.end; - underscoreCount += 1; - } - lexer.step(); - } + // Filter out underscores; + if (underscoreCount > 0) { + var bytes = lexer.allocator.alloc(u8, text.len - underscoreCount) catch unreachable; + var i: usize = 0; + for (text) |char| { + if (char != '_') { + bytes[i] = char; + i += 1; + } + } + } - // Fractional digits; - if (first != '.' and lexer.code_point == '.') { - // An underscore must not come last; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.end -= 1; - lexer.syntaxError(); + // Store bigints as text to avoid precision loss; + if (isBigIntegerLiteral) { + lexer.identifier = text; + } else if (isInvalidLegacyOctalLiteral) { + if (std.fmt.parseFloat(f64, text)) |num| { + lexer.number = num; + } else |err| { + lexer.addError(lexer.start, "Invalid number {s}", .{text}, true); + } + } } + } else { + // Floating-point literal; + var isInvalidLegacyOctalLiteral = first == '0' and (lexer.code_point == '8' or lexer.code_point == '9'); - hasDotOrExponent = true; - lexer.step(); - if (lexer.code_point == '_') { - lexer.syntaxError(); - } + // Initial digits; while (true) { if (lexer.code_point < '0' or lexer.code_point > '9') { if (lexer.code_point != '_') { @@ -1150,110 +1291,151 @@ pub const Lexer = struct { lexer.syntaxError(); } + // The specification forbids underscores in this case; + if (isInvalidLegacyOctalLiteral) { + lexer.syntaxError(); + } + lastUnderscoreEnd = lexer.end; underscoreCount += 1; } lexer.step(); } - } - // Exponent; - if (lexer.code_point == 'e' or lexer.code_point == 'E') { - // An underscore must not come last; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.end -= 1; - lexer.syntaxError(); - } + // Fractional digits; + if (first != '.' and lexer.code_point == '.') { + // An underscore must not come last; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.end -= 1; + lexer.syntaxError(); + } - hasDotOrExponent = true; - lexer.step(); - if (lexer.code_point == '+' or lexer.code_point == '-') { + hasDotOrExponent = true; lexer.step(); - } - if (lexer.code_point < '0' or lexer.code_point > '9') { - lexer.syntaxError(); - } - while (true) { - if (lexer.code_point < '0' or lexer.code_point > '9') { - if (lexer.code_point != '_') { - break; - } + if (lexer.code_point == '_') { + lexer.syntaxError(); + } + while (true) { + if (lexer.code_point < '0' or lexer.code_point > '9') { + if (lexer.code_point != '_') { + break; + } - // Cannot have multiple underscores in a row; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.syntaxError(); + // Cannot have multiple underscores in a row; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.syntaxError(); + } + + lastUnderscoreEnd = lexer.end; + underscoreCount += 1; } + lexer.step(); + } + } - lastUnderscoreEnd = lexer.end; - underscoreCount += 1; + // Exponent; + if (lexer.code_point == 'e' or lexer.code_point == 'E') { + // An underscore must not come last; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.end -= 1; + lexer.syntaxError(); } + + hasDotOrExponent = true; lexer.step(); - } - } + if (lexer.code_point == '+' or lexer.code_point == '-') { + lexer.step(); + } + if (lexer.code_point < '0' or lexer.code_point > '9') { + lexer.syntaxError(); + } + while (true) { + if (lexer.code_point < '0' or lexer.code_point > '9') { + if (lexer.code_point != '_') { + break; + } - // Take a slice of the text to parse; - var text = lexer.raw(); + // Cannot have multiple underscores in a row; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.syntaxError(); + } - // Filter out underscores; - if (underscoreCount > 0) { - var i: usize = 0; - if (lexer.allocator.alloc(u8, text.len - underscoreCount)) |bytes| { - for (text) |char| { - if (char != '_') { - bytes[i] = char; - i += 1; + lastUnderscoreEnd = lexer.end; + underscoreCount += 1; } + lexer.step(); } - text = bytes; - } else |err| { - lexer.addError(lexer.start, "Out of Memory Wah Wah Wah", .{}, true); - return; } - } - if (lexer.code_point == 'n' and !hasDotOrExponent) { - // The only bigint literal that can start with 0 is "0n" - if (text.len > 1 and first == '0') { - lexer.syntaxError(); - } + // Take a slice of the text to parse; + var text = lexer.raw(); - // Store bigints as text to avoid precision loss; - lexer.identifier = text; - } else if (!hasDotOrExponent and lexer.end - lexer.start < 10) { - // Parse a 32-bit integer (very fast path); - var number: u32 = 0; - for (text) |c| { - number = number * 10 + @intCast(u32, c - '0'); + // Filter out underscores; + if (underscoreCount > 0) { + var i: usize = 0; + if (lexer.allocator.alloc(u8, text.len - underscoreCount)) |bytes| { + for (text) |char| { + if (char != '_') { + bytes[i] = char; + i += 1; + } + } + text = bytes; + } else |err| { + lexer.addError(lexer.start, "Out of Memory Wah Wah Wah", .{}, true); + return; + } } - lexer.number = @intToFloat(f64, number); - } else { - // Parse a double-precision floating-point number; - if (std.fmt.parseFloat(f64, text)) |num| { - lexer.number = num; - } else |err| { - lexer.addError(lexer.start, "Invalid number", .{}, true); + + if (lexer.code_point == 'n' and !hasDotOrExponent) { + // The only bigint literal that can start with 0 is "0n" + if (text.len > 1 and first == '0') { + lexer.syntaxError(); + } + + // Store bigints as text to avoid precision loss; + lexer.identifier = text; + } else if (!hasDotOrExponent and lexer.end - lexer.start < 10) { + // Parse a 32-bit integer (very fast path); + var number: u32 = 0; + for (text) |c| { + number = number * 10 + @intCast(u32, c - '0'); + } + lexer.number = @intToFloat(f64, number); + } else { + // Parse a double-precision floating-point number; + if (std.fmt.parseFloat(f64, text)) |num| { + lexer.number = num; + } else |err| { + lexer.addError(lexer.start, "Invalid number", .{}, true); + } } } - } - // An underscore must not come last; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.end -= 1; - lexer.syntaxError(); - } + // An underscore must not come last; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.end -= 1; + lexer.syntaxError(); + } - // Handle bigint literals after the underscore-at-end check above; - if (lexer.code_point == 'n' and !hasDotOrExponent) { - lexer.token = T.t_big_integer_literal; - lexer.step(); - } + // Handle bigint literals after the underscore-at-end check above; + if (lexer.code_point == 'n' and !hasDotOrExponent) { + lexer.token = T.t_big_integer_literal; + lexer.step(); + } - // Identifiers can't occur immediately after numbers; - if (isIdentifierStart(lexer.code_point)) { - lexer.syntaxError(); + // Identifiers can't occur immediately after numbers; + if (isIdentifierStart(lexer.code_point)) { + lexer.syntaxError(); + } } - } -}; + }; +} + +// JS/TS lexer +pub const Lexer = NewLexerType(null); +pub const JSONLexer = NewLexerType(JSONOptions{ .allow_comments = false, .allow_trailing_commas = false }); +pub const TSConfigJSONLexer = NewLexerType(JSONOptions{ .allow_comments = true, .allow_trailing_commas = true }); fn isIdentifierStart(codepoint: CodePoint) bool { switch (codepoint) { @@ -1309,10 +1491,10 @@ fn isWhitespace(codepoint: CodePoint) bool { 0x202F, // narrow no-break space 0x205F, // medium mathematical space 0x3000, // ideographic space - 0xFEFF, + 0xFEFF, // zero width non-breaking space => { return true; - }, // zero width non-breaking space + }, else => { return false; }, @@ -1346,7 +1528,7 @@ fn float64(num: anytype) callconv(.Inline) f64 { return @intToFloat(f64, num); } -fn test_lexer(contents: []const u8) Lexer { +fn test_lexer(contents: []const u8) @This() { alloc.setup(std.heap.page_allocator) catch unreachable; const msgs = std.ArrayList(logger.Msg).init(alloc.dynamic); const log = logger.Log{ @@ -1357,10 +1539,10 @@ fn test_lexer(contents: []const u8) Lexer { "index.js", contents, ); - return Lexer.init(log, source, alloc.dynamic) catch unreachable; + return @This().init(log, source, alloc.dynamic) catch unreachable; } -// test "Lexer.next()" { +// test "@This().next()" { // try alloc.setup(std.heap.page_allocator); // const msgs = std.ArrayList(logger.Msg).init(alloc.dynamic); // const log = logger.Log{ @@ -1368,7 +1550,7 @@ fn test_lexer(contents: []const u8) Lexer { // }; // const source = logger.Source.initPathString("index.js", "for (let i = 0; i < 100; i++) { console.log('hi'); }", std.heap.page_allocator); -// var lex = try Lexer.init(log, source, alloc.dynamic); +// var lex = try @This().init(log, source, alloc.dynamic); // lex.next(); // } @@ -1411,7 +1593,7 @@ pub fn test_stringLiteralEquals(expected: string, source_text: string) void { source_text, ); - var lex = try Lexer.init(log, source, std.heap.page_allocator); + var lex = try @This().init(log, source, std.heap.page_allocator); while (!lex.token.isString() and lex.token != .t_end_of_file) { lex.next(); } @@ -1420,7 +1602,7 @@ pub fn test_stringLiteralEquals(expected: string, source_text: string) void { std.testing.expectEqualStrings(expected, lit); } -pub fn test_skipTo(lexer: *Lexer, n: string) void { +pub fn test_skipTo(lexer: *@This(), n: string) void { var i: usize = 0; while (i < n.len) { lexer.next(); @@ -1428,7 +1610,7 @@ pub fn test_skipTo(lexer: *Lexer, n: string) void { } } -test "Lexer.rawTemplateContents" { +test "@This().rawTemplateContents" { test_stringLiteralEquals("hello!", "const a = 'hello!';"); test_stringLiteralEquals("hello!hi", "const b = 'hello!hi';"); test_stringLiteralEquals("hello!\n\nhi", "const b = `hello!\n\nhi`;"); diff --git a/src/js_parser.zig b/src/js_parser.zig index e7692c76f..460e0d275 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -3173,12 +3173,10 @@ const P = struct { var isDirectivePrologue = true; run: while (true) { - if (p.lexer.comments_to_preserve_before) |comments| { - for (comments) |comment| { - try stmts.append(p.s(S.Comment{ - .text = comment.text, - }, p.lexer.loc())); - } + for (p.lexer.comments_to_preserve_before.items) |comment| { + try stmts.append(p.s(S.Comment{ + .text = comment.text, + }, p.lexer.loc())); } if (p.lexer.token == .t_end_of_file) { @@ -5737,14 +5735,14 @@ const P = struct { p.lexer.preserve_all_comments_before = true; p.lexer.expect(.t_open_paren); - const comments = p.lexer.comments_to_preserve_before; + const comments = p.lexer.comments_to_preserve_before.toOwnedSlice(); p.lexer.preserve_all_comments_before = false; const value = p.parseExpr(.comma); p.lexer.expect(.t_close_paren); p.allow_in = old_allow_in; - return p.e(E.Import{ .expr = value, .leading_interior_comments = comments orelse &([_]G.Comment{}), .import_record_index = 0 }, loc); + return p.e(E.Import{ .expr = value, .leading_interior_comments = comments, .import_record_index = 0 }, loc); } pub fn parseJSXElement(loc: logger.Loc) Expr { @@ -5866,7 +5864,7 @@ const P = struct { switch (expr.data) { .e_null, .e_super, .e_boolean, .e_big_int, .e_reg_exp, .e_new_target, .e_undefined => {}, .e_string => |e_| { - // idc about legacy octal loc + // If you're using this, you're probably not using 0-prefixed legacy octal notation // if e.LegacyOctalLoc.Start > 0 { }, .e_number => |e_| { @@ -6013,27 +6011,27 @@ const P = struct { switch (data.value) { .expr => |*expr| { const was_anonymous_named_expr = expr.isAnonymousNamed(); - data.value.expr = p.m(p.visitExpr(expr.*)); - - // Optionally preserve the name - data.value.expr = p.maybeKeepExprSymbolName(expr, "default", was_anonymous_named_expr); - - // Discard type-only export default statements - if (p.options.ts) { - switch (expr.data) { - .e_identifier => |ident| { - const symbol = p.symbols.items[ident.ref.inner_index]; - if (symbol.kind == .unbound) { - if (p.local_type_names.get(symbol.original_name)) |local_type| { - if (local_type.value) { - return; - } - } - } - }, - else => {}, - } - } + // data.value.expr = p.m(p.visitExpr(expr.*)); + + // // Optionally preserve the name + // data.value.expr = p.maybeKeepExprSymbolName(expr, "default", was_anonymous_named_expr); + + // // Discard type-only export default statements + // if (p.options.ts) { + // switch (expr.data) { + // .e_identifier => |ident| { + // const symbol = p.symbols.items[ident.ref.inner_index]; + // if (symbol.kind == .unbound) { + // if (p.local_type_names.get(symbol.original_name)) |local_type| { + // if (local_type.value) { + // return; + // } + // } + // } + // }, + // else => {}, + // } + // } }, .stmt => |st| {}, diff --git a/src/json_parser.zig b/src/json_parser.zig new file mode 100644 index 000000000..8a025864f --- /dev/null +++ b/src/json_parser.zig @@ -0,0 +1,212 @@ +const std = @import("std"); +const logger = @import("logger.zig"); +const js_lexer = @import("js_lexer.zig"); +const importRecord = @import("import_record.zig"); +const js_ast = @import("js_ast.zig"); +const options = @import("options.zig"); +const alloc = @import("alloc.zig"); + +const fs = @import("fs.zig"); +usingnamespace @import("strings.zig"); +usingnamespace @import("ast/base.zig"); +usingnamespace js_ast.G; + +const ImportKind = importRecord.ImportKind; +const BindingNodeIndex = js_ast.BindingNodeIndex; + +const StmtNodeIndex = js_ast.StmtNodeIndex; +const ExprNodeIndex = js_ast.ExprNodeIndex; +const ExprNodeList = js_ast.ExprNodeList; +const StmtNodeList = js_ast.StmtNodeList; +const BindingNodeList = js_ast.BindingNodeList; +const assert = std.debug.assert; + +const Ref = js_ast.Ref; +const LocRef = js_ast.LocRef; +const S = js_ast.S; +const B = js_ast.B; +const G = js_ast.G; +const T = js_lexer.T; +const E = js_ast.E; +const Stmt = js_ast.Stmt; +const Expr = js_ast.Expr; +const Binding = js_ast.Binding; +const Symbol = js_ast.Symbol; +const Level = js_ast.Op.Level; +const Op = js_ast.Op; +const Scope = js_ast.Scope; +const locModuleScope = logger.Loc.Empty; + +fn JSONLikeParser(opts: js_lexer.JSONOptions) type { + const Lexer = if (opts.allow_comments) js_lexer.TSConfigJSONLexer else js_lexer.JSONLexer; + return struct { + lexer: Lexer, + source: logger.Source, + log: logger.Log, + allocator: *std.mem.Allocator, + + pub fn init(allocator: *std.mem.Allocator, source: logger.Source, log: logger.Log) Parser { + return Parser{ + .lexer = Lexer.init(log, source, allocator), + .allocator = allocator, + .log = log, + .source = source, + }; + } + + const Parser = @This(); + + pub fn e(p: *Parser, t: anytype, loc: logger.Loc) Expr { + if (@typeInfo(@TypeOf(t)) == .Pointer) { + return Expr.init(t, loc); + } else { + return Expr.alloc(p.allocator, t, loc); + } + } + pub fn parseExpr(p: *Parser) ?Expr { + const loc = p.lexer.loc(); + + switch (p.lexer.token) { + .t_false => { + p.lexer.next(); + return p.e(E.Boolean{ + .value = false, + }, loc); + }, + .t_true => { + p.lexer.next(); + return p.e(E.Boolean{ + .value = true, + }, loc); + }, + .t_null => { + p.lexer.next(); + return p.e(E.Null{}, loc); + }, + .t_string_literal => { + const value = p.lexer.string_literal; + p.lexer.next(); + return p.e(E.String{ + .value = value, + }, loc); + }, + .t_numeric_literal => { + const value = p.lexer.number; + p.lexer.next(); + return p.e(E.Number{ .value = value }, loc); + }, + .t_minus => { + p.lexer.next(); + const value = p.lexer.number; + p.lexer.expect(.t_numeric_literal); + return p.e(E.Number{ .value = -value }, loc); + }, + .t_open_bracket => { + p.lexer.next(); + var is_single_line = !p.lexer.has_newline_before; + var exprs = List(Expr).init(p.allocator); + + while (p.lexer.token != .t_close_bracket) { + if (exprs.items.len > 0) { + if (p.lexer.has_newline_before) { + is_single_line = false; + } + + if (!p.parseMaybeTrailingComma(.t_close_bracket)) { + break; + } + + if (p.lexer.has_newline_before) { + is_single_line = false; + } + } + + if (p.parseExpr()) |expr| { + try exprs.append(expr); + } else { + break; + } + } + + if (p.lexer.has_newline_before) { + is_single_line = false; + } + p.lexer.expect(.t_close_bracket); + return p.e(E.Array{ .items = exprs.toOwnedSlice() }, loc); + }, + .t_open_brace => { + p.lexer.next(); + var is_single_line = !p.lexer.has_newline_before; + var properties = List(G.Property).init(p.allocator); + var duplicates = std.StringHashMap(u0).init(p.allocator); + + while (p.lexer.token != .t_close_brace) { + if (properties.items.len > 0) { + is_single_line = if (p.lexer.has_newline_before) false else is_single_line; + if (!p.parseMaybeTrailingComma(.t_close_brace)) { + break; + } + is_single_line = if (p.lexer.has_newline_before) false else is_single_line; + } + + var key_string = p.lexer.string_literal; + var key_range = p.lexer.range(); + var key = p.e(E.String{ .value = key_string }, key_range.loc); + p.lexer.expect(.t_string_literal); + var key_text = p.lexer.utf16ToString(); + // Warn about duplicate keys + if (duplicates.contains(key_text)) { + p.log.addRangeWarningFmt(p.source, r, "Duplicate key \"{s}\" in object literal", .{key_text}) catch unreachable; + } else { + duplicates.put(key_text, 0) catch unreachable; + } + + p.lexer.expect(.t_colon); + var value = p.parseExpr() orelse return null; + try properties.append(G.Property{ .key = key, .value = value }); + } + + is_single_line = if (p.lexer.has_newline_before) false else is_single_line; + p.lexer.expect(.t_close_brace); + return p.e(E.Object{ + .properties = properties.toOwnedSlice(), + .is_single_line = is_single_line, + }, loc); + }, + else => { + p.lexer.unexpected(); + return null; + }, + } + } + + pub fn parseMaybeTrailingComma(p: *Parser, closer: T) bool { + const comma_range = p.lexer.range(); + p.lexer.expect(.t_comma); + + if (p.lexer.token == closer) { + if (!opts.allow_trailing_commas) { + p.log.addRangeError(p.source, comma_range, "JSON does not support trailing commas") catch unreachable; + } + return false; + } + + return true; + } + }; +} + +const JSONParser = JSONLikeParser(js_lexer.JSONOptions{}); +const TSConfigParser = JSONLikeParser(js_lexer.JSONOptions{ .allow_comments = true, .allow_trailing_commas = true }); + +pub fn ParseJSON(log: logger.Log, source: logger.Source) !?Expr { + var parser = JSONParser.init(allocator, log, source); + + return try parser.parseExpr(); +} + +pub fn ParseTSConfig(log: logger.Loc, source: logger.Source) !?Expr { + var parser = TSConfigParser.init(allocator, log, source); + + return try parser.parseExpr(); +} diff --git a/src/logger.zig b/src/logger.zig index 47253728c..bf7bcb21c 100644 --- a/src/logger.zig +++ b/src/logger.zig @@ -173,11 +173,27 @@ pub const Log = struct { pub fn addRangeWarning(log: *Log, source: ?Source, r: Range, text: string) !void { log.warnings += 1; try log.addMsg(Msg{ - .kind = .warning, + .kind = .warn, .data = rangeData(source, r, text), }); } + pub fn addWarningFmt(log: *Log, source: ?Source, l: Loc, allocator: *std.mem.Allocator, comptime text: string, args: anytype) !void { + log.errors += 1; + try log.addMsg(Msg{ + .kind = .err, + .data = rangeData(source, Range{ .loc = l }, std.fmt.allocPrint(allocator, text, args) catch unreachable), + }); + } + + pub fn addRangeWarningFmt(log: *Log, source: ?Source, r: Range, allocator: *std.mem.Allocator, comptime text: string, args: anytype) !void { + log.errors += 1; + try log.addMsg(Msg{ + .kind = .err, + .data = rangeData(source, r, std.fmt.allocPrint(allocator, text, args) catch unreachable), + }); + } + pub fn addWarning(log: *Log, source: ?Source, l: Loc, text: string) !void { log.warnings += 1; try log.addMsg(Msg{ |