aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorGravatar Jarred Sumner <jarred@jarredsumner.com> 2021-04-25 00:56:48 -0700
committerGravatar Jarred Sumner <jarred@jarredsumner.com> 2021-04-25 00:56:48 -0700
commitc0b7f71b9ace0bfd91fb5e0364f104b5093e6c37 (patch)
tree7da69736690eb11b974800ad947d986938e3bb50 /src
parent322b5cbd382c9a59deb20dd307dd6d09bd844482 (diff)
downloadbun-c0b7f71b9ace0bfd91fb5e0364f104b5093e6c37.tar.gz
bun-c0b7f71b9ace0bfd91fb5e0364f104b5093e6c37.tar.zst
bun-c0b7f71b9ace0bfd91fb5e0364f104b5093e6c37.zip
little defines, little readme, json parser
Diffstat (limited to 'src')
-rw-r--r--src/defines-table.zig798
-rw-r--r--src/defines.zig12
-rw-r--r--src/js_lexer.zig2202
-rw-r--r--src/js_parser.zig58
-rw-r--r--src/json_parser.zig212
-rw-r--r--src/logger.zig18
6 files changed, 2259 insertions, 1041 deletions
diff --git a/src/defines-table.zig b/src/defines-table.zig
new file mode 100644
index 000000000..3e3a720a3
--- /dev/null
+++ b/src/defines-table.zig
@@ -0,0 +1,798 @@
+usingnamespace @import("strings.zig");
+
+// If something is in this list, then a direct identifier expression or property
+// access chain matching this will be assumed to have no side effects and will
+// be removed.
+//
+// This also means code is allowed to be reordered past things in this list. For
+// example, if "console.log" is in this list, permitting reordering allows for
+// "if (a) console.log(b); else console.log(c)" to be reordered and transformed
+// into "console.log(a ? b : c)". Notice that "a" and "console.log" are in a
+// different order, which can only happen if evaluating the "console.log"
+// property access can be assumed to not change the value of "a".
+//
+// Note that membership in this list says nothing about whether calling any of
+// these functions has any side effects. It only says something about
+// referencing these function without calling them.
+pub const GlobalDefinesKey = [][]string{
+ // These global identifiers should exist in all JavaScript environments. This
+ // deliberately omits "NaN", "Infinity", and "undefined" because these are
+ // treated as automatically-inlined constants instead of identifiers.
+ []string{"Array"},
+ []string{"Boolean"},
+ []string{"Function"},
+ []string{"Math"},
+ []string{"Number"},
+ []string{"Object"},
+ []string{"RegExp"},
+ []string{"String"},
+
+ // Object: Static methods
+ // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object#Static_methods
+ []string{ "Object", "assign" },
+ []string{ "Object", "create" },
+ []string{ "Object", "defineProperties" },
+ []string{ "Object", "defineProperty" },
+ []string{ "Object", "entries" },
+ []string{ "Object", "freeze" },
+ []string{ "Object", "fromEntries" },
+ []string{ "Object", "getOwnPropertyDescriptor" },
+ []string{ "Object", "getOwnPropertyDescriptors" },
+ []string{ "Object", "getOwnPropertyNames" },
+ []string{ "Object", "getOwnPropertySymbols" },
+ []string{ "Object", "getPrototypeOf" },
+ []string{ "Object", "is" },
+ []string{ "Object", "isExtensible" },
+ []string{ "Object", "isFrozen" },
+ []string{ "Object", "isSealed" },
+ []string{ "Object", "keys" },
+ []string{ "Object", "preventExtensions" },
+ []string{ "Object", "seal" },
+ []string{ "Object", "setPrototypeOf" },
+ []string{ "Object", "values" },
+
+ // Object: Instance methods
+ // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object#Instance_methods
+ []string{ "Object", "prototype", "__defineGetter__" },
+ []string{ "Object", "prototype", "__defineSetter__" },
+ []string{ "Object", "prototype", "__lookupGetter__" },
+ []string{ "Object", "prototype", "__lookupSetter__" },
+ []string{ "Object", "prototype", "hasOwnProperty" },
+ []string{ "Object", "prototype", "isPrototypeOf" },
+ []string{ "Object", "prototype", "propertyIsEnumerable" },
+ []string{ "Object", "prototype", "toLocaleString" },
+ []string{ "Object", "prototype", "toString" },
+ []string{ "Object", "prototype", "unwatch" },
+ []string{ "Object", "prototype", "valueOf" },
+ []string{ "Object", "prototype", "watch" },
+
+ // Math: Static properties
+ // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math#Static_properties
+ []string{ "Math", "E" },
+ []string{ "Math", "LN10" },
+ []string{ "Math", "LN2" },
+ []string{ "Math", "LOG10E" },
+ []string{ "Math", "LOG2E" },
+ []string{ "Math", "PI" },
+ []string{ "Math", "SQRT1_2" },
+ []string{ "Math", "SQRT2" },
+
+ // Math: Static methods
+ // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math#Static_methods
+ []string{ "Math", "abs" },
+ []string{ "Math", "acos" },
+ []string{ "Math", "acosh" },
+ []string{ "Math", "asin" },
+ []string{ "Math", "asinh" },
+ []string{ "Math", "atan" },
+ []string{ "Math", "atan2" },
+ []string{ "Math", "atanh" },
+ []string{ "Math", "cbrt" },
+ []string{ "Math", "ceil" },
+ []string{ "Math", "clz32" },
+ []string{ "Math", "cos" },
+ []string{ "Math", "cosh" },
+ []string{ "Math", "exp" },
+ []string{ "Math", "expm1" },
+ []string{ "Math", "floor" },
+ []string{ "Math", "fround" },
+ []string{ "Math", "hypot" },
+ []string{ "Math", "imul" },
+ []string{ "Math", "log" },
+ []string{ "Math", "log10" },
+ []string{ "Math", "log1p" },
+ []string{ "Math", "log2" },
+ []string{ "Math", "max" },
+ []string{ "Math", "min" },
+ []string{ "Math", "pow" },
+ []string{ "Math", "random" },
+ []string{ "Math", "round" },
+ []string{ "Math", "sign" },
+ []string{ "Math", "sin" },
+ []string{ "Math", "sinh" },
+ []string{ "Math", "sqrt" },
+ []string{ "Math", "tan" },
+ []string{ "Math", "tanh" },
+ []string{ "Math", "trunc" },
+
+ // Other globals present in both the browser and node (except "eval" because
+ // it has special behavior)
+ []string{"AbortController"},
+ []string{"AbortSignal"},
+ []string{"AggregateError"},
+ []string{"ArrayBuffer"},
+ []string{"BigInt"},
+ []string{"DataView"},
+ []string{"Date"},
+ []string{"Error"},
+ []string{"EvalError"},
+ []string{"Event"},
+ []string{"EventTarget"},
+ []string{"Float32Array"},
+ []string{"Float64Array"},
+ []string{"Int16Array"},
+ []string{"Int32Array"},
+ []string{"Int8Array"},
+ []string{"Intl"},
+ []string{"JSON"},
+ []string{"Map"},
+ []string{"MessageChannel"},
+ []string{"MessageEvent"},
+ []string{"MessagePort"},
+ []string{"Promise"},
+ []string{"Proxy"},
+ []string{"RangeError"},
+ []string{"ReferenceError"},
+ []string{"Reflect"},
+ []string{"Set"},
+ []string{"Symbol"},
+ []string{"SyntaxError"},
+ []string{"TextDecoder"},
+ []string{"TextEncoder"},
+ []string{"TypeError"},
+ []string{"URIError"},
+ []string{"URL"},
+ []string{"URLSearchParams"},
+ []string{"Uint16Array"},
+ []string{"Uint32Array"},
+ []string{"Uint8Array"},
+ []string{"Uint8ClampedArray"},
+ []string{"WeakMap"},
+ []string{"WeakSet"},
+ []string{"WebAssembly"},
+ []string{"clearInterval"},
+ []string{"clearTimeout"},
+ []string{"console"},
+ []string{"decodeURI"},
+ []string{"decodeURIComponent"},
+ []string{"encodeURI"},
+ []string{"encodeURIComponent"},
+ []string{"escape"},
+ []string{"globalThis"},
+ []string{"isFinite"},
+ []string{"isNaN"},
+ []string{"parseFloat"},
+ []string{"parseInt"},
+ []string{"queueMicrotask"},
+ []string{"setInterval"},
+ []string{"setTimeout"},
+ []string{"unescape"},
+
+ // Console method references are assumed to have no side effects
+ // https://developer.mozilla.org/en-US/docs/Web/API/console
+ []string{ "console", "assert" },
+ []string{ "console", "clear" },
+ []string{ "console", "count" },
+ []string{ "console", "countReset" },
+ []string{ "console", "debug" },
+ []string{ "console", "dir" },
+ []string{ "console", "dirxml" },
+ []string{ "console", "error" },
+ []string{ "console", "group" },
+ []string{ "console", "groupCollapsed" },
+ []string{ "console", "groupEnd" },
+ []string{ "console", "info" },
+ []string{ "console", "log" },
+ []string{ "console", "table" },
+ []string{ "console", "time" },
+ []string{ "console", "timeEnd" },
+ []string{ "console", "timeLog" },
+ []string{ "console", "trace" },
+ []string{ "console", "warn" },
+
+ // CSSOM APIs
+ []string{"CSSAnimation"},
+ []string{"CSSFontFaceRule"},
+ []string{"CSSImportRule"},
+ []string{"CSSKeyframeRule"},
+ []string{"CSSKeyframesRule"},
+ []string{"CSSMediaRule"},
+ []string{"CSSNamespaceRule"},
+ []string{"CSSPageRule"},
+ []string{"CSSRule"},
+ []string{"CSSRuleList"},
+ []string{"CSSStyleDeclaration"},
+ []string{"CSSStyleRule"},
+ []string{"CSSStyleSheet"},
+ []string{"CSSSupportsRule"},
+ []string{"CSSTransition"},
+
+ // SVG DOM
+ []string{"SVGAElement"},
+ []string{"SVGAngle"},
+ []string{"SVGAnimateElement"},
+ []string{"SVGAnimateMotionElement"},
+ []string{"SVGAnimateTransformElement"},
+ []string{"SVGAnimatedAngle"},
+ []string{"SVGAnimatedBoolean"},
+ []string{"SVGAnimatedEnumeration"},
+ []string{"SVGAnimatedInteger"},
+ []string{"SVGAnimatedLength"},
+ []string{"SVGAnimatedLengthList"},
+ []string{"SVGAnimatedNumber"},
+ []string{"SVGAnimatedNumberList"},
+ []string{"SVGAnimatedPreserveAspectRatio"},
+ []string{"SVGAnimatedRect"},
+ []string{"SVGAnimatedString"},
+ []string{"SVGAnimatedTransformList"},
+ []string{"SVGAnimationElement"},
+ []string{"SVGCircleElement"},
+ []string{"SVGClipPathElement"},
+ []string{"SVGComponentTransferFunctionElement"},
+ []string{"SVGDefsElement"},
+ []string{"SVGDescElement"},
+ []string{"SVGElement"},
+ []string{"SVGEllipseElement"},
+ []string{"SVGFEBlendElement"},
+ []string{"SVGFEColorMatrixElement"},
+ []string{"SVGFEComponentTransferElement"},
+ []string{"SVGFECompositeElement"},
+ []string{"SVGFEConvolveMatrixElement"},
+ []string{"SVGFEDiffuseLightingElement"},
+ []string{"SVGFEDisplacementMapElement"},
+ []string{"SVGFEDistantLightElement"},
+ []string{"SVGFEDropShadowElement"},
+ []string{"SVGFEFloodElement"},
+ []string{"SVGFEFuncAElement"},
+ []string{"SVGFEFuncBElement"},
+ []string{"SVGFEFuncGElement"},
+ []string{"SVGFEFuncRElement"},
+ []string{"SVGFEGaussianBlurElement"},
+ []string{"SVGFEImageElement"},
+ []string{"SVGFEMergeElement"},
+ []string{"SVGFEMergeNodeElement"},
+ []string{"SVGFEMorphologyElement"},
+ []string{"SVGFEOffsetElement"},
+ []string{"SVGFEPointLightElement"},
+ []string{"SVGFESpecularLightingElement"},
+ []string{"SVGFESpotLightElement"},
+ []string{"SVGFETileElement"},
+ []string{"SVGFETurbulenceElement"},
+ []string{"SVGFilterElement"},
+ []string{"SVGForeignObjectElement"},
+ []string{"SVGGElement"},
+ []string{"SVGGeometryElement"},
+ []string{"SVGGradientElement"},
+ []string{"SVGGraphicsElement"},
+ []string{"SVGImageElement"},
+ []string{"SVGLength"},
+ []string{"SVGLengthList"},
+ []string{"SVGLineElement"},
+ []string{"SVGLinearGradientElement"},
+ []string{"SVGMPathElement"},
+ []string{"SVGMarkerElement"},
+ []string{"SVGMaskElement"},
+ []string{"SVGMatrix"},
+ []string{"SVGMetadataElement"},
+ []string{"SVGNumber"},
+ []string{"SVGNumberList"},
+ []string{"SVGPathElement"},
+ []string{"SVGPatternElement"},
+ []string{"SVGPoint"},
+ []string{"SVGPointList"},
+ []string{"SVGPolygonElement"},
+ []string{"SVGPolylineElement"},
+ []string{"SVGPreserveAspectRatio"},
+ []string{"SVGRadialGradientElement"},
+ []string{"SVGRect"},
+ []string{"SVGRectElement"},
+ []string{"SVGSVGElement"},
+ []string{"SVGScriptElement"},
+ []string{"SVGSetElement"},
+ []string{"SVGStopElement"},
+ []string{"SVGStringList"},
+ []string{"SVGStyleElement"},
+ []string{"SVGSwitchElement"},
+ []string{"SVGSymbolElement"},
+ []string{"SVGTSpanElement"},
+ []string{"SVGTextContentElement"},
+ []string{"SVGTextElement"},
+ []string{"SVGTextPathElement"},
+ []string{"SVGTextPositioningElement"},
+ []string{"SVGTitleElement"},
+ []string{"SVGTransform"},
+ []string{"SVGTransformList"},
+ []string{"SVGUnitTypes"},
+ []string{"SVGUseElement"},
+ []string{"SVGViewElement"},
+
+ // Other browser APIs
+ //
+ // This list contains all globals present in modern versions of Chrome, Safari,
+ // and Firefox except for the following properties, since they have a side effect
+ // of triggering layout (https://gist.github.com/paulirish/5d52fb081b3570c81e3a):
+ //
+ // - scrollX
+ // - scrollY
+ // - innerWidth
+ // - innerHeight
+ // - pageXOffset
+ // - pageYOffset
+ //
+ // The following globals have also been removed since they sometimes throw an
+ // exception when accessed, which is a side effect (for more information see
+ // https://stackoverflow.com/a/33047477):
+ //
+ // - localStorage
+ // - sessionStorage
+ //
+ []string{"AnalyserNode"},
+ []string{"Animation"},
+ []string{"AnimationEffect"},
+ []string{"AnimationEvent"},
+ []string{"AnimationPlaybackEvent"},
+ []string{"AnimationTimeline"},
+ []string{"Attr"},
+ []string{"Audio"},
+ []string{"AudioBuffer"},
+ []string{"AudioBufferSourceNode"},
+ []string{"AudioDestinationNode"},
+ []string{"AudioListener"},
+ []string{"AudioNode"},
+ []string{"AudioParam"},
+ []string{"AudioProcessingEvent"},
+ []string{"AudioScheduledSourceNode"},
+ []string{"BarProp"},
+ []string{"BeforeUnloadEvent"},
+ []string{"BiquadFilterNode"},
+ []string{"Blob"},
+ []string{"BlobEvent"},
+ []string{"ByteLengthQueuingStrategy"},
+ []string{"CDATASection"},
+ []string{"CSS"},
+ []string{"CanvasGradient"},
+ []string{"CanvasPattern"},
+ []string{"CanvasRenderingContext2D"},
+ []string{"ChannelMergerNode"},
+ []string{"ChannelSplitterNode"},
+ []string{"CharacterData"},
+ []string{"ClipboardEvent"},
+ []string{"CloseEvent"},
+ []string{"Comment"},
+ []string{"CompositionEvent"},
+ []string{"ConvolverNode"},
+ []string{"CountQueuingStrategy"},
+ []string{"Crypto"},
+ []string{"CustomElementRegistry"},
+ []string{"CustomEvent"},
+ []string{"DOMException"},
+ []string{"DOMImplementation"},
+ []string{"DOMMatrix"},
+ []string{"DOMMatrixReadOnly"},
+ []string{"DOMParser"},
+ []string{"DOMPoint"},
+ []string{"DOMPointReadOnly"},
+ []string{"DOMQuad"},
+ []string{"DOMRect"},
+ []string{"DOMRectList"},
+ []string{"DOMRectReadOnly"},
+ []string{"DOMStringList"},
+ []string{"DOMStringMap"},
+ []string{"DOMTokenList"},
+ []string{"DataTransfer"},
+ []string{"DataTransferItem"},
+ []string{"DataTransferItemList"},
+ []string{"DelayNode"},
+ []string{"Document"},
+ []string{"DocumentFragment"},
+ []string{"DocumentTimeline"},
+ []string{"DocumentType"},
+ []string{"DragEvent"},
+ []string{"DynamicsCompressorNode"},
+ []string{"Element"},
+ []string{"ErrorEvent"},
+ []string{"EventSource"},
+ []string{"File"},
+ []string{"FileList"},
+ []string{"FileReader"},
+ []string{"FocusEvent"},
+ []string{"FontFace"},
+ []string{"FormData"},
+ []string{"GainNode"},
+ []string{"Gamepad"},
+ []string{"GamepadButton"},
+ []string{"GamepadEvent"},
+ []string{"Geolocation"},
+ []string{"GeolocationPositionError"},
+ []string{"HTMLAllCollection"},
+ []string{"HTMLAnchorElement"},
+ []string{"HTMLAreaElement"},
+ []string{"HTMLAudioElement"},
+ []string{"HTMLBRElement"},
+ []string{"HTMLBaseElement"},
+ []string{"HTMLBodyElement"},
+ []string{"HTMLButtonElement"},
+ []string{"HTMLCanvasElement"},
+ []string{"HTMLCollection"},
+ []string{"HTMLDListElement"},
+ []string{"HTMLDataElement"},
+ []string{"HTMLDataListElement"},
+ []string{"HTMLDetailsElement"},
+ []string{"HTMLDirectoryElement"},
+ []string{"HTMLDivElement"},
+ []string{"HTMLDocument"},
+ []string{"HTMLElement"},
+ []string{"HTMLEmbedElement"},
+ []string{"HTMLFieldSetElement"},
+ []string{"HTMLFontElement"},
+ []string{"HTMLFormControlsCollection"},
+ []string{"HTMLFormElement"},
+ []string{"HTMLFrameElement"},
+ []string{"HTMLFrameSetElement"},
+ []string{"HTMLHRElement"},
+ []string{"HTMLHeadElement"},
+ []string{"HTMLHeadingElement"},
+ []string{"HTMLHtmlElement"},
+ []string{"HTMLIFrameElement"},
+ []string{"HTMLImageElement"},
+ []string{"HTMLInputElement"},
+ []string{"HTMLLIElement"},
+ []string{"HTMLLabelElement"},
+ []string{"HTMLLegendElement"},
+ []string{"HTMLLinkElement"},
+ []string{"HTMLMapElement"},
+ []string{"HTMLMarqueeElement"},
+ []string{"HTMLMediaElement"},
+ []string{"HTMLMenuElement"},
+ []string{"HTMLMetaElement"},
+ []string{"HTMLMeterElement"},
+ []string{"HTMLModElement"},
+ []string{"HTMLOListElement"},
+ []string{"HTMLObjectElement"},
+ []string{"HTMLOptGroupElement"},
+ []string{"HTMLOptionElement"},
+ []string{"HTMLOptionsCollection"},
+ []string{"HTMLOutputElement"},
+ []string{"HTMLParagraphElement"},
+ []string{"HTMLParamElement"},
+ []string{"HTMLPictureElement"},
+ []string{"HTMLPreElement"},
+ []string{"HTMLProgressElement"},
+ []string{"HTMLQuoteElement"},
+ []string{"HTMLScriptElement"},
+ []string{"HTMLSelectElement"},
+ []string{"HTMLSlotElement"},
+ []string{"HTMLSourceElement"},
+ []string{"HTMLSpanElement"},
+ []string{"HTMLStyleElement"},
+ []string{"HTMLTableCaptionElement"},
+ []string{"HTMLTableCellElement"},
+ []string{"HTMLTableColElement"},
+ []string{"HTMLTableElement"},
+ []string{"HTMLTableRowElement"},
+ []string{"HTMLTableSectionElement"},
+ []string{"HTMLTemplateElement"},
+ []string{"HTMLTextAreaElement"},
+ []string{"HTMLTimeElement"},
+ []string{"HTMLTitleElement"},
+ []string{"HTMLTrackElement"},
+ []string{"HTMLUListElement"},
+ []string{"HTMLUnknownElement"},
+ []string{"HTMLVideoElement"},
+ []string{"HashChangeEvent"},
+ []string{"Headers"},
+ []string{"History"},
+ []string{"IDBCursor"},
+ []string{"IDBCursorWithValue"},
+ []string{"IDBDatabase"},
+ []string{"IDBFactory"},
+ []string{"IDBIndex"},
+ []string{"IDBKeyRange"},
+ []string{"IDBObjectStore"},
+ []string{"IDBOpenDBRequest"},
+ []string{"IDBRequest"},
+ []string{"IDBTransaction"},
+ []string{"IDBVersionChangeEvent"},
+ []string{"Image"},
+ []string{"ImageData"},
+ []string{"InputEvent"},
+ []string{"IntersectionObserver"},
+ []string{"IntersectionObserverEntry"},
+ []string{"KeyboardEvent"},
+ []string{"KeyframeEffect"},
+ []string{"Location"},
+ []string{"MediaCapabilities"},
+ []string{"MediaElementAudioSourceNode"},
+ []string{"MediaEncryptedEvent"},
+ []string{"MediaError"},
+ []string{"MediaList"},
+ []string{"MediaQueryList"},
+ []string{"MediaQueryListEvent"},
+ []string{"MediaRecorder"},
+ []string{"MediaSource"},
+ []string{"MediaStream"},
+ []string{"MediaStreamAudioDestinationNode"},
+ []string{"MediaStreamAudioSourceNode"},
+ []string{"MediaStreamTrack"},
+ []string{"MediaStreamTrackEvent"},
+ []string{"MimeType"},
+ []string{"MimeTypeArray"},
+ []string{"MouseEvent"},
+ []string{"MutationEvent"},
+ []string{"MutationObserver"},
+ []string{"MutationRecord"},
+ []string{"NamedNodeMap"},
+ []string{"Navigator"},
+ []string{"Node"},
+ []string{"NodeFilter"},
+ []string{"NodeIterator"},
+ []string{"NodeList"},
+ []string{"Notification"},
+ []string{"OfflineAudioCompletionEvent"},
+ []string{"Option"},
+ []string{"OscillatorNode"},
+ []string{"PageTransitionEvent"},
+ []string{"Path2D"},
+ []string{"Performance"},
+ []string{"PerformanceEntry"},
+ []string{"PerformanceMark"},
+ []string{"PerformanceMeasure"},
+ []string{"PerformanceNavigation"},
+ []string{"PerformanceObserver"},
+ []string{"PerformanceObserverEntryList"},
+ []string{"PerformanceResourceTiming"},
+ []string{"PerformanceTiming"},
+ []string{"PeriodicWave"},
+ []string{"Plugin"},
+ []string{"PluginArray"},
+ []string{"PointerEvent"},
+ []string{"PopStateEvent"},
+ []string{"ProcessingInstruction"},
+ []string{"ProgressEvent"},
+ []string{"PromiseRejectionEvent"},
+ []string{"RTCCertificate"},
+ []string{"RTCDTMFSender"},
+ []string{"RTCDTMFToneChangeEvent"},
+ []string{"RTCDataChannel"},
+ []string{"RTCDataChannelEvent"},
+ []string{"RTCIceCandidate"},
+ []string{"RTCPeerConnection"},
+ []string{"RTCPeerConnectionIceEvent"},
+ []string{"RTCRtpReceiver"},
+ []string{"RTCRtpSender"},
+ []string{"RTCRtpTransceiver"},
+ []string{"RTCSessionDescription"},
+ []string{"RTCStatsReport"},
+ []string{"RTCTrackEvent"},
+ []string{"RadioNodeList"},
+ []string{"Range"},
+ []string{"ReadableStream"},
+ []string{"Request"},
+ []string{"ResizeObserver"},
+ []string{"ResizeObserverEntry"},
+ []string{"Response"},
+ []string{"Screen"},
+ []string{"ScriptProcessorNode"},
+ []string{"SecurityPolicyViolationEvent"},
+ []string{"Selection"},
+ []string{"ShadowRoot"},
+ []string{"SourceBuffer"},
+ []string{"SourceBufferList"},
+ []string{"SpeechSynthesisEvent"},
+ []string{"SpeechSynthesisUtterance"},
+ []string{"StaticRange"},
+ []string{"Storage"},
+ []string{"StorageEvent"},
+ []string{"StyleSheet"},
+ []string{"StyleSheetList"},
+ []string{"Text"},
+ []string{"TextMetrics"},
+ []string{"TextTrack"},
+ []string{"TextTrackCue"},
+ []string{"TextTrackCueList"},
+ []string{"TextTrackList"},
+ []string{"TimeRanges"},
+ []string{"TrackEvent"},
+ []string{"TransitionEvent"},
+ []string{"TreeWalker"},
+ []string{"UIEvent"},
+ []string{"VTTCue"},
+ []string{"ValidityState"},
+ []string{"VisualViewport"},
+ []string{"WaveShaperNode"},
+ []string{"WebGLActiveInfo"},
+ []string{"WebGLBuffer"},
+ []string{"WebGLContextEvent"},
+ []string{"WebGLFramebuffer"},
+ []string{"WebGLProgram"},
+ []string{"WebGLQuery"},
+ []string{"WebGLRenderbuffer"},
+ []string{"WebGLRenderingContext"},
+ []string{"WebGLSampler"},
+ []string{"WebGLShader"},
+ []string{"WebGLShaderPrecisionFormat"},
+ []string{"WebGLSync"},
+ []string{"WebGLTexture"},
+ []string{"WebGLUniformLocation"},
+ []string{"WebKitCSSMatrix"},
+ []string{"WebSocket"},
+ []string{"WheelEvent"},
+ []string{"Window"},
+ []string{"Worker"},
+ []string{"XMLDocument"},
+ []string{"XMLHttpRequest"},
+ []string{"XMLHttpRequestEventTarget"},
+ []string{"XMLHttpRequestUpload"},
+ []string{"XMLSerializer"},
+ []string{"XPathEvaluator"},
+ []string{"XPathExpression"},
+ []string{"XPathResult"},
+ []string{"XSLTProcessor"},
+ []string{"alert"},
+ []string{"atob"},
+ []string{"blur"},
+ []string{"btoa"},
+ []string{"cancelAnimationFrame"},
+ []string{"captureEvents"},
+ []string{"close"},
+ []string{"closed"},
+ []string{"confirm"},
+ []string{"customElements"},
+ []string{"devicePixelRatio"},
+ []string{"document"},
+ []string{"event"},
+ []string{"fetch"},
+ []string{"find"},
+ []string{"focus"},
+ []string{"frameElement"},
+ []string{"frames"},
+ []string{"getComputedStyle"},
+ []string{"getSelection"},
+ []string{"history"},
+ []string{"indexedDB"},
+ []string{"isSecureContext"},
+ []string{"length"},
+ []string{"location"},
+ []string{"locationbar"},
+ []string{"matchMedia"},
+ []string{"menubar"},
+ []string{"moveBy"},
+ []string{"moveTo"},
+ []string{"name"},
+ []string{"navigator"},
+ []string{"onabort"},
+ []string{"onafterprint"},
+ []string{"onanimationend"},
+ []string{"onanimationiteration"},
+ []string{"onanimationstart"},
+ []string{"onbeforeprint"},
+ []string{"onbeforeunload"},
+ []string{"onblur"},
+ []string{"oncanplay"},
+ []string{"oncanplaythrough"},
+ []string{"onchange"},
+ []string{"onclick"},
+ []string{"oncontextmenu"},
+ []string{"oncuechange"},
+ []string{"ondblclick"},
+ []string{"ondrag"},
+ []string{"ondragend"},
+ []string{"ondragenter"},
+ []string{"ondragleave"},
+ []string{"ondragover"},
+ []string{"ondragstart"},
+ []string{"ondrop"},
+ []string{"ondurationchange"},
+ []string{"onemptied"},
+ []string{"onended"},
+ []string{"onerror"},
+ []string{"onfocus"},
+ []string{"ongotpointercapture"},
+ []string{"onhashchange"},
+ []string{"oninput"},
+ []string{"oninvalid"},
+ []string{"onkeydown"},
+ []string{"onkeypress"},
+ []string{"onkeyup"},
+ []string{"onlanguagechange"},
+ []string{"onload"},
+ []string{"onloadeddata"},
+ []string{"onloadedmetadata"},
+ []string{"onloadstart"},
+ []string{"onlostpointercapture"},
+ []string{"onmessage"},
+ []string{"onmousedown"},
+ []string{"onmouseenter"},
+ []string{"onmouseleave"},
+ []string{"onmousemove"},
+ []string{"onmouseout"},
+ []string{"onmouseover"},
+ []string{"onmouseup"},
+ []string{"onoffline"},
+ []string{"ononline"},
+ []string{"onpagehide"},
+ []string{"onpageshow"},
+ []string{"onpause"},
+ []string{"onplay"},
+ []string{"onplaying"},
+ []string{"onpointercancel"},
+ []string{"onpointerdown"},
+ []string{"onpointerenter"},
+ []string{"onpointerleave"},
+ []string{"onpointermove"},
+ []string{"onpointerout"},
+ []string{"onpointerover"},
+ []string{"onpointerup"},
+ []string{"onpopstate"},
+ []string{"onprogress"},
+ []string{"onratechange"},
+ []string{"onrejectionhandled"},
+ []string{"onreset"},
+ []string{"onresize"},
+ []string{"onscroll"},
+ []string{"onseeked"},
+ []string{"onseeking"},
+ []string{"onselect"},
+ []string{"onstalled"},
+ []string{"onstorage"},
+ []string{"onsubmit"},
+ []string{"onsuspend"},
+ []string{"ontimeupdate"},
+ []string{"ontoggle"},
+ []string{"ontransitioncancel"},
+ []string{"ontransitionend"},
+ []string{"ontransitionrun"},
+ []string{"ontransitionstart"},
+ []string{"onunhandledrejection"},
+ []string{"onunload"},
+ []string{"onvolumechange"},
+ []string{"onwaiting"},
+ []string{"onwebkitanimationend"},
+ []string{"onwebkitanimationiteration"},
+ []string{"onwebkitanimationstart"},
+ []string{"onwebkittransitionend"},
+ []string{"onwheel"},
+ []string{"open"},
+ []string{"opener"},
+ []string{"origin"},
+ []string{"outerHeight"},
+ []string{"outerWidth"},
+ []string{"parent"},
+ []string{"performance"},
+ []string{"personalbar"},
+ []string{"postMessage"},
+ []string{"print"},
+ []string{"prompt"},
+ []string{"releaseEvents"},
+ []string{"requestAnimationFrame"},
+ []string{"resizeBy"},
+ []string{"resizeTo"},
+ []string{"screen"},
+ []string{"screenLeft"},
+ []string{"screenTop"},
+ []string{"screenX"},
+ []string{"screenY"},
+ []string{"scroll"},
+ []string{"scrollBy"},
+ []string{"scrollTo"},
+ []string{"scrollbars"},
+ []string{"self"},
+ []string{"speechSynthesis"},
+ []string{"status"},
+ []string{"statusbar"},
+ []string{"stop"},
+ []string{"toolbar"},
+ []string{"top"},
+ []string{"webkitURL"},
+ []string{"window"},
+};
diff --git a/src/defines.zig b/src/defines.zig
new file mode 100644
index 000000000..6021df57f
--- /dev/null
+++ b/src/defines.zig
@@ -0,0 +1,12 @@
+const std = @import("std");
+const js_ast = @import("./js_ast.zig");
+
+const GlobalDefinesKey = @import("./defines-table.zig").GlobalDefinesKey;
+
+pub const defaultIdentifierDefines = comptime {};
+
+pub const IdentifierDefine = struct {};
+
+pub const DotDefine = struct {};
+
+pub const Defines = struct {};
diff --git a/src/js_lexer.zig b/src/js_lexer.zig
index a16c72c19..fa95341a5 100644
--- a/src/js_lexer.zig
+++ b/src/js_lexer.zig
@@ -22,1123 +22,1264 @@ pub const StrictModeReservedWords = tables.StrictModeReservedWords;
pub const PropertyModifierKeyword = tables.PropertyModifierKeyword;
pub const TypescriptStmtKeyword = tables.TypescriptStmtKeyword;
-// TODO: JSON
-const IS_JSON_FILE = false;
-
-pub const Lexer = struct {
- // pub const Error = error{
- // UnexpectedToken,
- // EndOfFile,
- // };
-
- // err: ?Lexer.Error,
- log: logger.Log,
- source: logger.Source,
- current: usize = 0,
- start: usize = 0,
- end: usize = 0,
- approximate_newline_count: i32 = 0,
- legacy_octal_loc: logger.Loc = logger.Loc.Empty,
- previous_backslash_quote_in_jsx: logger.Range = logger.Range.None,
- token: T = T.t_end_of_file,
- has_newline_before: bool = false,
- has_pure_comment_before: bool = false,
- preserve_all_comments_before: bool = false,
- is_legacy_octal_literal: bool = false,
- comments_to_preserve_before: ?[]js_ast.G.Comment = null,
- all_original_comments: ?[]js_ast.G.Comment = null,
- code_point: CodePoint = -1,
- string_literal: JavascriptString,
- identifier: []const u8 = "",
- jsx_factory_pragma_comment: ?js_ast.Span = null,
- jsx_fragment_pragma_comment: ?js_ast.Span = null,
- source_mapping_url: ?js_ast.Span = null,
- number: f64 = 0.0,
- rescan_close_brace_as_template_token: bool = false,
- for_global_name: bool = false,
- prev_error_loc: logger.Loc = logger.Loc.Empty,
- allocator: *std.mem.Allocator,
-
- pub fn loc(self: *Lexer) logger.Loc {
- return logger.usize2Loc(self.start);
- }
+pub const JSONOptions = struct {
+ allow_comments: bool = false,
+ allow_trailing_commas: bool = false,
+};
- fn nextCodepointSlice(it: *Lexer) callconv(.Inline) ?[]const u8 {
- if (it.current >= it.source.contents.len) {
- return null;
+pub fn NewLexerType(comptime jsonOptions: ?JSONOptions) type {
+ return struct {
+ // pub const Error = error{
+ // UnexpectedToken,
+ // EndOfFile,
+ // };
+
+ // err: ?@This().Error,
+ log: logger.Log,
+ source: logger.Source,
+ current: usize = 0,
+ start: usize = 0,
+ end: usize = 0,
+ approximate_newline_count: i32 = 0,
+ legacy_octal_loc: logger.Loc = logger.Loc.Empty,
+ previous_backslash_quote_in_jsx: logger.Range = logger.Range.None,
+ token: T = T.t_end_of_file,
+ has_newline_before: bool = false,
+ has_pure_comment_before: bool = false,
+ preserve_all_comments_before: bool = false,
+ is_legacy_octal_literal: bool = false,
+ comments_to_preserve_before: std.ArrayList(js_ast.G.Comment),
+ all_original_comments: ?[]js_ast.G.Comment = null,
+ code_point: CodePoint = -1,
+ string_literal: JavascriptString,
+ identifier: []const u8 = "",
+ jsx_factory_pragma_comment: ?js_ast.Span = null,
+ jsx_fragment_pragma_comment: ?js_ast.Span = null,
+ source_mapping_url: ?js_ast.Span = null,
+ number: f64 = 0.0,
+ rescan_close_brace_as_template_token: bool = false,
+ for_global_name: bool = false,
+ prev_error_loc: logger.Loc = logger.Loc.Empty,
+ allocator: *std.mem.Allocator,
+
+ pub fn loc(self: *@This()) logger.Loc {
+ return logger.usize2Loc(self.start);
}
- const cp_len = unicode.utf8ByteSequenceLength(it.source.contents[it.current]) catch unreachable;
- it.end = it.current;
- it.current += cp_len;
-
- return it.source.contents[it.current - cp_len .. it.current];
- }
-
- pub fn syntaxError(self: *Lexer) void {
- self.addError(self.start, "Syntax Error!!", .{}, true);
- }
+ fn nextCodepointSlice(it: *@This()) callconv(.Inline) ?[]const u8 {
+ if (it.current >= it.source.contents.len) {
+ return null;
+ }
- pub fn addDefaultError(self: *Lexer, msg: []const u8) void {
- self.addError(self.start, "{s}", .{msg}, true);
- }
+ const cp_len = unicode.utf8ByteSequenceLength(it.source.contents[it.current]) catch unreachable;
+ it.end = it.current;
+ it.current += cp_len;
- pub fn addError(self: *Lexer, _loc: usize, comptime format: []const u8, args: anytype, panic: bool) void {
- var __loc = logger.usize2Loc(_loc);
- if (__loc.eql(self.prev_error_loc)) {
- return;
+ return it.source.contents[it.current - cp_len .. it.current];
}
- const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable;
- self.log.addError(self.source, __loc, errorMessage) catch unreachable;
- self.prev_error_loc = __loc;
- var msg = self.log.msgs.items[self.log.msgs.items.len - 1];
- msg.formatNoWriter(std.debug.panic);
- }
+ pub fn syntaxError(self: *@This()) void {
+ self.addError(self.start, "Syntax Error!!", .{}, true);
+ }
- pub fn addRangeError(self: *Lexer, r: logger.Range, comptime format: []const u8, args: anytype, panic: bool) void {
- if (self.prev_error_loc.eql(r.loc)) {
- return;
+ pub fn addDefaultError(self: *@This(), msg: []const u8) void {
+ self.addError(self.start, "{s}", .{msg}, true);
}
- const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable;
- var msg = self.log.addRangeError(self.source, r, errorMessage);
- self.prev_error_loc = r.loc;
+ pub fn addError(self: *@This(), _loc: usize, comptime format: []const u8, args: anytype, panic: bool) void {
+ var __loc = logger.usize2Loc(_loc);
+ if (__loc.eql(self.prev_error_loc)) {
+ return;
+ }
- if (panic) {
- self.doPanic(errorMessage);
+ const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable;
+ self.log.addError(self.source, __loc, errorMessage) catch unreachable;
+ self.prev_error_loc = __loc;
+ var msg = self.log.msgs.items[self.log.msgs.items.len - 1];
+ msg.formatNoWriter(std.debug.panic);
}
- }
- fn doPanic(self: *Lexer, content: []const u8) void {
- std.debug.panic("{s}", .{content});
- }
+ pub fn addRangeError(self: *@This(), r: logger.Range, comptime format: []const u8, args: anytype, panic: bool) void {
+ if (self.prev_error_loc.eql(r.loc)) {
+ return;
+ }
- pub fn codePointEql(self: *Lexer, a: u8) bool {
- return @intCast(CodePoint, a) == self.code_point;
- }
+ const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable;
+ var msg = self.log.addRangeError(self.source, r, errorMessage);
+ self.prev_error_loc = r.loc;
- fn nextCodepoint(it: *Lexer) callconv(.Inline) CodePoint {
- const slice = it.nextCodepointSlice() orelse return @as(CodePoint, -1);
+ if (panic) {
+ self.doPanic(errorMessage);
+ }
+ }
- switch (slice.len) {
- 1 => return @as(CodePoint, slice[0]),
- 2 => return @as(CodePoint, unicode.utf8Decode2(slice) catch unreachable),
- 3 => return @as(CodePoint, unicode.utf8Decode3(slice) catch unreachable),
- 4 => return @as(CodePoint, unicode.utf8Decode4(slice) catch unreachable),
- else => unreachable,
+ fn doPanic(self: *@This(), content: []const u8) void {
+ std.debug.panic("{s}", .{content});
}
- }
- /// Look ahead at the next n codepoints without advancing the iterator.
- /// If fewer than n codepoints are available, then return the remainder of the string.
- fn peek(it: *Lexer, n: usize) []const u8 {
- const original_i = it.current;
- defer it.current = original_i;
-
- var end_ix = original_i;
- var found: usize = 0;
- while (found < n) : (found += 1) {
- const next_codepoint = it.nextCodepointSlice() orelse return it.source.contents[original_i..];
- end_ix += next_codepoint.len;
+ pub fn codePointEql(self: *@This(), a: u8) bool {
+ return @intCast(CodePoint, a) == self.code_point;
}
- return it.source.contents[original_i..end_ix];
- }
+ fn nextCodepoint(it: *@This()) callconv(.Inline) CodePoint {
+ const slice = it.nextCodepointSlice() orelse return @as(CodePoint, -1);
- pub fn isIdentifierOrKeyword(lexer: Lexer) bool {
- return @enumToInt(lexer.token) >= @enumToInt(T.t_identifier);
- }
+ switch (slice.len) {
+ 1 => return @as(CodePoint, slice[0]),
+ 2 => return @as(CodePoint, unicode.utf8Decode2(slice) catch unreachable),
+ 3 => return @as(CodePoint, unicode.utf8Decode3(slice) catch unreachable),
+ 4 => return @as(CodePoint, unicode.utf8Decode4(slice) catch unreachable),
+ else => unreachable,
+ }
+ }
- fn parseStringLiteral(lexer: *Lexer) void {
- var quote: CodePoint = lexer.code_point;
- var needs_slow_path = false;
- var suffixLen: usize = 1;
-
- if (quote != '`') {
- lexer.token = T.t_string_literal;
- } else if (lexer.rescan_close_brace_as_template_token) {
- lexer.token = T.t_template_tail;
- } else {
- lexer.token = T.t_no_substitution_template_literal;
+ /// Look ahead at the next n codepoints without advancing the iterator.
+ /// If fewer than n codepoints are available, then return the remainder of the string.
+ fn peek(it: *@This(), n: usize) []const u8 {
+ const original_i = it.current;
+ defer it.current = original_i;
+
+ var end_ix = original_i;
+ var found: usize = 0;
+ while (found < n) : (found += 1) {
+ const next_codepoint = it.nextCodepointSlice() orelse return it.source.contents[original_i..];
+ end_ix += next_codepoint.len;
+ }
+
+ return it.source.contents[original_i..end_ix];
}
- lexer.step();
- stringLiteral: while (true) {
- switch (lexer.code_point) {
- '\\' => {
- needs_slow_path = true;
- lexer.step();
+ pub fn isIdentifierOrKeyword(lexer: @This()) bool {
+ return @enumToInt(lexer.token) >= @enumToInt(T.t_identifier);
+ }
- // Handle Windows CRLF
- if (lexer.code_point == '\r' and !IS_JSON_FILE) {
+ fn parseStringLiteral(lexer: *@This()) void {
+ var quote: CodePoint = lexer.code_point;
+ var needs_slow_path = false;
+ var suffixLen: usize = 1;
+
+ if (quote != '`') {
+ lexer.token = T.t_string_literal;
+ } else if (lexer.rescan_close_brace_as_template_token) {
+ lexer.token = T.t_template_tail;
+ } else {
+ lexer.token = T.t_no_substitution_template_literal;
+ }
+ lexer.step();
+
+ stringLiteral: while (true) {
+ switch (lexer.code_point) {
+ '\\' => {
+ needs_slow_path = true;
lexer.step();
- if (lexer.code_point == '\n') {
+
+ // Handle Windows CRLF
+ if (lexer.code_point == '\r' and jsonOptions != null) {
lexer.step();
+ if (lexer.code_point == '\n') {
+ lexer.step();
+ }
+ continue :stringLiteral;
}
- continue :stringLiteral;
- }
- },
- // This indicates the end of the file
-
- -1 => {
- lexer.addDefaultError("Unterminated string literal");
- },
+ },
+ // This indicates the end of the file
- '\r' => {
- if (quote != '`') {
+ -1 => {
lexer.addDefaultError("Unterminated string literal");
- }
+ },
- // Template literals require newline normalization
- needs_slow_path = true;
- },
+ '\r' => {
+ if (quote != '`') {
+ lexer.addDefaultError("Unterminated string literal");
+ }
- '\n' => {
- if (quote != '`') {
- lexer.addDefaultError("Unterminated string literal");
- }
- },
+ // Template literals require newline normalization
+ needs_slow_path = true;
+ },
- '$' => {
- if (quote == '`') {
- lexer.step();
- if (lexer.code_point == '{') {
- suffixLen = 2;
+ '\n' => {
+ if (quote != '`') {
+ lexer.addDefaultError("Unterminated string literal");
+ }
+ },
+
+ '$' => {
+ if (quote == '`') {
lexer.step();
- if (lexer.rescan_close_brace_as_template_token) {
- lexer.token = T.t_template_middle;
- } else {
- lexer.token = T.t_template_head;
+ if (lexer.code_point == '{') {
+ suffixLen = 2;
+ lexer.step();
+ if (lexer.rescan_close_brace_as_template_token) {
+ lexer.token = T.t_template_middle;
+ } else {
+ lexer.token = T.t_template_head;
+ }
+ break :stringLiteral;
}
- break :stringLiteral;
+ continue :stringLiteral;
}
- continue :stringLiteral;
- }
- },
+ },
- else => {
- if (quote == lexer.code_point) {
- lexer.step();
- break :stringLiteral;
- }
- // Non-ASCII strings need the slow path
- if (lexer.code_point >= 0x80) {
- needs_slow_path = true;
- } else if (IS_JSON_FILE and lexer.code_point < 0x20) {
- lexer.syntaxError();
- }
- },
+ else => {
+ if (quote == lexer.code_point) {
+ lexer.step();
+ break :stringLiteral;
+ }
+ // Non-ASCII strings need the slow path
+ if (lexer.code_point >= 0x80) {
+ needs_slow_path = true;
+ } else if (jsonOptions != null and lexer.code_point < 0x20) {
+ lexer.syntaxError();
+ }
+ },
+ }
+ lexer.step();
}
- lexer.step();
- }
- const text = lexer.source.contents[lexer.start + 1 .. lexer.end - suffixLen];
- // TODO: actually implement proper utf16
- lexer.string_literal = lexer.allocator.alloc(u16, text.len) catch unreachable;
- var i: usize = 0;
- for (text) |byte| {
- lexer.string_literal[i] = byte;
- i += 1;
- }
- // for (text)
- // // if (needs_slow_path) {
- // // // Slow path
+ const text = lexer.source.contents[lexer.start + 1 .. lexer.end - suffixLen];
+ if (needs_slow_path) {
+ lexer.string_literal = lexer.stringToUTF16(text);
+ } else {
+ lexer.string_literal = lexer.allocator.alloc(u16, text.len) catch unreachable;
+ var i: usize = 0;
+ for (text) |byte| {
+ lexer.string_literal[i] = byte;
+ i += 1;
+ }
+ }
- // // // lexer.string_literal = lexer.(lexer.start + 1, text);
- // // } else {
- // // // Fast path
+ if (quote == '\'' and jsonOptions != null) {
+ lexer.addRangeError(lexer.range(), "JSON strings must use double quotes", .{}, true);
+ }
+ // for (text)
+ // // if (needs_slow_path) {
+ // // // Slow path
- // // }
- }
+ // // // lexer.string_literal = lexer.(lexer.start + 1, text);
+ // // } else {
+ // // // Fast path
- fn step(lexer: *Lexer) void {
- lexer.code_point = lexer.nextCodepoint();
-
- // Track the approximate number of newlines in the file so we can preallocate
- // the line offset table in the printer for source maps. The line offset table
- // is the #1 highest allocation in the heap profile, so this is worth doing.
- // This count is approximate because it handles "\n" and "\r\n" (the common
- // cases) but not "\r" or "\u2028" or "\u2029". Getting this wrong is harmless
- // because it's only a preallocation. The array will just grow if it's too small.
- if (lexer.code_point == '\n') {
- lexer.approximate_newline_count += 1;
+ // // }
}
- }
- pub fn expect(self: *Lexer, comptime token: T) void {
- if (self.token != token) {
- self.expected(token);
+ fn step(lexer: *@This()) void {
+ lexer.code_point = lexer.nextCodepoint();
+
+ // Track the approximate number of newlines in the file so we can preallocate
+ // the line offset table in the printer for source maps. The line offset table
+ // is the #1 highest allocation in the heap profile, so this is worth doing.
+ // This count is approximate because it handles "\n" and "\r\n" (the common
+ // cases) but not "\r" or "\u2028" or "\u2029". Getting this wrong is harmless
+ // because it's only a preallocation. The array will just grow if it's too small.
+ if (lexer.code_point == '\n') {
+ lexer.approximate_newline_count += 1;
+ }
}
- self.next();
- }
+ pub fn expect(self: *@This(), comptime token: T) void {
+ if (self.token != token) {
+ self.expected(token);
+ }
- pub fn expectOrInsertSemicolon(lexer: *Lexer) void {
- if (lexer.token == T.t_semicolon or (!lexer.has_newline_before and
- lexer.token != T.t_close_brace and lexer.token != T.t_end_of_file))
- {
- lexer.expect(T.t_semicolon);
+ self.next();
}
- }
- pub fn addUnsupportedSyntaxError(self: *Lexer, msg: []const u8) void {
- self.addError(self.end, "Unsupported syntax: {s}", .{msg}, true);
- }
+ pub fn expectOrInsertSemicolon(lexer: *@This()) void {
+ if (lexer.token == T.t_semicolon or (!lexer.has_newline_before and
+ lexer.token != T.t_close_brace and lexer.token != T.t_end_of_file))
+ {
+ lexer.expect(T.t_semicolon);
+ }
+ }
- pub fn scanIdentifierWithEscapes(self: *Lexer) void {
- self.addUnsupportedSyntaxError("escape sequence");
- return;
- }
+ pub fn addUnsupportedSyntaxError(self: *@This(), msg: []const u8) void {
+ self.addError(self.end, "Unsupported syntax: {s}", .{msg}, true);
+ }
+
+ pub fn scanIdentifierWithEscapes(self: *@This()) void {
+ self.addUnsupportedSyntaxError("escape sequence");
+ return;
+ }
- pub fn debugInfo(self: *Lexer) void {
- if (self.log.errors > 0) {
- const stderr = std.io.getStdErr().writer();
- self.log.print(stderr) catch unreachable;
- } else {
- if (self.token == T.t_identifier or self.token == T.t_string_literal) {
- std.debug.print(" {s} ", .{self.raw()});
+ pub fn debugInfo(self: *@This()) void {
+ if (self.log.errors > 0) {
+ const stderr = std.io.getStdErr().writer();
+ self.log.print(stderr) catch unreachable;
} else {
- std.debug.print(" <{s}> ", .{tokenToString.get(self.token)});
+ if (self.token == T.t_identifier or self.token == T.t_string_literal) {
+ std.debug.print(" {s} ", .{self.raw()});
+ } else {
+ std.debug.print(" <{s}> ", .{tokenToString.get(self.token)});
+ }
}
}
- }
- pub fn expectContextualKeyword(self: *Lexer, comptime keyword: string) void {
- if (!self.isContextualKeyword(keyword)) {
- self.addError(self.start, "\"{s}\"", .{keyword}, true);
+ pub fn expectContextualKeyword(self: *@This(), comptime keyword: string) void {
+ if (!self.isContextualKeyword(keyword)) {
+ self.addError(self.start, "\"{s}\"", .{keyword}, true);
+ }
+ self.next();
}
- self.next();
- }
-
- pub fn next(lexer: *Lexer) void {
- lexer.has_newline_before = lexer.end == 0;
- lex: while (lexer.log.errors == 0) {
- lexer.start = lexer.end;
- lexer.token = T.t_end_of_file;
+ pub fn next(lexer: *@This()) void {
+ lexer.has_newline_before = lexer.end == 0;
- switch (lexer.code_point) {
- -1 => {
- lexer.token = T.t_end_of_file;
- break :lex;
- },
+ lex: while (lexer.log.errors == 0) {
+ lexer.start = lexer.end;
+ lexer.token = T.t_end_of_file;
- '#' => {
- if (lexer.start == 0 and lexer.source.contents[1] == '!') {
- lexer.addUnsupportedSyntaxError("#!hashbang is not supported yet.");
- return;
- }
+ switch (lexer.code_point) {
+ -1 => {
+ lexer.token = T.t_end_of_file;
+ break :lex;
+ },
- lexer.step();
- if (!isIdentifierStart(lexer.code_point)) {
- lexer.syntaxError();
- }
- lexer.step();
+ '#' => {
+ if (lexer.start == 0 and lexer.source.contents[1] == '!') {
+ lexer.addUnsupportedSyntaxError("#!hashbang is not supported yet.");
+ return;
+ }
- if (isIdentifierStart(lexer.code_point)) {
lexer.step();
- while (isIdentifierContinue(lexer.code_point)) {
- lexer.step();
- }
- if (lexer.code_point == '\\') {
- lexer.scanIdentifierWithEscapes();
- lexer.token = T.t_private_identifier;
- // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier);
- } else {
- lexer.token = T.t_private_identifier;
- lexer.identifier = lexer.raw();
+ if (!isIdentifierStart(lexer.code_point)) {
+ lexer.syntaxError();
}
- break;
- }
- },
- '\r', '\n', 0x2028, 0x2029 => {
- lexer.step();
- lexer.has_newline_before = true;
- continue;
- },
-
- '\t', ' ' => {
- lexer.step();
- continue;
- },
-
- '(' => {
- lexer.step();
- lexer.token = T.t_open_paren;
- },
- ')' => {
- lexer.step();
- lexer.token = T.t_close_paren;
- },
- '[' => {
- lexer.step();
- lexer.token = T.t_open_bracket;
- },
- ']' => {
- lexer.step();
- lexer.token = T.t_close_bracket;
- },
- '{' => {
- lexer.step();
- lexer.token = T.t_open_brace;
- },
- '}' => {
- lexer.step();
- lexer.token = T.t_close_brace;
- },
- ',' => {
- lexer.step();
- lexer.token = T.t_comma;
- },
- ':' => {
- lexer.step();
- lexer.token = T.t_colon;
- },
- ';' => {
- lexer.step();
- lexer.token = T.t_semicolon;
- },
- '@' => {
- lexer.step();
- lexer.token = T.t_at;
- },
- '~' => {
- lexer.step();
- lexer.token = T.t_tilde;
- },
+ lexer.step();
- '?' => {
- // '?' or '?.' or '??' or '??='
- lexer.step();
- switch (lexer.code_point) {
- '?' => {
+ if (isIdentifierStart(lexer.code_point)) {
lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_question_question_equals;
- },
- else => {
- lexer.token = T.t_question_question;
- },
+ while (isIdentifierContinue(lexer.code_point)) {
+ lexer.step();
}
- },
-
- '.' => {
- lexer.token = T.t_question;
- const current = lexer.current;
- const contents = lexer.source.contents;
-
- // Lookahead to disambiguate with 'a?.1:b'
- if (current < contents.len) {
- const c = contents[current];
- if (c < '0' or c > '9') {
- lexer.step();
- lexer.token = T.t_question_dot;
- }
+ if (lexer.code_point == '\\') {
+ lexer.scanIdentifierWithEscapes();
+ lexer.token = T.t_private_identifier;
+ // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier);
+ } else {
+ lexer.token = T.t_private_identifier;
+ lexer.identifier = lexer.raw();
}
- },
- else => {
- lexer.token = T.t_question;
- },
- }
- },
-
- '%' => {
- // '%' or '%='
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_percent_equals;
- },
+ break;
+ }
+ },
+ '\r', '\n', 0x2028, 0x2029 => {
+ lexer.step();
+ lexer.has_newline_before = true;
+ continue;
+ },
+ '\t', ' ' => {
+ lexer.step();
+ continue;
+ },
+ '(' => {
+ lexer.step();
+ lexer.token = T.t_open_paren;
+ },
+ ')' => {
+ lexer.step();
+ lexer.token = T.t_close_paren;
+ },
+ '[' => {
+ lexer.step();
+ lexer.token = T.t_open_bracket;
+ },
+ ']' => {
+ lexer.step();
+ lexer.token = T.t_close_bracket;
+ },
+ '{' => {
+ lexer.step();
+ lexer.token = T.t_open_brace;
+ },
+ '}' => {
+ lexer.step();
+ lexer.token = T.t_close_brace;
+ },
+ ',' => {
+ lexer.step();
+ lexer.token = T.t_comma;
+ },
+ ':' => {
+ lexer.step();
+ lexer.token = T.t_colon;
+ },
+ ';' => {
+ lexer.step();
+ lexer.token = T.t_semicolon;
+ },
+ '@' => {
+ lexer.step();
+ lexer.token = T.t_at;
+ },
+ '~' => {
+ lexer.step();
+ lexer.token = T.t_tilde;
+ },
+ '?' => {
+ // '?' or '?.' or '??' or '??='
+ lexer.step();
+ switch (lexer.code_point) {
+ '?' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_question_question_equals;
+ },
+ else => {
+ lexer.token = T.t_question_question;
+ },
+ }
+ },
- else => {
- lexer.token = T.t_percent;
- },
- }
- },
+ '.' => {
+ lexer.token = T.t_question;
+ const current = lexer.current;
+ const contents = lexer.source.contents;
+
+ // Lookahead to disambiguate with 'a?.1:b'
+ if (current < contents.len) {
+ const c = contents[current];
+ if (c < '0' or c > '9') {
+ lexer.step();
+ lexer.token = T.t_question_dot;
+ }
+ }
+ },
+ else => {
+ lexer.token = T.t_question;
+ },
+ }
+ },
+ '%' => {
+ // '%' or '%='
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_percent_equals;
+ },
- '&' => {
- // '&' or '&=' or '&&' or '&&='
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_ampersand_equals;
- },
+ else => {
+ lexer.token = T.t_percent;
+ },
+ }
+ },
- '&' => {
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_ampersand_ampersand_equals;
- },
+ '&' => {
+ // '&' or '&=' or '&&' or '&&='
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_ampersand_equals;
+ },
- else => {
- lexer.token = T.t_ampersand_ampersand;
- },
- }
- },
- else => {
- lexer.token = T.t_ampersand;
- },
- }
- },
+ '&' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_ampersand_ampersand_equals;
+ },
+
+ else => {
+ lexer.token = T.t_ampersand_ampersand;
+ },
+ }
+ },
+ else => {
+ lexer.token = T.t_ampersand;
+ },
+ }
+ },
- '|' => {
+ '|' => {
- // '|' or '|=' or '||' or '||='
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_bar_equals;
- },
- '|' => {
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_bar_bar_equals;
- },
+ // '|' or '|=' or '||' or '||='
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_bar_equals;
+ },
+ '|' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_bar_bar_equals;
+ },
+
+ else => {
+ lexer.token = T.t_bar_bar;
+ },
+ }
+ },
+ else => {
+ lexer.token = T.t_bar;
+ },
+ }
+ },
- else => {
- lexer.token = T.t_bar_bar;
- },
- }
- },
- else => {
- lexer.token = T.t_bar;
- },
- }
- },
+ '^' => {
+ // '^' or '^='
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_caret_equals;
+ },
- '^' => {
- // '^' or '^='
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_caret_equals;
- },
+ else => {
+ lexer.token = T.t_caret;
+ },
+ }
+ },
- else => {
- lexer.token = T.t_caret;
- },
- }
- },
+ '+' => {
+ // '+' or '+=' or '++'
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_plus_equals;
+ },
- '+' => {
- // '+' or '+=' or '++'
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_plus_equals;
- },
+ '+' => {
+ lexer.step();
+ lexer.token = T.t_plus_plus;
+ },
- '+' => {
- lexer.step();
- lexer.token = T.t_plus_plus;
- },
+ else => {
+ lexer.token = T.t_plus;
+ },
+ }
+ },
- else => {
- lexer.token = T.t_plus;
- },
- }
- },
+ '-' => {
+ // '+' or '+=' or '++'
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_minus_equals;
+ },
- '=' => {
- // '=' or '=>' or '==' or '==='
- lexer.step();
- switch (lexer.code_point) {
- '>' => {
- lexer.step();
- lexer.token = T.t_equals_greater_than;
- },
+ '-' => {
+ lexer.step();
- '=' => {
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
+ if (lexer.code_point == '>' and lexer.has_newline_before) {
lexer.step();
- lexer.token = T.t_equals_equals_equals;
- },
-
- else => {
- lexer.token = T.t_equals_equals;
- },
- }
- },
+ lexer.log.addRangeWarning(lexer.source, lexer.range(), "Treating \"-->\" as the start of a legacy HTML single-line comment") catch unreachable;
+
+ singleLineHTMLCloseComment: while (true) {
+ switch (lexer.code_point) {
+ '\r', '\n', 0x2028, 0x2029 => {
+ break :singleLineHTMLCloseComment;
+ },
+ -1 => {
+ break :singleLineHTMLCloseComment;
+ },
+ else => {},
+ }
+ lexer.step();
+ }
+ continue;
+ }
- else => {
- lexer.token = T.t_equals;
- },
- }
- },
+ lexer.token = T.t_minus_minus;
+ },
- '<' => {
- // '<' or '<<' or '<=' or '<<=' or '<!--'
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_less_than_equals;
- },
+ else => {
+ lexer.token = T.t_plus;
+ },
+ }
+ },
- '<' => {
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_less_than_less_than_equals;
- },
+ '*' => {
+ // '*' or '*=' or '**' or '**='
- else => {
- lexer.token = T.t_less_than_less_than;
- },
- }
- },
- // Handle legacy HTML-style comments
- '!' => {
- if (std.mem.eql(u8, lexer.peek("--".len), "--")) {
- lexer.addUnsupportedSyntaxError("Legacy HTML comments not implemented yet!");
- return;
- }
-
- lexer.token = T.t_less_than;
- },
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = .t_asterisk_equals;
+ },
+ '*' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = .t_asterisk_asterisk_equals;
+ },
+ else => {
+ lexer.token = .t_asterisk_asterisk;
+ },
+ }
+ },
+ else => {
+ lexer.token = .t_asterisk;
+ },
+ }
+ },
+ '/' => {
+ // '/' or '/=' or '//' or '/* ... */'
+ lexer.step();
+ // TODO: forGlobalName
- else => {
- lexer.token = T.t_less_than;
- },
- }
- },
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = .t_slash_equals;
+ },
+ '/' => {
+ lexer.step();
+ singleLineComment: while (true) {
+ lexer.step();
+ switch (lexer.code_point) {
+ '\r', '\n', 0x2028, 0x2029 => {
+ break :singleLineComment;
+ },
+ -1 => {
+ break :singleLineComment;
+ },
+ else => {},
+ }
+ }
- '>' => {
- // '>' or '>>' or '>>>' or '>=' or '>>=' or '>>>='
- lexer.step();
+ if (jsonOptions) |json| {
+ if (!json.allow_comments) {
+ lexer.addRangeError(lexer.range(), "JSON does not support comments", .{}, true);
+ return;
+ }
+ }
+ lexer.scanCommentText();
+ continue;
+ },
+ '*' => {
+ lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_greater_than_equals;
- },
- '>' => {
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_greater_than_greater_than_equals;
- },
- '>' => {
- lexer.step();
+ multiLineComment: while (true) {
switch (lexer.code_point) {
- '=' => {
+ '*' => {
+ lexer.step();
+ if (lexer.code_point == '/') {
+ lexer.step();
+ break :multiLineComment;
+ }
+ },
+ '\r', '\n', 0x2028, 0x2029 => {
lexer.step();
- lexer.token = T.t_greater_than_greater_than_greater_than_equals;
+ lexer.has_newline_before = true;
+ },
+ -1 => {
+ lexer.start = lexer.end;
+ lexer.addError(lexer.start, "Expected \"*/\" to terminate multi-line comment", .{}, true);
},
else => {
- lexer.token = T.t_greater_than_greater_than_greater_than;
+ lexer.step();
},
}
- },
- else => {
- lexer.token = T.t_greater_than_greater_than;
- },
- }
- },
- else => {
- lexer.token = T.t_greater_than;
- },
- }
- },
+ }
+ if (jsonOptions) |json| {
+ if (!json.allow_comments) {
+ lexer.addRangeError(lexer.range(), "JSON does not support comments", .{}, true);
+ return;
+ }
+ }
+ lexer.scanCommentText();
+ continue;
+ },
+ else => {
+ lexer.token = .t_slash;
+ },
+ }
+ },
- '!' => {
- // '!' or '!=' or '!=='
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- switch (lexer.code_point) {
- '=' => {
- lexer.step();
- lexer.token = T.t_exclamation_equals_equals;
- },
+ '=' => {
+ // '=' or '=>' or '==' or '==='
+ lexer.step();
+ switch (lexer.code_point) {
+ '>' => {
+ lexer.step();
+ lexer.token = T.t_equals_greater_than;
+ },
- else => {
- lexer.token = T.t_exclamation_equals;
- },
- }
- },
- else => {
- lexer.token = T.t_exclamation;
- },
- }
- },
+ '=' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_equals_equals_equals;
+ },
+
+ else => {
+ lexer.token = T.t_equals_equals;
+ },
+ }
+ },
- '\'', '"', '`' => {
- lexer.parseStringLiteral();
- },
+ else => {
+ lexer.token = T.t_equals;
+ },
+ }
+ },
- '_', '$', 'a'...'z', 'A'...'Z' => {
- lexer.step();
- while (isIdentifierContinue(lexer.code_point)) {
+ '<' => {
+ // '<' or '<<' or '<=' or '<<=' or '<!--'
lexer.step();
- }
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_less_than_equals;
+ },
- if (lexer.code_point == '\\') {
- lexer.scanIdentifierWithEscapes();
- } else {
- const contents = lexer.raw();
- lexer.identifier = contents;
- if (Keywords.get(contents)) |keyword| {
- lexer.token = keyword;
- } else {
- lexer.token = T.t_identifier;
+ '<' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_less_than_less_than_equals;
+ },
+
+ else => {
+ lexer.token = T.t_less_than_less_than;
+ },
+ }
+ },
+ // Handle legacy HTML-style comments
+ '!' => {
+ if (std.mem.eql(u8, lexer.peek("--".len), "--")) {
+ lexer.addUnsupportedSyntaxError("Legacy HTML comments not implemented yet!");
+ return;
+ }
+
+ lexer.token = T.t_less_than;
+ },
+
+ else => {
+ lexer.token = T.t_less_than;
+ },
}
- }
- },
+ },
- '\\' => {
- // TODO: normal
- lexer.scanIdentifierWithEscapes();
- },
+ '>' => {
+ // '>' or '>>' or '>>>' or '>=' or '>>=' or '>>>='
+ lexer.step();
- '.', '0'...'9' => {
- lexer.parseNumericLiteralOrDot();
- },
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_greater_than_equals;
+ },
+ '>' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_greater_than_greater_than_equals;
+ },
+ '>' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_greater_than_greater_than_greater_than_equals;
+ },
+ else => {
+ lexer.token = T.t_greater_than_greater_than_greater_than;
+ },
+ }
+ },
+ else => {
+ lexer.token = T.t_greater_than_greater_than;
+ },
+ }
+ },
+ else => {
+ lexer.token = T.t_greater_than;
+ },
+ }
+ },
- else => {
- // Check for unusual whitespace characters
- if (isWhitespace(lexer.code_point)) {
+ '!' => {
+ // '!' or '!=' or '!=='
lexer.step();
- continue;
- }
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ switch (lexer.code_point) {
+ '=' => {
+ lexer.step();
+ lexer.token = T.t_exclamation_equals_equals;
+ },
+
+ else => {
+ lexer.token = T.t_exclamation_equals;
+ },
+ }
+ },
+ else => {
+ lexer.token = T.t_exclamation;
+ },
+ }
+ },
- if (isIdentifierStart(lexer.code_point)) {
+ '\'', '"', '`' => {
+ lexer.parseStringLiteral();
+ },
+
+ '_', '$', 'a'...'z', 'A'...'Z' => {
lexer.step();
while (isIdentifierContinue(lexer.code_point)) {
lexer.step();
}
- if (lexer.code_point == '\\') {
- // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier);
+ if (lexer.code_point == '\\') {
+ lexer.scanIdentifierWithEscapes();
} else {
- lexer.token = T.t_identifier;
- lexer.identifier = lexer.raw();
+ const contents = lexer.raw();
+ lexer.identifier = contents;
+ if (Keywords.get(contents)) |keyword| {
+ lexer.token = keyword;
+ } else {
+ lexer.token = T.t_identifier;
+ }
}
- break;
- }
-
- lexer.end = lexer.current;
- lexer.token = T.t_syntax_error;
- },
- }
+ },
- return;
- }
- }
+ '\\' => {
+ // TODO: normal
+ lexer.scanIdentifierWithEscapes();
+ },
- pub fn expected(self: *Lexer, token: T) void {
- if (tokenToString.get(token).len > 0) {
- self.expectedString(tokenToString.get(token));
- } else {
- self.unexpected();
- }
- }
+ '.', '0'...'9' => {
+ lexer.parseNumericLiteralOrDot();
+ },
- pub fn unexpected(lexer: *Lexer) void {
- var found: string = undefined;
- if (lexer.start == lexer.source.contents.len) {
- found = "end of file";
- } else {
- found = lexer.raw();
- }
+ else => {
+ // Check for unusual whitespace characters
+ if (isWhitespace(lexer.code_point)) {
+ lexer.step();
+ continue;
+ }
- lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}, true);
- }
+ if (isIdentifierStart(lexer.code_point)) {
+ lexer.step();
+ while (isIdentifierContinue(lexer.code_point)) {
+ lexer.step();
+ }
+ if (lexer.code_point == '\\') {
- pub fn raw(self: *Lexer) []const u8 {
- return self.source.contents[self.start..self.end];
- }
+ // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier);
+ } else {
+ lexer.token = T.t_identifier;
+ lexer.identifier = lexer.raw();
+ }
+ break;
+ }
- pub fn isContextualKeyword(self: *Lexer, comptime keyword: string) bool {
- return self.token == .t_identifier and strings.eql(self.raw(), keyword);
- }
+ lexer.end = lexer.current;
+ lexer.token = T.t_syntax_error;
+ },
+ }
- pub fn expectedString(self: *Lexer, text: string) void {
- var found = text;
- if (self.source.contents.len == self.start) {
- found = "end of file";
+ return;
+ }
}
- self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }, true);
- }
-
- pub fn range(self: *Lexer) logger.Range {
- return logger.Range{
- .loc = logger.usize2Loc(self.start),
- .len = std.math.lossyCast(i32, self.end - self.start),
- };
- }
- pub fn init(log: logger.Log, source: logger.Source, allocator: *std.mem.Allocator) !Lexer {
- var empty_string_literal: JavascriptString = undefined;
- var lex = Lexer{
- .log = log,
- .source = source,
- .string_literal = empty_string_literal,
- .prev_error_loc = logger.Loc.Empty,
- .allocator = allocator,
- };
- lex.step();
- lex.next();
-
- return lex;
- }
+ pub fn expected(self: *@This(), token: T) void {
+ if (tokenToString.get(token).len > 0) {
+ self.expectedString(tokenToString.get(token));
+ } else {
+ self.unexpected();
+ }
+ }
- pub fn scanRegExp(lexer: *Lexer) void {
- while (true) {
- switch (lexer.code_point) {
- '/' => {
- lexer.step();
- while (isIdentifierContinue(lexer.code_point)) {
- switch (lexer.code_point) {
- 'g', 'i', 'm', 's', 'u', 'y' => {
- lexer.step();
- },
- else => {
- lexer.syntaxError();
- },
- }
- }
- },
- '[' => {
- lexer.step();
- while (lexer.code_point != ']') {
- lexer.scanRegExpValidateAndStep();
- }
- lexer.step();
- },
- else => {
- lexer.scanRegExpValidateAndStep();
- },
+ pub fn unexpected(lexer: *@This()) void {
+ var found: string = undefined;
+ if (lexer.start == lexer.source.contents.len) {
+ found = "end of file";
+ } else {
+ found = lexer.raw();
}
+
+ lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}, true);
}
- }
- // TODO: use wtf-8 encoding.
- pub fn stringToUTF16(lexer: *Lexer, str: string) JavascriptString {
- var buf: JavascriptString = lexer.allocator.alloc(u16, std.mem.len(str)) catch unreachable;
- var i: usize = 0;
- // theres prob a faster/better way
- for (str) |char| {
- buf[i] = char;
- i += 1;
+ pub fn raw(self: *@This()) []const u8 {
+ return self.source.contents[self.start..self.end];
}
- return buf;
- }
+ pub fn isContextualKeyword(self: *@This(), comptime keyword: string) bool {
+ return self.token == .t_identifier and strings.eql(self.raw(), keyword);
+ }
- // TODO: use wtf-8 encoding.
- pub fn utf16ToStringWithValidation(lexer: *Lexer, js: JavascriptString) !string {
- return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js);
- }
+ pub fn expectedString(self: *@This(), text: string) void {
+ var found = text;
+ if (self.source.contents.len == self.start) {
+ found = "end of file";
+ }
+ self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }, true);
+ }
- // TODO: use wtf-8 encoding.
- pub fn utf16ToString(lexer: *Lexer, js: JavascriptString) string {
- return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js) catch unreachable;
- }
+ pub fn scanCommentText(lexer: *@This()) void {
+ var text = lexer.source.contents[lexer.start..lexer.end];
+ const has_preserve_annotation = text.len > 2 and text[2] == '!';
+ const is_multiline_comment = text[1] == '*';
- pub fn nextInsideJSXElement() void {
- std.debug.panic("JSX not implemented yet.", .{});
- }
+ // Omit the trailing "*/" from the checks below
+ var endCommentText = text.len;
+ if (is_multiline_comment) {
+ endCommentText -= 2;
+ }
- fn scanRegExpValidateAndStep(lexer: *Lexer) void {
- if (lexer.code_point == '\\') {
- lexer.step();
- }
+ if (has_preserve_annotation or lexer.preserve_all_comments_before) {
+ if (is_multiline_comment) {
+ // text = lexer.removeMultilineCommentIndent(lexer.source.contents[0..lexer.start], text);
+ }
- switch (lexer.code_point) {
- '\r', '\n', 0x2028, 0x2029 => {
- // Newlines aren't allowed in regular expressions
- lexer.syntaxError();
- },
- -1 => { // EOF
- lexer.syntaxError();
- },
- else => {
- lexer.step();
- },
+ lexer.comments_to_preserve_before.append(js_ast.G.Comment{
+ .text = text,
+ .loc = lexer.loc(),
+ }) catch unreachable;
+ }
}
- }
- pub fn rescanCloseBraceAsTemplateToken(lexer: *Lexer) void {
- if (lexer.token != .t_close_brace) {
- lexer.expected(.t_close_brace);
+ // TODO: implement this
+ // it's too complicated to handle all the edgecases right now given the state of Zig's standard library
+ pub fn removeMultilineCommentIndent(lexer: *@This(), _prefix: string, text: string) string {
+ return text;
}
- lexer.rescan_close_brace_as_template_token = true;
- lexer.code_point = '`';
- lexer.current = lexer.end;
- lexer.end -= 1;
- lexer.next();
- lexer.rescan_close_brace_as_template_token = false;
- }
-
- pub fn rawTemplateContents(lexer: *Lexer) string {
- var text: string = undefined;
-
- switch (lexer.token) {
- .t_no_substitution_template_literal, .t_template_tail => {
- text = lexer.source.contents[lexer.start + 1 .. lexer.end - 1];
- },
- .t_template_middle, .t_template_head => {
- text = lexer.source.contents[lexer.start + 1 .. lexer.end - 2];
- },
- else => {},
+ pub fn range(self: *@This()) logger.Range {
+ return logger.Range{
+ .loc = logger.usize2Loc(self.start),
+ .len = std.math.lossyCast(i32, self.end - self.start),
+ };
}
- if (strings.indexOfChar(text, '\r') == null) {
- return text;
+ pub fn init(log: logger.Log, source: logger.Source, allocator: *std.mem.Allocator) !@This() {
+ var empty_string_literal: JavascriptString = undefined;
+ var lex = @This(){
+ .log = log,
+ .source = source,
+ .string_literal = empty_string_literal,
+ .prev_error_loc = logger.Loc.Empty,
+ .allocator = allocator,
+ .comments_to_preserve_before = std.ArrayList(js_ast.G.Comment).init(allocator),
+ };
+ lex.step();
+ lex.next();
+
+ return lex;
}
- // From the specification:
- //
- // 11.8.6.1 Static Semantics: TV and TRV
- //
- // TV excludes the code units of LineContinuation while TRV includes
- // them. <CR><LF> and <CR> LineTerminatorSequences are normalized to
- // <LF> for both TV and TRV. An explicit EscapeSequence is needed to
- // include a <CR> or <CR><LF> sequence.
- var bytes = MutableString.initCopy(lexer.allocator, text) catch unreachable;
- var end: usize = 0;
- var i: usize = 0;
- var c: u8 = '0';
- while (i < bytes.list.items.len) {
- c = bytes.list.items[i];
- i += 1;
-
- if (c == '\r') {
- // Convert '\r\n' into '\n'
- if (i < bytes.list.items.len and bytes.list.items[i] == '\n') {
- i += 1;
+ pub fn scanRegExp(lexer: *@This()) void {
+ while (true) {
+ switch (lexer.code_point) {
+ '/' => {
+ lexer.step();
+ while (isIdentifierContinue(lexer.code_point)) {
+ switch (lexer.code_point) {
+ 'g', 'i', 'm', 's', 'u', 'y' => {
+ lexer.step();
+ },
+ else => {
+ lexer.syntaxError();
+ },
+ }
+ }
+ },
+ '[' => {
+ lexer.step();
+ while (lexer.code_point != ']') {
+ lexer.scanRegExpValidateAndStep();
+ }
+ lexer.step();
+ },
+ else => {
+ lexer.scanRegExpValidateAndStep();
+ },
}
+ }
+ }
- // Convert '\r' into '\n'
- c = '\n';
+ // TODO: use wtf-8 encoding.
+ pub fn stringToUTF16(lexer: *@This(), str: string) JavascriptString {
+ var buf: JavascriptString = lexer.allocator.alloc(u16, std.mem.len(str)) catch unreachable;
+ var i: usize = 0;
+ // theres prob a faster/better way
+ for (str) |char| {
+ buf[i] = char;
+ i += 1;
}
- bytes.list.items[end] = c;
- end += 1;
+ return buf;
}
- return bytes.toOwnedSliceLength(end + 1);
- }
-
- fn parseNumericLiteralOrDot(lexer: *Lexer) void {
- // Number or dot;
- var first = lexer.code_point;
- lexer.step();
-
- // Dot without a digit after it;
- if (first == '.' and (lexer.code_point < '0' or lexer.code_point > '9')) {
- // "..."
- if ((lexer.code_point == '.' and
- lexer.current < lexer.source.contents.len) and
- lexer.source.contents[lexer.current] == '.')
- {
- lexer.step();
- lexer.step();
- lexer.token = T.t_dot_dot_dot;
- return;
- }
+ // TODO: use wtf-8 encoding.
+ pub fn utf16ToStringWithValidation(lexer: *@This(), js: JavascriptString) !string {
+ return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js);
+ }
- // "."
- lexer.token = T.t_dot;
- return;
+ // TODO: use wtf-8 encoding.
+ pub fn utf16ToString(lexer: *@This(), js: JavascriptString) string {
+ return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js) catch unreachable;
}
- var underscoreCount: usize = 0;
- var lastUnderscoreEnd: usize = 0;
- var hasDotOrExponent = first == '.';
- var base: f32 = 0.0;
- lexer.is_legacy_octal_literal = false;
+ pub fn nextInsideJSXElement() void {
+ std.debug.panic("JSX not implemented yet.", .{});
+ }
- // Assume this is a number, but potentially change to a bigint later;
- lexer.token = T.t_numeric_literal;
+ fn scanRegExpValidateAndStep(lexer: *@This()) void {
+ if (lexer.code_point == '\\') {
+ lexer.step();
+ }
- // Check for binary, octal, or hexadecimal literal;
- if (first == '0') {
switch (lexer.code_point) {
- 'b', 'B' => {
- base = 2;
+ '\r', '\n', 0x2028, 0x2029 => {
+ // Newlines aren't allowed in regular expressions
+ lexer.syntaxError();
},
-
- 'o', 'O' => {
- base = 8;
+ -1 => { // EOF
+ lexer.syntaxError();
},
-
- 'x', 'X' => {
- base = 16;
+ else => {
+ lexer.step();
},
+ }
+ }
+
+ pub fn rescanCloseBraceAsTemplateToken(lexer: *@This()) void {
+ if (lexer.token != .t_close_brace) {
+ lexer.expected(.t_close_brace);
+ }
- '0'...'7', '_' => {
- base = 8;
- lexer.is_legacy_octal_literal = true;
+ lexer.rescan_close_brace_as_template_token = true;
+ lexer.code_point = '`';
+ lexer.current = lexer.end;
+ lexer.end -= 1;
+ lexer.next();
+ lexer.rescan_close_brace_as_template_token = false;
+ }
+
+ pub fn rawTemplateContents(lexer: *@This()) string {
+ var text: string = undefined;
+
+ switch (lexer.token) {
+ .t_no_substitution_template_literal, .t_template_tail => {
+ text = lexer.source.contents[lexer.start + 1 .. lexer.end - 1];
+ },
+ .t_template_middle, .t_template_head => {
+ text = lexer.source.contents[lexer.start + 1 .. lexer.end - 2];
},
else => {},
}
+
+ if (strings.indexOfChar(text, '\r') == null) {
+ return text;
+ }
+
+ // From the specification:
+ //
+ // 11.8.6.1 Static Semantics: TV and TRV
+ //
+ // TV excludes the code units of LineContinuation while TRV includes
+ // them. <CR><LF> and <CR> LineTerminatorSequences are normalized to
+ // <LF> for both TV and TRV. An explicit EscapeSequence is needed to
+ // include a <CR> or <CR><LF> sequence.
+ var bytes = MutableString.initCopy(lexer.allocator, text) catch unreachable;
+ var end: usize = 0;
+ var i: usize = 0;
+ var c: u8 = '0';
+ while (i < bytes.list.items.len) {
+ c = bytes.list.items[i];
+ i += 1;
+
+ if (c == '\r') {
+ // Convert '\r\n' into '\n'
+ if (i < bytes.list.items.len and bytes.list.items[i] == '\n') {
+ i += 1;
+ }
+
+ // Convert '\r' into '\n'
+ c = '\n';
+ }
+
+ bytes.list.items[end] = c;
+ end += 1;
+ }
+
+ return bytes.toOwnedSliceLength(end + 1);
}
- if (base != 0) {
- // Integer literal;
- var isFirst = true;
- var isInvalidLegacyOctalLiteral = false;
- lexer.number = 0;
- if (!lexer.is_legacy_octal_literal) {
- lexer.step();
+ fn parseNumericLiteralOrDot(lexer: *@This()) void {
+ // Number or dot;
+ var first = lexer.code_point;
+ lexer.step();
+
+ // Dot without a digit after it;
+ if (first == '.' and (lexer.code_point < '0' or lexer.code_point > '9')) {
+ // "..."
+ if ((lexer.code_point == '.' and
+ lexer.current < lexer.source.contents.len) and
+ lexer.source.contents[lexer.current] == '.')
+ {
+ lexer.step();
+ lexer.step();
+ lexer.token = T.t_dot_dot_dot;
+ return;
+ }
+
+ // "."
+ lexer.token = T.t_dot;
+ return;
}
- integerLiteral: while (true) {
- switch (lexer.code_point) {
- '_' => {
- // Cannot have multiple underscores in a row;
- if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
- lexer.syntaxError();
- }
+ var underscoreCount: usize = 0;
+ var lastUnderscoreEnd: usize = 0;
+ var hasDotOrExponent = first == '.';
+ var base: f32 = 0.0;
+ lexer.is_legacy_octal_literal = false;
- // The first digit must exist;
- if (isFirst or lexer.is_legacy_octal_literal) {
- lexer.syntaxError();
- }
+ // Assume this is a number, but potentially change to a bigint later;
+ lexer.token = T.t_numeric_literal;
- lastUnderscoreEnd = lexer.end;
- underscoreCount += 1;
+ // Check for binary, octal, or hexadecimal literal;
+ if (first == '0') {
+ switch (lexer.code_point) {
+ 'b', 'B' => {
+ base = 2;
},
- '0', '1' => {
- lexer.number = lexer.number * base + float64(lexer.code_point - '0');
+ 'o', 'O' => {
+ base = 8;
},
- '2', '3', '4', '5', '6', '7' => {
- if (base == 2) {
- lexer.syntaxError();
- }
- lexer.number = lexer.number * base + float64(lexer.code_point - '0');
- },
- '8', '9' => {
- if (lexer.is_legacy_octal_literal) {
- isInvalidLegacyOctalLiteral = true;
- } else if (base < 10) {
- lexer.syntaxError();
- }
- lexer.number = lexer.number * base + float64(lexer.code_point - '0');
- },
- 'A', 'B', 'C', 'D', 'E', 'F' => {
- if (base != 16) {
- lexer.syntaxError();
- }
- lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'A');
+ 'x', 'X' => {
+ base = 16;
},
- 'a', 'b', 'c', 'd', 'e', 'f' => {
- if (base != 16) {
- lexer.syntaxError();
- }
- lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'a');
+ '0'...'7', '_' => {
+ base = 8;
+ lexer.is_legacy_octal_literal = true;
},
- else => {
- // The first digit must exist;
- if (isFirst) {
- lexer.syntaxError();
- }
+ else => {},
+ }
+ }
- break :integerLiteral;
- },
+ if (base != 0) {
+ // Integer literal;
+ var isFirst = true;
+ var isInvalidLegacyOctalLiteral = false;
+ lexer.number = 0;
+ if (!lexer.is_legacy_octal_literal) {
+ lexer.step();
}
- lexer.step();
- isFirst = false;
- }
+ integerLiteral: while (true) {
+ switch (lexer.code_point) {
+ '_' => {
+ // Cannot have multiple underscores in a row;
+ if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
+ lexer.syntaxError();
+ }
- var isBigIntegerLiteral = lexer.code_point == 'n' and !hasDotOrExponent;
+ // The first digit must exist;
+ if (isFirst or lexer.is_legacy_octal_literal) {
+ lexer.syntaxError();
+ }
- // Slow path: do we need to re-scan the input as text?
- if (isBigIntegerLiteral or isInvalidLegacyOctalLiteral) {
- var text = lexer.raw();
+ lastUnderscoreEnd = lexer.end;
+ underscoreCount += 1;
+ },
- // Can't use a leading zero for bigint literals;
- if (isBigIntegerLiteral and lexer.is_legacy_octal_literal) {
- lexer.syntaxError();
- }
+ '0', '1' => {
+ lexer.number = lexer.number * base + float64(lexer.code_point - '0');
+ },
- // Filter out underscores;
- if (underscoreCount > 0) {
- var bytes = lexer.allocator.alloc(u8, text.len - underscoreCount) catch unreachable;
- var i: usize = 0;
- for (text) |char| {
- if (char != '_') {
- bytes[i] = char;
- i += 1;
- }
- }
- }
+ '2', '3', '4', '5', '6', '7' => {
+ if (base == 2) {
+ lexer.syntaxError();
+ }
+ lexer.number = lexer.number * base + float64(lexer.code_point - '0');
+ },
+ '8', '9' => {
+ if (lexer.is_legacy_octal_literal) {
+ isInvalidLegacyOctalLiteral = true;
+ } else if (base < 10) {
+ lexer.syntaxError();
+ }
+ lexer.number = lexer.number * base + float64(lexer.code_point - '0');
+ },
+ 'A', 'B', 'C', 'D', 'E', 'F' => {
+ if (base != 16) {
+ lexer.syntaxError();
+ }
+ lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'A');
+ },
- // Store bigints as text to avoid precision loss;
- if (isBigIntegerLiteral) {
- lexer.identifier = text;
- } else if (isInvalidLegacyOctalLiteral) {
- if (std.fmt.parseFloat(f64, text)) |num| {
- lexer.number = num;
- } else |err| {
- lexer.addError(lexer.start, "Invalid number {s}", .{text}, true);
+ 'a', 'b', 'c', 'd', 'e', 'f' => {
+ if (base != 16) {
+ lexer.syntaxError();
+ }
+ lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'a');
+ },
+ else => {
+ // The first digit must exist;
+ if (isFirst) {
+ lexer.syntaxError();
+ }
+
+ break :integerLiteral;
+ },
}
+
+ lexer.step();
+ isFirst = false;
}
- }
- } else {
- // Floating-point literal;
- var isInvalidLegacyOctalLiteral = first == '0' and (lexer.code_point == '8' or lexer.code_point == '9');
- // Initial digits;
- while (true) {
- if (lexer.code_point < '0' or lexer.code_point > '9') {
- if (lexer.code_point != '_') {
- break;
- }
+ var isBigIntegerLiteral = lexer.code_point == 'n' and !hasDotOrExponent;
- // Cannot have multiple underscores in a row;
- if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
- lexer.syntaxError();
- }
+ // Slow path: do we need to re-scan the input as text?
+ if (isBigIntegerLiteral or isInvalidLegacyOctalLiteral) {
+ var text = lexer.raw();
- // The specification forbids underscores in this case;
- if (isInvalidLegacyOctalLiteral) {
+ // Can't use a leading zero for bigint literals;
+ if (isBigIntegerLiteral and lexer.is_legacy_octal_literal) {
lexer.syntaxError();
}
- lastUnderscoreEnd = lexer.end;
- underscoreCount += 1;
- }
- lexer.step();
- }
+ // Filter out underscores;
+ if (underscoreCount > 0) {
+ var bytes = lexer.allocator.alloc(u8, text.len - underscoreCount) catch unreachable;
+ var i: usize = 0;
+ for (text) |char| {
+ if (char != '_') {
+ bytes[i] = char;
+ i += 1;
+ }
+ }
+ }
- // Fractional digits;
- if (first != '.' and lexer.code_point == '.') {
- // An underscore must not come last;
- if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
- lexer.end -= 1;
- lexer.syntaxError();
+ // Store bigints as text to avoid precision loss;
+ if (isBigIntegerLiteral) {
+ lexer.identifier = text;
+ } else if (isInvalidLegacyOctalLiteral) {
+ if (std.fmt.parseFloat(f64, text)) |num| {
+ lexer.number = num;
+ } else |err| {
+ lexer.addError(lexer.start, "Invalid number {s}", .{text}, true);
+ }
+ }
}
+ } else {
+ // Floating-point literal;
+ var isInvalidLegacyOctalLiteral = first == '0' and (lexer.code_point == '8' or lexer.code_point == '9');
- hasDotOrExponent = true;
- lexer.step();
- if (lexer.code_point == '_') {
- lexer.syntaxError();
- }
+ // Initial digits;
while (true) {
if (lexer.code_point < '0' or lexer.code_point > '9') {
if (lexer.code_point != '_') {
@@ -1150,110 +1291,151 @@ pub const Lexer = struct {
lexer.syntaxError();
}
+ // The specification forbids underscores in this case;
+ if (isInvalidLegacyOctalLiteral) {
+ lexer.syntaxError();
+ }
+
lastUnderscoreEnd = lexer.end;
underscoreCount += 1;
}
lexer.step();
}
- }
- // Exponent;
- if (lexer.code_point == 'e' or lexer.code_point == 'E') {
- // An underscore must not come last;
- if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
- lexer.end -= 1;
- lexer.syntaxError();
- }
+ // Fractional digits;
+ if (first != '.' and lexer.code_point == '.') {
+ // An underscore must not come last;
+ if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
+ lexer.end -= 1;
+ lexer.syntaxError();
+ }
- hasDotOrExponent = true;
- lexer.step();
- if (lexer.code_point == '+' or lexer.code_point == '-') {
+ hasDotOrExponent = true;
lexer.step();
- }
- if (lexer.code_point < '0' or lexer.code_point > '9') {
- lexer.syntaxError();
- }
- while (true) {
- if (lexer.code_point < '0' or lexer.code_point > '9') {
- if (lexer.code_point != '_') {
- break;
- }
+ if (lexer.code_point == '_') {
+ lexer.syntaxError();
+ }
+ while (true) {
+ if (lexer.code_point < '0' or lexer.code_point > '9') {
+ if (lexer.code_point != '_') {
+ break;
+ }
- // Cannot have multiple underscores in a row;
- if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
- lexer.syntaxError();
+ // Cannot have multiple underscores in a row;
+ if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
+ lexer.syntaxError();
+ }
+
+ lastUnderscoreEnd = lexer.end;
+ underscoreCount += 1;
}
+ lexer.step();
+ }
+ }
- lastUnderscoreEnd = lexer.end;
- underscoreCount += 1;
+ // Exponent;
+ if (lexer.code_point == 'e' or lexer.code_point == 'E') {
+ // An underscore must not come last;
+ if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
+ lexer.end -= 1;
+ lexer.syntaxError();
}
+
+ hasDotOrExponent = true;
lexer.step();
- }
- }
+ if (lexer.code_point == '+' or lexer.code_point == '-') {
+ lexer.step();
+ }
+ if (lexer.code_point < '0' or lexer.code_point > '9') {
+ lexer.syntaxError();
+ }
+ while (true) {
+ if (lexer.code_point < '0' or lexer.code_point > '9') {
+ if (lexer.code_point != '_') {
+ break;
+ }
- // Take a slice of the text to parse;
- var text = lexer.raw();
+ // Cannot have multiple underscores in a row;
+ if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
+ lexer.syntaxError();
+ }
- // Filter out underscores;
- if (underscoreCount > 0) {
- var i: usize = 0;
- if (lexer.allocator.alloc(u8, text.len - underscoreCount)) |bytes| {
- for (text) |char| {
- if (char != '_') {
- bytes[i] = char;
- i += 1;
+ lastUnderscoreEnd = lexer.end;
+ underscoreCount += 1;
}
+ lexer.step();
}
- text = bytes;
- } else |err| {
- lexer.addError(lexer.start, "Out of Memory Wah Wah Wah", .{}, true);
- return;
}
- }
- if (lexer.code_point == 'n' and !hasDotOrExponent) {
- // The only bigint literal that can start with 0 is "0n"
- if (text.len > 1 and first == '0') {
- lexer.syntaxError();
- }
+ // Take a slice of the text to parse;
+ var text = lexer.raw();
- // Store bigints as text to avoid precision loss;
- lexer.identifier = text;
- } else if (!hasDotOrExponent and lexer.end - lexer.start < 10) {
- // Parse a 32-bit integer (very fast path);
- var number: u32 = 0;
- for (text) |c| {
- number = number * 10 + @intCast(u32, c - '0');
+ // Filter out underscores;
+ if (underscoreCount > 0) {
+ var i: usize = 0;
+ if (lexer.allocator.alloc(u8, text.len - underscoreCount)) |bytes| {
+ for (text) |char| {
+ if (char != '_') {
+ bytes[i] = char;
+ i += 1;
+ }
+ }
+ text = bytes;
+ } else |err| {
+ lexer.addError(lexer.start, "Out of Memory Wah Wah Wah", .{}, true);
+ return;
+ }
}
- lexer.number = @intToFloat(f64, number);
- } else {
- // Parse a double-precision floating-point number;
- if (std.fmt.parseFloat(f64, text)) |num| {
- lexer.number = num;
- } else |err| {
- lexer.addError(lexer.start, "Invalid number", .{}, true);
+
+ if (lexer.code_point == 'n' and !hasDotOrExponent) {
+ // The only bigint literal that can start with 0 is "0n"
+ if (text.len > 1 and first == '0') {
+ lexer.syntaxError();
+ }
+
+ // Store bigints as text to avoid precision loss;
+ lexer.identifier = text;
+ } else if (!hasDotOrExponent and lexer.end - lexer.start < 10) {
+ // Parse a 32-bit integer (very fast path);
+ var number: u32 = 0;
+ for (text) |c| {
+ number = number * 10 + @intCast(u32, c - '0');
+ }
+ lexer.number = @intToFloat(f64, number);
+ } else {
+ // Parse a double-precision floating-point number;
+ if (std.fmt.parseFloat(f64, text)) |num| {
+ lexer.number = num;
+ } else |err| {
+ lexer.addError(lexer.start, "Invalid number", .{}, true);
+ }
}
}
- }
- // An underscore must not come last;
- if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
- lexer.end -= 1;
- lexer.syntaxError();
- }
+ // An underscore must not come last;
+ if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) {
+ lexer.end -= 1;
+ lexer.syntaxError();
+ }
- // Handle bigint literals after the underscore-at-end check above;
- if (lexer.code_point == 'n' and !hasDotOrExponent) {
- lexer.token = T.t_big_integer_literal;
- lexer.step();
- }
+ // Handle bigint literals after the underscore-at-end check above;
+ if (lexer.code_point == 'n' and !hasDotOrExponent) {
+ lexer.token = T.t_big_integer_literal;
+ lexer.step();
+ }
- // Identifiers can't occur immediately after numbers;
- if (isIdentifierStart(lexer.code_point)) {
- lexer.syntaxError();
+ // Identifiers can't occur immediately after numbers;
+ if (isIdentifierStart(lexer.code_point)) {
+ lexer.syntaxError();
+ }
}
- }
-};
+ };
+}
+
+// JS/TS lexer
+pub const Lexer = NewLexerType(null);
+pub const JSONLexer = NewLexerType(JSONOptions{ .allow_comments = false, .allow_trailing_commas = false });
+pub const TSConfigJSONLexer = NewLexerType(JSONOptions{ .allow_comments = true, .allow_trailing_commas = true });
fn isIdentifierStart(codepoint: CodePoint) bool {
switch (codepoint) {
@@ -1309,10 +1491,10 @@ fn isWhitespace(codepoint: CodePoint) bool {
0x202F, // narrow no-break space
0x205F, // medium mathematical space
0x3000, // ideographic space
- 0xFEFF,
+ 0xFEFF, // zero width non-breaking space
=> {
return true;
- }, // zero width non-breaking space
+ },
else => {
return false;
},
@@ -1346,7 +1528,7 @@ fn float64(num: anytype) callconv(.Inline) f64 {
return @intToFloat(f64, num);
}
-fn test_lexer(contents: []const u8) Lexer {
+fn test_lexer(contents: []const u8) @This() {
alloc.setup(std.heap.page_allocator) catch unreachable;
const msgs = std.ArrayList(logger.Msg).init(alloc.dynamic);
const log = logger.Log{
@@ -1357,10 +1539,10 @@ fn test_lexer(contents: []const u8) Lexer {
"index.js",
contents,
);
- return Lexer.init(log, source, alloc.dynamic) catch unreachable;
+ return @This().init(log, source, alloc.dynamic) catch unreachable;
}
-// test "Lexer.next()" {
+// test "@This().next()" {
// try alloc.setup(std.heap.page_allocator);
// const msgs = std.ArrayList(logger.Msg).init(alloc.dynamic);
// const log = logger.Log{
@@ -1368,7 +1550,7 @@ fn test_lexer(contents: []const u8) Lexer {
// };
// const source = logger.Source.initPathString("index.js", "for (let i = 0; i < 100; i++) { console.log('hi'); }", std.heap.page_allocator);
-// var lex = try Lexer.init(log, source, alloc.dynamic);
+// var lex = try @This().init(log, source, alloc.dynamic);
// lex.next();
// }
@@ -1411,7 +1593,7 @@ pub fn test_stringLiteralEquals(expected: string, source_text: string) void {
source_text,
);
- var lex = try Lexer.init(log, source, std.heap.page_allocator);
+ var lex = try @This().init(log, source, std.heap.page_allocator);
while (!lex.token.isString() and lex.token != .t_end_of_file) {
lex.next();
}
@@ -1420,7 +1602,7 @@ pub fn test_stringLiteralEquals(expected: string, source_text: string) void {
std.testing.expectEqualStrings(expected, lit);
}
-pub fn test_skipTo(lexer: *Lexer, n: string) void {
+pub fn test_skipTo(lexer: *@This(), n: string) void {
var i: usize = 0;
while (i < n.len) {
lexer.next();
@@ -1428,7 +1610,7 @@ pub fn test_skipTo(lexer: *Lexer, n: string) void {
}
}
-test "Lexer.rawTemplateContents" {
+test "@This().rawTemplateContents" {
test_stringLiteralEquals("hello!", "const a = 'hello!';");
test_stringLiteralEquals("hello!hi", "const b = 'hello!hi';");
test_stringLiteralEquals("hello!\n\nhi", "const b = `hello!\n\nhi`;");
diff --git a/src/js_parser.zig b/src/js_parser.zig
index e7692c76f..460e0d275 100644
--- a/src/js_parser.zig
+++ b/src/js_parser.zig
@@ -3173,12 +3173,10 @@ const P = struct {
var isDirectivePrologue = true;
run: while (true) {
- if (p.lexer.comments_to_preserve_before) |comments| {
- for (comments) |comment| {
- try stmts.append(p.s(S.Comment{
- .text = comment.text,
- }, p.lexer.loc()));
- }
+ for (p.lexer.comments_to_preserve_before.items) |comment| {
+ try stmts.append(p.s(S.Comment{
+ .text = comment.text,
+ }, p.lexer.loc()));
}
if (p.lexer.token == .t_end_of_file) {
@@ -5737,14 +5735,14 @@ const P = struct {
p.lexer.preserve_all_comments_before = true;
p.lexer.expect(.t_open_paren);
- const comments = p.lexer.comments_to_preserve_before;
+ const comments = p.lexer.comments_to_preserve_before.toOwnedSlice();
p.lexer.preserve_all_comments_before = false;
const value = p.parseExpr(.comma);
p.lexer.expect(.t_close_paren);
p.allow_in = old_allow_in;
- return p.e(E.Import{ .expr = value, .leading_interior_comments = comments orelse &([_]G.Comment{}), .import_record_index = 0 }, loc);
+ return p.e(E.Import{ .expr = value, .leading_interior_comments = comments, .import_record_index = 0 }, loc);
}
pub fn parseJSXElement(loc: logger.Loc) Expr {
@@ -5866,7 +5864,7 @@ const P = struct {
switch (expr.data) {
.e_null, .e_super, .e_boolean, .e_big_int, .e_reg_exp, .e_new_target, .e_undefined => {},
.e_string => |e_| {
- // idc about legacy octal loc
+ // If you're using this, you're probably not using 0-prefixed legacy octal notation
// if e.LegacyOctalLoc.Start > 0 {
},
.e_number => |e_| {
@@ -6013,27 +6011,27 @@ const P = struct {
switch (data.value) {
.expr => |*expr| {
const was_anonymous_named_expr = expr.isAnonymousNamed();
- data.value.expr = p.m(p.visitExpr(expr.*));
-
- // Optionally preserve the name
- data.value.expr = p.maybeKeepExprSymbolName(expr, "default", was_anonymous_named_expr);
-
- // Discard type-only export default statements
- if (p.options.ts) {
- switch (expr.data) {
- .e_identifier => |ident| {
- const symbol = p.symbols.items[ident.ref.inner_index];
- if (symbol.kind == .unbound) {
- if (p.local_type_names.get(symbol.original_name)) |local_type| {
- if (local_type.value) {
- return;
- }
- }
- }
- },
- else => {},
- }
- }
+ // data.value.expr = p.m(p.visitExpr(expr.*));
+
+ // // Optionally preserve the name
+ // data.value.expr = p.maybeKeepExprSymbolName(expr, "default", was_anonymous_named_expr);
+
+ // // Discard type-only export default statements
+ // if (p.options.ts) {
+ // switch (expr.data) {
+ // .e_identifier => |ident| {
+ // const symbol = p.symbols.items[ident.ref.inner_index];
+ // if (symbol.kind == .unbound) {
+ // if (p.local_type_names.get(symbol.original_name)) |local_type| {
+ // if (local_type.value) {
+ // return;
+ // }
+ // }
+ // }
+ // },
+ // else => {},
+ // }
+ // }
},
.stmt => |st| {},
diff --git a/src/json_parser.zig b/src/json_parser.zig
new file mode 100644
index 000000000..8a025864f
--- /dev/null
+++ b/src/json_parser.zig
@@ -0,0 +1,212 @@
+const std = @import("std");
+const logger = @import("logger.zig");
+const js_lexer = @import("js_lexer.zig");
+const importRecord = @import("import_record.zig");
+const js_ast = @import("js_ast.zig");
+const options = @import("options.zig");
+const alloc = @import("alloc.zig");
+
+const fs = @import("fs.zig");
+usingnamespace @import("strings.zig");
+usingnamespace @import("ast/base.zig");
+usingnamespace js_ast.G;
+
+const ImportKind = importRecord.ImportKind;
+const BindingNodeIndex = js_ast.BindingNodeIndex;
+
+const StmtNodeIndex = js_ast.StmtNodeIndex;
+const ExprNodeIndex = js_ast.ExprNodeIndex;
+const ExprNodeList = js_ast.ExprNodeList;
+const StmtNodeList = js_ast.StmtNodeList;
+const BindingNodeList = js_ast.BindingNodeList;
+const assert = std.debug.assert;
+
+const Ref = js_ast.Ref;
+const LocRef = js_ast.LocRef;
+const S = js_ast.S;
+const B = js_ast.B;
+const G = js_ast.G;
+const T = js_lexer.T;
+const E = js_ast.E;
+const Stmt = js_ast.Stmt;
+const Expr = js_ast.Expr;
+const Binding = js_ast.Binding;
+const Symbol = js_ast.Symbol;
+const Level = js_ast.Op.Level;
+const Op = js_ast.Op;
+const Scope = js_ast.Scope;
+const locModuleScope = logger.Loc.Empty;
+
+fn JSONLikeParser(opts: js_lexer.JSONOptions) type {
+ const Lexer = if (opts.allow_comments) js_lexer.TSConfigJSONLexer else js_lexer.JSONLexer;
+ return struct {
+ lexer: Lexer,
+ source: logger.Source,
+ log: logger.Log,
+ allocator: *std.mem.Allocator,
+
+ pub fn init(allocator: *std.mem.Allocator, source: logger.Source, log: logger.Log) Parser {
+ return Parser{
+ .lexer = Lexer.init(log, source, allocator),
+ .allocator = allocator,
+ .log = log,
+ .source = source,
+ };
+ }
+
+ const Parser = @This();
+
+ pub fn e(p: *Parser, t: anytype, loc: logger.Loc) Expr {
+ if (@typeInfo(@TypeOf(t)) == .Pointer) {
+ return Expr.init(t, loc);
+ } else {
+ return Expr.alloc(p.allocator, t, loc);
+ }
+ }
+ pub fn parseExpr(p: *Parser) ?Expr {
+ const loc = p.lexer.loc();
+
+ switch (p.lexer.token) {
+ .t_false => {
+ p.lexer.next();
+ return p.e(E.Boolean{
+ .value = false,
+ }, loc);
+ },
+ .t_true => {
+ p.lexer.next();
+ return p.e(E.Boolean{
+ .value = true,
+ }, loc);
+ },
+ .t_null => {
+ p.lexer.next();
+ return p.e(E.Null{}, loc);
+ },
+ .t_string_literal => {
+ const value = p.lexer.string_literal;
+ p.lexer.next();
+ return p.e(E.String{
+ .value = value,
+ }, loc);
+ },
+ .t_numeric_literal => {
+ const value = p.lexer.number;
+ p.lexer.next();
+ return p.e(E.Number{ .value = value }, loc);
+ },
+ .t_minus => {
+ p.lexer.next();
+ const value = p.lexer.number;
+ p.lexer.expect(.t_numeric_literal);
+ return p.e(E.Number{ .value = -value }, loc);
+ },
+ .t_open_bracket => {
+ p.lexer.next();
+ var is_single_line = !p.lexer.has_newline_before;
+ var exprs = List(Expr).init(p.allocator);
+
+ while (p.lexer.token != .t_close_bracket) {
+ if (exprs.items.len > 0) {
+ if (p.lexer.has_newline_before) {
+ is_single_line = false;
+ }
+
+ if (!p.parseMaybeTrailingComma(.t_close_bracket)) {
+ break;
+ }
+
+ if (p.lexer.has_newline_before) {
+ is_single_line = false;
+ }
+ }
+
+ if (p.parseExpr()) |expr| {
+ try exprs.append(expr);
+ } else {
+ break;
+ }
+ }
+
+ if (p.lexer.has_newline_before) {
+ is_single_line = false;
+ }
+ p.lexer.expect(.t_close_bracket);
+ return p.e(E.Array{ .items = exprs.toOwnedSlice() }, loc);
+ },
+ .t_open_brace => {
+ p.lexer.next();
+ var is_single_line = !p.lexer.has_newline_before;
+ var properties = List(G.Property).init(p.allocator);
+ var duplicates = std.StringHashMap(u0).init(p.allocator);
+
+ while (p.lexer.token != .t_close_brace) {
+ if (properties.items.len > 0) {
+ is_single_line = if (p.lexer.has_newline_before) false else is_single_line;
+ if (!p.parseMaybeTrailingComma(.t_close_brace)) {
+ break;
+ }
+ is_single_line = if (p.lexer.has_newline_before) false else is_single_line;
+ }
+
+ var key_string = p.lexer.string_literal;
+ var key_range = p.lexer.range();
+ var key = p.e(E.String{ .value = key_string }, key_range.loc);
+ p.lexer.expect(.t_string_literal);
+ var key_text = p.lexer.utf16ToString();
+ // Warn about duplicate keys
+ if (duplicates.contains(key_text)) {
+ p.log.addRangeWarningFmt(p.source, r, "Duplicate key \"{s}\" in object literal", .{key_text}) catch unreachable;
+ } else {
+ duplicates.put(key_text, 0) catch unreachable;
+ }
+
+ p.lexer.expect(.t_colon);
+ var value = p.parseExpr() orelse return null;
+ try properties.append(G.Property{ .key = key, .value = value });
+ }
+
+ is_single_line = if (p.lexer.has_newline_before) false else is_single_line;
+ p.lexer.expect(.t_close_brace);
+ return p.e(E.Object{
+ .properties = properties.toOwnedSlice(),
+ .is_single_line = is_single_line,
+ }, loc);
+ },
+ else => {
+ p.lexer.unexpected();
+ return null;
+ },
+ }
+ }
+
+ pub fn parseMaybeTrailingComma(p: *Parser, closer: T) bool {
+ const comma_range = p.lexer.range();
+ p.lexer.expect(.t_comma);
+
+ if (p.lexer.token == closer) {
+ if (!opts.allow_trailing_commas) {
+ p.log.addRangeError(p.source, comma_range, "JSON does not support trailing commas") catch unreachable;
+ }
+ return false;
+ }
+
+ return true;
+ }
+ };
+}
+
+const JSONParser = JSONLikeParser(js_lexer.JSONOptions{});
+const TSConfigParser = JSONLikeParser(js_lexer.JSONOptions{ .allow_comments = true, .allow_trailing_commas = true });
+
+pub fn ParseJSON(log: logger.Log, source: logger.Source) !?Expr {
+ var parser = JSONParser.init(allocator, log, source);
+
+ return try parser.parseExpr();
+}
+
+pub fn ParseTSConfig(log: logger.Loc, source: logger.Source) !?Expr {
+ var parser = TSConfigParser.init(allocator, log, source);
+
+ return try parser.parseExpr();
+}
diff --git a/src/logger.zig b/src/logger.zig
index 47253728c..bf7bcb21c 100644
--- a/src/logger.zig
+++ b/src/logger.zig
@@ -173,11 +173,27 @@ pub const Log = struct {
pub fn addRangeWarning(log: *Log, source: ?Source, r: Range, text: string) !void {
log.warnings += 1;
try log.addMsg(Msg{
- .kind = .warning,
+ .kind = .warn,
.data = rangeData(source, r, text),
});
}
+ pub fn addWarningFmt(log: *Log, source: ?Source, l: Loc, allocator: *std.mem.Allocator, comptime text: string, args: anytype) !void {
+ log.errors += 1;
+ try log.addMsg(Msg{
+ .kind = .err,
+ .data = rangeData(source, Range{ .loc = l }, std.fmt.allocPrint(allocator, text, args) catch unreachable),
+ });
+ }
+
+ pub fn addRangeWarningFmt(log: *Log, source: ?Source, r: Range, allocator: *std.mem.Allocator, comptime text: string, args: anytype) !void {
+ log.errors += 1;
+ try log.addMsg(Msg{
+ .kind = .err,
+ .data = rangeData(source, r, std.fmt.allocPrint(allocator, text, args) catch unreachable),
+ });
+ }
+
pub fn addWarning(log: *Log, source: ?Source, l: Loc, text: string) !void {
log.warnings += 1;
try log.addMsg(Msg{