diff options
author | 2021-04-25 00:56:48 -0700 | |
---|---|---|
committer | 2021-04-25 00:56:48 -0700 | |
commit | c0b7f71b9ace0bfd91fb5e0364f104b5093e6c37 (patch) | |
tree | 7da69736690eb11b974800ad947d986938e3bb50 /src | |
parent | 322b5cbd382c9a59deb20dd307dd6d09bd844482 (diff) | |
download | bun-c0b7f71b9ace0bfd91fb5e0364f104b5093e6c37.tar.gz bun-c0b7f71b9ace0bfd91fb5e0364f104b5093e6c37.tar.zst bun-c0b7f71b9ace0bfd91fb5e0364f104b5093e6c37.zip |
little defines, little readme, json parser
Diffstat (limited to 'src')
-rw-r--r-- | src/defines-table.zig | 798 | ||||
-rw-r--r-- | src/defines.zig | 12 | ||||
-rw-r--r-- | src/js_lexer.zig | 2202 | ||||
-rw-r--r-- | src/js_parser.zig | 58 | ||||
-rw-r--r-- | src/json_parser.zig | 212 | ||||
-rw-r--r-- | src/logger.zig | 18 |
6 files changed, 2259 insertions, 1041 deletions
diff --git a/src/defines-table.zig b/src/defines-table.zig new file mode 100644 index 000000000..3e3a720a3 --- /dev/null +++ b/src/defines-table.zig @@ -0,0 +1,798 @@ +usingnamespace @import("strings.zig"); + +// If something is in this list, then a direct identifier expression or property +// access chain matching this will be assumed to have no side effects and will +// be removed. +// +// This also means code is allowed to be reordered past things in this list. For +// example, if "console.log" is in this list, permitting reordering allows for +// "if (a) console.log(b); else console.log(c)" to be reordered and transformed +// into "console.log(a ? b : c)". Notice that "a" and "console.log" are in a +// different order, which can only happen if evaluating the "console.log" +// property access can be assumed to not change the value of "a". +// +// Note that membership in this list says nothing about whether calling any of +// these functions has any side effects. It only says something about +// referencing these function without calling them. +pub const GlobalDefinesKey = [][]string{ + // These global identifiers should exist in all JavaScript environments. This + // deliberately omits "NaN", "Infinity", and "undefined" because these are + // treated as automatically-inlined constants instead of identifiers. + []string{"Array"}, + []string{"Boolean"}, + []string{"Function"}, + []string{"Math"}, + []string{"Number"}, + []string{"Object"}, + []string{"RegExp"}, + []string{"String"}, + + // Object: Static methods + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object#Static_methods + []string{ "Object", "assign" }, + []string{ "Object", "create" }, + []string{ "Object", "defineProperties" }, + []string{ "Object", "defineProperty" }, + []string{ "Object", "entries" }, + []string{ "Object", "freeze" }, + []string{ "Object", "fromEntries" }, + []string{ "Object", "getOwnPropertyDescriptor" }, + []string{ "Object", "getOwnPropertyDescriptors" }, + []string{ "Object", "getOwnPropertyNames" }, + []string{ "Object", "getOwnPropertySymbols" }, + []string{ "Object", "getPrototypeOf" }, + []string{ "Object", "is" }, + []string{ "Object", "isExtensible" }, + []string{ "Object", "isFrozen" }, + []string{ "Object", "isSealed" }, + []string{ "Object", "keys" }, + []string{ "Object", "preventExtensions" }, + []string{ "Object", "seal" }, + []string{ "Object", "setPrototypeOf" }, + []string{ "Object", "values" }, + + // Object: Instance methods + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object#Instance_methods + []string{ "Object", "prototype", "__defineGetter__" }, + []string{ "Object", "prototype", "__defineSetter__" }, + []string{ "Object", "prototype", "__lookupGetter__" }, + []string{ "Object", "prototype", "__lookupSetter__" }, + []string{ "Object", "prototype", "hasOwnProperty" }, + []string{ "Object", "prototype", "isPrototypeOf" }, + []string{ "Object", "prototype", "propertyIsEnumerable" }, + []string{ "Object", "prototype", "toLocaleString" }, + []string{ "Object", "prototype", "toString" }, + []string{ "Object", "prototype", "unwatch" }, + []string{ "Object", "prototype", "valueOf" }, + []string{ "Object", "prototype", "watch" }, + + // Math: Static properties + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math#Static_properties + []string{ "Math", "E" }, + []string{ "Math", "LN10" }, + []string{ "Math", "LN2" }, + []string{ "Math", "LOG10E" }, + []string{ "Math", "LOG2E" }, + []string{ "Math", "PI" }, + []string{ "Math", "SQRT1_2" }, + []string{ "Math", "SQRT2" }, + + // Math: Static methods + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math#Static_methods + []string{ "Math", "abs" }, + []string{ "Math", "acos" }, + []string{ "Math", "acosh" }, + []string{ "Math", "asin" }, + []string{ "Math", "asinh" }, + []string{ "Math", "atan" }, + []string{ "Math", "atan2" }, + []string{ "Math", "atanh" }, + []string{ "Math", "cbrt" }, + []string{ "Math", "ceil" }, + []string{ "Math", "clz32" }, + []string{ "Math", "cos" }, + []string{ "Math", "cosh" }, + []string{ "Math", "exp" }, + []string{ "Math", "expm1" }, + []string{ "Math", "floor" }, + []string{ "Math", "fround" }, + []string{ "Math", "hypot" }, + []string{ "Math", "imul" }, + []string{ "Math", "log" }, + []string{ "Math", "log10" }, + []string{ "Math", "log1p" }, + []string{ "Math", "log2" }, + []string{ "Math", "max" }, + []string{ "Math", "min" }, + []string{ "Math", "pow" }, + []string{ "Math", "random" }, + []string{ "Math", "round" }, + []string{ "Math", "sign" }, + []string{ "Math", "sin" }, + []string{ "Math", "sinh" }, + []string{ "Math", "sqrt" }, + []string{ "Math", "tan" }, + []string{ "Math", "tanh" }, + []string{ "Math", "trunc" }, + + // Other globals present in both the browser and node (except "eval" because + // it has special behavior) + []string{"AbortController"}, + []string{"AbortSignal"}, + []string{"AggregateError"}, + []string{"ArrayBuffer"}, + []string{"BigInt"}, + []string{"DataView"}, + []string{"Date"}, + []string{"Error"}, + []string{"EvalError"}, + []string{"Event"}, + []string{"EventTarget"}, + []string{"Float32Array"}, + []string{"Float64Array"}, + []string{"Int16Array"}, + []string{"Int32Array"}, + []string{"Int8Array"}, + []string{"Intl"}, + []string{"JSON"}, + []string{"Map"}, + []string{"MessageChannel"}, + []string{"MessageEvent"}, + []string{"MessagePort"}, + []string{"Promise"}, + []string{"Proxy"}, + []string{"RangeError"}, + []string{"ReferenceError"}, + []string{"Reflect"}, + []string{"Set"}, + []string{"Symbol"}, + []string{"SyntaxError"}, + []string{"TextDecoder"}, + []string{"TextEncoder"}, + []string{"TypeError"}, + []string{"URIError"}, + []string{"URL"}, + []string{"URLSearchParams"}, + []string{"Uint16Array"}, + []string{"Uint32Array"}, + []string{"Uint8Array"}, + []string{"Uint8ClampedArray"}, + []string{"WeakMap"}, + []string{"WeakSet"}, + []string{"WebAssembly"}, + []string{"clearInterval"}, + []string{"clearTimeout"}, + []string{"console"}, + []string{"decodeURI"}, + []string{"decodeURIComponent"}, + []string{"encodeURI"}, + []string{"encodeURIComponent"}, + []string{"escape"}, + []string{"globalThis"}, + []string{"isFinite"}, + []string{"isNaN"}, + []string{"parseFloat"}, + []string{"parseInt"}, + []string{"queueMicrotask"}, + []string{"setInterval"}, + []string{"setTimeout"}, + []string{"unescape"}, + + // Console method references are assumed to have no side effects + // https://developer.mozilla.org/en-US/docs/Web/API/console + []string{ "console", "assert" }, + []string{ "console", "clear" }, + []string{ "console", "count" }, + []string{ "console", "countReset" }, + []string{ "console", "debug" }, + []string{ "console", "dir" }, + []string{ "console", "dirxml" }, + []string{ "console", "error" }, + []string{ "console", "group" }, + []string{ "console", "groupCollapsed" }, + []string{ "console", "groupEnd" }, + []string{ "console", "info" }, + []string{ "console", "log" }, + []string{ "console", "table" }, + []string{ "console", "time" }, + []string{ "console", "timeEnd" }, + []string{ "console", "timeLog" }, + []string{ "console", "trace" }, + []string{ "console", "warn" }, + + // CSSOM APIs + []string{"CSSAnimation"}, + []string{"CSSFontFaceRule"}, + []string{"CSSImportRule"}, + []string{"CSSKeyframeRule"}, + []string{"CSSKeyframesRule"}, + []string{"CSSMediaRule"}, + []string{"CSSNamespaceRule"}, + []string{"CSSPageRule"}, + []string{"CSSRule"}, + []string{"CSSRuleList"}, + []string{"CSSStyleDeclaration"}, + []string{"CSSStyleRule"}, + []string{"CSSStyleSheet"}, + []string{"CSSSupportsRule"}, + []string{"CSSTransition"}, + + // SVG DOM + []string{"SVGAElement"}, + []string{"SVGAngle"}, + []string{"SVGAnimateElement"}, + []string{"SVGAnimateMotionElement"}, + []string{"SVGAnimateTransformElement"}, + []string{"SVGAnimatedAngle"}, + []string{"SVGAnimatedBoolean"}, + []string{"SVGAnimatedEnumeration"}, + []string{"SVGAnimatedInteger"}, + []string{"SVGAnimatedLength"}, + []string{"SVGAnimatedLengthList"}, + []string{"SVGAnimatedNumber"}, + []string{"SVGAnimatedNumberList"}, + []string{"SVGAnimatedPreserveAspectRatio"}, + []string{"SVGAnimatedRect"}, + []string{"SVGAnimatedString"}, + []string{"SVGAnimatedTransformList"}, + []string{"SVGAnimationElement"}, + []string{"SVGCircleElement"}, + []string{"SVGClipPathElement"}, + []string{"SVGComponentTransferFunctionElement"}, + []string{"SVGDefsElement"}, + []string{"SVGDescElement"}, + []string{"SVGElement"}, + []string{"SVGEllipseElement"}, + []string{"SVGFEBlendElement"}, + []string{"SVGFEColorMatrixElement"}, + []string{"SVGFEComponentTransferElement"}, + []string{"SVGFECompositeElement"}, + []string{"SVGFEConvolveMatrixElement"}, + []string{"SVGFEDiffuseLightingElement"}, + []string{"SVGFEDisplacementMapElement"}, + []string{"SVGFEDistantLightElement"}, + []string{"SVGFEDropShadowElement"}, + []string{"SVGFEFloodElement"}, + []string{"SVGFEFuncAElement"}, + []string{"SVGFEFuncBElement"}, + []string{"SVGFEFuncGElement"}, + []string{"SVGFEFuncRElement"}, + []string{"SVGFEGaussianBlurElement"}, + []string{"SVGFEImageElement"}, + []string{"SVGFEMergeElement"}, + []string{"SVGFEMergeNodeElement"}, + []string{"SVGFEMorphologyElement"}, + []string{"SVGFEOffsetElement"}, + []string{"SVGFEPointLightElement"}, + []string{"SVGFESpecularLightingElement"}, + []string{"SVGFESpotLightElement"}, + []string{"SVGFETileElement"}, + []string{"SVGFETurbulenceElement"}, + []string{"SVGFilterElement"}, + []string{"SVGForeignObjectElement"}, + []string{"SVGGElement"}, + []string{"SVGGeometryElement"}, + []string{"SVGGradientElement"}, + []string{"SVGGraphicsElement"}, + []string{"SVGImageElement"}, + []string{"SVGLength"}, + []string{"SVGLengthList"}, + []string{"SVGLineElement"}, + []string{"SVGLinearGradientElement"}, + []string{"SVGMPathElement"}, + []string{"SVGMarkerElement"}, + []string{"SVGMaskElement"}, + []string{"SVGMatrix"}, + []string{"SVGMetadataElement"}, + []string{"SVGNumber"}, + []string{"SVGNumberList"}, + []string{"SVGPathElement"}, + []string{"SVGPatternElement"}, + []string{"SVGPoint"}, + []string{"SVGPointList"}, + []string{"SVGPolygonElement"}, + []string{"SVGPolylineElement"}, + []string{"SVGPreserveAspectRatio"}, + []string{"SVGRadialGradientElement"}, + []string{"SVGRect"}, + []string{"SVGRectElement"}, + []string{"SVGSVGElement"}, + []string{"SVGScriptElement"}, + []string{"SVGSetElement"}, + []string{"SVGStopElement"}, + []string{"SVGStringList"}, + []string{"SVGStyleElement"}, + []string{"SVGSwitchElement"}, + []string{"SVGSymbolElement"}, + []string{"SVGTSpanElement"}, + []string{"SVGTextContentElement"}, + []string{"SVGTextElement"}, + []string{"SVGTextPathElement"}, + []string{"SVGTextPositioningElement"}, + []string{"SVGTitleElement"}, + []string{"SVGTransform"}, + []string{"SVGTransformList"}, + []string{"SVGUnitTypes"}, + []string{"SVGUseElement"}, + []string{"SVGViewElement"}, + + // Other browser APIs + // + // This list contains all globals present in modern versions of Chrome, Safari, + // and Firefox except for the following properties, since they have a side effect + // of triggering layout (https://gist.github.com/paulirish/5d52fb081b3570c81e3a): + // + // - scrollX + // - scrollY + // - innerWidth + // - innerHeight + // - pageXOffset + // - pageYOffset + // + // The following globals have also been removed since they sometimes throw an + // exception when accessed, which is a side effect (for more information see + // https://stackoverflow.com/a/33047477): + // + // - localStorage + // - sessionStorage + // + []string{"AnalyserNode"}, + []string{"Animation"}, + []string{"AnimationEffect"}, + []string{"AnimationEvent"}, + []string{"AnimationPlaybackEvent"}, + []string{"AnimationTimeline"}, + []string{"Attr"}, + []string{"Audio"}, + []string{"AudioBuffer"}, + []string{"AudioBufferSourceNode"}, + []string{"AudioDestinationNode"}, + []string{"AudioListener"}, + []string{"AudioNode"}, + []string{"AudioParam"}, + []string{"AudioProcessingEvent"}, + []string{"AudioScheduledSourceNode"}, + []string{"BarProp"}, + []string{"BeforeUnloadEvent"}, + []string{"BiquadFilterNode"}, + []string{"Blob"}, + []string{"BlobEvent"}, + []string{"ByteLengthQueuingStrategy"}, + []string{"CDATASection"}, + []string{"CSS"}, + []string{"CanvasGradient"}, + []string{"CanvasPattern"}, + []string{"CanvasRenderingContext2D"}, + []string{"ChannelMergerNode"}, + []string{"ChannelSplitterNode"}, + []string{"CharacterData"}, + []string{"ClipboardEvent"}, + []string{"CloseEvent"}, + []string{"Comment"}, + []string{"CompositionEvent"}, + []string{"ConvolverNode"}, + []string{"CountQueuingStrategy"}, + []string{"Crypto"}, + []string{"CustomElementRegistry"}, + []string{"CustomEvent"}, + []string{"DOMException"}, + []string{"DOMImplementation"}, + []string{"DOMMatrix"}, + []string{"DOMMatrixReadOnly"}, + []string{"DOMParser"}, + []string{"DOMPoint"}, + []string{"DOMPointReadOnly"}, + []string{"DOMQuad"}, + []string{"DOMRect"}, + []string{"DOMRectList"}, + []string{"DOMRectReadOnly"}, + []string{"DOMStringList"}, + []string{"DOMStringMap"}, + []string{"DOMTokenList"}, + []string{"DataTransfer"}, + []string{"DataTransferItem"}, + []string{"DataTransferItemList"}, + []string{"DelayNode"}, + []string{"Document"}, + []string{"DocumentFragment"}, + []string{"DocumentTimeline"}, + []string{"DocumentType"}, + []string{"DragEvent"}, + []string{"DynamicsCompressorNode"}, + []string{"Element"}, + []string{"ErrorEvent"}, + []string{"EventSource"}, + []string{"File"}, + []string{"FileList"}, + []string{"FileReader"}, + []string{"FocusEvent"}, + []string{"FontFace"}, + []string{"FormData"}, + []string{"GainNode"}, + []string{"Gamepad"}, + []string{"GamepadButton"}, + []string{"GamepadEvent"}, + []string{"Geolocation"}, + []string{"GeolocationPositionError"}, + []string{"HTMLAllCollection"}, + []string{"HTMLAnchorElement"}, + []string{"HTMLAreaElement"}, + []string{"HTMLAudioElement"}, + []string{"HTMLBRElement"}, + []string{"HTMLBaseElement"}, + []string{"HTMLBodyElement"}, + []string{"HTMLButtonElement"}, + []string{"HTMLCanvasElement"}, + []string{"HTMLCollection"}, + []string{"HTMLDListElement"}, + []string{"HTMLDataElement"}, + []string{"HTMLDataListElement"}, + []string{"HTMLDetailsElement"}, + []string{"HTMLDirectoryElement"}, + []string{"HTMLDivElement"}, + []string{"HTMLDocument"}, + []string{"HTMLElement"}, + []string{"HTMLEmbedElement"}, + []string{"HTMLFieldSetElement"}, + []string{"HTMLFontElement"}, + []string{"HTMLFormControlsCollection"}, + []string{"HTMLFormElement"}, + []string{"HTMLFrameElement"}, + []string{"HTMLFrameSetElement"}, + []string{"HTMLHRElement"}, + []string{"HTMLHeadElement"}, + []string{"HTMLHeadingElement"}, + []string{"HTMLHtmlElement"}, + []string{"HTMLIFrameElement"}, + []string{"HTMLImageElement"}, + []string{"HTMLInputElement"}, + []string{"HTMLLIElement"}, + []string{"HTMLLabelElement"}, + []string{"HTMLLegendElement"}, + []string{"HTMLLinkElement"}, + []string{"HTMLMapElement"}, + []string{"HTMLMarqueeElement"}, + []string{"HTMLMediaElement"}, + []string{"HTMLMenuElement"}, + []string{"HTMLMetaElement"}, + []string{"HTMLMeterElement"}, + []string{"HTMLModElement"}, + []string{"HTMLOListElement"}, + []string{"HTMLObjectElement"}, + []string{"HTMLOptGroupElement"}, + []string{"HTMLOptionElement"}, + []string{"HTMLOptionsCollection"}, + []string{"HTMLOutputElement"}, + []string{"HTMLParagraphElement"}, + []string{"HTMLParamElement"}, + []string{"HTMLPictureElement"}, + []string{"HTMLPreElement"}, + []string{"HTMLProgressElement"}, + []string{"HTMLQuoteElement"}, + []string{"HTMLScriptElement"}, + []string{"HTMLSelectElement"}, + []string{"HTMLSlotElement"}, + []string{"HTMLSourceElement"}, + []string{"HTMLSpanElement"}, + []string{"HTMLStyleElement"}, + []string{"HTMLTableCaptionElement"}, + []string{"HTMLTableCellElement"}, + []string{"HTMLTableColElement"}, + []string{"HTMLTableElement"}, + []string{"HTMLTableRowElement"}, + []string{"HTMLTableSectionElement"}, + []string{"HTMLTemplateElement"}, + []string{"HTMLTextAreaElement"}, + []string{"HTMLTimeElement"}, + []string{"HTMLTitleElement"}, + []string{"HTMLTrackElement"}, + []string{"HTMLUListElement"}, + []string{"HTMLUnknownElement"}, + []string{"HTMLVideoElement"}, + []string{"HashChangeEvent"}, + []string{"Headers"}, + []string{"History"}, + []string{"IDBCursor"}, + []string{"IDBCursorWithValue"}, + []string{"IDBDatabase"}, + []string{"IDBFactory"}, + []string{"IDBIndex"}, + []string{"IDBKeyRange"}, + []string{"IDBObjectStore"}, + []string{"IDBOpenDBRequest"}, + []string{"IDBRequest"}, + []string{"IDBTransaction"}, + []string{"IDBVersionChangeEvent"}, + []string{"Image"}, + []string{"ImageData"}, + []string{"InputEvent"}, + []string{"IntersectionObserver"}, + []string{"IntersectionObserverEntry"}, + []string{"KeyboardEvent"}, + []string{"KeyframeEffect"}, + []string{"Location"}, + []string{"MediaCapabilities"}, + []string{"MediaElementAudioSourceNode"}, + []string{"MediaEncryptedEvent"}, + []string{"MediaError"}, + []string{"MediaList"}, + []string{"MediaQueryList"}, + []string{"MediaQueryListEvent"}, + []string{"MediaRecorder"}, + []string{"MediaSource"}, + []string{"MediaStream"}, + []string{"MediaStreamAudioDestinationNode"}, + []string{"MediaStreamAudioSourceNode"}, + []string{"MediaStreamTrack"}, + []string{"MediaStreamTrackEvent"}, + []string{"MimeType"}, + []string{"MimeTypeArray"}, + []string{"MouseEvent"}, + []string{"MutationEvent"}, + []string{"MutationObserver"}, + []string{"MutationRecord"}, + []string{"NamedNodeMap"}, + []string{"Navigator"}, + []string{"Node"}, + []string{"NodeFilter"}, + []string{"NodeIterator"}, + []string{"NodeList"}, + []string{"Notification"}, + []string{"OfflineAudioCompletionEvent"}, + []string{"Option"}, + []string{"OscillatorNode"}, + []string{"PageTransitionEvent"}, + []string{"Path2D"}, + []string{"Performance"}, + []string{"PerformanceEntry"}, + []string{"PerformanceMark"}, + []string{"PerformanceMeasure"}, + []string{"PerformanceNavigation"}, + []string{"PerformanceObserver"}, + []string{"PerformanceObserverEntryList"}, + []string{"PerformanceResourceTiming"}, + []string{"PerformanceTiming"}, + []string{"PeriodicWave"}, + []string{"Plugin"}, + []string{"PluginArray"}, + []string{"PointerEvent"}, + []string{"PopStateEvent"}, + []string{"ProcessingInstruction"}, + []string{"ProgressEvent"}, + []string{"PromiseRejectionEvent"}, + []string{"RTCCertificate"}, + []string{"RTCDTMFSender"}, + []string{"RTCDTMFToneChangeEvent"}, + []string{"RTCDataChannel"}, + []string{"RTCDataChannelEvent"}, + []string{"RTCIceCandidate"}, + []string{"RTCPeerConnection"}, + []string{"RTCPeerConnectionIceEvent"}, + []string{"RTCRtpReceiver"}, + []string{"RTCRtpSender"}, + []string{"RTCRtpTransceiver"}, + []string{"RTCSessionDescription"}, + []string{"RTCStatsReport"}, + []string{"RTCTrackEvent"}, + []string{"RadioNodeList"}, + []string{"Range"}, + []string{"ReadableStream"}, + []string{"Request"}, + []string{"ResizeObserver"}, + []string{"ResizeObserverEntry"}, + []string{"Response"}, + []string{"Screen"}, + []string{"ScriptProcessorNode"}, + []string{"SecurityPolicyViolationEvent"}, + []string{"Selection"}, + []string{"ShadowRoot"}, + []string{"SourceBuffer"}, + []string{"SourceBufferList"}, + []string{"SpeechSynthesisEvent"}, + []string{"SpeechSynthesisUtterance"}, + []string{"StaticRange"}, + []string{"Storage"}, + []string{"StorageEvent"}, + []string{"StyleSheet"}, + []string{"StyleSheetList"}, + []string{"Text"}, + []string{"TextMetrics"}, + []string{"TextTrack"}, + []string{"TextTrackCue"}, + []string{"TextTrackCueList"}, + []string{"TextTrackList"}, + []string{"TimeRanges"}, + []string{"TrackEvent"}, + []string{"TransitionEvent"}, + []string{"TreeWalker"}, + []string{"UIEvent"}, + []string{"VTTCue"}, + []string{"ValidityState"}, + []string{"VisualViewport"}, + []string{"WaveShaperNode"}, + []string{"WebGLActiveInfo"}, + []string{"WebGLBuffer"}, + []string{"WebGLContextEvent"}, + []string{"WebGLFramebuffer"}, + []string{"WebGLProgram"}, + []string{"WebGLQuery"}, + []string{"WebGLRenderbuffer"}, + []string{"WebGLRenderingContext"}, + []string{"WebGLSampler"}, + []string{"WebGLShader"}, + []string{"WebGLShaderPrecisionFormat"}, + []string{"WebGLSync"}, + []string{"WebGLTexture"}, + []string{"WebGLUniformLocation"}, + []string{"WebKitCSSMatrix"}, + []string{"WebSocket"}, + []string{"WheelEvent"}, + []string{"Window"}, + []string{"Worker"}, + []string{"XMLDocument"}, + []string{"XMLHttpRequest"}, + []string{"XMLHttpRequestEventTarget"}, + []string{"XMLHttpRequestUpload"}, + []string{"XMLSerializer"}, + []string{"XPathEvaluator"}, + []string{"XPathExpression"}, + []string{"XPathResult"}, + []string{"XSLTProcessor"}, + []string{"alert"}, + []string{"atob"}, + []string{"blur"}, + []string{"btoa"}, + []string{"cancelAnimationFrame"}, + []string{"captureEvents"}, + []string{"close"}, + []string{"closed"}, + []string{"confirm"}, + []string{"customElements"}, + []string{"devicePixelRatio"}, + []string{"document"}, + []string{"event"}, + []string{"fetch"}, + []string{"find"}, + []string{"focus"}, + []string{"frameElement"}, + []string{"frames"}, + []string{"getComputedStyle"}, + []string{"getSelection"}, + []string{"history"}, + []string{"indexedDB"}, + []string{"isSecureContext"}, + []string{"length"}, + []string{"location"}, + []string{"locationbar"}, + []string{"matchMedia"}, + []string{"menubar"}, + []string{"moveBy"}, + []string{"moveTo"}, + []string{"name"}, + []string{"navigator"}, + []string{"onabort"}, + []string{"onafterprint"}, + []string{"onanimationend"}, + []string{"onanimationiteration"}, + []string{"onanimationstart"}, + []string{"onbeforeprint"}, + []string{"onbeforeunload"}, + []string{"onblur"}, + []string{"oncanplay"}, + []string{"oncanplaythrough"}, + []string{"onchange"}, + []string{"onclick"}, + []string{"oncontextmenu"}, + []string{"oncuechange"}, + []string{"ondblclick"}, + []string{"ondrag"}, + []string{"ondragend"}, + []string{"ondragenter"}, + []string{"ondragleave"}, + []string{"ondragover"}, + []string{"ondragstart"}, + []string{"ondrop"}, + []string{"ondurationchange"}, + []string{"onemptied"}, + []string{"onended"}, + []string{"onerror"}, + []string{"onfocus"}, + []string{"ongotpointercapture"}, + []string{"onhashchange"}, + []string{"oninput"}, + []string{"oninvalid"}, + []string{"onkeydown"}, + []string{"onkeypress"}, + []string{"onkeyup"}, + []string{"onlanguagechange"}, + []string{"onload"}, + []string{"onloadeddata"}, + []string{"onloadedmetadata"}, + []string{"onloadstart"}, + []string{"onlostpointercapture"}, + []string{"onmessage"}, + []string{"onmousedown"}, + []string{"onmouseenter"}, + []string{"onmouseleave"}, + []string{"onmousemove"}, + []string{"onmouseout"}, + []string{"onmouseover"}, + []string{"onmouseup"}, + []string{"onoffline"}, + []string{"ononline"}, + []string{"onpagehide"}, + []string{"onpageshow"}, + []string{"onpause"}, + []string{"onplay"}, + []string{"onplaying"}, + []string{"onpointercancel"}, + []string{"onpointerdown"}, + []string{"onpointerenter"}, + []string{"onpointerleave"}, + []string{"onpointermove"}, + []string{"onpointerout"}, + []string{"onpointerover"}, + []string{"onpointerup"}, + []string{"onpopstate"}, + []string{"onprogress"}, + []string{"onratechange"}, + []string{"onrejectionhandled"}, + []string{"onreset"}, + []string{"onresize"}, + []string{"onscroll"}, + []string{"onseeked"}, + []string{"onseeking"}, + []string{"onselect"}, + []string{"onstalled"}, + []string{"onstorage"}, + []string{"onsubmit"}, + []string{"onsuspend"}, + []string{"ontimeupdate"}, + []string{"ontoggle"}, + []string{"ontransitioncancel"}, + []string{"ontransitionend"}, + []string{"ontransitionrun"}, + []string{"ontransitionstart"}, + []string{"onunhandledrejection"}, + []string{"onunload"}, + []string{"onvolumechange"}, + []string{"onwaiting"}, + []string{"onwebkitanimationend"}, + []string{"onwebkitanimationiteration"}, + []string{"onwebkitanimationstart"}, + []string{"onwebkittransitionend"}, + []string{"onwheel"}, + []string{"open"}, + []string{"opener"}, + []string{"origin"}, + []string{"outerHeight"}, + []string{"outerWidth"}, + []string{"parent"}, + []string{"performance"}, + []string{"personalbar"}, + []string{"postMessage"}, + []string{"print"}, + []string{"prompt"}, + []string{"releaseEvents"}, + []string{"requestAnimationFrame"}, + []string{"resizeBy"}, + []string{"resizeTo"}, + []string{"screen"}, + []string{"screenLeft"}, + []string{"screenTop"}, + []string{"screenX"}, + []string{"screenY"}, + []string{"scroll"}, + []string{"scrollBy"}, + []string{"scrollTo"}, + []string{"scrollbars"}, + []string{"self"}, + []string{"speechSynthesis"}, + []string{"status"}, + []string{"statusbar"}, + []string{"stop"}, + []string{"toolbar"}, + []string{"top"}, + []string{"webkitURL"}, + []string{"window"}, +}; diff --git a/src/defines.zig b/src/defines.zig new file mode 100644 index 000000000..6021df57f --- /dev/null +++ b/src/defines.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const js_ast = @import("./js_ast.zig"); + +const GlobalDefinesKey = @import("./defines-table.zig").GlobalDefinesKey; + +pub const defaultIdentifierDefines = comptime {}; + +pub const IdentifierDefine = struct {}; + +pub const DotDefine = struct {}; + +pub const Defines = struct {}; diff --git a/src/js_lexer.zig b/src/js_lexer.zig index a16c72c19..fa95341a5 100644 --- a/src/js_lexer.zig +++ b/src/js_lexer.zig @@ -22,1123 +22,1264 @@ pub const StrictModeReservedWords = tables.StrictModeReservedWords; pub const PropertyModifierKeyword = tables.PropertyModifierKeyword; pub const TypescriptStmtKeyword = tables.TypescriptStmtKeyword; -// TODO: JSON -const IS_JSON_FILE = false; - -pub const Lexer = struct { - // pub const Error = error{ - // UnexpectedToken, - // EndOfFile, - // }; - - // err: ?Lexer.Error, - log: logger.Log, - source: logger.Source, - current: usize = 0, - start: usize = 0, - end: usize = 0, - approximate_newline_count: i32 = 0, - legacy_octal_loc: logger.Loc = logger.Loc.Empty, - previous_backslash_quote_in_jsx: logger.Range = logger.Range.None, - token: T = T.t_end_of_file, - has_newline_before: bool = false, - has_pure_comment_before: bool = false, - preserve_all_comments_before: bool = false, - is_legacy_octal_literal: bool = false, - comments_to_preserve_before: ?[]js_ast.G.Comment = null, - all_original_comments: ?[]js_ast.G.Comment = null, - code_point: CodePoint = -1, - string_literal: JavascriptString, - identifier: []const u8 = "", - jsx_factory_pragma_comment: ?js_ast.Span = null, - jsx_fragment_pragma_comment: ?js_ast.Span = null, - source_mapping_url: ?js_ast.Span = null, - number: f64 = 0.0, - rescan_close_brace_as_template_token: bool = false, - for_global_name: bool = false, - prev_error_loc: logger.Loc = logger.Loc.Empty, - allocator: *std.mem.Allocator, - - pub fn loc(self: *Lexer) logger.Loc { - return logger.usize2Loc(self.start); - } +pub const JSONOptions = struct { + allow_comments: bool = false, + allow_trailing_commas: bool = false, +}; - fn nextCodepointSlice(it: *Lexer) callconv(.Inline) ?[]const u8 { - if (it.current >= it.source.contents.len) { - return null; +pub fn NewLexerType(comptime jsonOptions: ?JSONOptions) type { + return struct { + // pub const Error = error{ + // UnexpectedToken, + // EndOfFile, + // }; + + // err: ?@This().Error, + log: logger.Log, + source: logger.Source, + current: usize = 0, + start: usize = 0, + end: usize = 0, + approximate_newline_count: i32 = 0, + legacy_octal_loc: logger.Loc = logger.Loc.Empty, + previous_backslash_quote_in_jsx: logger.Range = logger.Range.None, + token: T = T.t_end_of_file, + has_newline_before: bool = false, + has_pure_comment_before: bool = false, + preserve_all_comments_before: bool = false, + is_legacy_octal_literal: bool = false, + comments_to_preserve_before: std.ArrayList(js_ast.G.Comment), + all_original_comments: ?[]js_ast.G.Comment = null, + code_point: CodePoint = -1, + string_literal: JavascriptString, + identifier: []const u8 = "", + jsx_factory_pragma_comment: ?js_ast.Span = null, + jsx_fragment_pragma_comment: ?js_ast.Span = null, + source_mapping_url: ?js_ast.Span = null, + number: f64 = 0.0, + rescan_close_brace_as_template_token: bool = false, + for_global_name: bool = false, + prev_error_loc: logger.Loc = logger.Loc.Empty, + allocator: *std.mem.Allocator, + + pub fn loc(self: *@This()) logger.Loc { + return logger.usize2Loc(self.start); } - const cp_len = unicode.utf8ByteSequenceLength(it.source.contents[it.current]) catch unreachable; - it.end = it.current; - it.current += cp_len; - - return it.source.contents[it.current - cp_len .. it.current]; - } - - pub fn syntaxError(self: *Lexer) void { - self.addError(self.start, "Syntax Error!!", .{}, true); - } + fn nextCodepointSlice(it: *@This()) callconv(.Inline) ?[]const u8 { + if (it.current >= it.source.contents.len) { + return null; + } - pub fn addDefaultError(self: *Lexer, msg: []const u8) void { - self.addError(self.start, "{s}", .{msg}, true); - } + const cp_len = unicode.utf8ByteSequenceLength(it.source.contents[it.current]) catch unreachable; + it.end = it.current; + it.current += cp_len; - pub fn addError(self: *Lexer, _loc: usize, comptime format: []const u8, args: anytype, panic: bool) void { - var __loc = logger.usize2Loc(_loc); - if (__loc.eql(self.prev_error_loc)) { - return; + return it.source.contents[it.current - cp_len .. it.current]; } - const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; - self.log.addError(self.source, __loc, errorMessage) catch unreachable; - self.prev_error_loc = __loc; - var msg = self.log.msgs.items[self.log.msgs.items.len - 1]; - msg.formatNoWriter(std.debug.panic); - } + pub fn syntaxError(self: *@This()) void { + self.addError(self.start, "Syntax Error!!", .{}, true); + } - pub fn addRangeError(self: *Lexer, r: logger.Range, comptime format: []const u8, args: anytype, panic: bool) void { - if (self.prev_error_loc.eql(r.loc)) { - return; + pub fn addDefaultError(self: *@This(), msg: []const u8) void { + self.addError(self.start, "{s}", .{msg}, true); } - const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; - var msg = self.log.addRangeError(self.source, r, errorMessage); - self.prev_error_loc = r.loc; + pub fn addError(self: *@This(), _loc: usize, comptime format: []const u8, args: anytype, panic: bool) void { + var __loc = logger.usize2Loc(_loc); + if (__loc.eql(self.prev_error_loc)) { + return; + } - if (panic) { - self.doPanic(errorMessage); + const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; + self.log.addError(self.source, __loc, errorMessage) catch unreachable; + self.prev_error_loc = __loc; + var msg = self.log.msgs.items[self.log.msgs.items.len - 1]; + msg.formatNoWriter(std.debug.panic); } - } - fn doPanic(self: *Lexer, content: []const u8) void { - std.debug.panic("{s}", .{content}); - } + pub fn addRangeError(self: *@This(), r: logger.Range, comptime format: []const u8, args: anytype, panic: bool) void { + if (self.prev_error_loc.eql(r.loc)) { + return; + } - pub fn codePointEql(self: *Lexer, a: u8) bool { - return @intCast(CodePoint, a) == self.code_point; - } + const errorMessage = std.fmt.allocPrint(self.allocator, format, args) catch unreachable; + var msg = self.log.addRangeError(self.source, r, errorMessage); + self.prev_error_loc = r.loc; - fn nextCodepoint(it: *Lexer) callconv(.Inline) CodePoint { - const slice = it.nextCodepointSlice() orelse return @as(CodePoint, -1); + if (panic) { + self.doPanic(errorMessage); + } + } - switch (slice.len) { - 1 => return @as(CodePoint, slice[0]), - 2 => return @as(CodePoint, unicode.utf8Decode2(slice) catch unreachable), - 3 => return @as(CodePoint, unicode.utf8Decode3(slice) catch unreachable), - 4 => return @as(CodePoint, unicode.utf8Decode4(slice) catch unreachable), - else => unreachable, + fn doPanic(self: *@This(), content: []const u8) void { + std.debug.panic("{s}", .{content}); } - } - /// Look ahead at the next n codepoints without advancing the iterator. - /// If fewer than n codepoints are available, then return the remainder of the string. - fn peek(it: *Lexer, n: usize) []const u8 { - const original_i = it.current; - defer it.current = original_i; - - var end_ix = original_i; - var found: usize = 0; - while (found < n) : (found += 1) { - const next_codepoint = it.nextCodepointSlice() orelse return it.source.contents[original_i..]; - end_ix += next_codepoint.len; + pub fn codePointEql(self: *@This(), a: u8) bool { + return @intCast(CodePoint, a) == self.code_point; } - return it.source.contents[original_i..end_ix]; - } + fn nextCodepoint(it: *@This()) callconv(.Inline) CodePoint { + const slice = it.nextCodepointSlice() orelse return @as(CodePoint, -1); - pub fn isIdentifierOrKeyword(lexer: Lexer) bool { - return @enumToInt(lexer.token) >= @enumToInt(T.t_identifier); - } + switch (slice.len) { + 1 => return @as(CodePoint, slice[0]), + 2 => return @as(CodePoint, unicode.utf8Decode2(slice) catch unreachable), + 3 => return @as(CodePoint, unicode.utf8Decode3(slice) catch unreachable), + 4 => return @as(CodePoint, unicode.utf8Decode4(slice) catch unreachable), + else => unreachable, + } + } - fn parseStringLiteral(lexer: *Lexer) void { - var quote: CodePoint = lexer.code_point; - var needs_slow_path = false; - var suffixLen: usize = 1; - - if (quote != '`') { - lexer.token = T.t_string_literal; - } else if (lexer.rescan_close_brace_as_template_token) { - lexer.token = T.t_template_tail; - } else { - lexer.token = T.t_no_substitution_template_literal; + /// Look ahead at the next n codepoints without advancing the iterator. + /// If fewer than n codepoints are available, then return the remainder of the string. + fn peek(it: *@This(), n: usize) []const u8 { + const original_i = it.current; + defer it.current = original_i; + + var end_ix = original_i; + var found: usize = 0; + while (found < n) : (found += 1) { + const next_codepoint = it.nextCodepointSlice() orelse return it.source.contents[original_i..]; + end_ix += next_codepoint.len; + } + + return it.source.contents[original_i..end_ix]; } - lexer.step(); - stringLiteral: while (true) { - switch (lexer.code_point) { - '\\' => { - needs_slow_path = true; - lexer.step(); + pub fn isIdentifierOrKeyword(lexer: @This()) bool { + return @enumToInt(lexer.token) >= @enumToInt(T.t_identifier); + } - // Handle Windows CRLF - if (lexer.code_point == '\r' and !IS_JSON_FILE) { + fn parseStringLiteral(lexer: *@This()) void { + var quote: CodePoint = lexer.code_point; + var needs_slow_path = false; + var suffixLen: usize = 1; + + if (quote != '`') { + lexer.token = T.t_string_literal; + } else if (lexer.rescan_close_brace_as_template_token) { + lexer.token = T.t_template_tail; + } else { + lexer.token = T.t_no_substitution_template_literal; + } + lexer.step(); + + stringLiteral: while (true) { + switch (lexer.code_point) { + '\\' => { + needs_slow_path = true; lexer.step(); - if (lexer.code_point == '\n') { + + // Handle Windows CRLF + if (lexer.code_point == '\r' and jsonOptions != null) { lexer.step(); + if (lexer.code_point == '\n') { + lexer.step(); + } + continue :stringLiteral; } - continue :stringLiteral; - } - }, - // This indicates the end of the file - - -1 => { - lexer.addDefaultError("Unterminated string literal"); - }, + }, + // This indicates the end of the file - '\r' => { - if (quote != '`') { + -1 => { lexer.addDefaultError("Unterminated string literal"); - } + }, - // Template literals require newline normalization - needs_slow_path = true; - }, + '\r' => { + if (quote != '`') { + lexer.addDefaultError("Unterminated string literal"); + } - '\n' => { - if (quote != '`') { - lexer.addDefaultError("Unterminated string literal"); - } - }, + // Template literals require newline normalization + needs_slow_path = true; + }, - '$' => { - if (quote == '`') { - lexer.step(); - if (lexer.code_point == '{') { - suffixLen = 2; + '\n' => { + if (quote != '`') { + lexer.addDefaultError("Unterminated string literal"); + } + }, + + '$' => { + if (quote == '`') { lexer.step(); - if (lexer.rescan_close_brace_as_template_token) { - lexer.token = T.t_template_middle; - } else { - lexer.token = T.t_template_head; + if (lexer.code_point == '{') { + suffixLen = 2; + lexer.step(); + if (lexer.rescan_close_brace_as_template_token) { + lexer.token = T.t_template_middle; + } else { + lexer.token = T.t_template_head; + } + break :stringLiteral; } - break :stringLiteral; + continue :stringLiteral; } - continue :stringLiteral; - } - }, + }, - else => { - if (quote == lexer.code_point) { - lexer.step(); - break :stringLiteral; - } - // Non-ASCII strings need the slow path - if (lexer.code_point >= 0x80) { - needs_slow_path = true; - } else if (IS_JSON_FILE and lexer.code_point < 0x20) { - lexer.syntaxError(); - } - }, + else => { + if (quote == lexer.code_point) { + lexer.step(); + break :stringLiteral; + } + // Non-ASCII strings need the slow path + if (lexer.code_point >= 0x80) { + needs_slow_path = true; + } else if (jsonOptions != null and lexer.code_point < 0x20) { + lexer.syntaxError(); + } + }, + } + lexer.step(); } - lexer.step(); - } - const text = lexer.source.contents[lexer.start + 1 .. lexer.end - suffixLen]; - // TODO: actually implement proper utf16 - lexer.string_literal = lexer.allocator.alloc(u16, text.len) catch unreachable; - var i: usize = 0; - for (text) |byte| { - lexer.string_literal[i] = byte; - i += 1; - } - // for (text) - // // if (needs_slow_path) { - // // // Slow path + const text = lexer.source.contents[lexer.start + 1 .. lexer.end - suffixLen]; + if (needs_slow_path) { + lexer.string_literal = lexer.stringToUTF16(text); + } else { + lexer.string_literal = lexer.allocator.alloc(u16, text.len) catch unreachable; + var i: usize = 0; + for (text) |byte| { + lexer.string_literal[i] = byte; + i += 1; + } + } - // // // lexer.string_literal = lexer.(lexer.start + 1, text); - // // } else { - // // // Fast path + if (quote == '\'' and jsonOptions != null) { + lexer.addRangeError(lexer.range(), "JSON strings must use double quotes", .{}, true); + } + // for (text) + // // if (needs_slow_path) { + // // // Slow path - // // } - } + // // // lexer.string_literal = lexer.(lexer.start + 1, text); + // // } else { + // // // Fast path - fn step(lexer: *Lexer) void { - lexer.code_point = lexer.nextCodepoint(); - - // Track the approximate number of newlines in the file so we can preallocate - // the line offset table in the printer for source maps. The line offset table - // is the #1 highest allocation in the heap profile, so this is worth doing. - // This count is approximate because it handles "\n" and "\r\n" (the common - // cases) but not "\r" or "\u2028" or "\u2029". Getting this wrong is harmless - // because it's only a preallocation. The array will just grow if it's too small. - if (lexer.code_point == '\n') { - lexer.approximate_newline_count += 1; + // // } } - } - pub fn expect(self: *Lexer, comptime token: T) void { - if (self.token != token) { - self.expected(token); + fn step(lexer: *@This()) void { + lexer.code_point = lexer.nextCodepoint(); + + // Track the approximate number of newlines in the file so we can preallocate + // the line offset table in the printer for source maps. The line offset table + // is the #1 highest allocation in the heap profile, so this is worth doing. + // This count is approximate because it handles "\n" and "\r\n" (the common + // cases) but not "\r" or "\u2028" or "\u2029". Getting this wrong is harmless + // because it's only a preallocation. The array will just grow if it's too small. + if (lexer.code_point == '\n') { + lexer.approximate_newline_count += 1; + } } - self.next(); - } + pub fn expect(self: *@This(), comptime token: T) void { + if (self.token != token) { + self.expected(token); + } - pub fn expectOrInsertSemicolon(lexer: *Lexer) void { - if (lexer.token == T.t_semicolon or (!lexer.has_newline_before and - lexer.token != T.t_close_brace and lexer.token != T.t_end_of_file)) - { - lexer.expect(T.t_semicolon); + self.next(); } - } - pub fn addUnsupportedSyntaxError(self: *Lexer, msg: []const u8) void { - self.addError(self.end, "Unsupported syntax: {s}", .{msg}, true); - } + pub fn expectOrInsertSemicolon(lexer: *@This()) void { + if (lexer.token == T.t_semicolon or (!lexer.has_newline_before and + lexer.token != T.t_close_brace and lexer.token != T.t_end_of_file)) + { + lexer.expect(T.t_semicolon); + } + } - pub fn scanIdentifierWithEscapes(self: *Lexer) void { - self.addUnsupportedSyntaxError("escape sequence"); - return; - } + pub fn addUnsupportedSyntaxError(self: *@This(), msg: []const u8) void { + self.addError(self.end, "Unsupported syntax: {s}", .{msg}, true); + } + + pub fn scanIdentifierWithEscapes(self: *@This()) void { + self.addUnsupportedSyntaxError("escape sequence"); + return; + } - pub fn debugInfo(self: *Lexer) void { - if (self.log.errors > 0) { - const stderr = std.io.getStdErr().writer(); - self.log.print(stderr) catch unreachable; - } else { - if (self.token == T.t_identifier or self.token == T.t_string_literal) { - std.debug.print(" {s} ", .{self.raw()}); + pub fn debugInfo(self: *@This()) void { + if (self.log.errors > 0) { + const stderr = std.io.getStdErr().writer(); + self.log.print(stderr) catch unreachable; } else { - std.debug.print(" <{s}> ", .{tokenToString.get(self.token)}); + if (self.token == T.t_identifier or self.token == T.t_string_literal) { + std.debug.print(" {s} ", .{self.raw()}); + } else { + std.debug.print(" <{s}> ", .{tokenToString.get(self.token)}); + } } } - } - pub fn expectContextualKeyword(self: *Lexer, comptime keyword: string) void { - if (!self.isContextualKeyword(keyword)) { - self.addError(self.start, "\"{s}\"", .{keyword}, true); + pub fn expectContextualKeyword(self: *@This(), comptime keyword: string) void { + if (!self.isContextualKeyword(keyword)) { + self.addError(self.start, "\"{s}\"", .{keyword}, true); + } + self.next(); } - self.next(); - } - - pub fn next(lexer: *Lexer) void { - lexer.has_newline_before = lexer.end == 0; - lex: while (lexer.log.errors == 0) { - lexer.start = lexer.end; - lexer.token = T.t_end_of_file; + pub fn next(lexer: *@This()) void { + lexer.has_newline_before = lexer.end == 0; - switch (lexer.code_point) { - -1 => { - lexer.token = T.t_end_of_file; - break :lex; - }, + lex: while (lexer.log.errors == 0) { + lexer.start = lexer.end; + lexer.token = T.t_end_of_file; - '#' => { - if (lexer.start == 0 and lexer.source.contents[1] == '!') { - lexer.addUnsupportedSyntaxError("#!hashbang is not supported yet."); - return; - } + switch (lexer.code_point) { + -1 => { + lexer.token = T.t_end_of_file; + break :lex; + }, - lexer.step(); - if (!isIdentifierStart(lexer.code_point)) { - lexer.syntaxError(); - } - lexer.step(); + '#' => { + if (lexer.start == 0 and lexer.source.contents[1] == '!') { + lexer.addUnsupportedSyntaxError("#!hashbang is not supported yet."); + return; + } - if (isIdentifierStart(lexer.code_point)) { lexer.step(); - while (isIdentifierContinue(lexer.code_point)) { - lexer.step(); - } - if (lexer.code_point == '\\') { - lexer.scanIdentifierWithEscapes(); - lexer.token = T.t_private_identifier; - // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); - } else { - lexer.token = T.t_private_identifier; - lexer.identifier = lexer.raw(); + if (!isIdentifierStart(lexer.code_point)) { + lexer.syntaxError(); } - break; - } - }, - '\r', '\n', 0x2028, 0x2029 => { - lexer.step(); - lexer.has_newline_before = true; - continue; - }, - - '\t', ' ' => { - lexer.step(); - continue; - }, - - '(' => { - lexer.step(); - lexer.token = T.t_open_paren; - }, - ')' => { - lexer.step(); - lexer.token = T.t_close_paren; - }, - '[' => { - lexer.step(); - lexer.token = T.t_open_bracket; - }, - ']' => { - lexer.step(); - lexer.token = T.t_close_bracket; - }, - '{' => { - lexer.step(); - lexer.token = T.t_open_brace; - }, - '}' => { - lexer.step(); - lexer.token = T.t_close_brace; - }, - ',' => { - lexer.step(); - lexer.token = T.t_comma; - }, - ':' => { - lexer.step(); - lexer.token = T.t_colon; - }, - ';' => { - lexer.step(); - lexer.token = T.t_semicolon; - }, - '@' => { - lexer.step(); - lexer.token = T.t_at; - }, - '~' => { - lexer.step(); - lexer.token = T.t_tilde; - }, + lexer.step(); - '?' => { - // '?' or '?.' or '??' or '??=' - lexer.step(); - switch (lexer.code_point) { - '?' => { + if (isIdentifierStart(lexer.code_point)) { lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_question_question_equals; - }, - else => { - lexer.token = T.t_question_question; - }, + while (isIdentifierContinue(lexer.code_point)) { + lexer.step(); } - }, - - '.' => { - lexer.token = T.t_question; - const current = lexer.current; - const contents = lexer.source.contents; - - // Lookahead to disambiguate with 'a?.1:b' - if (current < contents.len) { - const c = contents[current]; - if (c < '0' or c > '9') { - lexer.step(); - lexer.token = T.t_question_dot; - } + if (lexer.code_point == '\\') { + lexer.scanIdentifierWithEscapes(); + lexer.token = T.t_private_identifier; + // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); + } else { + lexer.token = T.t_private_identifier; + lexer.identifier = lexer.raw(); } - }, - else => { - lexer.token = T.t_question; - }, - } - }, - - '%' => { - // '%' or '%=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_percent_equals; - }, + break; + } + }, + '\r', '\n', 0x2028, 0x2029 => { + lexer.step(); + lexer.has_newline_before = true; + continue; + }, + '\t', ' ' => { + lexer.step(); + continue; + }, + '(' => { + lexer.step(); + lexer.token = T.t_open_paren; + }, + ')' => { + lexer.step(); + lexer.token = T.t_close_paren; + }, + '[' => { + lexer.step(); + lexer.token = T.t_open_bracket; + }, + ']' => { + lexer.step(); + lexer.token = T.t_close_bracket; + }, + '{' => { + lexer.step(); + lexer.token = T.t_open_brace; + }, + '}' => { + lexer.step(); + lexer.token = T.t_close_brace; + }, + ',' => { + lexer.step(); + lexer.token = T.t_comma; + }, + ':' => { + lexer.step(); + lexer.token = T.t_colon; + }, + ';' => { + lexer.step(); + lexer.token = T.t_semicolon; + }, + '@' => { + lexer.step(); + lexer.token = T.t_at; + }, + '~' => { + lexer.step(); + lexer.token = T.t_tilde; + }, + '?' => { + // '?' or '?.' or '??' or '??=' + lexer.step(); + switch (lexer.code_point) { + '?' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_question_question_equals; + }, + else => { + lexer.token = T.t_question_question; + }, + } + }, - else => { - lexer.token = T.t_percent; - }, - } - }, + '.' => { + lexer.token = T.t_question; + const current = lexer.current; + const contents = lexer.source.contents; + + // Lookahead to disambiguate with 'a?.1:b' + if (current < contents.len) { + const c = contents[current]; + if (c < '0' or c > '9') { + lexer.step(); + lexer.token = T.t_question_dot; + } + } + }, + else => { + lexer.token = T.t_question; + }, + } + }, + '%' => { + // '%' or '%=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_percent_equals; + }, - '&' => { - // '&' or '&=' or '&&' or '&&=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_ampersand_equals; - }, + else => { + lexer.token = T.t_percent; + }, + } + }, - '&' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_ampersand_ampersand_equals; - }, + '&' => { + // '&' or '&=' or '&&' or '&&=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_ampersand_equals; + }, - else => { - lexer.token = T.t_ampersand_ampersand; - }, - } - }, - else => { - lexer.token = T.t_ampersand; - }, - } - }, + '&' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_ampersand_ampersand_equals; + }, + + else => { + lexer.token = T.t_ampersand_ampersand; + }, + } + }, + else => { + lexer.token = T.t_ampersand; + }, + } + }, - '|' => { + '|' => { - // '|' or '|=' or '||' or '||=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_bar_equals; - }, - '|' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_bar_bar_equals; - }, + // '|' or '|=' or '||' or '||=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_bar_equals; + }, + '|' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_bar_bar_equals; + }, + + else => { + lexer.token = T.t_bar_bar; + }, + } + }, + else => { + lexer.token = T.t_bar; + }, + } + }, - else => { - lexer.token = T.t_bar_bar; - }, - } - }, - else => { - lexer.token = T.t_bar; - }, - } - }, + '^' => { + // '^' or '^=' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_caret_equals; + }, - '^' => { - // '^' or '^=' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_caret_equals; - }, + else => { + lexer.token = T.t_caret; + }, + } + }, - else => { - lexer.token = T.t_caret; - }, - } - }, + '+' => { + // '+' or '+=' or '++' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_plus_equals; + }, - '+' => { - // '+' or '+=' or '++' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_plus_equals; - }, + '+' => { + lexer.step(); + lexer.token = T.t_plus_plus; + }, - '+' => { - lexer.step(); - lexer.token = T.t_plus_plus; - }, + else => { + lexer.token = T.t_plus; + }, + } + }, - else => { - lexer.token = T.t_plus; - }, - } - }, + '-' => { + // '+' or '+=' or '++' + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_minus_equals; + }, - '=' => { - // '=' or '=>' or '==' or '===' - lexer.step(); - switch (lexer.code_point) { - '>' => { - lexer.step(); - lexer.token = T.t_equals_greater_than; - }, + '-' => { + lexer.step(); - '=' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { + if (lexer.code_point == '>' and lexer.has_newline_before) { lexer.step(); - lexer.token = T.t_equals_equals_equals; - }, - - else => { - lexer.token = T.t_equals_equals; - }, - } - }, + lexer.log.addRangeWarning(lexer.source, lexer.range(), "Treating \"-->\" as the start of a legacy HTML single-line comment") catch unreachable; + + singleLineHTMLCloseComment: while (true) { + switch (lexer.code_point) { + '\r', '\n', 0x2028, 0x2029 => { + break :singleLineHTMLCloseComment; + }, + -1 => { + break :singleLineHTMLCloseComment; + }, + else => {}, + } + lexer.step(); + } + continue; + } - else => { - lexer.token = T.t_equals; - }, - } - }, + lexer.token = T.t_minus_minus; + }, - '<' => { - // '<' or '<<' or '<=' or '<<=' or '<!--' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_less_than_equals; - }, + else => { + lexer.token = T.t_plus; + }, + } + }, - '<' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_less_than_less_than_equals; - }, + '*' => { + // '*' or '*=' or '**' or '**=' - else => { - lexer.token = T.t_less_than_less_than; - }, - } - }, - // Handle legacy HTML-style comments - '!' => { - if (std.mem.eql(u8, lexer.peek("--".len), "--")) { - lexer.addUnsupportedSyntaxError("Legacy HTML comments not implemented yet!"); - return; - } - - lexer.token = T.t_less_than; - }, + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = .t_asterisk_equals; + }, + '*' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = .t_asterisk_asterisk_equals; + }, + else => { + lexer.token = .t_asterisk_asterisk; + }, + } + }, + else => { + lexer.token = .t_asterisk; + }, + } + }, + '/' => { + // '/' or '/=' or '//' or '/* ... */' + lexer.step(); + // TODO: forGlobalName - else => { - lexer.token = T.t_less_than; - }, - } - }, + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = .t_slash_equals; + }, + '/' => { + lexer.step(); + singleLineComment: while (true) { + lexer.step(); + switch (lexer.code_point) { + '\r', '\n', 0x2028, 0x2029 => { + break :singleLineComment; + }, + -1 => { + break :singleLineComment; + }, + else => {}, + } + } - '>' => { - // '>' or '>>' or '>>>' or '>=' or '>>=' or '>>>=' - lexer.step(); + if (jsonOptions) |json| { + if (!json.allow_comments) { + lexer.addRangeError(lexer.range(), "JSON does not support comments", .{}, true); + return; + } + } + lexer.scanCommentText(); + continue; + }, + '*' => { + lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_greater_than_equals; - }, - '>' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_greater_than_greater_than_equals; - }, - '>' => { - lexer.step(); + multiLineComment: while (true) { switch (lexer.code_point) { - '=' => { + '*' => { + lexer.step(); + if (lexer.code_point == '/') { + lexer.step(); + break :multiLineComment; + } + }, + '\r', '\n', 0x2028, 0x2029 => { lexer.step(); - lexer.token = T.t_greater_than_greater_than_greater_than_equals; + lexer.has_newline_before = true; + }, + -1 => { + lexer.start = lexer.end; + lexer.addError(lexer.start, "Expected \"*/\" to terminate multi-line comment", .{}, true); }, else => { - lexer.token = T.t_greater_than_greater_than_greater_than; + lexer.step(); }, } - }, - else => { - lexer.token = T.t_greater_than_greater_than; - }, - } - }, - else => { - lexer.token = T.t_greater_than; - }, - } - }, + } + if (jsonOptions) |json| { + if (!json.allow_comments) { + lexer.addRangeError(lexer.range(), "JSON does not support comments", .{}, true); + return; + } + } + lexer.scanCommentText(); + continue; + }, + else => { + lexer.token = .t_slash; + }, + } + }, - '!' => { - // '!' or '!=' or '!==' - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - switch (lexer.code_point) { - '=' => { - lexer.step(); - lexer.token = T.t_exclamation_equals_equals; - }, + '=' => { + // '=' or '=>' or '==' or '===' + lexer.step(); + switch (lexer.code_point) { + '>' => { + lexer.step(); + lexer.token = T.t_equals_greater_than; + }, - else => { - lexer.token = T.t_exclamation_equals; - }, - } - }, - else => { - lexer.token = T.t_exclamation; - }, - } - }, + '=' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_equals_equals_equals; + }, + + else => { + lexer.token = T.t_equals_equals; + }, + } + }, - '\'', '"', '`' => { - lexer.parseStringLiteral(); - }, + else => { + lexer.token = T.t_equals; + }, + } + }, - '_', '$', 'a'...'z', 'A'...'Z' => { - lexer.step(); - while (isIdentifierContinue(lexer.code_point)) { + '<' => { + // '<' or '<<' or '<=' or '<<=' or '<!--' lexer.step(); - } + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_less_than_equals; + }, - if (lexer.code_point == '\\') { - lexer.scanIdentifierWithEscapes(); - } else { - const contents = lexer.raw(); - lexer.identifier = contents; - if (Keywords.get(contents)) |keyword| { - lexer.token = keyword; - } else { - lexer.token = T.t_identifier; + '<' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_less_than_less_than_equals; + }, + + else => { + lexer.token = T.t_less_than_less_than; + }, + } + }, + // Handle legacy HTML-style comments + '!' => { + if (std.mem.eql(u8, lexer.peek("--".len), "--")) { + lexer.addUnsupportedSyntaxError("Legacy HTML comments not implemented yet!"); + return; + } + + lexer.token = T.t_less_than; + }, + + else => { + lexer.token = T.t_less_than; + }, } - } - }, + }, - '\\' => { - // TODO: normal - lexer.scanIdentifierWithEscapes(); - }, + '>' => { + // '>' or '>>' or '>>>' or '>=' or '>>=' or '>>>=' + lexer.step(); - '.', '0'...'9' => { - lexer.parseNumericLiteralOrDot(); - }, + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_greater_than_equals; + }, + '>' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_greater_than_greater_than_equals; + }, + '>' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_greater_than_greater_than_greater_than_equals; + }, + else => { + lexer.token = T.t_greater_than_greater_than_greater_than; + }, + } + }, + else => { + lexer.token = T.t_greater_than_greater_than; + }, + } + }, + else => { + lexer.token = T.t_greater_than; + }, + } + }, - else => { - // Check for unusual whitespace characters - if (isWhitespace(lexer.code_point)) { + '!' => { + // '!' or '!=' or '!==' lexer.step(); - continue; - } + switch (lexer.code_point) { + '=' => { + lexer.step(); + switch (lexer.code_point) { + '=' => { + lexer.step(); + lexer.token = T.t_exclamation_equals_equals; + }, + + else => { + lexer.token = T.t_exclamation_equals; + }, + } + }, + else => { + lexer.token = T.t_exclamation; + }, + } + }, - if (isIdentifierStart(lexer.code_point)) { + '\'', '"', '`' => { + lexer.parseStringLiteral(); + }, + + '_', '$', 'a'...'z', 'A'...'Z' => { lexer.step(); while (isIdentifierContinue(lexer.code_point)) { lexer.step(); } - if (lexer.code_point == '\\') { - // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); + if (lexer.code_point == '\\') { + lexer.scanIdentifierWithEscapes(); } else { - lexer.token = T.t_identifier; - lexer.identifier = lexer.raw(); + const contents = lexer.raw(); + lexer.identifier = contents; + if (Keywords.get(contents)) |keyword| { + lexer.token = keyword; + } else { + lexer.token = T.t_identifier; + } } - break; - } - - lexer.end = lexer.current; - lexer.token = T.t_syntax_error; - }, - } + }, - return; - } - } + '\\' => { + // TODO: normal + lexer.scanIdentifierWithEscapes(); + }, - pub fn expected(self: *Lexer, token: T) void { - if (tokenToString.get(token).len > 0) { - self.expectedString(tokenToString.get(token)); - } else { - self.unexpected(); - } - } + '.', '0'...'9' => { + lexer.parseNumericLiteralOrDot(); + }, - pub fn unexpected(lexer: *Lexer) void { - var found: string = undefined; - if (lexer.start == lexer.source.contents.len) { - found = "end of file"; - } else { - found = lexer.raw(); - } + else => { + // Check for unusual whitespace characters + if (isWhitespace(lexer.code_point)) { + lexer.step(); + continue; + } - lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}, true); - } + if (isIdentifierStart(lexer.code_point)) { + lexer.step(); + while (isIdentifierContinue(lexer.code_point)) { + lexer.step(); + } + if (lexer.code_point == '\\') { - pub fn raw(self: *Lexer) []const u8 { - return self.source.contents[self.start..self.end]; - } + // lexer.Identifier, lexer.Token = lexer.scanIdentifierWithEscapes(normalIdentifier); + } else { + lexer.token = T.t_identifier; + lexer.identifier = lexer.raw(); + } + break; + } - pub fn isContextualKeyword(self: *Lexer, comptime keyword: string) bool { - return self.token == .t_identifier and strings.eql(self.raw(), keyword); - } + lexer.end = lexer.current; + lexer.token = T.t_syntax_error; + }, + } - pub fn expectedString(self: *Lexer, text: string) void { - var found = text; - if (self.source.contents.len == self.start) { - found = "end of file"; + return; + } } - self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }, true); - } - - pub fn range(self: *Lexer) logger.Range { - return logger.Range{ - .loc = logger.usize2Loc(self.start), - .len = std.math.lossyCast(i32, self.end - self.start), - }; - } - pub fn init(log: logger.Log, source: logger.Source, allocator: *std.mem.Allocator) !Lexer { - var empty_string_literal: JavascriptString = undefined; - var lex = Lexer{ - .log = log, - .source = source, - .string_literal = empty_string_literal, - .prev_error_loc = logger.Loc.Empty, - .allocator = allocator, - }; - lex.step(); - lex.next(); - - return lex; - } + pub fn expected(self: *@This(), token: T) void { + if (tokenToString.get(token).len > 0) { + self.expectedString(tokenToString.get(token)); + } else { + self.unexpected(); + } + } - pub fn scanRegExp(lexer: *Lexer) void { - while (true) { - switch (lexer.code_point) { - '/' => { - lexer.step(); - while (isIdentifierContinue(lexer.code_point)) { - switch (lexer.code_point) { - 'g', 'i', 'm', 's', 'u', 'y' => { - lexer.step(); - }, - else => { - lexer.syntaxError(); - }, - } - } - }, - '[' => { - lexer.step(); - while (lexer.code_point != ']') { - lexer.scanRegExpValidateAndStep(); - } - lexer.step(); - }, - else => { - lexer.scanRegExpValidateAndStep(); - }, + pub fn unexpected(lexer: *@This()) void { + var found: string = undefined; + if (lexer.start == lexer.source.contents.len) { + found = "end of file"; + } else { + found = lexer.raw(); } + + lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}, true); } - } - // TODO: use wtf-8 encoding. - pub fn stringToUTF16(lexer: *Lexer, str: string) JavascriptString { - var buf: JavascriptString = lexer.allocator.alloc(u16, std.mem.len(str)) catch unreachable; - var i: usize = 0; - // theres prob a faster/better way - for (str) |char| { - buf[i] = char; - i += 1; + pub fn raw(self: *@This()) []const u8 { + return self.source.contents[self.start..self.end]; } - return buf; - } + pub fn isContextualKeyword(self: *@This(), comptime keyword: string) bool { + return self.token == .t_identifier and strings.eql(self.raw(), keyword); + } - // TODO: use wtf-8 encoding. - pub fn utf16ToStringWithValidation(lexer: *Lexer, js: JavascriptString) !string { - return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js); - } + pub fn expectedString(self: *@This(), text: string) void { + var found = text; + if (self.source.contents.len == self.start) { + found = "end of file"; + } + self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }, true); + } - // TODO: use wtf-8 encoding. - pub fn utf16ToString(lexer: *Lexer, js: JavascriptString) string { - return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js) catch unreachable; - } + pub fn scanCommentText(lexer: *@This()) void { + var text = lexer.source.contents[lexer.start..lexer.end]; + const has_preserve_annotation = text.len > 2 and text[2] == '!'; + const is_multiline_comment = text[1] == '*'; - pub fn nextInsideJSXElement() void { - std.debug.panic("JSX not implemented yet.", .{}); - } + // Omit the trailing "*/" from the checks below + var endCommentText = text.len; + if (is_multiline_comment) { + endCommentText -= 2; + } - fn scanRegExpValidateAndStep(lexer: *Lexer) void { - if (lexer.code_point == '\\') { - lexer.step(); - } + if (has_preserve_annotation or lexer.preserve_all_comments_before) { + if (is_multiline_comment) { + // text = lexer.removeMultilineCommentIndent(lexer.source.contents[0..lexer.start], text); + } - switch (lexer.code_point) { - '\r', '\n', 0x2028, 0x2029 => { - // Newlines aren't allowed in regular expressions - lexer.syntaxError(); - }, - -1 => { // EOF - lexer.syntaxError(); - }, - else => { - lexer.step(); - }, + lexer.comments_to_preserve_before.append(js_ast.G.Comment{ + .text = text, + .loc = lexer.loc(), + }) catch unreachable; + } } - } - pub fn rescanCloseBraceAsTemplateToken(lexer: *Lexer) void { - if (lexer.token != .t_close_brace) { - lexer.expected(.t_close_brace); + // TODO: implement this + // it's too complicated to handle all the edgecases right now given the state of Zig's standard library + pub fn removeMultilineCommentIndent(lexer: *@This(), _prefix: string, text: string) string { + return text; } - lexer.rescan_close_brace_as_template_token = true; - lexer.code_point = '`'; - lexer.current = lexer.end; - lexer.end -= 1; - lexer.next(); - lexer.rescan_close_brace_as_template_token = false; - } - - pub fn rawTemplateContents(lexer: *Lexer) string { - var text: string = undefined; - - switch (lexer.token) { - .t_no_substitution_template_literal, .t_template_tail => { - text = lexer.source.contents[lexer.start + 1 .. lexer.end - 1]; - }, - .t_template_middle, .t_template_head => { - text = lexer.source.contents[lexer.start + 1 .. lexer.end - 2]; - }, - else => {}, + pub fn range(self: *@This()) logger.Range { + return logger.Range{ + .loc = logger.usize2Loc(self.start), + .len = std.math.lossyCast(i32, self.end - self.start), + }; } - if (strings.indexOfChar(text, '\r') == null) { - return text; + pub fn init(log: logger.Log, source: logger.Source, allocator: *std.mem.Allocator) !@This() { + var empty_string_literal: JavascriptString = undefined; + var lex = @This(){ + .log = log, + .source = source, + .string_literal = empty_string_literal, + .prev_error_loc = logger.Loc.Empty, + .allocator = allocator, + .comments_to_preserve_before = std.ArrayList(js_ast.G.Comment).init(allocator), + }; + lex.step(); + lex.next(); + + return lex; } - // From the specification: - // - // 11.8.6.1 Static Semantics: TV and TRV - // - // TV excludes the code units of LineContinuation while TRV includes - // them. <CR><LF> and <CR> LineTerminatorSequences are normalized to - // <LF> for both TV and TRV. An explicit EscapeSequence is needed to - // include a <CR> or <CR><LF> sequence. - var bytes = MutableString.initCopy(lexer.allocator, text) catch unreachable; - var end: usize = 0; - var i: usize = 0; - var c: u8 = '0'; - while (i < bytes.list.items.len) { - c = bytes.list.items[i]; - i += 1; - - if (c == '\r') { - // Convert '\r\n' into '\n' - if (i < bytes.list.items.len and bytes.list.items[i] == '\n') { - i += 1; + pub fn scanRegExp(lexer: *@This()) void { + while (true) { + switch (lexer.code_point) { + '/' => { + lexer.step(); + while (isIdentifierContinue(lexer.code_point)) { + switch (lexer.code_point) { + 'g', 'i', 'm', 's', 'u', 'y' => { + lexer.step(); + }, + else => { + lexer.syntaxError(); + }, + } + } + }, + '[' => { + lexer.step(); + while (lexer.code_point != ']') { + lexer.scanRegExpValidateAndStep(); + } + lexer.step(); + }, + else => { + lexer.scanRegExpValidateAndStep(); + }, } + } + } - // Convert '\r' into '\n' - c = '\n'; + // TODO: use wtf-8 encoding. + pub fn stringToUTF16(lexer: *@This(), str: string) JavascriptString { + var buf: JavascriptString = lexer.allocator.alloc(u16, std.mem.len(str)) catch unreachable; + var i: usize = 0; + // theres prob a faster/better way + for (str) |char| { + buf[i] = char; + i += 1; } - bytes.list.items[end] = c; - end += 1; + return buf; } - return bytes.toOwnedSliceLength(end + 1); - } - - fn parseNumericLiteralOrDot(lexer: *Lexer) void { - // Number or dot; - var first = lexer.code_point; - lexer.step(); - - // Dot without a digit after it; - if (first == '.' and (lexer.code_point < '0' or lexer.code_point > '9')) { - // "..." - if ((lexer.code_point == '.' and - lexer.current < lexer.source.contents.len) and - lexer.source.contents[lexer.current] == '.') - { - lexer.step(); - lexer.step(); - lexer.token = T.t_dot_dot_dot; - return; - } + // TODO: use wtf-8 encoding. + pub fn utf16ToStringWithValidation(lexer: *@This(), js: JavascriptString) !string { + return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js); + } - // "." - lexer.token = T.t_dot; - return; + // TODO: use wtf-8 encoding. + pub fn utf16ToString(lexer: *@This(), js: JavascriptString) string { + return std.unicode.utf16leToUtf8Alloc(lexer.allocator, js) catch unreachable; } - var underscoreCount: usize = 0; - var lastUnderscoreEnd: usize = 0; - var hasDotOrExponent = first == '.'; - var base: f32 = 0.0; - lexer.is_legacy_octal_literal = false; + pub fn nextInsideJSXElement() void { + std.debug.panic("JSX not implemented yet.", .{}); + } - // Assume this is a number, but potentially change to a bigint later; - lexer.token = T.t_numeric_literal; + fn scanRegExpValidateAndStep(lexer: *@This()) void { + if (lexer.code_point == '\\') { + lexer.step(); + } - // Check for binary, octal, or hexadecimal literal; - if (first == '0') { switch (lexer.code_point) { - 'b', 'B' => { - base = 2; + '\r', '\n', 0x2028, 0x2029 => { + // Newlines aren't allowed in regular expressions + lexer.syntaxError(); }, - - 'o', 'O' => { - base = 8; + -1 => { // EOF + lexer.syntaxError(); }, - - 'x', 'X' => { - base = 16; + else => { + lexer.step(); }, + } + } + + pub fn rescanCloseBraceAsTemplateToken(lexer: *@This()) void { + if (lexer.token != .t_close_brace) { + lexer.expected(.t_close_brace); + } - '0'...'7', '_' => { - base = 8; - lexer.is_legacy_octal_literal = true; + lexer.rescan_close_brace_as_template_token = true; + lexer.code_point = '`'; + lexer.current = lexer.end; + lexer.end -= 1; + lexer.next(); + lexer.rescan_close_brace_as_template_token = false; + } + + pub fn rawTemplateContents(lexer: *@This()) string { + var text: string = undefined; + + switch (lexer.token) { + .t_no_substitution_template_literal, .t_template_tail => { + text = lexer.source.contents[lexer.start + 1 .. lexer.end - 1]; + }, + .t_template_middle, .t_template_head => { + text = lexer.source.contents[lexer.start + 1 .. lexer.end - 2]; }, else => {}, } + + if (strings.indexOfChar(text, '\r') == null) { + return text; + } + + // From the specification: + // + // 11.8.6.1 Static Semantics: TV and TRV + // + // TV excludes the code units of LineContinuation while TRV includes + // them. <CR><LF> and <CR> LineTerminatorSequences are normalized to + // <LF> for both TV and TRV. An explicit EscapeSequence is needed to + // include a <CR> or <CR><LF> sequence. + var bytes = MutableString.initCopy(lexer.allocator, text) catch unreachable; + var end: usize = 0; + var i: usize = 0; + var c: u8 = '0'; + while (i < bytes.list.items.len) { + c = bytes.list.items[i]; + i += 1; + + if (c == '\r') { + // Convert '\r\n' into '\n' + if (i < bytes.list.items.len and bytes.list.items[i] == '\n') { + i += 1; + } + + // Convert '\r' into '\n' + c = '\n'; + } + + bytes.list.items[end] = c; + end += 1; + } + + return bytes.toOwnedSliceLength(end + 1); } - if (base != 0) { - // Integer literal; - var isFirst = true; - var isInvalidLegacyOctalLiteral = false; - lexer.number = 0; - if (!lexer.is_legacy_octal_literal) { - lexer.step(); + fn parseNumericLiteralOrDot(lexer: *@This()) void { + // Number or dot; + var first = lexer.code_point; + lexer.step(); + + // Dot without a digit after it; + if (first == '.' and (lexer.code_point < '0' or lexer.code_point > '9')) { + // "..." + if ((lexer.code_point == '.' and + lexer.current < lexer.source.contents.len) and + lexer.source.contents[lexer.current] == '.') + { + lexer.step(); + lexer.step(); + lexer.token = T.t_dot_dot_dot; + return; + } + + // "." + lexer.token = T.t_dot; + return; } - integerLiteral: while (true) { - switch (lexer.code_point) { - '_' => { - // Cannot have multiple underscores in a row; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.syntaxError(); - } + var underscoreCount: usize = 0; + var lastUnderscoreEnd: usize = 0; + var hasDotOrExponent = first == '.'; + var base: f32 = 0.0; + lexer.is_legacy_octal_literal = false; - // The first digit must exist; - if (isFirst or lexer.is_legacy_octal_literal) { - lexer.syntaxError(); - } + // Assume this is a number, but potentially change to a bigint later; + lexer.token = T.t_numeric_literal; - lastUnderscoreEnd = lexer.end; - underscoreCount += 1; + // Check for binary, octal, or hexadecimal literal; + if (first == '0') { + switch (lexer.code_point) { + 'b', 'B' => { + base = 2; }, - '0', '1' => { - lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + 'o', 'O' => { + base = 8; }, - '2', '3', '4', '5', '6', '7' => { - if (base == 2) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point - '0'); - }, - '8', '9' => { - if (lexer.is_legacy_octal_literal) { - isInvalidLegacyOctalLiteral = true; - } else if (base < 10) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point - '0'); - }, - 'A', 'B', 'C', 'D', 'E', 'F' => { - if (base != 16) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'A'); + 'x', 'X' => { + base = 16; }, - 'a', 'b', 'c', 'd', 'e', 'f' => { - if (base != 16) { - lexer.syntaxError(); - } - lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'a'); + '0'...'7', '_' => { + base = 8; + lexer.is_legacy_octal_literal = true; }, - else => { - // The first digit must exist; - if (isFirst) { - lexer.syntaxError(); - } + else => {}, + } + } - break :integerLiteral; - }, + if (base != 0) { + // Integer literal; + var isFirst = true; + var isInvalidLegacyOctalLiteral = false; + lexer.number = 0; + if (!lexer.is_legacy_octal_literal) { + lexer.step(); } - lexer.step(); - isFirst = false; - } + integerLiteral: while (true) { + switch (lexer.code_point) { + '_' => { + // Cannot have multiple underscores in a row; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.syntaxError(); + } - var isBigIntegerLiteral = lexer.code_point == 'n' and !hasDotOrExponent; + // The first digit must exist; + if (isFirst or lexer.is_legacy_octal_literal) { + lexer.syntaxError(); + } - // Slow path: do we need to re-scan the input as text? - if (isBigIntegerLiteral or isInvalidLegacyOctalLiteral) { - var text = lexer.raw(); + lastUnderscoreEnd = lexer.end; + underscoreCount += 1; + }, - // Can't use a leading zero for bigint literals; - if (isBigIntegerLiteral and lexer.is_legacy_octal_literal) { - lexer.syntaxError(); - } + '0', '1' => { + lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + }, - // Filter out underscores; - if (underscoreCount > 0) { - var bytes = lexer.allocator.alloc(u8, text.len - underscoreCount) catch unreachable; - var i: usize = 0; - for (text) |char| { - if (char != '_') { - bytes[i] = char; - i += 1; - } - } - } + '2', '3', '4', '5', '6', '7' => { + if (base == 2) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + }, + '8', '9' => { + if (lexer.is_legacy_octal_literal) { + isInvalidLegacyOctalLiteral = true; + } else if (base < 10) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point - '0'); + }, + 'A', 'B', 'C', 'D', 'E', 'F' => { + if (base != 16) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'A'); + }, - // Store bigints as text to avoid precision loss; - if (isBigIntegerLiteral) { - lexer.identifier = text; - } else if (isInvalidLegacyOctalLiteral) { - if (std.fmt.parseFloat(f64, text)) |num| { - lexer.number = num; - } else |err| { - lexer.addError(lexer.start, "Invalid number {s}", .{text}, true); + 'a', 'b', 'c', 'd', 'e', 'f' => { + if (base != 16) { + lexer.syntaxError(); + } + lexer.number = lexer.number * base + float64(lexer.code_point + 10 - 'a'); + }, + else => { + // The first digit must exist; + if (isFirst) { + lexer.syntaxError(); + } + + break :integerLiteral; + }, } + + lexer.step(); + isFirst = false; } - } - } else { - // Floating-point literal; - var isInvalidLegacyOctalLiteral = first == '0' and (lexer.code_point == '8' or lexer.code_point == '9'); - // Initial digits; - while (true) { - if (lexer.code_point < '0' or lexer.code_point > '9') { - if (lexer.code_point != '_') { - break; - } + var isBigIntegerLiteral = lexer.code_point == 'n' and !hasDotOrExponent; - // Cannot have multiple underscores in a row; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.syntaxError(); - } + // Slow path: do we need to re-scan the input as text? + if (isBigIntegerLiteral or isInvalidLegacyOctalLiteral) { + var text = lexer.raw(); - // The specification forbids underscores in this case; - if (isInvalidLegacyOctalLiteral) { + // Can't use a leading zero for bigint literals; + if (isBigIntegerLiteral and lexer.is_legacy_octal_literal) { lexer.syntaxError(); } - lastUnderscoreEnd = lexer.end; - underscoreCount += 1; - } - lexer.step(); - } + // Filter out underscores; + if (underscoreCount > 0) { + var bytes = lexer.allocator.alloc(u8, text.len - underscoreCount) catch unreachable; + var i: usize = 0; + for (text) |char| { + if (char != '_') { + bytes[i] = char; + i += 1; + } + } + } - // Fractional digits; - if (first != '.' and lexer.code_point == '.') { - // An underscore must not come last; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.end -= 1; - lexer.syntaxError(); + // Store bigints as text to avoid precision loss; + if (isBigIntegerLiteral) { + lexer.identifier = text; + } else if (isInvalidLegacyOctalLiteral) { + if (std.fmt.parseFloat(f64, text)) |num| { + lexer.number = num; + } else |err| { + lexer.addError(lexer.start, "Invalid number {s}", .{text}, true); + } + } } + } else { + // Floating-point literal; + var isInvalidLegacyOctalLiteral = first == '0' and (lexer.code_point == '8' or lexer.code_point == '9'); - hasDotOrExponent = true; - lexer.step(); - if (lexer.code_point == '_') { - lexer.syntaxError(); - } + // Initial digits; while (true) { if (lexer.code_point < '0' or lexer.code_point > '9') { if (lexer.code_point != '_') { @@ -1150,110 +1291,151 @@ pub const Lexer = struct { lexer.syntaxError(); } + // The specification forbids underscores in this case; + if (isInvalidLegacyOctalLiteral) { + lexer.syntaxError(); + } + lastUnderscoreEnd = lexer.end; underscoreCount += 1; } lexer.step(); } - } - // Exponent; - if (lexer.code_point == 'e' or lexer.code_point == 'E') { - // An underscore must not come last; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.end -= 1; - lexer.syntaxError(); - } + // Fractional digits; + if (first != '.' and lexer.code_point == '.') { + // An underscore must not come last; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.end -= 1; + lexer.syntaxError(); + } - hasDotOrExponent = true; - lexer.step(); - if (lexer.code_point == '+' or lexer.code_point == '-') { + hasDotOrExponent = true; lexer.step(); - } - if (lexer.code_point < '0' or lexer.code_point > '9') { - lexer.syntaxError(); - } - while (true) { - if (lexer.code_point < '0' or lexer.code_point > '9') { - if (lexer.code_point != '_') { - break; - } + if (lexer.code_point == '_') { + lexer.syntaxError(); + } + while (true) { + if (lexer.code_point < '0' or lexer.code_point > '9') { + if (lexer.code_point != '_') { + break; + } - // Cannot have multiple underscores in a row; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.syntaxError(); + // Cannot have multiple underscores in a row; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.syntaxError(); + } + + lastUnderscoreEnd = lexer.end; + underscoreCount += 1; } + lexer.step(); + } + } - lastUnderscoreEnd = lexer.end; - underscoreCount += 1; + // Exponent; + if (lexer.code_point == 'e' or lexer.code_point == 'E') { + // An underscore must not come last; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.end -= 1; + lexer.syntaxError(); } + + hasDotOrExponent = true; lexer.step(); - } - } + if (lexer.code_point == '+' or lexer.code_point == '-') { + lexer.step(); + } + if (lexer.code_point < '0' or lexer.code_point > '9') { + lexer.syntaxError(); + } + while (true) { + if (lexer.code_point < '0' or lexer.code_point > '9') { + if (lexer.code_point != '_') { + break; + } - // Take a slice of the text to parse; - var text = lexer.raw(); + // Cannot have multiple underscores in a row; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.syntaxError(); + } - // Filter out underscores; - if (underscoreCount > 0) { - var i: usize = 0; - if (lexer.allocator.alloc(u8, text.len - underscoreCount)) |bytes| { - for (text) |char| { - if (char != '_') { - bytes[i] = char; - i += 1; + lastUnderscoreEnd = lexer.end; + underscoreCount += 1; } + lexer.step(); } - text = bytes; - } else |err| { - lexer.addError(lexer.start, "Out of Memory Wah Wah Wah", .{}, true); - return; } - } - if (lexer.code_point == 'n' and !hasDotOrExponent) { - // The only bigint literal that can start with 0 is "0n" - if (text.len > 1 and first == '0') { - lexer.syntaxError(); - } + // Take a slice of the text to parse; + var text = lexer.raw(); - // Store bigints as text to avoid precision loss; - lexer.identifier = text; - } else if (!hasDotOrExponent and lexer.end - lexer.start < 10) { - // Parse a 32-bit integer (very fast path); - var number: u32 = 0; - for (text) |c| { - number = number * 10 + @intCast(u32, c - '0'); + // Filter out underscores; + if (underscoreCount > 0) { + var i: usize = 0; + if (lexer.allocator.alloc(u8, text.len - underscoreCount)) |bytes| { + for (text) |char| { + if (char != '_') { + bytes[i] = char; + i += 1; + } + } + text = bytes; + } else |err| { + lexer.addError(lexer.start, "Out of Memory Wah Wah Wah", .{}, true); + return; + } } - lexer.number = @intToFloat(f64, number); - } else { - // Parse a double-precision floating-point number; - if (std.fmt.parseFloat(f64, text)) |num| { - lexer.number = num; - } else |err| { - lexer.addError(lexer.start, "Invalid number", .{}, true); + + if (lexer.code_point == 'n' and !hasDotOrExponent) { + // The only bigint literal that can start with 0 is "0n" + if (text.len > 1 and first == '0') { + lexer.syntaxError(); + } + + // Store bigints as text to avoid precision loss; + lexer.identifier = text; + } else if (!hasDotOrExponent and lexer.end - lexer.start < 10) { + // Parse a 32-bit integer (very fast path); + var number: u32 = 0; + for (text) |c| { + number = number * 10 + @intCast(u32, c - '0'); + } + lexer.number = @intToFloat(f64, number); + } else { + // Parse a double-precision floating-point number; + if (std.fmt.parseFloat(f64, text)) |num| { + lexer.number = num; + } else |err| { + lexer.addError(lexer.start, "Invalid number", .{}, true); + } } } - } - // An underscore must not come last; - if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { - lexer.end -= 1; - lexer.syntaxError(); - } + // An underscore must not come last; + if (lastUnderscoreEnd > 0 and lexer.end == lastUnderscoreEnd + 1) { + lexer.end -= 1; + lexer.syntaxError(); + } - // Handle bigint literals after the underscore-at-end check above; - if (lexer.code_point == 'n' and !hasDotOrExponent) { - lexer.token = T.t_big_integer_literal; - lexer.step(); - } + // Handle bigint literals after the underscore-at-end check above; + if (lexer.code_point == 'n' and !hasDotOrExponent) { + lexer.token = T.t_big_integer_literal; + lexer.step(); + } - // Identifiers can't occur immediately after numbers; - if (isIdentifierStart(lexer.code_point)) { - lexer.syntaxError(); + // Identifiers can't occur immediately after numbers; + if (isIdentifierStart(lexer.code_point)) { + lexer.syntaxError(); + } } - } -}; + }; +} + +// JS/TS lexer +pub const Lexer = NewLexerType(null); +pub const JSONLexer = NewLexerType(JSONOptions{ .allow_comments = false, .allow_trailing_commas = false }); +pub const TSConfigJSONLexer = NewLexerType(JSONOptions{ .allow_comments = true, .allow_trailing_commas = true }); fn isIdentifierStart(codepoint: CodePoint) bool { switch (codepoint) { @@ -1309,10 +1491,10 @@ fn isWhitespace(codepoint: CodePoint) bool { 0x202F, // narrow no-break space 0x205F, // medium mathematical space 0x3000, // ideographic space - 0xFEFF, + 0xFEFF, // zero width non-breaking space => { return true; - }, // zero width non-breaking space + }, else => { return false; }, @@ -1346,7 +1528,7 @@ fn float64(num: anytype) callconv(.Inline) f64 { return @intToFloat(f64, num); } -fn test_lexer(contents: []const u8) Lexer { +fn test_lexer(contents: []const u8) @This() { alloc.setup(std.heap.page_allocator) catch unreachable; const msgs = std.ArrayList(logger.Msg).init(alloc.dynamic); const log = logger.Log{ @@ -1357,10 +1539,10 @@ fn test_lexer(contents: []const u8) Lexer { "index.js", contents, ); - return Lexer.init(log, source, alloc.dynamic) catch unreachable; + return @This().init(log, source, alloc.dynamic) catch unreachable; } -// test "Lexer.next()" { +// test "@This().next()" { // try alloc.setup(std.heap.page_allocator); // const msgs = std.ArrayList(logger.Msg).init(alloc.dynamic); // const log = logger.Log{ @@ -1368,7 +1550,7 @@ fn test_lexer(contents: []const u8) Lexer { // }; // const source = logger.Source.initPathString("index.js", "for (let i = 0; i < 100; i++) { console.log('hi'); }", std.heap.page_allocator); -// var lex = try Lexer.init(log, source, alloc.dynamic); +// var lex = try @This().init(log, source, alloc.dynamic); // lex.next(); // } @@ -1411,7 +1593,7 @@ pub fn test_stringLiteralEquals(expected: string, source_text: string) void { source_text, ); - var lex = try Lexer.init(log, source, std.heap.page_allocator); + var lex = try @This().init(log, source, std.heap.page_allocator); while (!lex.token.isString() and lex.token != .t_end_of_file) { lex.next(); } @@ -1420,7 +1602,7 @@ pub fn test_stringLiteralEquals(expected: string, source_text: string) void { std.testing.expectEqualStrings(expected, lit); } -pub fn test_skipTo(lexer: *Lexer, n: string) void { +pub fn test_skipTo(lexer: *@This(), n: string) void { var i: usize = 0; while (i < n.len) { lexer.next(); @@ -1428,7 +1610,7 @@ pub fn test_skipTo(lexer: *Lexer, n: string) void { } } -test "Lexer.rawTemplateContents" { +test "@This().rawTemplateContents" { test_stringLiteralEquals("hello!", "const a = 'hello!';"); test_stringLiteralEquals("hello!hi", "const b = 'hello!hi';"); test_stringLiteralEquals("hello!\n\nhi", "const b = `hello!\n\nhi`;"); diff --git a/src/js_parser.zig b/src/js_parser.zig index e7692c76f..460e0d275 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -3173,12 +3173,10 @@ const P = struct { var isDirectivePrologue = true; run: while (true) { - if (p.lexer.comments_to_preserve_before) |comments| { - for (comments) |comment| { - try stmts.append(p.s(S.Comment{ - .text = comment.text, - }, p.lexer.loc())); - } + for (p.lexer.comments_to_preserve_before.items) |comment| { + try stmts.append(p.s(S.Comment{ + .text = comment.text, + }, p.lexer.loc())); } if (p.lexer.token == .t_end_of_file) { @@ -5737,14 +5735,14 @@ const P = struct { p.lexer.preserve_all_comments_before = true; p.lexer.expect(.t_open_paren); - const comments = p.lexer.comments_to_preserve_before; + const comments = p.lexer.comments_to_preserve_before.toOwnedSlice(); p.lexer.preserve_all_comments_before = false; const value = p.parseExpr(.comma); p.lexer.expect(.t_close_paren); p.allow_in = old_allow_in; - return p.e(E.Import{ .expr = value, .leading_interior_comments = comments orelse &([_]G.Comment{}), .import_record_index = 0 }, loc); + return p.e(E.Import{ .expr = value, .leading_interior_comments = comments, .import_record_index = 0 }, loc); } pub fn parseJSXElement(loc: logger.Loc) Expr { @@ -5866,7 +5864,7 @@ const P = struct { switch (expr.data) { .e_null, .e_super, .e_boolean, .e_big_int, .e_reg_exp, .e_new_target, .e_undefined => {}, .e_string => |e_| { - // idc about legacy octal loc + // If you're using this, you're probably not using 0-prefixed legacy octal notation // if e.LegacyOctalLoc.Start > 0 { }, .e_number => |e_| { @@ -6013,27 +6011,27 @@ const P = struct { switch (data.value) { .expr => |*expr| { const was_anonymous_named_expr = expr.isAnonymousNamed(); - data.value.expr = p.m(p.visitExpr(expr.*)); - - // Optionally preserve the name - data.value.expr = p.maybeKeepExprSymbolName(expr, "default", was_anonymous_named_expr); - - // Discard type-only export default statements - if (p.options.ts) { - switch (expr.data) { - .e_identifier => |ident| { - const symbol = p.symbols.items[ident.ref.inner_index]; - if (symbol.kind == .unbound) { - if (p.local_type_names.get(symbol.original_name)) |local_type| { - if (local_type.value) { - return; - } - } - } - }, - else => {}, - } - } + // data.value.expr = p.m(p.visitExpr(expr.*)); + + // // Optionally preserve the name + // data.value.expr = p.maybeKeepExprSymbolName(expr, "default", was_anonymous_named_expr); + + // // Discard type-only export default statements + // if (p.options.ts) { + // switch (expr.data) { + // .e_identifier => |ident| { + // const symbol = p.symbols.items[ident.ref.inner_index]; + // if (symbol.kind == .unbound) { + // if (p.local_type_names.get(symbol.original_name)) |local_type| { + // if (local_type.value) { + // return; + // } + // } + // } + // }, + // else => {}, + // } + // } }, .stmt => |st| {}, diff --git a/src/json_parser.zig b/src/json_parser.zig new file mode 100644 index 000000000..8a025864f --- /dev/null +++ b/src/json_parser.zig @@ -0,0 +1,212 @@ +const std = @import("std"); +const logger = @import("logger.zig"); +const js_lexer = @import("js_lexer.zig"); +const importRecord = @import("import_record.zig"); +const js_ast = @import("js_ast.zig"); +const options = @import("options.zig"); +const alloc = @import("alloc.zig"); + +const fs = @import("fs.zig"); +usingnamespace @import("strings.zig"); +usingnamespace @import("ast/base.zig"); +usingnamespace js_ast.G; + +const ImportKind = importRecord.ImportKind; +const BindingNodeIndex = js_ast.BindingNodeIndex; + +const StmtNodeIndex = js_ast.StmtNodeIndex; +const ExprNodeIndex = js_ast.ExprNodeIndex; +const ExprNodeList = js_ast.ExprNodeList; +const StmtNodeList = js_ast.StmtNodeList; +const BindingNodeList = js_ast.BindingNodeList; +const assert = std.debug.assert; + +const Ref = js_ast.Ref; +const LocRef = js_ast.LocRef; +const S = js_ast.S; +const B = js_ast.B; +const G = js_ast.G; +const T = js_lexer.T; +const E = js_ast.E; +const Stmt = js_ast.Stmt; +const Expr = js_ast.Expr; +const Binding = js_ast.Binding; +const Symbol = js_ast.Symbol; +const Level = js_ast.Op.Level; +const Op = js_ast.Op; +const Scope = js_ast.Scope; +const locModuleScope = logger.Loc.Empty; + +fn JSONLikeParser(opts: js_lexer.JSONOptions) type { + const Lexer = if (opts.allow_comments) js_lexer.TSConfigJSONLexer else js_lexer.JSONLexer; + return struct { + lexer: Lexer, + source: logger.Source, + log: logger.Log, + allocator: *std.mem.Allocator, + + pub fn init(allocator: *std.mem.Allocator, source: logger.Source, log: logger.Log) Parser { + return Parser{ + .lexer = Lexer.init(log, source, allocator), + .allocator = allocator, + .log = log, + .source = source, + }; + } + + const Parser = @This(); + + pub fn e(p: *Parser, t: anytype, loc: logger.Loc) Expr { + if (@typeInfo(@TypeOf(t)) == .Pointer) { + return Expr.init(t, loc); + } else { + return Expr.alloc(p.allocator, t, loc); + } + } + pub fn parseExpr(p: *Parser) ?Expr { + const loc = p.lexer.loc(); + + switch (p.lexer.token) { + .t_false => { + p.lexer.next(); + return p.e(E.Boolean{ + .value = false, + }, loc); + }, + .t_true => { + p.lexer.next(); + return p.e(E.Boolean{ + .value = true, + }, loc); + }, + .t_null => { + p.lexer.next(); + return p.e(E.Null{}, loc); + }, + .t_string_literal => { + const value = p.lexer.string_literal; + p.lexer.next(); + return p.e(E.String{ + .value = value, + }, loc); + }, + .t_numeric_literal => { + const value = p.lexer.number; + p.lexer.next(); + return p.e(E.Number{ .value = value }, loc); + }, + .t_minus => { + p.lexer.next(); + const value = p.lexer.number; + p.lexer.expect(.t_numeric_literal); + return p.e(E.Number{ .value = -value }, loc); + }, + .t_open_bracket => { + p.lexer.next(); + var is_single_line = !p.lexer.has_newline_before; + var exprs = List(Expr).init(p.allocator); + + while (p.lexer.token != .t_close_bracket) { + if (exprs.items.len > 0) { + if (p.lexer.has_newline_before) { + is_single_line = false; + } + + if (!p.parseMaybeTrailingComma(.t_close_bracket)) { + break; + } + + if (p.lexer.has_newline_before) { + is_single_line = false; + } + } + + if (p.parseExpr()) |expr| { + try exprs.append(expr); + } else { + break; + } + } + + if (p.lexer.has_newline_before) { + is_single_line = false; + } + p.lexer.expect(.t_close_bracket); + return p.e(E.Array{ .items = exprs.toOwnedSlice() }, loc); + }, + .t_open_brace => { + p.lexer.next(); + var is_single_line = !p.lexer.has_newline_before; + var properties = List(G.Property).init(p.allocator); + var duplicates = std.StringHashMap(u0).init(p.allocator); + + while (p.lexer.token != .t_close_brace) { + if (properties.items.len > 0) { + is_single_line = if (p.lexer.has_newline_before) false else is_single_line; + if (!p.parseMaybeTrailingComma(.t_close_brace)) { + break; + } + is_single_line = if (p.lexer.has_newline_before) false else is_single_line; + } + + var key_string = p.lexer.string_literal; + var key_range = p.lexer.range(); + var key = p.e(E.String{ .value = key_string }, key_range.loc); + p.lexer.expect(.t_string_literal); + var key_text = p.lexer.utf16ToString(); + // Warn about duplicate keys + if (duplicates.contains(key_text)) { + p.log.addRangeWarningFmt(p.source, r, "Duplicate key \"{s}\" in object literal", .{key_text}) catch unreachable; + } else { + duplicates.put(key_text, 0) catch unreachable; + } + + p.lexer.expect(.t_colon); + var value = p.parseExpr() orelse return null; + try properties.append(G.Property{ .key = key, .value = value }); + } + + is_single_line = if (p.lexer.has_newline_before) false else is_single_line; + p.lexer.expect(.t_close_brace); + return p.e(E.Object{ + .properties = properties.toOwnedSlice(), + .is_single_line = is_single_line, + }, loc); + }, + else => { + p.lexer.unexpected(); + return null; + }, + } + } + + pub fn parseMaybeTrailingComma(p: *Parser, closer: T) bool { + const comma_range = p.lexer.range(); + p.lexer.expect(.t_comma); + + if (p.lexer.token == closer) { + if (!opts.allow_trailing_commas) { + p.log.addRangeError(p.source, comma_range, "JSON does not support trailing commas") catch unreachable; + } + return false; + } + + return true; + } + }; +} + +const JSONParser = JSONLikeParser(js_lexer.JSONOptions{}); +const TSConfigParser = JSONLikeParser(js_lexer.JSONOptions{ .allow_comments = true, .allow_trailing_commas = true }); + +pub fn ParseJSON(log: logger.Log, source: logger.Source) !?Expr { + var parser = JSONParser.init(allocator, log, source); + + return try parser.parseExpr(); +} + +pub fn ParseTSConfig(log: logger.Loc, source: logger.Source) !?Expr { + var parser = TSConfigParser.init(allocator, log, source); + + return try parser.parseExpr(); +} diff --git a/src/logger.zig b/src/logger.zig index 47253728c..bf7bcb21c 100644 --- a/src/logger.zig +++ b/src/logger.zig @@ -173,11 +173,27 @@ pub const Log = struct { pub fn addRangeWarning(log: *Log, source: ?Source, r: Range, text: string) !void { log.warnings += 1; try log.addMsg(Msg{ - .kind = .warning, + .kind = .warn, .data = rangeData(source, r, text), }); } + pub fn addWarningFmt(log: *Log, source: ?Source, l: Loc, allocator: *std.mem.Allocator, comptime text: string, args: anytype) !void { + log.errors += 1; + try log.addMsg(Msg{ + .kind = .err, + .data = rangeData(source, Range{ .loc = l }, std.fmt.allocPrint(allocator, text, args) catch unreachable), + }); + } + + pub fn addRangeWarningFmt(log: *Log, source: ?Source, r: Range, allocator: *std.mem.Allocator, comptime text: string, args: anytype) !void { + log.errors += 1; + try log.addMsg(Msg{ + .kind = .err, + .data = rangeData(source, r, std.fmt.allocPrint(allocator, text, args) catch unreachable), + }); + } + pub fn addWarning(log: *Log, source: ?Source, l: Loc, text: string) !void { log.warnings += 1; try log.addMsg(Msg{ |