Start reimplementation in typescript

src/vm/lexer.ts

@@ -0,0 +1,95 @@
+import {BehaviorSubject} from '../util/subject.js'
+import {Input} from './input.js'
+import {log} from '../log.js'
+import {StreamLogger} from '../util/log.js'
+
+export type LexTerminator = { type: 'terminator' }
+export type LexInput = { type: 'input', value: string, literal?: true }
+
+export type LexToken = LexTerminator | LexInput
+
+const logger = log.getStreamLogger('lexer')
+
+export class Lexer extends BehaviorSubject<LexToken> {
+    private isEscape: boolean = false
+    private inQuote?: '"'|"'"
+    private tokenAccumulator: string = ''
+
+    private logger: StreamLogger
+
+    constructor(input: Input) {
+        super()
+        this.logger = log.getStreamLogger('lexer')
+        input.subscribe(input => this.lexInput(input))
+    }
+
+    private logState(c: string): void {
+        this.logger.verbose({
+            c,
+            isEscape: this.isEscape,
+            inQuote: this.inQuote,
+            tokenAccumulator: this.tokenAccumulator,
+        })
+    }
+
+    private async emitToken(reason: string, literal?: true): Promise<void> {
+        logger.verbose({ emitToken: reason })
+        await this.next({ type: 'input', value: this.tokenAccumulator, literal })
+        this.tokenAccumulator = ''
+    }
+
+    private async lexInput(input: string): Promise<void> {
+        logger.debug({ input })
+
+        let inputChars = input.split('')
+
+        while ( inputChars.length ) {
+            const c = inputChars.shift()!
+            this.logState(c)
+
+            // We got the 2nd character after an escape
+            if ( this.isEscape ) {
+                this.tokenAccumulator += c
+                this.isEscape = false
+                continue
+            }
+
+            // We are about to get an escape character
+            if ( c === '\\' ) {
+                this.isEscape = true
+                continue
+            }
+
+            // We got a statement terminator
+            if ( (c === ';' || c === '\n') && !this.inQuote ) {
+                if ( this.tokenAccumulator ) {
+                    await this.emitToken('terminator')
+                }
+                await this.next({ type: 'terminator' })
+                continue
+            }
+
+            // Whitespace separates tokens
+            if ( (c === ' ' || c === '\t' || c === '\r') && !this.inQuote ) {
+                if ( this.tokenAccumulator ) {
+                    await this.emitToken('whitespace')
+                }
+                continue
+            }
+
+            // We are either starting or ending an unescaped matching quote
+            if ( c === `'` || c === `"` ) {
+                if ( c === this.inQuote ) {
+                    this.inQuote = undefined
+                    await this.emitToken('quote', true)
+                    continue
+                } else if ( !this.inQuote ) {
+                    this.inQuote = c
+                    continue
+                }
+            }
+
+            this.tokenAccumulator += c
+        }
+    }
+}