Start reimplementation in typescript

src/index.ts

@@ -0,0 +1,21 @@
+import {log} from './log.js'
+import {Lifecycle} from './util/lifecycle.js'
+import {Input} from './vm/input.js'
+import {Lexer} from "./vm/lexer.js";
+import {Parser} from "./vm/parser.js";
+import {commands} from "./vm/commands/index.js";
+
+const lifecycle = new Lifecycle()
+const input = new Input()
+input.adoptLifecycle(lifecycle)
+input.subscribe(line => log.verbose('input', { line }))
+
+const lexer = new Lexer(input)
+lexer.subscribe(token => log.verbose('token', token))
+
+const parser = new Parser(lexer, commands)
+parser.subscribe(exec => log.verbose('exec', exec))
+
+input.setupPrompt()
+
+process.on('SIGINT', () => lifecycle.close())

src/log.ts

@@ -0,0 +1,5 @@
+import {ConsoleLogger, Logger, LogLevel} from './util/log.js'
+
+export const log: Logger = new ConsoleLogger(LogLevel.VERBOSE)
+log.setStreamLevel('lexer', LogLevel.INFO)
+log.setStreamLevel('token', LogLevel.INFO)

src/util/lifecycle.ts

@@ -0,0 +1,19 @@
+import {Awaitable} from './types.js'
+
+export type LifecycleCallback = () => Awaitable<unknown>
+
+export type LifecycleAware = {
+	adoptLifecycle(lifecycle: Lifecycle): void;
+}
+
+export class Lifecycle {
+	private onCloses: LifecycleCallback[] = []
+
+	onClose(closure: LifecycleCallback): void {
+		this.onCloses.push(closure)
+	}
+
+	close() {
+		this.onCloses.map(x => x())
+	}
+}

src/util/log.ts

@@ -0,0 +1,122 @@
+import {Awaitable} from './types.js'
+
+export enum LogLevel {
+    VERBOSE = 0,
+    DEBUG = 1,
+    INFO = 2,
+    WARN = 3,
+    ERROR = 4,
+}
+
+export const logLevelDisplay: Record<LogLevel, string> = {
+    [LogLevel.VERBOSE]: 'verb',
+    [LogLevel.DEBUG]: 'debug',
+    [LogLevel.INFO]: 'info',
+    [LogLevel.WARN]: 'warn',
+    [LogLevel.ERROR]: 'error',
+}
+
+export type LogLevels = {
+    default: LogLevel,
+    streams: Record<string, LogLevel>,
+}
+
+export type LogMessage = {
+    level: LogLevel,
+    timestamp: Date,
+    stream: string,
+    data: unknown,
+}
+
+export type StreamLogger = {
+    verbose(data: unknown): void,
+    debug(data: unknown): void,
+    info(data: unknown): void,
+    warn(data: unknown): void,
+    error(data: unknown): void,
+}
+
+export abstract class Logger {
+    protected logLevels: LogLevels
+
+    constructor(
+        defaultLevel: LogLevel,
+    ) {
+        this.logLevels = {
+            default: defaultLevel,
+            streams: {},
+        }
+    }
+
+    setDefaultLevel(level: LogLevel) {
+        this.logLevels.default = level
+    }
+
+    setStreamLevel(stream: string, level: LogLevel) {
+        this.logLevels.streams[stream] = level
+    }
+
+    getStreamLogger(stream: string): StreamLogger {
+        return {
+            verbose: (data: unknown) => this.verbose(stream, data),
+            debug: (data: unknown) => this.debug(stream, data),
+            info: (data: unknown) => this.info(stream, data),
+            warn: (data: unknown) => this.warn(stream, data),
+            error: (data: unknown) => this.error(stream, data),
+        }
+    }
+
+    verbose(stream: string, data: unknown): Awaitable<void> {
+        return this.logAtLevel(LogLevel.VERBOSE, stream, data)
+    }
+
+    debug(stream: string, data: unknown): Awaitable<void> {
+        return this.logAtLevel(LogLevel.DEBUG, stream, data)
+    }
+
+    info(stream: string, data: unknown): Awaitable<void> {
+        return this.logAtLevel(LogLevel.INFO, stream, data)
+    }
+
+    warn(stream: string, data: unknown): Awaitable<void> {
+        return this.logAtLevel(LogLevel.WARN, stream, data)
+    }
+
+    error(stream: string, data: unknown): Awaitable<void> {
+        return this.logAtLevel(LogLevel.ERROR, stream, data)
+    }
+
+    logAtLevel(level: LogLevel, stream: string, data: unknown): Awaitable<void> {
+        return this.log({
+            timestamp: new Date,
+            level,
+            stream,
+            data,
+        })
+    }
+
+    shouldLog(message: LogMessage): boolean {
+        return message.level >= this.getLevelForStream(message.stream)
+    }
+
+    log(message: LogMessage): Awaitable<void> {
+        if ( this.shouldLog(message) ) {
+            return this.write(message)
+        }
+    }
+
+    getLevelForStream(stream: string): LogLevel {
+        if ( stream in this.logLevels.streams ) {
+            return this.logLevels.streams[stream]
+        }
+        return this.logLevels.default
+    }
+
+    protected abstract write(message: LogMessage): Awaitable<void>
+}
+
+export class ConsoleLogger extends Logger {
+    protected write(message: LogMessage): Awaitable<void> {
+        console.log(`[${message.stream}] [${logLevelDisplay[message.level]}] [${message.timestamp.toISOString()}]`, message.data)
+    }
+}

src/util/subject.ts

@@ -0,0 +1,225 @@
+/**
+ * Base error used to trigger an unsubscribe action from a subscriber.
+ * @extends Error
+ */
+export class UnsubscribeError extends Error {}
+
+/**
+ * Thrown when a closed observable is pushed to.
+ * @extends Error
+ */
+export class CompletedObservableError extends Error {
+    constructor() {
+        super('This observable can no longer be pushed to, as it has been completed.')
+    }
+}
+
+/**
+ * Type of a basic subscriber function.
+ */
+export type SubscriberFunction<T> = (val: T) => any
+
+/**
+ * Type of a basic subscriber function that handles errors.
+ */
+export type SubscriberErrorFunction = (error: Error) => any
+
+/**
+ * Type of a basic subscriber function that handles completed events.
+ */
+export type SubscriberCompleteFunction<T> = (val?: T) => any
+
+/**
+ * Subscribers that define multiple handler methods.
+ */
+export type ComplexSubscriber<T> = {
+    next?: SubscriberFunction<T>,
+    error?: SubscriberErrorFunction,
+    complete?: SubscriberCompleteFunction<T>,
+}
+
+/**
+ * Subscription to a behavior subject.
+ */
+export type Subscription<T> = SubscriberFunction<T> | ComplexSubscriber<T>
+
+/**
+ * Object providing helpers for unsubscribing from a subscription.
+ */
+export type Unsubscribe = { unsubscribe: () => void }
+
+/**
+ * A stream-based state class.
+ */
+export class BehaviorSubject<T> {
+    /**
+     * Subscribers to this subject.
+     * @type Array<ComplexSubscriber>
+     */
+    protected subscribers: ComplexSubscriber<T>[] = []
+
+    /**
+     * True if this subject has been marked complete.
+     * @type boolean
+     */
+    protected subjectIsComplete = false
+
+    /**
+     * The current value of this subject.
+     */
+    protected currentValue?: T
+
+    /**
+     * True if any value has been pushed to this subject.
+     * @type boolean
+     */
+    protected hasPush = false
+
+    /**
+     * Register a new subscription to this subject.
+     * @param {Subscription} subscriber
+     * @return Unsubscribe
+     */
+    public subscribe(subscriber: Subscription<T>): Unsubscribe {
+        if ( typeof subscriber === 'function' ) {
+            this.subscribers.push({ next: subscriber })
+        } else {
+            this.subscribers.push(subscriber)
+        }
+
+        return {
+            unsubscribe: () => {
+                this.subscribers = this.subscribers.filter(x => x !== subscriber)
+            },
+        }
+    }
+
+    public pipe<T2>(mapper: (val: T) => T2|Promise<T2>): BehaviorSubject<T2> {
+        const sub = new BehaviorSubject<T2>()
+        this.subscribe(async val => sub.next(await mapper(val)))
+        return sub
+    }
+
+    public pipeFlat<T2>(mapper: (val: T) => T2[]|Promise<T2[]>): BehaviorSubject<T2> {
+        const sub = new BehaviorSubject<T2>()
+        this.subscribe(async val => {
+            const vals = await mapper(val)
+            return Promise.all(vals.map(val => sub.next(val)))
+        })
+        return sub
+    }
+
+    /**
+     * Cast this subject to a promise, which resolves on the output of the next value.
+     * @return Promise
+     */
+    public toPromise(): Promise<T> {
+        return new Promise((resolve, reject) => {
+            const { unsubscribe } = this.subscribe({
+                next: (val: T) => {
+                    resolve(val)
+                    unsubscribe()
+                },
+                error: (error: Error) => {
+                    reject(error)
+                    unsubscribe()
+                },
+                complete: (val?: T) => {
+                    if ( typeof val !== 'undefined' ) {
+                        resolve(val)
+                    }
+                    unsubscribe()
+                },
+            })
+        })
+    }
+
+    /**
+     * Push a new value to this subject. The promise resolves when all subscribers have been pushed to.
+     * @param val
+     * @return Promise<void>
+     */
+    public async next(val: T): Promise<void> {
+        if ( this.subjectIsComplete ) {
+            throw new CompletedObservableError()
+        }
+        this.currentValue = val
+        this.hasPush = true
+        for ( const subscriber of this.subscribers ) {
+            if ( subscriber.next ) {
+                try {
+                    await subscriber.next(val)
+                } catch (e) {
+                    if ( e instanceof UnsubscribeError ) {
+                        this.subscribers = this.subscribers.filter(x => x !== subscriber)
+                    } else if (subscriber.error && e instanceof Error) {
+                        await subscriber.error(e)
+                    } else {
+                        throw e
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Push the given array of values to this subject in order.
+     * The promise resolves when all subscribers have been pushed to for all values.
+     * @param {Array} vals
+     * @return Promise<void>
+     */
+    public async push(vals: T[]): Promise<void> {
+        if ( this.subjectIsComplete ) {
+            throw new CompletedObservableError()
+        }
+        await Promise.all(vals.map(val => this.next(val)))
+    }
+
+    /**
+     * Mark this subject as complete.
+     * The promise resolves when all subscribers have been pushed to.
+     * @param [finalValue] - optionally, a final value to set
+     * @return Promise<void>
+     */
+    public async complete(finalValue?: T): Promise<void> {
+        if ( this.subjectIsComplete ) {
+            throw new CompletedObservableError()
+        }
+        if ( typeof finalValue === 'undefined' ) {
+            finalValue = this.value()
+        } else {
+            this.currentValue = finalValue
+        }
+
+        for ( const subscriber of this.subscribers ) {
+            if ( subscriber.complete ) {
+                try {
+                    await subscriber.complete(finalValue)
+                } catch (e) {
+                    if ( subscriber.error && e instanceof Error ) {
+                        await subscriber.error(e)
+                    } else {
+                        throw e
+                    }
+                }
+            }
+        }
+
+        this.subjectIsComplete = true
+    }
+
+    /**
+     * Get the current value of this subject.
+     */
+    public value(): T | undefined {
+        return this.currentValue
+    }
+
+    /**
+     * True if this subject is marked as complete.
+     * @return boolean
+     */
+    public isComplete(): boolean {
+        return this.subjectIsComplete
+    }
+}

src/util/types.ts

@@ -0,0 +1,9 @@
+export type Awaitable<T> = T | Promise<T>
+
+export type JSONScalar = string | boolean | number | undefined
+export type JSONData = JSONScalar | Array<JSONScalar | JSONData> | { [key: string]: JSONScalar | JSONData }
+
+/** A typescript-compatible version of Object.hasOwnProperty. */
+export function hasOwnProperty<X extends {}, Y extends PropertyKey>(obj: X, prop: Y): obj is X & Record<Y, unknown> {  // eslint-disable-line @typescript-eslint/ban-types
+    return Object.hasOwnProperty.call(obj, prop)
+}

src/vm/commands/command.ts

@@ -0,0 +1,73 @@
+import {LexInput} from '../lexer.js'
+import {ExpectedEndOfInputError, InvalidVariableNameError, UnexpectedEndOfInputError} from "../parse.js";
+
+export type StrLVal = { term: 'variable', name: string }
+
+export type StrTerm =
+    { term: 'string', value: string }
+    | StrLVal
+
+export class ParseContext {
+    constructor(
+        private inputs: LexInput[],
+    ) {}
+
+    assertEmpty() {
+        if ( this.inputs.length ) {
+            throw new ExpectedEndOfInputError(`Expected end of input. Found: ${this.inputs[0].value}`)
+        }
+    }
+
+    popOptionalTerm(): StrTerm|undefined {
+        if ( this.inputs.length ) return this.popTerm()
+        return undefined
+    }
+
+    popTerm(): StrTerm {
+        if ( !this.inputs.length ) {
+            throw new UnexpectedEndOfInputError('Unexpected end of input. Expected term.');
+        }
+
+        const input = this.inputs.shift()!
+
+        // Check if the token is a literal variable name:
+        if ( !input.literal && input.value.startsWith('$') ) {
+            if ( !input.value.match(/^\$[a-zA-Z0-9_]+$/) ) {
+                throw new InvalidVariableNameError(`Invalid variable name: ${input.value}`)
+            }
+
+            return { term: 'variable', name: input.value }
+        }
+
+        // Otherwise, parse it as a string literal:
+        return { term: 'string', value: input.value }
+    }
+
+    popLVal(): StrLVal {
+        if ( !this.inputs.length ) {
+            throw new UnexpectedEndOfInputError('Unexpected end of input. Expected lval.');
+        }
+
+        const input = this.inputs.shift()!
+
+        if ( input.literal || !input.value.match(/^\$[a-zA-Z0-9_]+$/) ) {
+            throw new InvalidVariableNameError(`Expected variable name. Found: ${input.value}`)
+        }
+
+        return { term: 'variable', name: input.value }
+    }
+}
+
+export type CommandData = Record<string, unknown>
+
+export abstract class Command<TData extends CommandData> {
+    abstract isParseCandidate(token: LexInput): boolean
+
+    abstract attemptParse(context: ParseContext): TData
+
+    abstract getDisplayName(): string
+
+    protected isKeyword(token: LexInput, keyword: string): boolean {
+        return !token.literal && token.value === keyword
+    }
+}

src/vm/commands/copy.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class Copy extends Command<{}> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'copy')
+    }
+
+    attemptParse(context: ParseContext): {} {
+        return {}
+    }
+
+    getDisplayName(): string {
+        return 'copy'
+    }
+}

src/vm/commands/edit.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class Edit extends Command<{}> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'edit')
+    }
+
+    attemptParse(context: ParseContext): {} {
+        return {}
+    }
+
+    getDisplayName(): string {
+        return 'edit'
+    }
+}

src/vm/commands/exit.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class Exit extends Command<{}> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'exit')
+    }
+
+    attemptParse(context: ParseContext): {} {
+        return {}
+    }
+
+    getDisplayName(): string {
+        return 'exit'
+    }
+}

src/vm/commands/from.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext, StrLVal} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class From extends Command<{ var: StrLVal }> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'from')
+    }
+
+    attemptParse(context: ParseContext): { var: StrLVal } {
+        return { var: context.popLVal() }
+    }
+
+    getDisplayName(): string {
+        return 'from'
+    }
+}

src/vm/commands/history.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class History extends Command<{}> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'history')
+    }
+
+    attemptParse(context: ParseContext): {} {
+        return {}
+    }
+
+    getDisplayName(): string {
+        return 'history'
+    }
+}

src/vm/commands/index.ts

@@ -0,0 +1,29 @@
+import {Command, CommandData} from './command.js'
+import {Exit} from "./exit.js";
+import {InFile} from "./infile.js";
+import {Copy} from "./copy.js";
+import {Edit} from "./edit.js";
+import {From} from "./from.js";
+import {History} from "./history.js";
+import {Load} from "./load.js";
+import {OutFile} from "./outfile.js";
+import {Paste} from "./paste.js";
+import {RunFile} from "./runfile.js";
+import {Save} from "./save.js";
+import {To} from "./to.js";
+
+export type Commands = Command<CommandData>[]
+export const commands: Commands = [
+    new Copy,
+    new Edit,
+    new Exit,
+    new From,
+    new History,
+    new InFile,
+    new Load,
+    new OutFile,
+    new Paste,
+    new RunFile,
+    new Save,
+    new To,
+]

src/vm/commands/infile.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext, StrTerm} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class InFile extends Command<{ path: StrTerm }> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'infile')
+    }
+
+    attemptParse(context: ParseContext): { path: StrTerm } {
+        return { path: context.popTerm() }
+    }
+
+    getDisplayName(): string {
+        return 'infile'
+    }
+}

src/vm/commands/load.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext, StrTerm} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class Load extends Command<{ path?: StrTerm }> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'load')
+    }
+
+    attemptParse(context: ParseContext): { path?: StrTerm } {
+        return { path: context.popOptionalTerm() }
+    }
+
+    getDisplayName(): string {
+        return 'load'
+    }
+}

src/vm/commands/outfile.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext, StrTerm} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class OutFile extends Command<{ path: StrTerm }> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'outfile')
+    }
+
+    attemptParse(context: ParseContext): { path: StrTerm } {
+        return { path: context.popTerm() }
+    }
+
+    getDisplayName(): string {
+        return 'outfile'
+    }
+}

src/vm/commands/paste.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class Paste extends Command<{}> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'paste')
+    }
+
+    attemptParse(context: ParseContext): {} {
+        return {}
+    }
+
+    getDisplayName(): string {
+        return 'paste'
+    }
+}

src/vm/commands/runfile.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext, StrTerm} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class RunFile extends Command<{ path: StrTerm }> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'runfile')
+    }
+
+    attemptParse(context: ParseContext): { path: StrTerm } {
+        return { path: context.popTerm() }
+    }
+
+    getDisplayName(): string {
+        return 'runfile'
+    }
+}

src/vm/commands/save.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext, StrTerm} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class Save extends Command<{ path?: StrTerm }> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'save')
+    }
+
+    attemptParse(context: ParseContext): { path?: StrTerm } {
+        return { path: context.popOptionalTerm() }
+    }
+
+    getDisplayName(): string {
+        return 'save'
+    }
+}

src/vm/commands/to.ts

@@ -0,0 +1,16 @@
+import {Command, ParseContext, StrLVal} from "./command.js";
+import {LexInput} from "../lexer.js";
+
+export class To extends Command<{ var: StrLVal }> {
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'to')
+    }
+
+    attemptParse(context: ParseContext): { var: StrLVal } {
+        return { var: context.popLVal() }
+    }
+
+    getDisplayName(): string {
+        return 'to'
+    }
+}

src/vm/index.ts

@@ -0,0 +1,7 @@
+import {Input} from './input.js'
+
+export class StrVM {
+	constructor(
+		private input: Input,
+	) {}
+}

src/vm/input.ts

@@ -0,0 +1,35 @@
+import * as readline from 'node:readline'
+import {BehaviorSubject} from "../util/subject.js";
+import {Lifecycle, LifecycleAware} from "../util/lifecycle.js";
+
+export class Input extends BehaviorSubject<string> implements LifecycleAware {
+	private rl?: readline.Interface
+
+	public setupPrompt(): void {
+		if ( this.rl ) {
+			this.closePrompt()
+		}
+
+		this.rl = readline.createInterface({
+			input: process.stdin,
+			output: process.stdout,
+			prompt: 'str %> ',
+		})
+
+		this.rl.prompt()
+
+		this.rl.on('line', async (line) => {
+			await this.next(line + '\n')
+			this.rl?.prompt(true)
+		})
+	}
+
+	public closePrompt(): void {
+		this.rl?.close()
+		this.rl = undefined
+	}
+
+	adoptLifecycle(lifecycle: Lifecycle): void {
+		lifecycle.onClose(() => this.closePrompt())
+	}
+}

src/vm/lexer.ts

@@ -0,0 +1,95 @@
+import {BehaviorSubject} from '../util/subject.js'
+import {Input} from './input.js'
+import {log} from '../log.js'
+import {StreamLogger} from '../util/log.js'
+
+export type LexTerminator = { type: 'terminator' }
+export type LexInput = { type: 'input', value: string, literal?: true }
+
+export type LexToken = LexTerminator | LexInput
+
+const logger = log.getStreamLogger('lexer')
+
+export class Lexer extends BehaviorSubject<LexToken> {
+    private isEscape: boolean = false
+    private inQuote?: '"'|"'"
+    private tokenAccumulator: string = ''
+
+    private logger: StreamLogger
+
+    constructor(input: Input) {
+        super()
+        this.logger = log.getStreamLogger('lexer')
+        input.subscribe(input => this.lexInput(input))
+    }
+
+    private logState(c: string): void {
+        this.logger.verbose({
+            c,
+            isEscape: this.isEscape,
+            inQuote: this.inQuote,
+            tokenAccumulator: this.tokenAccumulator,
+        })
+    }
+
+    private async emitToken(reason: string, literal?: true): Promise<void> {
+        logger.verbose({ emitToken: reason })
+        await this.next({ type: 'input', value: this.tokenAccumulator, literal })
+        this.tokenAccumulator = ''
+    }
+
+    private async lexInput(input: string): Promise<void> {
+        logger.debug({ input })
+
+        let inputChars = input.split('')
+
+        while ( inputChars.length ) {
+            const c = inputChars.shift()!
+            this.logState(c)
+
+            // We got the 2nd character after an escape
+            if ( this.isEscape ) {
+                this.tokenAccumulator += c
+                this.isEscape = false
+                continue
+            }
+
+            // We are about to get an escape character
+            if ( c === '\\' ) {
+                this.isEscape = true
+                continue
+            }
+
+            // We got a statement terminator
+            if ( (c === ';' || c === '\n') && !this.inQuote ) {
+                if ( this.tokenAccumulator ) {
+                    await this.emitToken('terminator')
+                }
+                await this.next({ type: 'terminator' })
+                continue
+            }
+
+            // Whitespace separates tokens
+            if ( (c === ' ' || c === '\t' || c === '\r') && !this.inQuote ) {
+                if ( this.tokenAccumulator ) {
+                    await this.emitToken('whitespace')
+                }
+                continue
+            }
+
+            // We are either starting or ending an unescaped matching quote
+            if ( c === `'` || c === `"` ) {
+                if ( c === this.inQuote ) {
+                    this.inQuote = undefined
+                    await this.emitToken('quote', true)
+                    continue
+                } else if ( !this.inQuote ) {
+                    this.inQuote = c
+                    continue
+                }
+            }
+
+            this.tokenAccumulator += c
+        }
+    }
+}

src/vm/parse.ts

@@ -0,0 +1,14 @@
+import {Command, CommandData} from './commands/command.js'
+
+export type Executable<TData extends CommandData> = {
+    command: Command<TData>,
+    data: TData,
+}
+
+export class ParseError extends Error {}
+export class InternalParseError extends ParseError {}
+export class IsNotKeywordError extends ParseError {}
+export class InvalidCommandError extends ParseError {}
+export class UnexpectedEndOfInputError extends ParseError {}
+export class ExpectedEndOfInputError extends InvalidCommandError {}
+export class InvalidVariableNameError extends ParseError {}

src/vm/parser.ts

@@ -0,0 +1,94 @@
+import {BehaviorSubject} from '../util/subject.js'
+import {Lexer, LexInput, LexToken} from './lexer.js'
+import {StreamLogger} from '../util/log.js'
+import {log} from '../log.js'
+import {Commands} from './commands/index.js'
+import {Command, CommandData, ParseContext} from './commands/command.js'
+import {Executable, InternalParseError, InvalidCommandError, IsNotKeywordError} from './parse.js'
+
+export class Parser extends BehaviorSubject<Executable<CommandData>> {
+    private logger: StreamLogger
+
+    private parseCandidate?: Command<CommandData>
+    private inputForCandidate: LexInput[] = []
+
+    constructor(lexer: Lexer, private commands: Commands) {
+        super()
+        this.logger = log.getStreamLogger('parser')
+        lexer.subscribe(token => this.handleToken(token))
+    }
+
+    async handleToken(token: LexToken) {
+        // We are in between full commands, so try to identify a new parse candidate:
+        if ( !this.parseCandidate ) {
+            // Ignore duplicated terminators between commands
+            if ( token.type === 'terminator' ) {
+                return
+            }
+
+            this.logger.verbose({ identifyParseCandidate: token })
+            if ( !this.isKeyword(token) ) {
+                throw new IsNotKeywordError('Expected keyword, found: ' + this.displayToken(token))
+            }
+
+            this.parseCandidate = this.getParseCandidate(token)
+            return
+        }
+
+        // We have already identified a parse candidate:
+        // If this is normal input token, collect it so we can give it to the candidate to parse:
+        if ( token.type === 'input' ) {
+            this.inputForCandidate.push(token)
+            return
+        }
+
+        // If we got a terminator, then ask the candidate to actually perform its parse:
+        if ( token.type === 'terminator' ) {
+            try {
+                // Have the candidate attempt to parse itself from the collecte data:
+                const context = new ParseContext(this.inputForCandidate)
+                this.logger.verbose({ parsing: this.parseCandidate.getDisplayName(), context })
+                const data = this.parseCandidate.attemptParse(context)
+
+                // The candidate must consume every token in the context:
+                context.assertEmpty()
+
+                // Emit the parsed command:
+                this.logger.debug({ parsed: this.parseCandidate.getDisplayName() })
+                await this.next({
+                    command: this.parseCandidate,
+                    data,
+                })
+                return
+            } finally {
+                this.parseCandidate = undefined
+                this.inputForCandidate = []
+            }
+        }
+
+        throw new InternalParseError('Encountered invalid token.')
+    }
+
+    private isKeyword(token: LexToken): token is (LexInput & {literal: undefined}) {
+        return token.type === 'input' && !token.literal
+    }
+
+    private getParseCandidate(token: LexInput): Command<CommandData> {
+        for ( const command of this.commands ) {
+            if ( command.isParseCandidate(token) ) {
+                this.logger.debug({ foundParseCandidate: command.getDisplayName(), token })
+                return command
+            }
+        }
+
+        throw new InvalidCommandError('Could not find parser for: ' + this.displayToken(token))
+    }
+
+    private displayToken(token: LexToken) {
+        if ( token.type === 'terminator' ) {
+            return '(TERMINATOR)'
+        }
+
+        return `(${token.literal ? 'LITERAL' : 'INPUT'}) ${token.value}`
+    }
+}

src/vm/string.ts

@@ -0,0 +1,51 @@
+export type Word = { type: 'word', value: string }
+export type Whitespace = { type: 'space', value: string }
+
+export type Component = Word | Whitespace
+
+export const isWord = (cmp: Component): cmp is Word =>
+    cmp.type === 'word'
+
+export const isWhitespace = (cmp: Component): cmp is Whitespace =>
+    cmp.type === 'space'
+
+export type Line = {
+    components: Component[],
+}
+
+export type SString = {
+    lines: Line[],
+}
+
+export const toNativeString = (value: SString): string =>
+    value.lines
+        .map(line =>
+            line.components
+                .map(cmp => cmp.value)
+                .join(''))
+        .join('\n')
+
+export const fromNativeString = (value: string): SString => ({
+    lines: value.split('\n')
+        .map(rawLine => {
+            const whitespace = [...rawLine.matchAll(/\s+/g)]
+            const words = rawLine.split(/\s+/g)
+            const line: Line = { components: [] }
+
+            for ( let i = 0; i < words.length; i += 1 ) {
+                line.components.push({
+                    type: 'word',
+                    value: words[i],
+                })
+
+                if ( i < whitespace.length ) {
+                    line.components.push({
+                        type: 'space',
+                        value: whitespace[i][0],
+                    })
+                }
+            }
+
+            return line
+        }),
+})