Add flatten command, accessing destructured elements by index, WIP support for regular expressions

src/vm/commands/command.ts

@@ -2,7 +2,7 @@ import {createHash} from 'node:crypto';
 import {LexInput, LexToken, tokenIsLVal} from '../lexer.js'
 import {
     Executable,
-    ExpectedEndOfInputError, InvalidSubcontextError,
+    ExpectedEndOfInputError, InvalidRegularExpressionLiteralError, InvalidSubcontextError,
     InvalidVariableNameError,
     IsNotKeywordError,
     UnexpectedEndOfInputError, UnexpectedEndofStatementError
@@ -10,10 +10,11 @@ import {
 import {Awaitable, ElementType, hasOwnProperty} from "../../util/types.js";
 import {StrVM} from "../vm.js";
 import os from "node:os";
+import {log} from "../../log.js";
 
 export class TypeError extends Error {}
 
-export type StrLVal = { term: 'variable', name: string }
+export type StrLVal = { term: 'variable', name: string, index?: number }
 
 export const isStrLVal = (val: unknown): val is StrLVal =>
     !!(typeof val === 'object'
@@ -41,6 +42,8 @@ export type StrString = { term: 'string', value: string, literal?: true }
 
 export type StrInt = { term: 'int', value: number }
 
+export type StrRex = { term: 'rex', value: string, flags: string }
+
 export type StrLamba = {
     term: 'lambda',
     value: {
@@ -49,7 +52,7 @@ export type StrLamba = {
     },
 }
 
-export type StrRVal = StrString | StrInt | StrDestructured | StrLamba
+export type StrRVal = StrString | StrInt | StrDestructured | StrLamba | StrRex
 
 export type StrDestructuredTable = {
     term: 'destructured',
@@ -91,6 +94,10 @@ export const hashStrRVal = (val: StrRVal): string => {
         return toHex(`s:int:${val.value}`)
     }
 
+    if ( val.term === 'rex' ) {
+        return toHex(`s:rex:${val.value}/${val.flags}`)
+    }
+
     if ( val.term === 'lambda' ) {
         throw new Error('Cannot hash lambda')  // todo
     }
@@ -110,20 +117,28 @@ export const isStrTerm = (val: unknown): val is StrTerm =>
                 && hasOwnProperty(val, 'value')))
 
 export const isStrRVal = (term: StrTerm): term is StrRVal =>
-    term.term === 'string' || term.term === 'int' || term.term === 'destructured' || term.term === 'lambda'
+    ['string', 'int', 'destructured', 'lambda', 'rex'].includes(term.term)
 
 export const unwrapString = (term: StrRVal): string => {
     if ( term.term === 'int' ) {
         return String(term.value)
     }
 
-    if ( term.term === 'destructured' || term.term === 'lambda' ) {
+    if ( term.term === 'destructured' || term.term === 'lambda' || term.term === 'rex' ) {
         throw new TypeError(`Found unexpected ${term.term} (expected: string|int)`)
     }
 
     return term.value
 }
 
+export const unwrapRex = (term: StrRVal): RegExp => {
+    if ( term.term !== 'rex' ) {
+        throw new TypeError(`Found unexpected ${term.term} (expected: rex)`)
+    }
+
+    return new RegExp(term.value, term.flags)
+}
+
 export const coerceString = (term: StrRVal): string => {
     if ( term.term === 'destructured' ) {
         return joinDestructured(term.value)
@@ -176,6 +191,8 @@ export interface ParseSubContext {
 }
 
 export class ParseContext {
+    private log = log.getStreamLogger('parseContext')
+
     constructor(
         private inputs: LexToken[],
         private childParser: (tokens: LexToken[]) => Awaitable<[Executable<CommandData>, LexToken[]]>,
@@ -246,11 +263,38 @@ export class ParseContext {
     private parseInputToTerm(input: LexInput): StrTerm {
         // Check if the token is a literal variable name:
         if ( !input.literal && input.value.startsWith('$') ) {
-            if ( !input.value.match(/^\$[a-zA-Z0-9_]+$/) ) {
+            if ( !input.value.match(/^\$[a-zA-Z0-9_]+(?:\.\d+)?$/) ) {
                 throw new InvalidVariableNameError(`Invalid variable name: ${input.value}`)
             }
 
-            return { term: 'variable', name: input.value }
+            const parts = input.value.split('.')
+            let index: number|undefined = undefined
+            if ( parts.length > 1 ) {
+                index = parseInt(parts[1], 10)
+            }
+
+            return { term: 'variable', name: parts[0], index }
+        }
+
+        // Check if the token is a literal regular expression:
+        if ( !input.literal && input.value.startsWith('/') ) {
+            const parts = input.value
+                .substring(1)  // trim the leading /
+                .split('')
+                .reverse()
+
+            let flags = ''
+            while ( parts.length && parts[0] !== '/' ) {
+                flags += parts.shift()
+            }
+
+            if ( !parts.length ) {
+                throw new InvalidRegularExpressionLiteralError(`Invalid regular expression literal: ${input.value}`)
+            }
+
+            parts.shift()  // trim the trailing /
+            const pattern = parts.reverse().join('')
+            return { term: 'rex', value: pattern, flags }
         }
 
         // Check if the token is a valid integer:
@@ -379,6 +423,8 @@ export class ParseContext {
                 }))
         }
 
+        this.log.debug({ lambdaBody: sc.inputs })
+
         // Now, the remainder of the subcontext inputs should be a series of executables
         // separated by `terminator` tokens -- e.g. (split _; join |), so parse executables
         // from the subcontext until it is empty:
@@ -457,7 +503,10 @@ export class ParseContext {
                 value: last.value.substring(0, last.value.length - 1),
             }
         }
-        sc.inputs.push(last)
+        if ( last.type !== 'input' || last.value || last.literal ) {
+            // Avoid pushing an empty input if the last input contained ONLY the empty right-paren
+            sc.inputs.push(last)
+        }
 
         return [sc, tokenIdx]
     }

src/vm/commands/contains.ts

@@ -20,9 +20,16 @@ export class Contains extends Command<{ find: StrTerm }> {
 
     execute(vm: StrVM, data: { find: StrTerm }): Awaitable<StrVM> {
         return vm.replaceContextMatchingTerm(ctx => ({
-            string: sub => sub.includes(ctx.resolveString(data.find)) ? sub : '',
-            destructuredOfStrings: parts => parts.filter(part =>
-                part.includes(ctx.resolveString(data.find))),
+            string: sub =>
+                ctx.applyStringOrRex(data.find, {
+                    string: find => sub.includes(find) ? sub : '',
+                    rex: find => find.test(sub) ? sub : '',
+                }),
+            destructuredOfStrings: parts =>
+                ctx.applyStringOrRex(data.find, {
+                    string: find => parts.filter(part => part.includes(find)),
+                    rex: find => parts.filter(part => find.test(part)),
+                }),
         }))
     }
 }

src/vm/commands/flatten.ts

@@ -0,0 +1,60 @@
+import {Command, ParseContext, StrDestructured, StrTerm} from "./command.js";
+import {LexInput} from "../lexer.js";
+import {StrVM} from "../vm.js";
+
+export type FlattenData = {
+    index?: StrTerm,
+}
+
+/**
+ * [[a,b],c,[d,e]] -> flatten 0 -> [a,b,c,[d,e]]
+ */
+export class Flatten extends Command<FlattenData> {
+    async attemptParse(context: ParseContext): Promise<FlattenData> {
+        return {
+            index: await context.popOptionalTerm(),
+        }
+    }
+
+    getDisplayName(): string {
+        return 'flatten'
+    }
+
+    isParseCandidate(token: LexInput): boolean {
+        return this.isKeyword(token, 'flatten')
+    }
+
+    async execute(vm: StrVM, data: FlattenData): Promise<StrVM> {
+        return vm.replaceContextMatchingTerm(ctx => ({
+            destructured: async (parts: StrDestructured['value']) => {
+                let index: number|undefined
+                if ( data.index ) {
+                    index = ctx.resolveInt(data.index)
+                }
+
+                const newParts: StrDestructured['value'] = []
+                for ( let i = 0; i < parts.length; i += 1 ) {
+                    const part = parts[i]!
+
+                    if ( typeof index !== 'undefined' && index !== i ) {
+                        // We're targeting a specific index, and we're not it -- so preserve the value.
+                        newParts.push(part)
+                        continue
+                    }
+
+                    if ( part.value.term !== 'destructured' ) {
+                        // This item is not a nested destructured, so nothing to flatten.
+                        newParts.push(part)
+                        continue
+                    }
+
+                    for ( const child of part.value.value ) {
+                        newParts.push(child)
+                    }
+                }
+
+                return newParts
+            },
+        }))
+    }
+}

src/vm/commands/index.ts

@@ -53,6 +53,7 @@ import {Chunk} from "./chunk.js";
 import {Script} from "./script.js";
 import {Take} from "./take.js";
 import {Group} from "./group.js";
+import {Flatten} from "./flatten.js";
 
 export type Commands = Command<CommandData>[]
 export const commands: Commands = [
@@ -68,6 +69,7 @@ export const commands: Commands = [
     new Edit,
     new Enclose,
     new Exit,
+    new Flatten,
     new From,
     new Group,
     new Help,

src/vm/commands/missing.ts

@@ -20,9 +20,16 @@ export class Missing extends Command<{ find: StrTerm }> {
 
     execute(vm: StrVM, data: { find: StrTerm }): Awaitable<StrVM> {
         return vm.replaceContextMatchingTerm(ctx => ({
-            string: sub => sub.includes(ctx.resolveString(data.find)) ? '' : sub,
-            destructuredOfStrings: parts => parts.filter(part =>
-                !part.includes(ctx.resolveString(data.find))),
+            string: sub =>
+                ctx.applyStringOrRex(data.find, {
+                    string: find => !sub.includes(find) ? sub : '',
+                    rex: find => !find.test(sub) ? sub : '',
+                }),
+            destructuredOfStrings: parts =>
+                ctx.applyStringOrRex(data.find, {
+                    string: find => parts.filter(part => !part.includes(find)),
+                    rex: find => parts.filter(part => !find.test(part)),
+                }),
         }))
     }
 }