Add mechanism for NPM package auto-discovery

package.json

@@ -67,5 +67,11 @@
     "@typescript-eslint/eslint-plugin": "^4.26.0",
     "@typescript-eslint/parser": "^4.26.0",
     "eslint": "^7.27.0"
+  },
+  "extollo": {
+    "discover": true,
+    "units": {
+      "discover": false
+    }
   }
 }

src/index.ts

@@ -71,6 +71,7 @@ export * from './service/Middlewares'
 
 export * from './support/cache/MemoryCache'
 export * from './support/cache/CacheFactory'
+export * from './support/NodeModules'
 
 export * from './views/ViewEngine'
 export * from './views/ViewEngineFactory'

src/lifecycle/Application.ts

@@ -48,6 +48,12 @@ export function appPath(...parts: PathLike[]): UniversalPath {
  * The main application container.
  */
 export class Application extends Container {
+    public static readonly NODE_MODULES_INJECTION = 'extollo/npm'
+
+    public static get NODE_MODULES_PROVIDER(): string {
+        return process.env.EXTOLLO_NPM || 'pnpm'
+    }
+
     public static getContainer(): Container {
         const existing = <Container | undefined> globalRegistry.getGlobal('extollo/injector')
         if ( !existing ) {
@@ -205,6 +211,7 @@ export class Application extends Container {
         this.setupLogging()
 
         this.registerFactory(new CacheFactory())  // FIXME move this somewhere else?
+        this.registerSingleton(Application.NODE_MODULES_INJECTION, Application.NODE_MODULES_PROVIDER)
 
         this.make<Logging>(Logging).debug(`Application root: ${this.baseDir}`)
     }

src/support/NodeModules.ts

@@ -0,0 +1,112 @@
+import * as childProcess from 'child_process'
+import {UniversalPath} from '../util'
+import {Inject, Injectable, InjectParam} from '../di'
+import {Application} from '../lifecycle/Application'
+import {Logging} from '../service/Logging'
+import {NodeModule, ExtolloAwareNodeModule} from './types'
+import {EventBus} from '../event/EventBus'
+import {PackageDiscovered} from './PackageDiscovered'
+
+/**
+ * A helper class for discovering and interacting with
+ * NPM-style modules.
+ */
+@Injectable()
+export class NodeModules {
+    @Inject()
+    protected readonly logging!: Logging
+
+    @Inject()
+    protected readonly bus!: EventBus
+
+    constructor(
+        @InjectParam(Application.NODE_MODULES_INJECTION)
+        protected readonly manager: string,
+    ) { }
+
+    /**
+     * Get the NodeModule entry for the base application.
+     */
+    async app(): Promise<NodeModule> {
+        return new Promise<NodeModule>((res, rej) => {
+            childProcess.exec(`${this.manager} ls --json`, (error, stdout) => {
+                if ( error ) {
+                    return rej(error)
+                }
+
+                res(JSON.parse(stdout)[0])
+            })
+        })
+    }
+
+    /**
+     * Get the path to the node_modules folder for the base application.
+     */
+    async root(): Promise<UniversalPath> {
+        return new Promise<UniversalPath>((res, rej) => {
+            childProcess.exec(`${this.manager} root`, (error, stdout) => {
+                if ( error ) {
+                    return rej(error)
+                }
+
+                res(new UniversalPath(stdout.trim()))
+            })
+        })
+    }
+
+    /**
+     * Iterate over packages, recursively, starting with the base application's
+     * package.json and fire PackageDiscovered events for any that have a valid
+     * Extollo discovery entry.
+     */
+    async discover(): Promise<void> {
+        const root = await this.root()
+        const module = await this.app()
+        return this.discoverRoot(root, module)
+    }
+
+    /**
+     * Recursively discover child-packages from the node_modules root for the
+     * given module.
+     *
+     * Fires PackageDiscovered events for valid, discovery-enabled packages.
+     *
+     * @param root - the path to node_modules
+     * @param module - the module whose children we are discovering
+     * @protected
+     */
+    protected async discoverRoot(root: UniversalPath, module: NodeModule): Promise<void> {
+        for ( const key in module.dependencies ) {
+            if ( !Object.prototype.hasOwnProperty.call(module.dependencies, key) ) {
+                continue
+            }
+
+            this.logging.verbose(`Auto-discovery considering package: ${key}`)
+
+            try {
+                const packageJson = root.concat(key, 'package.json')
+                this.logging.verbose(`Auto-discovery package path: ${packageJson}`)
+                if ( await packageJson.exists() ) {
+                    const packageJsonString: string = await packageJson.read()
+                    const packageJsonData: ExtolloAwareNodeModule = JSON.parse(packageJsonString)
+                    if ( !packageJsonData?.extollo?.discover ) {
+                        this.logging.debug(`Skipping non-discoverable package: ${key}`)
+                        continue
+                    }
+
+                    this.logging.info(`Auto-discovering package: ${key}`)
+                    await this.bus.dispatch(new PackageDiscovered(packageJsonData, packageJson.clone()))
+
+                    const packageNodeModules = packageJson.concat('..', 'node_modules')
+                    if ( await packageNodeModules.exists() && packageJsonData?.extollo?.recursiveDependencies?.discover ) {
+                        this.logging.debug(`Recursing: ${packageNodeModules}`)
+                        await this.discoverRoot(packageNodeModules, packageJsonData)
+                    }
+                }
+            } catch (e: unknown) {
+                this.logging.error(`Encountered error while discovering package: ${key}`)
+                this.logging.error(e)
+            }
+        }
+    }
+}

src/support/PackageDiscovered.ts

@@ -0,0 +1,33 @@
+import {Event} from '../event/Event'
+import {Awaitable, JSONState, UniversalPath} from '../util'
+import {ExtolloAwareNodeModule} from './types'
+
+/**
+ * An event indicating that an NPM package has been discovered
+ * by the framework.
+ *
+ * Application services can listen for this event to register
+ * various discovery logic (e.g. automatically boot units
+ */
+export class PackageDiscovered extends Event {
+    constructor(
+        public packageConfig: ExtolloAwareNodeModule,
+        public packageJson: UniversalPath,
+    ) {
+        super()
+    }
+
+    dehydrate(): Awaitable<JSONState> {
+        return {
+            packageConfig: this.packageConfig as JSONState,
+            packageJson: this.packageJson.toString(),
+        }
+    }
+
+    rehydrate(state: JSONState): Awaitable<void> {
+        if ( typeof state === 'object' ) {
+            this.packageConfig = (state.packageConfig as ExtolloAwareNodeModule)
+            this.packageJson = new UniversalPath(String(state.packageJson))
+        }
+    }
+}

src/support/types.ts

@@ -0,0 +1,45 @@
+/**
+ * Partial package.json that may contain a partial Extollo discovery config.
+ */
+export interface ExtolloPackageDiscoveryConfig {
+    extollo?: {
+        discover?: boolean,
+        units?: {
+            discover?: boolean,
+            paths?: string[],
+        },
+        recursiveDependencies?: {
+            discover?: boolean,
+        },
+    },
+}
+
+/**
+ * Interface that defines a NodeModule dependency.
+ */
+export interface NodeDependencySpecEntry {
+    from: string,
+    version: string,
+    resolved?: string,
+    dependencies?: {[key: string]: NodeDependencySpecEntry},
+    devDependencies?: {[key: string]: NodeDependencySpecEntry},
+    unsavedDependencies?: {[key: string]: NodeDependencySpecEntry},
+    optionalDependencies?: {[key: string]: NodeDependencySpecEntry},
+}
+
+/**
+ * Defines information and dependencies of an NPM package.
+ */
+export interface NodeModule {
+    name?: string,
+    version?: string,
+    dependencies?: {[key: string]: NodeDependencySpecEntry},
+    devDependencies?: {[key: string]: NodeDependencySpecEntry},
+    unsavedDependencies?: {[key: string]: NodeDependencySpecEntry},
+    optionalDependencies?: {[key: string]: NodeDependencySpecEntry},
+}
+
+/**
+ * Type alias for a NodeModule that contains an ExtolloPackageDiscoveryConfig.
+ */
+export type ExtolloAwareNodeModule = NodeModule & ExtolloPackageDiscoveryConfig