enhancement(extollo/extollo#2): add kernel module for parsing request body contents, uploading files

package.json

@@ -10,9 +10,11 @@
   "dependencies": {
     "@extollo/di": "^0.4.4",
     "@extollo/util": "^0.3.2",
+    "@types/busboy": "^0.2.3",
     "@types/negotiator": "^0.6.1",
     "@types/node": "^14.14.37",
     "@types/pug": "^2.0.4",
+    "busboy": "^0.3.1",
     "colors": "^1.4.0",
     "dotenv": "^8.2.0",
     "negotiator": "^0.6.2",

pnpm-lock.yaml

@@ -1,9 +1,11 @@
 dependencies:
   '@extollo/di': 0.4.4
   '@extollo/util': 0.3.2
+  '@types/busboy': 0.2.3
   '@types/negotiator': 0.6.1
   '@types/node': 14.14.37
   '@types/pug': 2.0.4
+  busboy: 0.3.1
   colors: 1.4.0
   dotenv: 8.2.0
   negotiator: 0.6.2
@@ -56,6 +58,12 @@ packages:
     dev: false
     resolution:
       integrity: sha512-cL50wHrIiRHn6W3niQZftYDgFB8K8x0dxbJPZtnt4/iy32m1aWxEx9UL8Ttldas6zDt4Ws1zBp0fInSKOKcQnQ==
+  /@types/busboy/0.2.3:
+    dependencies:
+      '@types/node': 14.14.37
+    dev: false
+    resolution:
+      integrity: sha1-ZpetKYcyRsUw8Jo/9aQIYYJCMNU=
   /@types/glob/7.1.3:
     dependencies:
       '@types/minimatch': 3.0.4
@@ -163,6 +171,14 @@ packages:
     dev: false
     resolution:
       integrity: sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
+  /busboy/0.3.1:
+    dependencies:
+      dicer: 0.3.0
+    dev: false
+    engines:
+      node: '>=4.5.0'
+    resolution:
+      integrity: sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==
   /call-bind/1.0.2:
     dependencies:
       function-bind: 1.1.1
@@ -197,6 +213,14 @@ packages:
     dev: false
     resolution:
       integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==
+  /dicer/0.3.0:
+    dependencies:
+      streamsearch: 0.1.2
+    dev: false
+    engines:
+      node: '>=4.5.0'
+    resolution:
+      integrity: sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==
   /diff/4.0.2:
     dev: false
     engines:
@@ -580,9 +604,11 @@ packages:
 specifiers:
   '@extollo/di': ^0.4.4
   '@extollo/util': ^0.3.2
+  '@types/busboy': ^0.2.3
   '@types/negotiator': ^0.6.1
   '@types/node': ^14.14.37
   '@types/pug': ^2.0.4
+  busboy: ^0.3.1
   colors: ^1.4.0
   dotenv: ^8.2.0
   negotiator: ^0.6.2

src/http/kernel/module/ParseIncomingBodyHTTPModule.ts

@@ -0,0 +1,93 @@
+import {HTTPKernelModule} from "../HTTPKernelModule"
+import {HTTPKernel} from "../HTTPKernel"
+import * as Busboy from "busboy"
+import {Request} from "../../lifecycle/Request"
+import {infer, uuid_v4} from "@extollo/util"
+import {Files} from "../../../service/Files"
+import {Config} from "../../../service/Config"
+import {Logging} from "../../../service/Logging"
+import {Injectable, Inject, Container} from "@extollo/di"
+
+@Injectable()
+export class ParseIncomingBodyHTTPModule extends HTTPKernelModule {
+    static register(kernel: HTTPKernel) {
+        const files = <Files> Container.getContainer().make(Files)
+        const logging = <Logging> Container.getContainer().make(Logging)
+        if ( !files.hasFilesystem() ) {
+            logging.warn(`No default filesystem is configured. This means request files will not be uploaded.`)
+        }
+
+        kernel.register(this).first()
+    }
+
+    @Inject()
+    protected readonly files!: Files
+
+    @Inject()
+    protected readonly config!: Config
+
+    @Inject()
+    protected readonly logging!: Logging
+
+    public async apply(request: Request): Promise<Request> {
+        if ( !request.getHeader('content-type') ) return request
+
+        const config = this.config.get('server.uploads', {})
+
+        await new Promise<void>((res, rej) => {
+            const busboy = new Busboy({
+                headers: request.toNative().headers,
+            })
+
+            busboy.on('field', (field, val) => {
+                request.parsedInput[field] = infer(val)
+            })
+
+            busboy.on('file', async (field, file, filename, encoding, mimetype) => {
+                if ( !this.files.hasFilesystem() ) return
+
+                if ( !config?.enable ) {
+                    this.logging.warn(`Skipping uploaded file '${filename}' because uploading is disabled. Set the server.uploads.enable config to allow uploads.`)
+                    file.resume()
+                    return
+                }
+
+                if ( config?.filter?.mimetype ) {
+                    const rex = new RegExp(config.filter.mimetype)
+                    if ( !rex.test(mimetype) ) {
+                        this.logging.debug(`Skipping uploaded file '${filename}' because the mimetype does not match the configured filter: (${config.filter.mimetype} -> ${mimetype})`)
+                        file.resume()
+                        return
+                    }
+                }
+
+                if ( config?.filter?.filename ) {
+                    const rex = new RegExp(config.filter.filename)
+                    if ( !rex.test(filename) ) {
+                        this.logging.debug(`Skipping uploaded file '${filename}' because the file name does not match the configured filter: (${config.filter.filename} -> ${filename})`)
+                        file.resume()
+                        return
+                    }
+                }
+
+                const fs = this.files.getFilesystem()
+                const storePath = `${config.filesystemPrefix ? config.filesystemPrefix : ''}${(config.filesystemPrefix && !config.filesystemPrefix.endsWith('/')) ? '/' : ''}${field}-${uuid_v4()}`
+                this.logging.verbose(`Uploading file in field ${field} to ${fs.getPrefix()}${storePath}`)
+                file.pipe(await fs.putStoreFileAsStream({ storePath }))  // FIXME might need to revisit this to ensure we don't res() before pipe finishes
+
+                request.uploadedFiles[field] = request.parsedInput[field] = fs.getPath(storePath)
+            })
+
+            busboy.on('finish', () => {
+                this.logging.debug(`Parsed body input: ${JSON.stringify(request.parsedInput)}`)
+                res()
+            })
+
+            busboy.on('error', rej)
+
+            request.toNative().pipe(busboy)
+        })
+
+        return request
+    }
+}

src/http/lifecycle/Request.ts

@@ -1,5 +1,5 @@
 import {Injectable, ScopedContainer, Container} from "@extollo/di"
-import {infer} from "@extollo/util"
+import {infer, UniversalPath} from "@extollo/util"
 import {IncomingMessage, ServerResponse} from "http"
 import {HTTPCookieJar} from "../kernel/HTTPCookieJar";
 import {TLSSocket} from "tls";
@@ -84,6 +84,12 @@ export class Request extends ScopedContainer {
     /** The media types accepted by the client. */
     public readonly mediaTypes: string[];
 
+    /** Input parsed from the request */
+    public readonly parsedInput: {[key: string]: any} = {}
+
+    /** Files parsed from the request. */
+    public readonly uploadedFiles: {[key: string]: UniversalPath} = {}
+
     constructor(
         /** The native Node.js request. */
         protected clientRequest: IncomingMessage,
@@ -162,11 +168,22 @@ export class Request extends ScopedContainer {
      * @param key
      */
     public input(key: string) {
+        if ( key in this.parsedInput ) {
+            return this.parsedInput[key]
+        }
+
         if ( key in this.query ) {
             return this.query[key]
         }
     }
 
+    /**
+     * Get the UniversalPath instance for a file uploaded in the given field on the request.
+     */
+    public file(key: string): UniversalPath | undefined {
+        return this.uploadedFiles[key]
+    }
+
     /**
      * Returns true if the request accepts the given media type.
      * @param type - a mimetype, or the short forms json, xml, or html

src/service/Files.ts

@@ -89,6 +89,23 @@ export class Files extends Unit {
         }
 
         await Promise.all(promises)
+
+        // Once they have initialized, register the DI token for the default filesystem
+        this.app().registerProducer(Filesystem, () => {
+            // This will throw FilesystemDoesNotExistError if no default filesystem was created
+            // Such behavior is desired as it is clearer than an invalid injection error, e.g.
+            return this.getFilesystem()
+        })
+
+        // If file uploads are enabled, ensure that the default upload prefix exists
+        if ( this.defaultFilesystem ) {
+            const upload = this.config.get('server.uploads', {})
+            if ( upload?.enable && upload?.filesystemPrefix ) {
+                await this.defaultFilesystem.mkdir({
+                    storePath: upload.filesystemPrefix,
+                })
+            }
+        }
     }
 
     async down() {

src/service/HTTPServer.ts

@@ -14,6 +14,7 @@ import {ExecuteResolvedRouteHandlerHTTPModule} from "../http/kernel/module/Execu
 import {error} from "../http/response/ErrorResponseFactory";
 import {ExecuteResolvedRoutePreflightHTTPModule} from "../http/kernel/module/ExecuteResolvedRoutePreflightHTTPModule";
 import {ExecuteResolvedRoutePostflightHTTPModule} from "../http/kernel/module/ExecuteResolvedRoutePostflightHTTPModule";
+import {ParseIncomingBodyHTTPModule} from "../http/kernel/module/ParseIncomingBodyHTTPModule";
 
 /**
  * Application unit that starts the HTTP/S server, creates Request and Response objects
@@ -42,6 +43,7 @@ export class HTTPServer extends Unit {
         ExecuteResolvedRouteHandlerHTTPModule.register(this.kernel)
         ExecuteResolvedRoutePreflightHTTPModule.register(this.kernel)
         ExecuteResolvedRoutePostflightHTTPModule.register(this.kernel)
+        ParseIncomingBodyHTTPModule.register(this.kernel)
 
         await new Promise<void>((res, rej) => {
             this.server = createServer(this.handler)