File-based response support & static server

- Clean up UniversalPath implementation
    - Use Readable/Writable types correctly for stream methods
    - Add .list() methods for getting child files

- Make Response body specify explicit types and support
  writing Readable streams to the body

- Create a static file server that supports directory listing

src/util/support/path/Filesystem.ts

@@ -2,9 +2,10 @@ import {UniversalPath} from '../path'
 import * as path from 'path'
 import * as os from 'os'
 import {uuid4} from '../data'
-import ReadableStream = NodeJS.ReadableStream;
-import WritableStream = NodeJS.WritableStream;
 import {ErrorWithContext} from '../../error/ErrorWithContext'
+import {Readable, Writable} from 'stream'
+import {Awaitable} from '../types'
+import {Collection} from '../../collection/Collection'
 
 /**
  * Error thrown when an operation is attempted on a non-existent file.
@@ -65,6 +66,16 @@ export interface Stat {
      */
     tags: string[],
 
+    /**
+     * True if the resource exists as a directory.
+     */
+    isDirectory: boolean,
+
+    /**
+     * True if the resource exists as a regular file.
+     */
+    isFile: boolean,
+
     accessed?: Date,
     modified?: Date,
     created?: Date,
@@ -77,12 +88,12 @@ export abstract class Filesystem {
     /**
      * Called when the Filesystem driver is initialized. Do any standup here.
      */
-    public open(): void | Promise<void> {}  // eslint-disable-line @typescript-eslint/no-empty-function
+    public open(): Awaitable<void> {}  // eslint-disable-line @typescript-eslint/no-empty-function
 
     /**
      * Called when the Filesystem driver is destroyed. Do any cleanup here.
      */
-    public close(): void | Promise<void> {}  // eslint-disable-line @typescript-eslint/no-empty-function
+    public close(): Awaitable<void> {}  // eslint-disable-line @typescript-eslint/no-empty-function
 
     /**
      * Get the URI prefix for this filesystem.
@@ -114,62 +125,67 @@ export abstract class Filesystem {
      *
      * @param args
      */
-    public abstract putLocalFile(args: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}): void | Promise<void>
+    public abstract putLocalFile(args: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}): Awaitable<void>
 
     /**
      * Download a file in the remote filesystem to the local filesystem and return it as a UniversalPath.
      * @param args
      */
-    public abstract getStoreFileAsTemp(args: {storePath: string}): UniversalPath | Promise<UniversalPath>
+    public abstract getStoreFileAsTemp(args: {storePath: string}): Awaitable<UniversalPath>
 
     /**
      * Open a readable stream for a file in the remote filesystem.
      * @param args
      */
-    public abstract getStoreFileAsStream(args: {storePath: string}): ReadableStream | Promise<ReadableStream>
+    public abstract getStoreFileAsStream(args: {storePath: string}): Awaitable<Readable>
 
     /**
      * Open a writable stream for a file in the remote filesystem.
      * @param args
      */
-    public abstract putStoreFileAsStream(args: {storePath: string}): WritableStream | Promise<WritableStream>
+    public abstract putStoreFileAsStream(args: {storePath: string}): Awaitable<Writable>
 
     /**
      * Fetch some information about a file that may or may not be in the remote filesystem without fetching the entire file.
      * @param args
      */
-    public abstract stat(args: {storePath: string}): Stat | Promise<Stat>
+    public abstract stat(args: {storePath: string}): Awaitable<Stat>
 
     /**
      * If the file does not exist in the remote filesystem, create it. If it does exist, update the modify timestamps.
      * @param args
      */
-    public abstract touch(args: {storePath: string}): void | Promise<void>
+    public abstract touch(args: {storePath: string}): Awaitable<void>
 
     /**
      * Remove the given resource(s) from the remote filesystem.
      * @param args
      */
-    public abstract remove(args: {storePath: string, recursive?: boolean }): void | Promise<void>
+    public abstract remove(args: {storePath: string, recursive?: boolean }): Awaitable<void>
 
     /**
      * Create the given path on the store as a directory, recursively.
      * @param args
      */
-    public abstract mkdir(args: {storePath: string}): void | Promise<void>
+    public abstract mkdir(args: {storePath: string}): Awaitable<void>
 
     /**
      * Get the metadata object for the given file, if it exists.
      * @param storePath
      */
-    public abstract getMetadata(storePath: string): FileMetadata | Promise<FileMetadata>
+    public abstract getMetadata(storePath: string): Awaitable<FileMetadata>
 
     /**
      * Set the metadata object for the given file, if the file exists.
      * @param storePath
      * @param meta
      */
-    public abstract setMetadata(storePath: string, meta: FileMetadata): void | Promise<void>
+    public abstract setMetadata(storePath: string, meta: FileMetadata): Awaitable<void>
+
+    /**
+     * List direct children of this resource.
+     */
+    public abstract list(storePath: string): Awaitable<Collection<string>>
 
     /**
      * Normalize the input tags into a single array of strings. This is useful for implementing the fluent

src/util/support/path/LocalFilesystem.ts

@@ -4,6 +4,7 @@ import * as path from 'path'
 import {UniversalPath} from '../path'
 import * as rimraf from 'rimraf'
 import * as mkdirp from 'mkdirp'
+import { Collection } from '../../collection/Collection'
 
 export interface LocalFilesystemConfig {
     baseDir: string
@@ -87,6 +88,8 @@ export class LocalFilesystem extends Filesystem {
                 accessed: stat.atime,
                 modified: stat.mtime,
                 created: stat.ctime,
+                isDirectory: stat.isDirectory(),
+                isFile: stat.isFile(),
             }
         } catch (e) {
             if ( e?.code === 'ENOENT' ) {
@@ -95,6 +98,8 @@ export class LocalFilesystem extends Filesystem {
                     exists: false,
                     sizeInBytes: 0,
                     tags: [],
+                    isFile: false,
+                    isDirectory: false,
                 }
             }
 
@@ -166,4 +171,13 @@ export class LocalFilesystem extends Filesystem {
     protected metadataPath(storePath: string): string {
         return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
     }
+
+    /**
+     * List all immediate children of the given path.
+     * @param storePath
+     */
+    public async list(storePath: string): Promise<Collection<string>> {
+        const paths = await fs.promises.readdir(this.storePath(storePath))
+        return Collection.collect<string>(paths)
+    }
 }

src/util/support/path/SSHFilesystem.ts

@@ -2,7 +2,8 @@ import {FileMetadata, Filesystem, Stat} from './Filesystem'
 import * as ssh2 from 'ssh2'
 import * as path from 'path'
 import * as fs from 'fs'
-import ReadableStream = NodeJS.ReadableStream
+import {Readable, Writable} from 'stream'
+import {Collection} from '../../collection/Collection'
 import {UniversalPath} from '../path'
 
 /**
@@ -37,7 +38,7 @@ export class SSHFilesystem extends Filesystem {
         })
     }
 
-    async getStoreFileAsStream(args: { storePath: string }): Promise<ReadableStream> {
+    async getStoreFileAsStream(args: { storePath: string }): Promise<Readable> {
         const sftp = await this.getSFTP()
         return sftp.createReadStream(this.storePath(args.storePath))
     }
@@ -62,7 +63,7 @@ export class SSHFilesystem extends Filesystem {
         })
     }
 
-    async putStoreFileAsStream(args: { storePath: string }): Promise<NodeJS.WritableStream> {
+    async putStoreFileAsStream(args: { storePath: string }): Promise<Writable> {
         const sftp = await this.getSFTP()
         return sftp.createWriteStream(this.storePath(args.storePath))
     }
@@ -126,6 +127,8 @@ export class SSHFilesystem extends Filesystem {
                 accessed: stat.atime,
                 modified: stat.mtime,
                 created: stat.ctime,
+                isFile: stat.isFile(),
+                isDirectory: stat.isDirectory(),
             }
         } catch (e) {
             return {
@@ -133,6 +136,8 @@ export class SSHFilesystem extends Filesystem {
                 exists: false,
                 sizeInBytes: 0,
                 tags: [],
+                isFile: false,
+                isDirectory: false,
             }
         }
     }
@@ -243,7 +248,7 @@ export class SSHFilesystem extends Filesystem {
      * @protected
      */
     protected storePath(storePath: string): string {
-        return path.resolve(this.baseConfig.baseDir, 'data', storePath)
+        return path.join(this.baseConfig.baseDir, 'data', storePath)
     }
 
     /**
@@ -252,7 +257,7 @@ export class SSHFilesystem extends Filesystem {
      * @protected
      */
     protected metadataPath(storePath: string): string {
-        return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
+        return path.join(this.baseConfig.baseDir, 'meta', storePath + '.json')
     }
 
     /**
@@ -268,4 +273,18 @@ export class SSHFilesystem extends Filesystem {
             stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
         })
     }
+
+    async list(storePath: string): Promise<Collection<string>> {
+        const sftp = await this.getSFTP()
+
+        return new Promise<Collection<string>>((res, rej) => {
+            sftp.readdir(this.storePath(storePath), (error, files) => {
+                if ( error ) {
+                    rej(error)
+                } else {
+                    res(Collection.collect(files).map(x => x.filename))
+                }
+            })
+        })
+    }
 }