Extollo/lib
2
0
Fork 0
Code Issues 15 Pull Requests Projects Releases 21 Wiki Activity

File-based response support & static server
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source 
- Clean up UniversalPath implementation
    - Use Readable/Writable types correctly for stream methods
    - Add .list() methods for getting child files

- Make Response body specify explicit types and support
  writing Readable streams to the body

- Create a static file server that supports directory listing
This commit is contained in:
Garrett Mills
2021-07-07 20:13:23 -05:00
parent b3b5b169e8
commit f496046461
14 changed files with 893 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/util/collection/Collection.ts
View File

@@ -50,6 +50,20 @@ class Collection<T> {
return new Collection(items)
}
/**
* Create a new collection from an item or array of items.
* Filters out undefined items.
* @param itemOrItems
*/
public static normalize<T2>(itemOrItems: (CollectionItem<T2> | undefined)[] | CollectionItem<T2> | undefined): Collection<T2> {
if ( !Array.isArray(itemOrItems) ) {
itemOrItems = [itemOrItems]
}
const items = itemOrItems.filter(x => typeof x !== 'undefined') as CollectionItem<T2>[]
return new Collection<T2>(items)
}
/**
* Create a collection of "undefined" elements of a given size.
* @param size

206
src/util/support/path.ts
View File

@@ -1,9 +1,10 @@
import * as nodePath from 'path'
import * as fs from 'fs'
import * as mkdirp from 'mkdirp'
import { Filesystem } from './path/Filesystem'
import ReadableStream = NodeJS.ReadableStream;
import WritableStream = NodeJS.WritableStream;
import * as mime from 'mime-types'
import {FileNotFoundError, Filesystem} from './path/Filesystem'
import {Collection} from '../collection/Collection'
import {Readable, Writable} from 'stream'
/**
* An item that could represent a path.
@@ -22,6 +23,36 @@ export function universalPath(...parts: PathLike[]): UniversalPath {
return main.concat(...concats)
}
/**
* Format bytes as human-readable text.
*
* @param bytes Number of bytes.
* @param si True to use metric (SI) units, aka powers of 1000. False to use
* binary (IEC), aka powers of 1024.
* @param dp Number of decimal places to display.
* @see https://stackoverflow.com/a/14919494/4971138
*/
export function bytesToHumanFileSize(bytes: number, si = false, dp = 1): string {
const thresh = si ? 1000 : 1024
if (Math.abs(bytes) < thresh) {
return bytes + ' B'
}
const units = si
? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
: ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']
let u = -1
const r = 10 ** dp
do {
bytes /= thresh
++u
} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1)
return bytes.toFixed(dp) + ' ' + units[u]
}
/**
* Walk recursively over entries in a directory.
*
@@ -155,6 +186,13 @@ export class UniversalPath {
return `${this.prefix}${this.resourceLocalPath}`
}
/**
* Get the basename of the path.
*/
get toBase(): string {
return nodePath.basename(this.resourceLocalPath)
}
/**
* Append and resolve the given paths to this resource and return a new UniversalPath.
*
@@ -224,6 +262,44 @@ export class UniversalPath {
return walk(this.resourceLocalPath)
}
/**
* Resolves true if this resource is a directory.
*/
async isDirectory(): Promise<boolean> {
if ( this.filesystem ) {
const stat = await this.filesystem.stat({
storePath: this.resourceLocalPath,
})
return stat.isDirectory
}
try {
return (await fs.promises.stat(this.resourceLocalPath)).isDirectory()
} catch (e) {
return false
}
}
/**
* Resolves true if this resource is a regular file.
*/
async isFile(): Promise<boolean> {
if ( this.filesystem ) {
const stat = await this.filesystem.stat({
storePath: this.resourceLocalPath,
})
return stat.isFile
}
try {
return (await fs.promises.stat(this.resourceLocalPath)).isFile()
} catch (e) {
return false
}
}
/**
* Returns true if the given resource exists at the path.
*/
@@ -244,6 +320,20 @@ export class UniversalPath {
}
}
/**
* List any immediate children of this resource.
*/
async list(): Promise<Collection<UniversalPath>> {
if ( this.filesystem ) {
const files = await this.filesystem.list(this.resourceLocalPath)
return files.map(x => this.concat(x))
}
const paths = await fs.promises.readdir(this.resourceLocalPath)
return Collection.collect<string>(paths)
.map(x => this.concat(x))
}
/**
* Recursively create this path as a directory. Equivalent to `mkdir -p` on Linux.
*/
@@ -290,7 +380,7 @@ export class UniversalPath {
/**
* Get a writable stream to this file's contents.
*/
async writeStream(): Promise<WritableStream> {
async writeStream(): Promise<Writable> {
if ( this.filesystem ) {
return this.filesystem.putStoreFileAsStream({
storePath: this.resourceLocalPath,
@@ -304,7 +394,7 @@ export class UniversalPath {
* Read the data from this resource's file as a string.
*/
async read(): Promise<string> {
let stream: ReadableStream
let stream: Readable
if ( this.filesystem ) {
stream = await this.filesystem.getStoreFileAsStream({
storePath: this.resourceLocalPath,
@@ -321,10 +411,37 @@ export class UniversalPath {
})
}
/**
* Get the size of this resource in bytes.
*/
async sizeInBytes(): Promise<number> {
if ( this.filesystem ) {
const stat = await this.filesystem.stat({
storePath: this.resourceLocalPath,
})
if ( stat.exists ) {
return stat.sizeInBytes
}
throw new FileNotFoundError(this.toString())
}
const stat = await fs.promises.stat(this.resourceLocalPath)
return stat.size
}
/**
* Get the size of this resource, formatted in a human-readable string.
*/
async sizeForHumans(): Promise<string> {
return bytesToHumanFileSize(await this.sizeInBytes())
}
/**
* Get a readable stream of this file's contents.
*/
async readStream(): Promise<ReadableStream> {
async readStream(): Promise<Readable> {
if ( this.filesystem ) {
return this.filesystem.getStoreFileAsStream({
storePath: this.resourceLocalPath,
@@ -334,17 +451,70 @@ export class UniversalPath {
}
}
/* get mime_type() {
return Mime.lookup(this.ext)
}
get content_type() {
return Mime.contentType(this.ext)
}
get charset() {
if ( this.mime_type ) {
return Mime.charset(this.mime_type)
/**
* Returns true if this path exists in the subtree of the given path.
* @param otherPath
*/
isChildOf(otherPath: UniversalPath): boolean {
if ( (this.filesystem || otherPath.filesystem) && otherPath.filesystem !== this.filesystem ) {
return false
}
}*/
if ( this.prefix !== otherPath.prefix ) {
return false
}
const relative = nodePath.relative(otherPath.toLocal, this.toLocal)
return Boolean(relative && !relative.startsWith('..') && !nodePath.isAbsolute(relative))
}
/**
* Returns true if the given path exists in the subtree of this path.
* @param otherPath
*/
isParentOf(otherPath: UniversalPath): boolean {
return otherPath.isChildOf(this)
}
/**
* Returns true if the given path refers to the same resource as this path.
* @param otherPath
*/
is(otherPath: UniversalPath): boolean {
if ( (this.filesystem || otherPath.filesystem) && otherPath.filesystem !== this.filesystem ) {
return false
}
if ( this.prefix !== otherPath.prefix ) {
return false
}
const relative = nodePath.relative(otherPath.toLocal, this.toLocal)
return relative === ''
}
/**
* Get the mime-type of this resource.
*/
get mimeType(): string | false {
return mime.lookup(this.resourceLocalPath)
}
/**
* Get the content-type header of this resource.
*/
get contentType(): string | false {
return mime.contentType(this.resourceLocalPath)
}
/**
* Get the charset of this resource.
*/
get charset(): string | false {
if ( this.mimeType ) {
return mime.charset(this.mimeType)
}
return false
}
}

44
src/util/support/path/Filesystem.ts
View File

@@ -2,9 +2,10 @@ import {UniversalPath} from '../path'
import * as path from 'path'
import * as os from 'os'
import {uuid4} from '../data'
import ReadableStream = NodeJS.ReadableStream;
import WritableStream = NodeJS.WritableStream;
import {ErrorWithContext} from '../../error/ErrorWithContext'
import {Readable, Writable} from 'stream'
import {Awaitable} from '../types'
import {Collection} from '../../collection/Collection'
/**
* Error thrown when an operation is attempted on a non-existent file.
@@ -65,6 +66,16 @@ export interface Stat {
*/
tags: string[],
/**
* True if the resource exists as a directory.
*/
isDirectory: boolean,
/**
* True if the resource exists as a regular file.
*/
isFile: boolean,
accessed?: Date,
modified?: Date,
created?: Date,
@@ -77,12 +88,12 @@ export abstract class Filesystem {
/**
* Called when the Filesystem driver is initialized. Do any standup here.
*/
public open(): void | Promise<void> {} // eslint-disable-line @typescript-eslint/no-empty-function
public open(): Awaitable<void> {} // eslint-disable-line @typescript-eslint/no-empty-function
/**
* Called when the Filesystem driver is destroyed. Do any cleanup here.
*/
public close(): void | Promise<void> {} // eslint-disable-line @typescript-eslint/no-empty-function
public close(): Awaitable<void> {} // eslint-disable-line @typescript-eslint/no-empty-function
/**
* Get the URI prefix for this filesystem.
@@ -114,62 +125,67 @@ export abstract class Filesystem {
*
* @param args
*/
public abstract putLocalFile(args: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}): void | Promise<void>
public abstract putLocalFile(args: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}): Awaitable<void>
/**
* Download a file in the remote filesystem to the local filesystem and return it as a UniversalPath.
* @param args
*/
public abstract getStoreFileAsTemp(args: {storePath: string}): UniversalPath | Promise<UniversalPath>
public abstract getStoreFileAsTemp(args: {storePath: string}): Awaitable<UniversalPath>
/**
* Open a readable stream for a file in the remote filesystem.
* @param args
*/
public abstract getStoreFileAsStream(args: {storePath: string}): ReadableStream | Promise<ReadableStream>
public abstract getStoreFileAsStream(args: {storePath: string}): Awaitable<Readable>
/**
* Open a writable stream for a file in the remote filesystem.
* @param args
*/
public abstract putStoreFileAsStream(args: {storePath: string}): WritableStream | Promise<WritableStream>
public abstract putStoreFileAsStream(args: {storePath: string}): Awaitable<Writable>
/**
* Fetch some information about a file that may or may not be in the remote filesystem without fetching the entire file.
* @param args
*/
public abstract stat(args: {storePath: string}): Stat | Promise<Stat>
public abstract stat(args: {storePath: string}): Awaitable<Stat>
/**
* If the file does not exist in the remote filesystem, create it. If it does exist, update the modify timestamps.
* @param args
*/
public abstract touch(args: {storePath: string}): void | Promise<void>
public abstract touch(args: {storePath: string}): Awaitable<void>
/**
* Remove the given resource(s) from the remote filesystem.
* @param args
*/
public abstract remove(args: {storePath: string, recursive?: boolean }): void | Promise<void>
public abstract remove(args: {storePath: string, recursive?: boolean }): Awaitable<void>
/**
* Create the given path on the store as a directory, recursively.
* @param args
*/
public abstract mkdir(args: {storePath: string}): void | Promise<void>
public abstract mkdir(args: {storePath: string}): Awaitable<void>
/**
* Get the metadata object for the given file, if it exists.
* @param storePath
*/
public abstract getMetadata(storePath: string): FileMetadata | Promise<FileMetadata>
public abstract getMetadata(storePath: string): Awaitable<FileMetadata>
/**
* Set the metadata object for the given file, if the file exists.
* @param storePath
* @param meta
*/
public abstract setMetadata(storePath: string, meta: FileMetadata): void | Promise<void>
public abstract setMetadata(storePath: string, meta: FileMetadata): Awaitable<void>
/**
* List direct children of this resource.
*/
public abstract list(storePath: string): Awaitable<Collection<string>>
/**
* Normalize the input tags into a single array of strings. This is useful for implementing the fluent

14
src/util/support/path/LocalFilesystem.ts
View File

@@ -4,6 +4,7 @@ import * as path from 'path'
import {UniversalPath} from '../path'
import * as rimraf from 'rimraf'
import * as mkdirp from 'mkdirp'
import { Collection } from '../../collection/Collection'
export interface LocalFilesystemConfig {
baseDir: string
@@ -87,6 +88,8 @@ export class LocalFilesystem extends Filesystem {
accessed: stat.atime,
modified: stat.mtime,
created: stat.ctime,
isDirectory: stat.isDirectory(),
isFile: stat.isFile(),
}
} catch (e) {
if ( e?.code === 'ENOENT' ) {
@@ -95,6 +98,8 @@ export class LocalFilesystem extends Filesystem {
exists: false,
sizeInBytes: 0,
tags: [],
isFile: false,
isDirectory: false,
}
}
@@ -166,4 +171,13 @@ export class LocalFilesystem extends Filesystem {
protected metadataPath(storePath: string): string {
return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
}
/**
* List all immediate children of the given path.
* @param storePath
*/
public async list(storePath: string): Promise<Collection<string>> {
const paths = await fs.promises.readdir(this.storePath(storePath))
return Collection.collect<string>(paths)
}
}

29
src/util/support/path/SSHFilesystem.ts
View File

@@ -2,7 +2,8 @@ import {FileMetadata, Filesystem, Stat} from './Filesystem'
import * as ssh2 from 'ssh2'
import * as path from 'path'
import * as fs from 'fs'
import ReadableStream = NodeJS.ReadableStream
import {Readable, Writable} from 'stream'
import {Collection} from '../../collection/Collection'
import {UniversalPath} from '../path'
/**
@@ -37,7 +38,7 @@ export class SSHFilesystem extends Filesystem {
})
}
async getStoreFileAsStream(args: { storePath: string }): Promise<ReadableStream> {
async getStoreFileAsStream(args: { storePath: string }): Promise<Readable> {
const sftp = await this.getSFTP()
return sftp.createReadStream(this.storePath(args.storePath))
}
@@ -62,7 +63,7 @@ export class SSHFilesystem extends Filesystem {
})
}
async putStoreFileAsStream(args: { storePath: string }): Promise<NodeJS.WritableStream> {
async putStoreFileAsStream(args: { storePath: string }): Promise<Writable> {
const sftp = await this.getSFTP()
return sftp.createWriteStream(this.storePath(args.storePath))
}
@@ -126,6 +127,8 @@ export class SSHFilesystem extends Filesystem {
accessed: stat.atime,
modified: stat.mtime,
created: stat.ctime,
isFile: stat.isFile(),
isDirectory: stat.isDirectory(),
}
} catch (e) {
return {
@@ -133,6 +136,8 @@ export class SSHFilesystem extends Filesystem {
exists: false,
sizeInBytes: 0,
tags: [],
isFile: false,
isDirectory: false,
}
}
}
@@ -243,7 +248,7 @@ export class SSHFilesystem extends Filesystem {
* @protected
*/
protected storePath(storePath: string): string {
return path.resolve(this.baseConfig.baseDir, 'data', storePath)
return path.join(this.baseConfig.baseDir, 'data', storePath)
}
/**
@@ -252,7 +257,7 @@ export class SSHFilesystem extends Filesystem {
* @protected
*/
protected metadataPath(storePath: string): string {
return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
return path.join(this.baseConfig.baseDir, 'meta', storePath + '.json')
}
/**
@@ -268,4 +273,18 @@ export class SSHFilesystem extends Filesystem {
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
})
}
async list(storePath: string): Promise<Collection<string>> {
const sftp = await this.getSFTP()
return new Promise<Collection<string>>((res, rej) => {
sftp.readdir(this.storePath(storePath), (error, files) => {
if ( error ) {
rej(error)
} else {
res(Collection.collect(files).map(x => x.filename))
}
})
})
}
}