Import other modules into monorepo
Some checks failed
continuous-integration/drone/push Build is failing

This commit is contained in:
2021-06-01 20:59:40 -05:00
parent 26d54033af
commit 9be9c44a32
138 changed files with 11544 additions and 139 deletions

33
src/util/cache/Cache.ts vendored Normal file
View File

@@ -0,0 +1,33 @@
import {Awaitable} from "../support/types"
/**
* Abstract interface class for a cached object.
*/
export abstract class Cache {
/**
* Fetch a value from the cache by its key.
* @param {string} key
* @return Promise<any|undefined>
*/
public abstract fetch(key: string): Awaitable<string|undefined>;
/**
* Store the given value in the cache by key.
* @param {string} key
* @param {string} value
*/
public abstract put(key: string, value: string): Awaitable<void>;
/**
* Check if the cache has the given key.
* @param {string} key
* @return Promise<boolean>
*/
public abstract has(key: string): Awaitable<boolean>;
/**
* Drop the given key from the cache.
* @param {string} key
*/
public abstract drop(key: string): Awaitable<void>;
}

41
src/util/cache/InMemCache.ts vendored Normal file
View File

@@ -0,0 +1,41 @@
import { Cache } from './Cache'
import { Collection } from '../collection/Collection'
/**
* Base interface for an item stored in a memory cache.
*/
export interface InMemCacheItem {
key: string,
item: string,
}
/**
* A cache implementation stored in memory.
* @extends Cache
*/
export class InMemCache extends Cache {
/**
* The stored cache items.
* @type Collection<InMemCacheItem>
*/
protected items: Collection<InMemCacheItem> = new Collection<InMemCacheItem>()
public async fetch(key: string) {
const item = this.items.firstWhere('key', '=', key)
if ( item ) return item.item
}
public async put(key: string, item: string) {
const existing = this.items.firstWhere('key', '=', key)
if ( existing ) existing.item = item
else this.items.push({ key, item })
}
public async has(key: string) {
return this.items.where('key', '=', key).length > 0
}
public async drop(key: string) {
this.items = this.items.whereNot('key', '=', key)
}
}

View File

@@ -0,0 +1,33 @@
import { Iterable } from './Iterable'
import { collect } from './Collection'
/**
* A basic Iterable implementation that uses an array as a backend.
* @extends Iterable
*/
export class ArrayIterable<T> extends Iterable<T> {
constructor(
/**
* Items to use for this iterable.
*/
protected items: T[],
) {
super()
}
async at(i: number) {
return this.items[i]
}
async range(start: number, end: number) {
return collect(this.items.slice(start, end + 1))
}
async count() {
return this.items.length
}
clone() {
return new ArrayIterable([...this.items])
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,119 @@
import {Collection} from './Collection'
export type MaybeIterationItem<T> = { done: boolean, value?: T }
export type ChunkCallback<T> = (items: Collection<T>) => any
export class StopIteration extends Error {}
/**
* Abstract class representing an iterable, lazy-loaded dataset.
* @abstract
*/
export abstract class Iterable<T> {
/**
* The current index of the iterable.
* @type number
*/
protected index = 0
/**
* Get the item of this iterable at the given index, if one exists.
* @param {number} i
* @return Promise<any|undefined>
*/
abstract at(i: number): Promise<T | undefined>
/**
* Get the collection of items in the given range of this iterable.
* @param {number} start
* @param {number} end
* @return Promise<Collection>
*/
abstract range(start: number, end: number): Promise<Collection<T>>
/**
* Count the number of items in this collection.
* @return Promise<number>
*/
abstract count(): Promise<number>
/**
* Get a copy of this iterable.
* @return Iterable
*/
abstract clone(): Iterable<T>
/**
* Return a collection of all items in this iterable.
* @return Promise<Collection>
*/
public async all(): Promise<Collection<T>> {
return this.range(0, (await this.count()) + 1)
}
/**
* Advance to the next value of this iterable.
* @return Promise<MaybeIterationItem>
*/
public async next(): Promise<MaybeIterationItem<T>> {
const i = this.index
if ( i >= await this.count() ) {
return { done: true }
}
this.index = i + 1
return { done: false, value: await this.at(i) }
}
/**
* Chunk the iterable into the given size and call the callback passing the chunk along.
* @param {number} size
* @param {ChunkCallback} callback
* @return Promise<void>
*/
public async chunk(size: number, callback: ChunkCallback<T>) {
const total = await this.count()
while ( this.index < total ) {
const items = await this.range(this.index, this.index + size - 1)
try {
await callback(items)
} catch ( error ) {
if ( error instanceof StopIteration ) break
else throw error
}
this.index += size
}
}
/**
* Advance the iterable to the given index.
* @param {number} index
* @return Promise<void>
*/
public async seek(index: number) {
if ( index < 0 ) throw new TypeError('Cannot seek to negative index.')
else if ( index >= await this.count() ) throw new TypeError('Cannot seek past last item.')
this.index = index
}
/**
* Peek at the next value of the iterable, without advancing.
* @return Promise<any|undefined>
*/
public async peek(): Promise<T | undefined> {
if ( this.index + 1 >= await this.count() ) return undefined
else return this.at(this.index + 1)
}
/**
* Reset the iterable to the first index.
* @return Promise<any>
*/
public async reset() {
this.index = 0
}
}

View File

@@ -0,0 +1,86 @@
/**
* Type representing a valid where operator.
*/
type WhereOperator = '&' | '>' | '>=' | '<' | '<=' | '!=' | '<=>' | '%' | '|' | '!' | '~' | '=' | '^'
/**
* Type associating search items with a key.
*/
type AssociatedSearchItem = { key: any, item: any }
/**
* Type representing the result of a where.
*/
type WhereResult = any[]
/**
* Returns true if the given item satisfies the given where clause.
* @param {AssociatedSearchItem} item
* @param {WhereOperator} operator
* @param [operand]
* @return boolean
*/
const whereMatch = (item: AssociatedSearchItem, operator: WhereOperator, operand?: any): boolean => {
switch ( operator ) {
case '&':
if ( item.key & operand ) return true
break
case '>':
if ( item.key > operand ) return true
break
case '>=':
if ( item.key >= operand ) return true
break
case '<':
if ( item.key < operand ) return true
break
case '<=':
if ( item.key <= operand ) return true
break
case '!=':
if ( item.key !== operand ) return true
break
case '<=>':
if ( item.key === operand && typeof item.key !== 'undefined' && item.key !== null )
return true
break
case '%':
if ( item.key % operand ) return true
break
case '|':
if ( item.key | operand ) return true
break
case '!':
if ( !item.key ) return true
break
case '~':
if ( ~item.key ) return true
break
case '=':
if ( item.key === operand ) return true
break
case '^':
if ( item.key ^ operand ) return true
break
}
return false
}
/**
* Apply the given where clause to the items and return those that match.
* @param {Array<AssociatedSearchItem>} items
* @param {WhereOperator} operator
* @param [operand]
*/
const applyWhere = (items: AssociatedSearchItem[], operator: WhereOperator, operand?: any): WhereResult => {
const matches: WhereResult = []
for ( const item of items ) {
if ( whereMatch(item, operator, operand) )
matches.push(item.item)
}
return matches
}
export { WhereOperator, WhereResult, AssociatedSearchItem, applyWhere, whereMatch }

179
src/util/const/http.ts Normal file
View File

@@ -0,0 +1,179 @@
/**
* Enum of HTTP statuses.
* @example
* HTTPStatus.http200 // => 200
*
* @example
* HTTPStatus.REQUEST_TIMEOUT // => 408
*/
export enum HTTPStatus {
http100 = 100,
http101 = 101,
http102 = 102,
http200 = 200,
http201 = 201,
http202 = 202,
http203 = 203,
http204 = 204,
http205 = 205,
http206 = 206,
http207 = 207,
http300 = 300,
http301 = 301,
http302 = 302,
http303 = 303,
http304 = 304,
http305 = 305,
http307 = 307,
http308 = 308,
http400 = 400,
http401 = 401,
http402 = 402,
http403 = 403,
http404 = 404,
http405 = 405,
http406 = 406,
http407 = 407,
http408 = 408,
http409 = 409,
http410 = 410,
http411 = 411,
http412 = 412,
http413 = 413,
http414 = 414,
http415 = 415,
http416 = 416,
http417 = 417,
http418 = 418,
http419 = 419,
http420 = 420,
http422 = 422,
http423 = 423,
http424 = 424,
http428 = 428,
http429 = 429,
http431 = 431,
http500 = 500,
http501 = 501,
http502 = 502,
http503 = 503,
http504 = 504,
http505 = 505,
http507 = 507,
http511 = 511,
CONTINUE = 100,
SWITCHING_PROTOCOLS = 101,
PROCESSING = 102,
OK = 200,
CREATED = 201,
ACCEPTED = 202,
NON_AUTHORITATIVE_INFORMATION = 203,
NO_CONTENT = 204,
RESET_CONTENT = 205,
PARTIAL_CONTENT = 206,
MULTI_STATUS = 207,
MULTIPLE_CHOICES = 300,
MOVED_PERMANENTLY = 301,
MOVED_TEMPORARILY = 302,
SEE_OTHER = 303,
NOT_MODIFIED = 304,
USE_PROXY = 305,
TEMPORARY_REDIRECT = 307,
PERMANENT_REDIRECT = 308,
BAD_REQUEST = 400,
UNAUTHORIZED = 401,
PAYMENT_REQUIRED = 402,
FORBIDDEN = 403,
NOT_FOUND = 404,
METHOD_NOT_ALLOWED = 405,
NOT_ACCEPTABLE = 406,
PROXY_AUTHENTICATION_REQUIRED = 407,
REQUEST_TIMEOUT = 408,
CONFLICT = 409,
GONE = 410,
LENGTH_REQUIRED = 411,
PRECONDITION_FAILED = 412,
REQUEST_TOO_LONG = 413,
REQUEST_URI_TOO_LONG = 414,
UNSUPPORTED_MEDIA_TYPE = 415,
REQUESTED_RANGE_NOT_SATISFIABLE = 416,
EXPECTATION_FAILED = 417,
IM_A_TEAPOT = 418,
INSUFFICIENT_SPACE_ON_RESOURCE = 419,
METHOD_FAILURE = 420,
UNPROCESSABLE_ENTITY = 422,
LOCKED = 423,
FAILED_DEPENDENCY = 424,
PRECONDITION_REQUIRED = 428,
TOO_MANY_REQUESTS = 429,
REQUEST_HEADER_FIELDS_TOO_LARGE = 431,
INTERNAL_SERVER_ERROR = 500,
NOT_IMPLEMENTED = 501,
BAD_GATEWAY = 502,
SERVICE_UNAVAILABLE = 503,
GATEWAY_TIMEOUT = 504,
HTTP_VERSION_NOT_SUPPORTED = 505,
INSUFFICIENT_STORAGE = 507,
NETWORK_AUTHENTICATION_REQUIRED = 511,
}
/**
* Maps HTTP status code to default status message.
*/
export const HTTPMessage = {
100: 'Continue',
101: 'Switching Protocols',
102: 'Processing',
200: 'OK',
201: 'Created',
202: 'Accepted',
203: 'Non Authoritative Information',
204: 'No Content',
205: 'Reset Content',
206: 'Partial Content',
207: 'Multi-Status',
300: 'Multiple Choices',
301: 'Moved Permanently',
302: 'Moved Temporarily',
303: 'See Other',
304: 'Not Modified',
305: 'Use Proxy',
307: 'Temporary Redirect',
308: 'Permanent Redirect',
400: 'Bad Request',
401: 'Unauthorized',
402: 'Payment Required',
403: 'Forbidden',
404: 'Not Found',
405: 'Method Not Allowed',
406: 'Not Acceptable',
407: 'Proxy Authentication Required',
408: 'Request Timeout',
409: 'Conflict',
410: 'Gone',
411: 'Length Required',
412: 'Precondition Failed',
413: 'Request Entity Too Large',
414: 'Request-URI Too Long',
415: 'Unsupported Media Type',
416: 'Request Range Not Satisfiable',
417: 'Expectation Failed',
418: 'I\'m a teapot',
419: 'Insufficient Space on Resource',
420: 'Method Failure',
422: 'Unprocessable Entity',
423: 'Locked',
424: 'Failed Dependency',
428: 'Precondition Required',
429: 'Too Many Requests',
431: 'Request Header Fields Too Large',
500: 'Server Error',
501: 'Not Implemented',
502: 'Bad Gateway',
503: 'Service Unavailable',
504: 'Gateway Timeout',
505: 'HTTP Version Not Supported',
507: 'Insufficient Storage',
511: 'Network Authentication Required',
}

View File

@@ -0,0 +1,26 @@
/**
* An Error base-class that also provides some additional context.
*
* All first-party error handlers in Extollo can render the context as part of
* the display of the error (e.g. in the console, in the HTML response, &c.)
*
* @example
* ```typescript
* function myFunc(arg1, arg2) {
* // ...do something...
* throw new ErrorWithContext('Something went wrong!', { arg1, arg2 })
* }
* ```
*/
export class ErrorWithContext extends Error {
public context: {[key: string]: any} = {}
public originalError?: Error
constructor(
message: string,
context?: {[key: string]: any}
) {
super(message)
if ( context ) this.context = context
}
}

34
src/util/index.ts Normal file
View File

@@ -0,0 +1,34 @@
export * from './cache/Cache'
export * from './cache/InMemCache'
export * from './collection/ArrayIterable'
export * from './collection/AsyncCollection'
export * from './collection/Collection'
export * from './collection/Iterable'
export * from './collection/where'
export * from './const/http'
export * from './error/ErrorWithContext'
export * from './logging/Logger'
export * from './logging/StandardLogger'
export * from './logging/types'
export * from './support/BehaviorSubject'
export * from './support/data'
export * from './support/mixin'
export * from './support/path'
export * from './support/debug'
export * from './support/path/Filesystem'
export * from './support/path/LocalFilesystem'
export * from './support/path/SSHFilesystem'
export * from './support/Rehydratable'
export * from './support/string'
export * from './support/timeout'
export * from './support/global'
export * from './support/Pipe'
export * from './support/Messages'
export * from './support/types'

View File

@@ -0,0 +1,50 @@
import {LoggingLevel, LogMessage} from './types'
import * as color from 'colors/safe'
/**
* Base class for an application logger.
*/
export abstract class Logger {
/**
* Write the given message to the log destination.
* @param {LogMessage} message
* @return Promise<void>
*/
public abstract write(message: LogMessage): Promise<void> | void;
/**
* Format the date object to the string output format.
* @param {Date} date
* @return string
*/
protected formatDate(date: Date): string {
const hours = date.getHours()
const minutes = date.getMinutes()
const seconds = date.getSeconds()
return `${date.getFullYear()}-${date.getMonth()+1}-${date.getDate()} ${hours > 9 ? hours : '0' + hours}:${minutes > 9 ? minutes : '0' + minutes}:${seconds > 9 ? seconds : '0' + seconds}`
}
/**
* Given a logging level, get the display string of that level.
* @param {LoggingLevel} level
* @return string
*/
protected levelDisplay(level: LoggingLevel): string {
switch(level) {
case LoggingLevel.Success:
return color.green('success')
case LoggingLevel.Error:
return color.red(' error')
case LoggingLevel.Warning:
return color.yellow('warning')
case LoggingLevel.Info:
return color.blue(' info')
case LoggingLevel.Debug:
return color.cyan(' debug')
case LoggingLevel.Verbose:
return color.gray('verbose')
case LoggingLevel.Silent:
return color.gray(' silent')
}
}
}

View File

@@ -0,0 +1,14 @@
import {Logger} from "./Logger";
import {LogMessage} from "./types";
import * as color from 'colors/safe'
/**
* A Logger implementation that writes to the console.
*/
export class StandardLogger extends Logger {
public write(message: LogMessage) {
const prefix = this.levelDisplay(message.level)
const text = `${prefix} ${color.gray(this.formatDate(message.date))} (${color.cyan(message.callerName || 'Unknown')})`
console.log(text, message.output)
}
}

50
src/util/logging/types.ts Normal file
View File

@@ -0,0 +1,50 @@
/**
* Base type for logging levels.
*/
enum LoggingLevel {
Silent = 0,
Success = 1,
Error = 2,
Warning = 3,
Info = 4,
Debug = 5,
Verbose = 6,
}
/**
* Returns true if the given element is a logging level.
* @param something
* @return boolean
*/
const isLoggingLevel = (something: any): something is LoggingLevel => {
return [
LoggingLevel.Silent,
LoggingLevel.Success,
LoggingLevel.Error,
LoggingLevel.Warning,
LoggingLevel.Info,
LoggingLevel.Debug,
LoggingLevel.Verbose
].includes(something)
}
/**
* Base type for a message written to the log.
*/
interface LogMessage {
level: LoggingLevel,
date: Date,
output: any,
callerName: string,
}
/**
* Returns true if the given object is a log message.
* @param something
* @return boolean
*/
const isLogMessage = (something: any): something is LogMessage => {
return isLoggingLevel(something?.level) && something?.date instanceof Date;
}
export { LoggingLevel, LogMessage, isLoggingLevel, isLogMessage }

View File

@@ -0,0 +1,199 @@
/**
* Base error used to trigger an unsubscribe action from a subscriber.
* @extends Error
*/
export class UnsubscribeError extends Error {}
/**
* Thrown when a closed observable is pushed to.
* @extends Error
*/
export class CompletedObservableError extends Error {
constructor() {
super('This observable can no longer be pushed to, as it has been completed.')
}
}
/**
* Type of a basic subscriber function.
*/
export type SubscriberFunction<T> = (val: T) => any
/**
* Type of a basic subscriber function that handles errors.
*/
export type SubscriberErrorFunction = (error: Error) => any
/**
* Type of a basic subscriber function that handles completed events.
*/
export type SubscriberCompleteFunction<T> = (val?: T) => any
/**
* Subscribers that define multiple handler methods.
*/
export type ComplexSubscriber<T> = {
next?: SubscriberFunction<T>,
error?: SubscriberErrorFunction,
complete?: SubscriberCompleteFunction<T>,
}
/**
* Subscription to a behavior subject.
*/
export type Subscription<T> = SubscriberFunction<T> | ComplexSubscriber<T>
/**
* Object providing helpers for unsubscribing from a subscription.
*/
export type Unsubscribe = { unsubscribe: () => void }
/**
* A stream-based state class.
*/
export class BehaviorSubject<T> {
/**
* Subscribers to this subject.
* @type Array<ComplexSubscriber>
*/
protected subscribers: ComplexSubscriber<T>[] = []
/**
* True if this subject has been marked complete.
* @type boolean
*/
protected _isComplete: boolean = false
/**
* The current value of this subject.
*/
protected _value?: T
/**
* True if any value has been pushed to this subject.
* @type boolean
*/
protected _hasPush: boolean = false
/**
* Register a new subscription to this subject.
* @param {Subscription} subscriber
* @return Unsubscribe
*/
public subscribe(subscriber: Subscription<T>): Unsubscribe {
if ( typeof subscriber === 'function' ) {
this.subscribers.push({ next: subscriber })
} else {
this.subscribers.push(subscriber)
}
return {
unsubscribe: () => {
this.subscribers = this.subscribers.filter(x => x !== subscriber)
}
}
}
/**
* Cast this subject to a promise, which resolves on the output of the next value.
* @return Promise
*/
public toPromise(): Promise<T> {
return new Promise((resolve, reject) => {
const { unsubscribe } = this.subscribe({
next: (val: T) => {
resolve(val)
unsubscribe()
},
error: (error: Error) => {
reject(error)
unsubscribe()
},
complete: (val?: T) => {
if ( typeof val !== 'undefined' ) resolve(val)
unsubscribe()
}
})
})
}
/**
* Push a new value to this subject. The promise resolves when all subscribers have been pushed to.
* @param val
* @return Promise<void>
*/
public async next(val: T): Promise<void> {
if ( this._isComplete ) throw new CompletedObservableError()
this._value = val
this._hasPush = true
for ( const subscriber of this.subscribers ) {
if ( subscriber.next ) {
try {
await subscriber.next(val)
} catch (e) {
if ( e instanceof UnsubscribeError ) {
this.subscribers = this.subscribers.filter(x => x !== subscriber)
} else if (subscriber.error) {
await subscriber.error(e)
} else {
throw e
}
}
}
}
}
/**
* Push the given array of values to this subject in order.
* The promise resolves when all subscribers have been pushed to for all values.
* @param {Array} vals
* @return Promise<void>
*/
public async push(vals: T[]): Promise<void> {
if ( this._isComplete ) throw new CompletedObservableError()
await Promise.all(vals.map(val => this.next(val)))
}
/**
* Mark this subject as complete.
* The promise resolves when all subscribers have been pushed to.
* @param [final_val] - optionally, a final value to set
* @return Promise<void>
*/
public async complete(final_val?: T): Promise<void> {
if ( this._isComplete ) throw new CompletedObservableError()
if ( typeof final_val === 'undefined' ) final_val = this.value()
else this._value = final_val
for ( const subscriber of this.subscribers ) {
if ( subscriber.complete ) {
try {
await subscriber.complete(final_val)
} catch (e) {
if ( subscriber.error ) {
await subscriber.error(e)
} else {
throw e
}
}
}
}
this._isComplete = true
}
/**
* Get the current value of this subject.
*/
public value(): T | undefined {
return this._value
}
/**
* True if this subject is marked as complete.
* @return boolean
*/
public isComplete(): boolean {
return this._isComplete
}
}

View File

@@ -0,0 +1,132 @@
import {collect} from "../collection/Collection";
import {InvalidJSONStateError, JSONState, Rehydratable} from "./Rehydratable";
import {Pipe} from "./Pipe";
/**
* A class for building and working with messages grouped by keys.
*/
export class Messages implements Rehydratable {
protected messages: {[key: string]: string[]} = {}
/**
* @param [allMessages] an initial array of messages to put in the "all" key
*/
constructor(allMessages?: string[]) {
if ( allMessages ) {
this.messages.all = allMessages
}
}
/** All group keys of messages. */
public keys(): string[] {
return Object.keys(this.messages)
}
/** Add a message to a group. */
public put(key: string, message: string): this {
if ( !this.messages[key] ) {
this.messages[key] = []
}
this.messages[key].push(message)
return this
}
/** Returns true if the given group has the message. */
public has(key: string, message: string): boolean {
return !!this.messages[key]?.includes(message)
}
/**
* Returns true if any messages are found.
* @param [forKeys] if provided, only search the given keys
*/
public any(forKeys?: string[]) {
if ( forKeys ) {
return forKeys.map(key => this.messages[key])
.filter(Boolean)
.some((bag: string[]) => bag.length)
}
return Object.values(this.messages).some((bag: string[]) => bag.length)
}
/**
* Returns the first message.
* @param [forKey] if provided, only search the given key
*/
public first(forKey?: string) {
if ( !forKey ) forKey = Object.keys(this.messages)[0]
if ( forKey && this.messages[forKey].length ) {
return this.messages[forKey][0]
}
}
/**
* Return all messages in a flat array.
* @param [forKey] if provided, only search the given key
*/
public all(forKey?: string) {
if ( forKey ) {
return this.messages[forKey] || []
}
return collect<string[]>(Object.values(this.messages)).collapse().all() as string[]
}
/**
* Flat array of only distinct messages.
* @param [forKey] if provided, only search the given key
*/
public unique(forKey?: string) {
return collect<string>(this.all(forKey)).unique<string>().all()
}
/**
* Get the total number of messages.
* @param [forKey] if provided, only search the given key
*/
public count(forKey?: string): number {
return this.all(forKey).length
}
async dehydrate(): Promise<JSONState> {
return this.messages
}
rehydrate(state: JSONState): void {
if ( typeof state === 'object' && !Array.isArray(state) ) {
let all = true
for ( const key in state ) {
if ( !state.hasOwnProperty(key) ) continue;
const set = state[key]
if ( !(Array.isArray(set) && set.every(x => typeof x === 'string')) ) {
all = false;
break;
}
}
if ( all ) {
// @ts-ignore
this.messages = state;
}
}
throw new InvalidJSONStateError('Invalid message state object.', { state });
}
toJSON() {
return this.messages
}
toString() {
return JSON.stringify(this)
}
/**
* Get a new Pipe object wrapping this instance.
*/
pipe(): Pipe<Messages> {
return Pipe.wrap<Messages>(this)
}
}

121
src/util/support/Pipe.ts Normal file
View File

@@ -0,0 +1,121 @@
/**
* A closure that maps a given pipe item to a different type.
*/
export type PipeOperator<T, T2> = (subject: T) => T2
/**
* A closure that maps a given pipe item to an item of the same type.
*/
export type ReflexivePipeOperator<T> = (subject: T) => T
/**
* A class for writing chained/conditional operations in a data-flow manner.
*
* This is useful when you need to do a series of operations on an object, perhaps conditionally.
*
* @example
* Say we have a Collection of items, and want to apply some transformations and filtering based on arguments:
*
* ```typescript
* const collection = collect([1, 2, 3, 4, 5, 6, 7, 8, 9])
*
* function transform(collection, evensOnly = false, returnEntireCollection = false) {
* return Pipe.wrap(collection)
* .when(evensOnly, coll => {
* return coll.filter(x => !(x % 2))
* })
* .unless(returnEntireCollection, coll => {
* return coll.take(3)
* })
* .tap(coll => {
* return coll.map(x => x * 2))
* })
* .get()
* }
*
* transform(collection) // => Collection[2, 4, 6]
*
* transform(collection, true) // => Collection[4, 8, 12]
*
* transform(collection, false, true) // => Collection[2, 4, 6, 8, 10, 12, 14, 16, 18]
* ```
*/
export class Pipe<T> {
/**
* Return a new Pipe containing the given subject.
* @param subject
*/
static wrap<subject_t>(subject: subject_t) {
return new Pipe<subject_t>(subject)
}
constructor(
/**
* The item being operated on.
*/
private subject: T
) {}
/**
* Apply the given operator to the item in the pipe, and return a new pipe with the result.
*
* @example
* ```typescript
* Pipe.wrap(2)
* .tap(x => x * 4)
* .get() // => 8
* ```
*
* @param op
*/
tap<T2>(op: PipeOperator<T, T2>) {
return new Pipe(op(this.subject))
}
/**
* If `check` is truthy, apply the given operator to the item in the pipe and return the result.
* Otherwise, just return the current pipe unchanged.
*
* @param check
* @param op
*/
when(check: boolean, op: ReflexivePipeOperator<T>) {
if ( check ) {
return Pipe.wrap(op(this.subject))
}
return this
}
/**
* If `check` is falsy, apply the given operator to the item in the pipe and return the result.
* Otherwise, just return the current pipe unchanged.
*
* @param check
* @param op
*/
unless(check: boolean, op: ReflexivePipeOperator<T>) {
return this.when(!check, op)
}
/**
* Alias of `unless()`.
* @param check
* @param op
*/
whenNot(check: boolean, op: ReflexivePipeOperator<T>) {
return this.unless(check, op)
}
/**
* Get the item in the pipe.
*
* @example
* ```typescript
* Pipe.wrap(4).get() // => 4
* ```
*/
get(): T {
return this.subject
}
}

View File

@@ -0,0 +1,43 @@
/**
* Type representing a JSON serializable object.
*/
import {ErrorWithContext} from "../error/ErrorWithContext";
export type JSONState = { [key: string]: string | boolean | number | undefined | JSONState | Array<string | boolean | number | undefined | JSONState> }
/**
* An error thrown when the state passed into a class is invalid for that class.
*/
export class InvalidJSONStateError extends ErrorWithContext {}
/**
* Returns true if the given object can be JSON serialized.
* @param what
* @return boolean
*/
export function isJSONState(what: any): what is JSONState {
try {
JSON.stringify(what)
return true
} catch (e) {
return false
}
}
/**
* Base interface for a class that can be rehydrated and restored.
*/
export interface Rehydratable {
/**
* Dehydrate this class' state and get it.
* @return Promise<JSONState>
*/
dehydrate(): Promise<JSONState>
/**
* Rehydrate a state into this class.
* @param {JSONState} state
* @return void|Promise<void>
*/
rehydrate(state: JSONState): void | Promise<void>
}

272
src/util/support/data.ts Normal file
View File

@@ -0,0 +1,272 @@
import * as node_uuid from 'uuid'
import {ErrorWithContext} from "../error/ErrorWithContext";
/**
* Make a deep copy of an object.
* @param target
*/
export function deepCopy<T>(target: T): T {
if ( target === null )
return target
if ( target instanceof Date )
return new Date(target.getTime()) as any
if ( target instanceof Array ) {
const copy = [] as any[]
(target as any[]).forEach(item => { copy.push(item) })
return copy.map((item: any) => deepCopy<any>(item)) as any
}
if ( typeof target === 'object' && target !== {} ) {
const copy = { ...(target as {[key: string]: any }) } as { [key: string]: any }
Object.keys(copy).forEach(key => {
copy[key] = deepCopy<any>(copy[key])
})
}
return target
}
/**
* Given a string of a value, try to infer the JavaScript type.
* @param {string} val
*/
export function infer(val: string) {
if ( !val ) return undefined
else if ( val.toLowerCase() === 'true' ) return true
else if ( val.toLowerCase() === 'false' ) return false
else if ( !isNaN(Number(val)) ) return Number(val)
else if ( isJSON(val) ) return JSON.parse(val)
else if ( val.toLowerCase() === 'null' ) return null
else if ( val.toLowerCase() === 'undefined' ) return undefined
else return val
}
/**
* Given an inferred value, try to convert back to the string form.
* @param val
*/
export function uninfer(val: any) {
if ( typeof val === 'undefined' ) return 'undefined'
else if ( val === true ) return 'true'
else if ( val === false ) return 'false'
else if ( !isNaN(Number(val)) ) return `${Number(val)}`
else if ( typeof val === 'string' ) return val
else return JSON.stringify(val)
}
/**
* Returns true if the given value is valid JSON.
* @param {string} val
*/
export function isJSON(val: string): boolean {
try {
JSON.parse(val)
return true
} catch (e) {
return false
}
}
/**
* Get a universally-unique ID string.
* @return string
*/
export function uuid_v4() {
return node_uuid.v4()
}
/**
* Walks the given data object down to the given path, yielding subkeys.
* This is NOT typesafe and is intended for framework use only.
* @param data
* @param path
* @param defaultValue
* @param currentPath
*/
export function* dataWalkUnsafe<T>(
data: {[key: string]: any} | any[] | undefined,
path: string, defaultValue?: T, currentPath: string = ''
): IterableIterator<[T | T[] | undefined, string]> {
if ( !data ) {
yield [defaultValue, currentPath]
return
} else if ( !path ) {
yield [data as T, currentPath]
return
}
const [part, ...subpath] = path.split('.')
if ( Array.isArray(data) ) {
if ( part === '*' ) {
if ( subpath.length ) {
// @ts-ignore
for ( const val of dataWalkUnsafe<T>(data, subpath.join('.'), defaultValue, currentPath) ) {
yield val
}
} else {
for ( const key in data ) {
yield [data[key], `${currentPath ? currentPath + '.' : ''}${key}`]
}
}
} else if ( !isNaN(parseInt(part)) ) {
const subdata = data[parseInt(part)]
if ( subpath.length ) {
// @ts-ignore
for ( const val of dataWalkUnsafe<T>(subdata, subpath, defaultValue, `${currentPath ? currentPath + '.' : ''}${parseInt(part)}`) ) {
yield val
}
} else {
yield [subdata, `${currentPath ? currentPath + '.' : ''}${parseInt(part)}`]
}
} else {
const subdata = data.map(x => x[part])
if ( subpath.length ) {
// @ts-ignore
for ( const val of dataWalkUnsafe<T>(subdata, subpath, defaultValue, `${currentPath ? currentPath + '.' : ''}${part}`) ) {
yield val
}
} else {
yield [subdata, `${currentPath ? currentPath + '.' : ''}*.${part}`]
}
}
} else {
if ( part === '*' ) {
const subdata = Object.values(data)
if ( subpath.length ) {
// @ts-ignore
for ( const val of dataWalkUnsafe<T>(subdata, subpath, defaultValue, `${currentPath ? currentPath + '.' : ''}*`) ) {
yield val
}
} else {
yield [subdata, `${currentPath ? currentPath + '.' : ''}*`]
}
} else {
const subdata = data[part]
if ( subpath.length ) {
// @ts-ignore
for ( const val of dataWalkUnsafe<T>(subdata, subpath, defaultValue, `${currentPath ? currentPath + '.' : ''}${part}`) ) {
yield val
}
} else {
yield [subdata, `${currentPath ? currentPath + '.' : ''}${part}`]
}
}
}
}
/**
* Set a deep-nested property on the given object.
* This is NOT typesafe and is meant for internal use only.
* @param path
* @param value
* @param data
*/
export function dataSetUnsafe(path: string, value: any, data: {[key: string]: any} | any[] | undefined) {
data = data || ((!isNaN(parseInt(path.split('.')[0]))) ? [] : {})
let current = data
const parts = path.split('.')
parts.forEach((part, idx) => {
if ( idx === path.split('.').length - 1 ) {
current[Array.isArray(current) ? parseInt(part) : part] = value
return
}
let next = Array.isArray(current) ? parseInt(parts[idx + 1]) : current?.[parts[idx + 1]]
if ( !next ) {
next = (!isNaN(parseInt(parts[idx + 1])) ? [] : {})
}
if ( Array.isArray(current) ) {
if ( isNaN(parseInt(part)) ) {
throw new ErrorWithContext(`Invalid property name "${part}" of array-type.`, {part, path})
}
current[parseInt(part)] = next
current = next
} else {
current[part] = next
current = next
}
})
return data
}
/**
* Get a deep-nested property on the given data object.
* This is NOT typesafe and is meant for internal use only.
* @param data
* @param path
* @param defaultValue
*/
export function dataGetUnsafe<T>(data: {[key: string]: any} | any[] | undefined, path: string, defaultValue?: T): T | T[] | undefined {
if ( !data ) return
if ( !path ) return data as T
const [part, ...subpath] = path.split('.')
if ( Array.isArray(data) ) {
if ( part === '*' ) {
return dataGetUnsafe<T>(data, subpath.join('.'), defaultValue)
} else if ( !isNaN(parseInt(part)) ) {
const subdata = data[parseInt(part)]
if ( subpath.length ) return dataGetUnsafe<T>(subdata, subpath.join('.'), defaultValue)
else return subdata
} else {
const subdata = data.map(x => x[part])
if ( subpath.length ) return dataGetUnsafe<T>(subdata, subpath.join('.'), defaultValue)
else return subdata
}
} else {
if ( part === '*' ) {
const subdata = Object.values(data)
if ( subpath.length ) return dataGetUnsafe<T>(subdata, subpath.join('.'), defaultValue)
else return subdata
} else {
const subdata = data[part]
if ( subpath.length ) return dataGetUnsafe<T>(subdata, subpath.join('.'), defaultValue)
else return subdata
}
}
}
function dataGet<
T,
P1 extends keyof NonNullable<T>
>(obj: T, prop1: P1): NonNullable<T>[P1] | undefined;
function dataGet<
T,
P1 extends keyof NonNullable<T>,
P2 extends keyof NonNullable<NonNullable<T>[P1]>
>(obj: T, prop1: P1, prop2: P2): NonNullable<NonNullable<T>[P1]>[P2] | undefined;
function dataGet<
T,
P1 extends keyof NonNullable<T>,
P2 extends keyof NonNullable<NonNullable<T>[P1]>,
P3 extends keyof NonNullable<NonNullable<NonNullable<T>[P1]>[P2]>
>(obj: T, prop1: P1, prop2: P2, prop3: P3): NonNullable<NonNullable<NonNullable<T>[P1]>[P2]>[P3] | undefined;
function dataGet<
T,
P1 extends keyof NonNullable<T>,
P2 extends keyof NonNullable<NonNullable<T>[P1]>,
P3 extends keyof NonNullable<NonNullable<NonNullable<T>[P1]>[P2]>,
P4 extends keyof NonNullable<NonNullable<NonNullable<NonNullable<T>[P1]>[P2]>[P3]>
>(obj: T, prop1: P1, prop2: P2, prop3: P3, prop4: P4): NonNullable<NonNullable<NonNullable<NonNullable<T>[P1]>[P2]>[P3]>[P4] | undefined;
/**
* A type-safe way to get the value of a key nested up to 4 levels deep in an object.
* @param obj
* @param props
*/
function dataGet(obj: any, ...props: string[]): any {
return obj && props.reduce(
(result, prop) => result == null ? undefined : result[prop],
obj
);
}

13
src/util/support/debug.ts Normal file
View File

@@ -0,0 +1,13 @@
export function isDebugging(key: string): boolean {
const env = 'EXTOLLO_DEBUG_' + key.split(/(?:\s|\.)+/).join('_').toUpperCase()
return process.env[env] === 'yes'
}
export function ifDebugging(key: string, run: () => any) {
if ( isDebugging(key) ) run()
}
export function logIfDebugging(key: string, ...output: any[]) {
ifDebugging(key, () => console.log(`[debug: ${key}]`, ...output))
}

View File

@@ -0,0 +1,48 @@
import {Collection} from "../collection/Collection";
import {uuid_v4} from "./data";
/**
* Type structure for a single item in the global registry.
*/
export type GlobalRegistrant = { key: string | symbol, value: any }
/**
* A convenient class to manage global variables.
*/
export class GlobalRegistry extends Collection<GlobalRegistrant> {
constructor() {
super()
this.setGlobal('registry_uuid', uuid_v4())
}
/**
* Store the given `value` associated by `key`.
* @param key
* @param value
*/
public setGlobal(key: string | symbol, value: any): this {
const existing = this.firstWhere('key', '=', key)
if ( existing ) {
existing.value = value
} else {
this.push({ key, value })
}
return this
}
/**
* Retrieve the value of the given `key`, if it exists in the store.
* @param key
*/
public getGlobal(key: string | symbol): any {
return this.firstWhere('key', '=', key)?.value
}
}
/**
* Export a singleton global registry for all code to share.
*
* Exporting an instance here guarantees that all code that import it will get the same instance.
*/
export const globalRegistry = new GlobalRegistry()

19
src/util/support/mixin.ts Normal file
View File

@@ -0,0 +1,19 @@
/**
* Apply the given mixin classes to the given constructor.
* @param derivedCtor
* @param {array} baseCtors
*/
export function applyMixins(derivedCtor: any, baseCtors: any[]) {
baseCtors.forEach(baseCtor => {
Object.getOwnPropertyNames(baseCtor.prototype).forEach(name => {
const desc = Object.getOwnPropertyDescriptor(baseCtor.prototype, name)
if ( typeof desc !== 'undefined' )
Object.defineProperty(derivedCtor.prototype, name, desc)
})
})
}
/**
* Base type for a constructor function.
*/
export type Constructor<T = {}> = new (...args: any[]) => T

348
src/util/support/path.ts Normal file
View File

@@ -0,0 +1,348 @@
import * as nodePath from 'path'
import * as fs from 'fs'
import * as mkdirp from 'mkdirp'
import { Filesystem } from "./path/Filesystem"
import ReadableStream = NodeJS.ReadableStream;
import WritableStream = NodeJS.WritableStream;
/**
* Possible prefixes for files referenced by a UniversalPath.
*/
export enum UniversalPathPrefix {
HTTP = 'http://',
HTTPS = 'https://',
Local = 'file://',
}
/**
* An item that could represent a path.
*/
export type PathLike = string | UniversalPath
/**
* Create a new UniversalPath from the given path-like segments.
* @param parts
*/
export function universalPath(...parts: PathLike[]): UniversalPath {
let [main, ...concats] = parts
if ( !(main instanceof UniversalPath) ) main = new UniversalPath(main)
return main.concat(...concats)
}
/**
* Walk recursively over entries in a directory.
*
* Right now the types are kinda weird for async iterables. This is like an async
* IterableIterable that resolves a string or another IterableIterable of the same type.
*
* Hence why it's separate from the UniversalPath class.
*
* @param dir
*/
export async function* walk(dir: string): any {
for await (const sub of await fs.promises.opendir(dir) ) {
const entry = nodePath.join(dir, sub.name)
if ( sub.isDirectory() ) yield* walk(entry)
else if ( sub.isFile() ) yield entry
}
}
/**
* Class representing some kind of filesystem resource.
*/
export class UniversalPath {
protected _prefix!: string
protected _local!: string
constructor(
/**
* The path string this path refers to.
*
* @example /home/user/file.txt
* @example https://site.com/file.txt
*/
protected readonly initial: string,
protected readonly filesystem?: Filesystem,
) {
this.setPrefix()
this.setLocal()
}
/**
* Determine the correct prefix for this path.
* @protected
*/
protected setPrefix() {
if ( this.initial.toLowerCase().startsWith('http://') ) {
this._prefix = UniversalPathPrefix.HTTP
} else if ( this.initial.toLowerCase().startsWith('https://') ) {
this._prefix = UniversalPathPrefix.HTTPS
} else if ( this.filesystem ) {
this._prefix = this.filesystem.getPrefix()
} else {
this._prefix = UniversalPathPrefix.Local
}
}
/**
* Determine the "localized" string of this path.
*
* This is the normalized path WITHOUT the prefix.
*
* @example
* The normalized path of "file:///home/user/file.txt" is "/home/user/file.txt".
*
* @protected
*/
protected setLocal() {
this._local = this.initial
if ( this.initial.toLowerCase().startsWith(this._prefix) ) {
this._local = this._local.slice(this._prefix.length)
}
if ( this._prefix === UniversalPathPrefix.Local && !this._local.startsWith('/') && !this.filesystem ) {
this._local = nodePath.resolve(this._local)
}
}
/**
* Return a new copy of this UniversalPath instance.
*/
clone() {
return new UniversalPath(this.initial)
}
/**
* Get the UniversalPathPrefix of this resource.
*/
get prefix() {
return this._prefix
}
/**
* Returns true if this resource refers to a file on the local filesystem.
*/
get isLocal() {
return this._prefix === UniversalPathPrefix.Local && !this.filesystem
}
/**
* Returns true if this resource refers to a file on a remote filesystem.
*/
get isRemote() {
return this._prefix !== UniversalPathPrefix.Local || this.filesystem
}
/**
* Get the non-prefixed path to this resource.
*/
get unqualified() {
return this._local
}
/**
* Get the path to this resource as it would be accessed from the current filesystem.
*/
get toLocal() {
if ( this.isLocal ) {
return this._local
} else {
return `${this.prefix}${this._local}`
}
}
/**
* Get the fully-prefixed path to this resource.
*/
get toRemote() {
return `${this.prefix}${this._local}`
}
/**
* Append and resolve the given paths to this resource and return a new UniversalPath.
*
* @example
* ```typescript
* const homeDir = universalPath('home', 'user')
*
* homeDir.concat('file.txt').toLocal // => /home/user/file.txt
*
* homeDir.concat('..', 'other_user').toLocal // => /home/other_user
* ```
*
* @param paths
*/
public concat(...paths: PathLike[]): UniversalPath {
const resolved = nodePath.join(this.unqualified, ...(paths.map(p => typeof p === 'string' ? p : p.unqualified)))
return new UniversalPath(`${this.prefix}${resolved}`)
}
/**
* Append the given path-like item to this resource's path.
* Unlike `concat`, this mutates the current instance.
* @param path
*/
public append(path: PathLike): this {
this._local += String(path)
return this
}
/**
* Cast the path to a string (fully-prefixed).
*/
toString() {
return `${this.prefix}${this._local}`
}
/**
* Get the extension of the resource referred to by this instance.
*
* @example
* ```typescript
* const myFile = universalPath('home', 'user', 'file.txt')
*
* myFile.ext // => 'txt'
* ```
*/
get ext() {
return nodePath.extname(this._local)
}
/**
* Recursively walk all files in this directory. Must be a local resource.
*
* This returns an async generator function.
*
* @example
* ```typescript
* const configFiles = universalPath('home', 'user', '.config')
*
* for await (const configFile of configFiles.walk()) {
* // configFile is a string
* // ... do something ...
* }
* ```
*/
walk() {
return walk(this._local)
}
/**
* Returns true if the given resource exists at the path.
*/
async exists() {
if ( this.filesystem ) {
const stat = await this.filesystem.stat({
storePath: this._local
})
return stat.exists
}
try {
await fs.promises.stat(this._local)
return true
} catch(e) {
return false
}
}
/**
* Recursively create this path as a directory. Equivalent to `mkdir -p` on Linux.
*/
async mkdir() {
if ( this.filesystem ) {
await this.filesystem.mkdir({
storePath: this._local
})
} else {
await mkdirp(this._local)
}
}
/**
* Write the given data to this resource as a file.
* @param data
*/
async write(data: string | Buffer) {
if ( typeof data === 'string' ) data = Buffer.from(data, 'utf8')
if ( this.filesystem ) {
const stream = await this.filesystem.putStoreFileAsStream({
storePath: this._local
})
await new Promise<void>((res, rej) => {
stream.write(data, err => {
if ( err ) rej(err)
else res()
})
})
} else {
const fd = await fs.promises.open(this._local, 'w')
await fd.write(data)
await fd.close()
}
}
/**
* Get a writable stream to this file's contents.
*/
async writeStream(): Promise<WritableStream> {
if ( this.filesystem ) {
return this.filesystem.putStoreFileAsStream({
storePath: this._local
})
} else {
return fs.createWriteStream(this._local)
}
}
/**
* Read the data from this resource's file as a string.
*/
async read() {
let stream: ReadableStream
if ( this.filesystem ) {
stream = await this.filesystem.getStoreFileAsStream({
storePath: this._local
})
} else {
stream = fs.createReadStream(this._local)
}
const chunks: any[] = []
return new Promise<string>((res, rej) => {
stream.on('data', chunk => chunks.push(Buffer.from(chunk)))
stream.on('error', rej)
stream.on('end', () => res(Buffer.concat(chunks).toString('utf-8')))
})
}
/**
* Get a readable stream of this file's contents.
*/
async readStream(): Promise<ReadableStream> {
if ( this.filesystem ) {
return this.filesystem.getStoreFileAsStream({
storePath: this._local
})
} else {
return fs.createReadStream(this._local)
}
}
/*get mime_type() {
return Mime.lookup(this.ext)
}
get content_type() {
return Mime.contentType(this.ext)
}
get charset() {
if ( this.mime_type ) {
return Mime.charset(this.mime_type)
}
}*/
}

View File

@@ -0,0 +1,200 @@
import {UniversalPath} from "../path"
import * as path from "path"
import * as os from "os"
import {uuid_v4} from "../data"
import ReadableStream = NodeJS.ReadableStream;
import WritableStream = NodeJS.WritableStream;
import {ErrorWithContext} from "../../error/ErrorWithContext";
/**
* Error thrown when an operation is attempted on a non-existent file.
*/
export class FileNotFoundError extends ErrorWithContext {
constructor(filename: string, context: any = {}) {
super(`The specified file does not exist: ${filename}`, context)
}
}
/**
* Interface representing metadata that can be stored about a given file.
*/
export interface FileMetadata {
/**
* Tags associated with this file.
*/
tags?: string[],
/**
* The mime-type of this file.
*/
mimeType?: string,
/**
* Miscellaneous metadata about the file.
*/
misc?: {[key: string]: string},
}
/**
* Interface defining information about a file.
*/
export interface Stat {
/**
* UniversalPath resource pointing to the file in its filesystem.
*/
path: UniversalPath,
/**
* True if the file exists. False otherwise.
*/
exists: boolean,
/**
* The size, in bytes, of the file on the remote filesystem.
* If `exists` is false, this number is undefined.
*/
sizeInBytes: number,
/**
* If specified, the mime-type of the remote file.
*/
mimeType?: string,
/**
* Tags associated with the remote file.
*/
tags: string[],
accessed?: Date,
modified?: Date,
created?: Date,
}
/**
* Abstract base-class for remote filesystem implementations.
*/
export abstract class Filesystem {
/**
* Called when the Filesystem driver is initialized. Do any standup here.
*/
public open(): void | Promise<void> {}
/**
* Called when the Filesystem driver is destroyed. Do any cleanup here.
*/
public close(): void | Promise<void> {}
/**
* Get the URI prefix for this filesystem.
* @example `file://`
* @example `s3://`
*/
public abstract getPrefix(): string
/**
* Get a UniversalPath that refers to a file on this filesystem.
* @param storePath
*/
public getPath(storePath: string): UniversalPath {
return new UniversalPath(storePath, this)
}
/**
* Store a file from the local filesystem into the remote filesystem.
*
* @example
* ```ts
* await store.putLocalFile({
* localPath: '/tmp/temp.file',
* storePath: 'my/upload-key/temp.file',
* mimeType: 'application/json',
* tags: ['json', 'user-data'],
* })
* ```
*
* @param args
*/
public abstract putLocalFile(args: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}): void | Promise<void>
/**
* Download a file in the remote filesystem to the local filesystem and return it as a UniversalPath.
* @param args
*/
public abstract getStoreFileAsTemp(args: {storePath: string}): UniversalPath | Promise<UniversalPath>
/**
* Open a readable stream for a file in the remote filesystem.
* @param args
*/
public abstract getStoreFileAsStream(args: {storePath: string}): ReadableStream | Promise<ReadableStream>
/**
* Open a writable stream for a file in the remote filesystem.
* @param args
*/
public abstract putStoreFileAsStream(args: {storePath: string}): WritableStream | Promise<WritableStream>
/**
* Fetch some information about a file that may or may not be in the remote filesystem without fetching the entire file.
* @param args
*/
public abstract stat(args: {storePath: string}): Stat | Promise<Stat>
/**
* If the file does not exist in the remote filesystem, create it. If it does exist, update the modify timestamps.
* @param args
*/
public abstract touch(args: {storePath: string}): void | Promise<void>
/**
* Remove the given resource(s) from the remote filesystem.
* @param args
*/
public abstract remove(args: {storePath: string, recursive?: boolean }): void | Promise<void>
/**
* Create the given path on the store as a directory, recursively.
* @param args
*/
public abstract mkdir(args: {storePath: string}): void | Promise<void>
/**
* Get the metadata object for the given file, if it exists.
* @param storePath
*/
public abstract getMetadata(storePath: string): FileMetadata | Promise<FileMetadata>
/**
* Set the metadata object for the given file, if the file exists.
* @param storePath
* @param meta
*/
public abstract setMetadata(storePath: string, meta: FileMetadata): void | Promise<void>
/**
* Normalize the input tags into a single array of strings. This is useful for implementing the fluent
* interface for `putLocalFile()`.
*
* @example
* ```typescript
* const tags: string[] = this._normalizeTags(args.tag, args.tags)
* ```
*
* @param tag
* @param tags
* @protected
*/
protected _normalizeTags(tag?: string, tags?: string[]): string[] {
if ( !tags ) tags = []
if ( tag ) tags.push(tag)
return tags
}
/**
* Generate the name of a temp-file on the LOCAL filesystem.
* @protected
*/
protected tempName(): string {
return path.resolve(os.tmpdir(), uuid_v4())
}
}

View File

@@ -0,0 +1,161 @@
import {FileMetadata, FileNotFoundError, Filesystem, Stat} from "./Filesystem"
import * as fs from "fs"
import * as path from "path"
import {UniversalPath} from "../path"
import * as rimraf from "rimraf"
import * as mkdirp from "mkdirp"
export interface LocalFilesystemConfig {
baseDir: string
}
/**
* A Filesystem implementation that stores files on the local disk.
* @todo walk
*/
export class LocalFilesystem extends Filesystem {
constructor(
protected readonly baseConfig: LocalFilesystemConfig
) { super() }
async open(): Promise<void> {
// Make sure the base directory exists
await mkdirp(this.baseConfig.baseDir)
await mkdirp(this.storePath(''))
await mkdirp(this.metadataPath(''))
}
public getPrefix(): string {
return 'local://'
}
public async putLocalFile({localPath, storePath, ...args}: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}) {
await fs.promises.copyFile(localPath, this.storePath(storePath))
await fs.promises.writeFile(this.metadataPath(storePath), JSON.stringify({
mimeType: args.mimeType,
tags: this._normalizeTags(args.tag, args.tags),
}))
}
public async getStoreFileAsTemp({ storePath }: {storePath: string}): Promise<UniversalPath> {
const tempName = this.tempName()
await fs.promises.copyFile(this.storePath(storePath), tempName)
return new UniversalPath(tempName)
}
public getStoreFileAsStream(args: { storePath: string }): fs.ReadStream | Promise<fs.ReadStream> {
return fs.createReadStream(this.storePath(args.storePath))
}
public putStoreFileAsStream(args: { storePath: string }): fs.WriteStream | Promise<fs.WriteStream> {
return fs.createWriteStream(this.storePath(args.storePath))
}
public async getMetadata(storePath: string) {
try {
const json = (await fs.promises.readFile(this.metadataPath(storePath))).toString('utf-8')
return JSON.parse(json)
} catch (e) {
return {
tags: []
}
}
}
async setMetadata(storePath: string, meta: FileMetadata): Promise<void> {
if ( !(await this.stat({storePath})).exists ) {
throw new FileNotFoundError(storePath)
}
const metaPath = this.metadataPath(storePath)
await fs.promises.writeFile(metaPath, JSON.stringify(meta))
}
public async stat(args: { storePath: string }): Promise<Stat> {
try {
const stat = await fs.promises.stat(this.storePath(args.storePath))
const meta = await this.getMetadata(args.storePath)
return {
path: new UniversalPath(args.storePath, this),
exists: true,
sizeInBytes: stat.size,
mimeType: meta.mimeType,
tags: meta.tags,
accessed: stat.atime,
modified: stat.mtime,
created: stat.ctime,
}
} catch (e) {
if ( e?.code === 'ENOENT' ) {
return {
path: new UniversalPath(args.storePath, this),
exists: false,
sizeInBytes: 0,
tags: []
}
}
throw e
}
}
public async touch(args: {storePath: string}): Promise<void> {
return new Promise<void>((res, rej) => {
const storePath = this.storePath(args.storePath)
const time = new Date()
fs.utimes(storePath, time, time, err => {
if ( err ) {
fs.open(storePath, 'w', (err2, fd) => {
if ( err2 ) return rej(err2)
fs.close(fd, err3 => {
if ( err3 ) return rej(err3)
res()
})
})
} else {
res()
}
})
})
}
public async remove(args: { storePath: string; recursive?: boolean }): Promise<void> {
if ( !args.recursive ) {
await fs.promises.unlink(this.storePath(args.storePath))
await fs.promises.unlink(this.metadataPath(args.storePath))
} else {
await new Promise<void>((res, rej) => {
rimraf(this.storePath(args.storePath), err => {
if ( err ) return rej(err)
else {
fs.promises.unlink(this.metadataPath(args.storePath)).then(() => res()).catch(rej)
}
})
})
}
}
public async mkdir(args: {storePath: string}): Promise<void> {
await mkdirp(this.storePath(args.storePath))
}
/**
* Given a relative path in the store, resolve it to an absolute path.
* @param storePath
* @protected
*/
protected storePath(storePath: string): string {
return path.resolve(this.baseConfig.baseDir, 'data', storePath)
}
/**
* Given a relative path in the store, resolve it to an absolute path to the metadata JSON
* file for that path.
* @param storePath
* @protected
*/
protected metadataPath(storePath: string): string {
return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
}
}

View File

@@ -0,0 +1,246 @@
import {FileMetadata, Filesystem, Stat} from "./Filesystem"
import * as ssh2 from "ssh2"
import * as path from "path"
import * as fs from "fs"
import ReadableStream = NodeJS.ReadableStream
import {UniversalPath} from "../path"
/**
* A Filesystem implementation that stores files on remote hosts via SFTP/SSH.
*/
export class SSHFilesystem extends Filesystem {
private _ssh?: ssh2.Client
constructor(
protected readonly baseConfig: { ssh: ssh2.ConnectConfig, baseDir: string },
) { super() }
getPrefix(): string {
return `sftp+${this.baseConfig.ssh.host}://`
}
async getStoreFileAsTemp(args: { storePath: string }): Promise<UniversalPath> {
const temp = this.tempName()
const write = fs.createWriteStream(temp)
const read = await this.getStoreFileAsStream(args)
return new Promise<UniversalPath>((res, rej) => {
write.on('finish', () => {
res(new UniversalPath(temp))
})
write.on('error', rej)
read.on('error', rej)
read.pipe(write)
})
}
async getStoreFileAsStream(args: { storePath: string }): Promise<ReadableStream> {
const sftp = await this.getSFTP()
return sftp.createReadStream(this.storePath(args.storePath))
}
async putLocalFile(args: { localPath: string; storePath: string; mimeType?: string; tags?: string[]; tag?: string }): Promise<void> {
const read = fs.createReadStream(args.localPath)
const write = await this.putStoreFileAsStream({storePath: args.storePath})
// write the metadata first
const sftp = await this.getSFTP()
await sftp.writeFile(this.metadataPath(args.storePath), JSON.stringify({
mimeType: args.mimeType,
tags: this._normalizeTags(args.tag, args.tags)
}))
// pipe the local file to the store
await new Promise<void>((res, rej) => {
write.on('finish', () => res())
write.on('error', rej)
read.on('error', rej)
read.pipe(write)
})
}
async putStoreFileAsStream(args: { storePath: string }): Promise<NodeJS.WritableStream> {
const sftp = await this.getSFTP()
return sftp.createWriteStream(this.storePath(args.storePath))
}
async mkdir(args: { storePath: string }): Promise<void> {
const sftp = await this.getSFTP()
await new Promise<void>((res, rej) => {
sftp.mkdir(this.storePath(args.storePath), err => {
if ( err ) rej(err)
else res()
})
})
}
async remove(args: { storePath: string; recursive?: boolean }): Promise<void> {
const sftp = await this.getSFTP()
await new Promise<void>((res, rej) => {
sftp.unlink(this.storePath(args.storePath), err => {
if ( err ) return rej(err)
else {
sftp.unlink(this.metadataPath(args.storePath), err2 => {
if ( err2 ) rej(err2)
else res()
})
}
})
})
}
async stat(args: { storePath: string }): Promise<Stat> {
const sftp = await this.getSFTP()
try {
const stat = await new Promise<any>((res, rej) => {
sftp.stat(this.storePath(args.storePath), (err, stat) => {
if ( err ) return rej(err)
res(stat)
})
})
const jsonStream = sftp.createReadStream(this.metadataPath(args.storePath))
const json = await this.streamToString(jsonStream)
const meta = JSON.parse(json)
return {
path: new UniversalPath(args.storePath, this),
exists: true,
sizeInBytes: stat.size,
mimeType: meta.mimeType,
tags: meta.tags,
accessed: stat.atime,
modified: stat.mtime,
created: stat.ctime,
}
} catch (e) {
return {
path: new UniversalPath(args.storePath, this),
exists: false,
sizeInBytes: 0,
tags: []
}
}
}
async touch(args: { storePath: string }): Promise<void> {
const sftp = await this.getSFTP()
return new Promise<void>((res, rej) => {
const storePath = this.storePath(args.storePath)
const time = new Date()
sftp.utimes(storePath, time, time, err => {
if ( err ) {
sftp.open(storePath, 'w', (err2, fd) => {
if ( err2 ) return rej(err2)
sftp.close(fd, err3 => {
if ( err3 ) return rej(err3)
res()
})
})
} else {
res()
}
})
})
}
async getMetadata(storePath: string): Promise<FileMetadata> {
try {
const sftp = await this.getSFTP()
return new Promise((res, rej) => {
sftp.readFile(this.metadataPath(storePath), (err, buffer) => {
if ( err ) rej(err)
res(JSON.parse(buffer.toString('utf-8')))
})
})
} catch (e) {
return {
tags: []
}
}
}
async setMetadata(storePath: string, meta: FileMetadata): Promise<void> {
const sftp = await this.getSFTP()
const metaPath = this.metadataPath(storePath)
await new Promise<void>((res, rej) => {
sftp.writeFile(metaPath, JSON.stringify(meta), err => {
if ( err ) rej(err)
else res()
})
})
}
async close(): Promise<void> {
await this._ssh?.end()
}
/**
* Using the provided `hostConfig`, create and cache the SSH connection to the host.
* If a connection already exists, re-use it.
*/
async getSSH(): Promise<ssh2.Client> {
if ( this._ssh ) return this._ssh
return new Promise((res, rej) => {
const client = new ssh2.Client()
client.on('ready', () => {
this._ssh = client
res(client)
}).connect(this.baseConfig.ssh)
client.on('error', rej)
})
}
/**
* Using `getSSH()`, create a new SFTP helper based on that connection.
*/
async getSFTP(): Promise<ssh2.SFTPWrapper> {
const ssh = await this.getSSH()
return new Promise((res, rej) => {
ssh.sftp((err, sftp) => {
if ( err ) rej(err)
else res(sftp)
})
})
}
/**
* Resolve the given store path to an absolute path on the remote filesystem.
* @param storePath
* @protected
*/
protected storePath(storePath: string): string {
return path.resolve(this.baseConfig.baseDir, 'data', storePath)
}
/**
* Resolve the given store path to an absolute path of a metadata file on the remote filesystem.
* @param storePath
* @protected
*/
protected metadataPath(storePath: string): string {
return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
}
/**
* Given a readable stream, cast it to a string.
* @param stream
* @protected
*/
protected streamToString(stream: NodeJS.ReadableStream): Promise<string> {
const chunks: Buffer[] = [];
return new Promise((resolve, reject) => {
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)))
stream.on('error', (err) => reject(err))
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
})
}
}

View File

@@ -0,0 +1,49 @@
/**
* If `string` is less than `length` characters long, add however many `padWith` characters
* are necessary to make up the difference to the END of the string.
* @param string
* @param length
* @param padWith
*/
export function padRight(string: string, length: number, padWith: string = ' ') {
while ( string.length < length ) {
string += padWith
}
return string
}
/**
* If `string` is less than `length` characters long, add however many `padWith` characters
* are necessary to make up the difference to the BEGINNING of the string.
* @param string
* @param length
* @param padWith
*/
export function padLeft(string: string, length: number, padWith: string = ' ') {
while ( string.length < length ) {
string = `${padWith}${string}`
}
return string
}
/**
* If `string` is less than `length` characters long, add however many `padWith` characters
* are necessary to make up the difference evenly split between the beginning and end of the string.
* @param string
* @param length
* @param padWith
*/
export function padCenter(string: string, length: number, padWith: string = ' ') {
let bit = false
while ( string.length < length ) {
if ( bit ) {
string = `${padWith}${string}`
} else {
string += padWith
}
}
return string
}

View File

@@ -0,0 +1,74 @@
/**
* Base interface representing a timeout subscriber.
*/
export interface TimeoutSubscriber<T> {
/**
* Handler will execute if the promise resolves before the timeout occurs.
* @param handler
*/
onTime: (handler: (arg: T) => any) => TimeoutSubscriber<T>,
/**
* Handler will execute if the promise resolves after the timeout occurs.
* @param handler
*/
late: (handler: (arg: T) => any) => TimeoutSubscriber<T>,
/**
* Handler will execute if the promise has not resolved when the timeout occurs.
* @param handler
*/
timeout: (handler: () => any) => TimeoutSubscriber<T>,
/**
* Start the timer.
*/
run: () => Promise<T>,
}
/**
* Subscribe to a promise with a timeout.
* @param {number} timeout - timeout in milliseconds
* @param {Promise} promise - the promise to subscribe to
*/
export function withTimeout<T>(timeout: number, promise: Promise<T>) {
let onTimeHandler: (arg: T) => any = (arg) => {}
let lateHandler: (arg: T) => any = (arg) => {}
let timeoutHandler: () => any = () => {}
const sub = {
onTime: handler => {
onTimeHandler = handler
return sub
},
late: handler => {
lateHandler = handler
return sub
},
timeout: handler => {
timeoutHandler = handler
return sub
},
run: async () => {
let expired = false
let resolved = false
setTimeout(() => {
expired = true
if ( !resolved ) timeoutHandler()
}, timeout)
const result: T = await promise
resolved = true
if ( !expired ) {
await onTimeHandler(result)
} else {
await lateHandler(result)
}
return result
}
} as TimeoutSubscriber<T>
return sub
}

View File

@@ -0,0 +1 @@
export type Awaitable<T> = T | Promise<T>