0.14.14: Misc bugfixes in migrations & AsyncCollection keys

.drone.yml

@@ -20,7 +20,9 @@ steps:
   - name: typedoc build
     image: node:18
     commands:
+      - "node -v"
       - "npm add --global pnpm"
+      - "pnpm --version"
       - pnpm i
       - pnpm run docs:build
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@extollo/lib",
-  "version": "0.14.11",
+  "version": "0.14.14",
   "description": "The framework library that lifts up your code.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -11,45 +11,45 @@
     "@atao60/fse-cli": "^0.1.7",
     "@extollo/ui": "^0.1.0",
     "@types/bcrypt": "^5.0.0",
-    "@types/busboy": "^0.2.3",
+    "@types/busboy": "^0.2.4",
     "@types/cli-table": "^0.3.1",
-    "@types/ioredis": "^4.26.6",
+    "@types/ioredis": "^4.28.10",
     "@types/jsonwebtoken": "^8.5.9",
     "@types/mime-types": "^2.1.1",
     "@types/mkdirp": "^1.0.2",
     "@types/negotiator": "^0.6.1",
-    "@types/node": "^14.17.4",
-    "@types/pg": "^8.6.5",
+    "@types/node": "^14.18.51",
+    "@types/pg": "^8.10.2",
     "@types/pluralize": "^0.0.29",
     "@types/pug": "^2.0.6",
     "@types/rimraf": "^3.0.2",
-    "@types/ssh2": "^0.5.46",
+    "@types/ssh2": "^0.5.52",
     "@types/uuid": "^8.3.4",
-    "@types/ws": "^8.5.3",
-    "bcrypt": "^5.0.1",
+    "@types/ws": "^8.5.5",
+    "bcrypt": "^5.1.0",
     "busboy": "^0.3.1",
     "cli-table": "^0.3.11",
     "colors": "^1.4.0",
-    "dotenv": "^8.2.0",
-    "ioredis": "^4.27.6",
+    "dotenv": "^8.6.0",
+    "ioredis": "^4.28.5",
     "jsonwebtoken": "^8.5.1",
     "mime-types": "^2.1.35",
     "mkdirp": "^1.0.4",
     "negotiator": "^0.6.3",
-    "node-fetch": "^3.2.10",
-    "pg": "^8.8.0",
+    "node-fetch": "^3.3.1",
+    "pg": "^8.11.0",
     "pluralize": "^8.0.0",
     "pug": "^3.0.2",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
-    "sqlite": "^4.1.2",
-    "sqlite3": "^5.1.1",
-    "ssh2": "^1.11.0",
+    "sqlite": "^4.2.1",
+    "sqlite3": "^5.1.6",
+    "ssh2": "^1.13.0",
     "ts-node": "^10.9.1",
-    "typescript": "^4.8.4",
+    "typescript": "^4.9.5",
     "uuid": "^8.3.2",
-    "ws": "^8.9.0",
-    "zod": "^3.11.6"
+    "ws": "^8.13.0",
+    "zod": "^3.21.4"
   },
   "scripts": {
     "test": "env TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\" }' mocha -r ts-node/register 'tests/**/*.ts'",
@@ -73,18 +73,19 @@
   "author": "garrettmills <shout@garrettmills.dev>",
   "license": "MIT",
   "devDependencies": {
-    "@knodes/typedoc-plugin-pages": "^0.23.1",
-    "@types/chai": "^4.3.3",
-    "@types/mocha": "^9.0.0",
-    "@types/sinon": "^10.0.13",
+    "@knodes/typedoc-plugin-pages": "^0.23.4",
+    "@types/chai": "^4.3.5",
+    "@types/mocha": "^9.1.1",
+    "@types/sinon": "^10.0.15",
     "@types/wtfnode": "^0.7.0",
-    "@typescript-eslint/eslint-plugin": "^5.38.1",
-    "@typescript-eslint/parser": "^5.38.1",
-    "chai": "^4.3.6",
-    "eslint": "^8.24.0",
-    "mocha": "^9.1.3",
+    "@typescript-eslint/eslint-plugin": "^5.59.11",
+    "@typescript-eslint/parser": "^5.59.11",
+    "chai": "^4.3.7",
+    "eslint": "^8.42.0",
+    "lunr": "^2.3.9",
+    "mocha": "^9.2.2",
     "sinon": "^12.0.1",
-    "typedoc": "^0.23.21",
+    "typedoc": "^0.23.28",
     "wtfnode": "^0.9.1"
   },
   "extollo": {

src/auth/provider/oauth/CoreIDLoginProvider.ts

@@ -92,5 +92,8 @@ export class CoreIDLoginProvider extends OAuth2LoginProvider<OAuth2LoginProvider
         user.email = data.email
         user.tagline = data.tagline
         user.photoUrl = data.profile_photo
+        if ( typeof user.save === 'function' ) {
+            user.save()
+        }
     }
 }

src/di/Container.ts

@@ -48,7 +48,7 @@ export interface AwareOfContainerLifecycle {
 
 export function isAwareOfContainerLifecycle(what: unknown): what is AwareOfContainerLifecycle {
     return Boolean(
-        typeof what === 'object'
+        (typeof what === 'object' || typeof what === 'function')
         && what !== null
         && hasOwnProperty(what, 'awareOfContainerLifecycle')
         && what.awareOfContainerLifecycle,

src/orm/dialect/PostgreSQLDialect.ts

@@ -544,7 +544,7 @@ export class PostgreSQLDialect extends SQLDialect {
         return parts.join('\n')
     }
 
-    public renderAlterTable(builder: TableBuilder): string {
+    public renderAlterTable(builder: TableBuilder): Maybe<string> {
         const alters: string[] = []
         const columns = builder.getColumns()
 
@@ -628,6 +628,10 @@ export class PostgreSQLDialect extends SQLDialect {
             alters.push(`    RENAME TO "${builder.getRename()}"`)
         }
 
+        if ( !alters.length ) {
+            return undefined
+        }
+
         return 'ALTER TABLE ' + builder.name + '\n' + alters.join(',\n')
     }
 

src/orm/dialect/SQLDialect.ts

@@ -2,7 +2,7 @@ import {Constraint, QuerySource} from '../types'
 import {AbstractBuilder} from '../builder/AbstractBuilder'
 import {AppClass} from '../../lifecycle/AppClass'
 import {ColumnBuilder, IndexBuilder, TableBuilder} from '../schema/TableBuilder'
-import {Collectable, Collection} from '../../util'
+import {Collectable, Collection, Maybe} from '../../util'
 
 /** A scalar value which can be interpolated safely into an SQL query. */
 export type ScalarEscapeValue = null | undefined | string | number | boolean | Date | QuerySafeValue;
@@ -198,7 +198,7 @@ export abstract class SQLDialect extends AppClass {
      * Given a table schema-builder, render an `ALTER TABLE...` query.
      * @param builder
      */
-    public abstract renderAlterTable(builder: TableBuilder): string;
+    public abstract renderAlterTable(builder: TableBuilder): Maybe<string>;
 
     /**
      * Given a table schema-builder, render a `DROP TABLE...` query.
@@ -314,7 +314,10 @@ export abstract class SQLDialect extends AppClass {
         if ( !builder.isExisting() && builder.isDirty() ) {
             parts.push(this.renderCreateTable(builder))
         } else if ( builder.isExisting() && builder.isDirty() ) {
-            parts.push(this.renderAlterTable(builder))
+            const alterTable = this.renderAlterTable(builder)
+            if ( alterTable ) {
+                parts.push(alterTable)
+            }
         }
 
         // Render the various schema queries as a single transaction

src/orm/schema/TableBuilder.ts

@@ -223,7 +223,16 @@ export class ColumnBuilder extends SchemaBuilderBase {
      * @param type
      */
     public type(type: FieldType): this {
-        if ( this.targetType === type ) {
+        if (
+            this.targetType === type
+            || (
+                this.existsInSchema
+                && (
+                    (this.targetType === FieldType.integer && type === FieldType.serial)
+                    || (this.targetType === FieldType.bigint && type === FieldType.bigserial)
+                )
+            )
+        ) {
             return this
         }
 

src/service/WebsocketServer.ts

@@ -68,7 +68,7 @@ export class WebsocketServer extends Unit {
 
         // Start the websocket server
         this.logging.info('Starting WebSocket server...')
-        this.server = new WebSocket.Server<WebSocket.WebSocket>({
+        this.server = new WebSocket.Server<typeof WebSocket.WebSocket>({
             server: this.http.getServer(),
         })
 

src/util/collection/AsyncCollection.ts

@@ -43,7 +43,8 @@ export class AsyncCollection<T> {
             if ( typeof key !== 'function' ) {
                 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
                 // @ts-ignore
-                key = x => x[key]
+                await callback(items.map(x => x[key]))
+                return
             }
 
             await callback(items.map(key))
@@ -56,7 +57,9 @@ export class AsyncCollection<T> {
             if ( typeof key !== 'function' ) {
                 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
                 // @ts-ignore
-                key = x => x[key]
+                await callback(items.map(x => x[key]).map(x => Number(x))
+                    .all())
+                return
             }
 
             await callback(items.map(key).map(x => Number(x))

src/util/index.ts

@@ -33,6 +33,8 @@ export * from './support/debug'
 export * from './support/path/Filesystem'
 export * from './support/path/LocalFilesystem'
 export * from './support/path/SSHFilesystem'
+export * from './support/path/ReadOnlyFilesystem'
+export * from './support/path/HTTPFilesystem'
 
 export * from './support/Safe'
 
@@ -43,3 +45,5 @@ export * from './support/global'
 export * from './support/Pipe'
 export * from './support/Messages'
 export * from './support/types'
+
+export * from './support/path-helpers'

src/util/logging/FileLogger.ts

@@ -1,7 +1,7 @@
 import {Logger} from './Logger'
 import {LogMessage} from './types'
 import {Injectable} from '../../di'
-import {universalPath} from '../support/path'
+import {universalPath} from '../support/path-helpers'
 import {appPath, env} from '../../lifecycle/Application'
 import {Writable} from 'stream'
 

src/util/support/path-helpers.ts

@@ -0,0 +1,21 @@
+import {PathLike, UniversalPath} from './path'
+import {HTTPFilesystem} from './path/HTTPFilesystem'
+import {make} from '../../make'
+import {Filesystem} from './path/Filesystem'
+import {Maybe} from './types'
+
+/**
+ * Create a new UniversalPath from the given path-like segments.
+ * @param parts
+ */
+export function universalPath(...parts: PathLike[]): UniversalPath {
+    let [main, ...concats] = parts  // eslint-disable-line prefer-const
+    if ( !(main instanceof UniversalPath) ) {
+        let fs: Maybe<Filesystem> = undefined
+        if ( main.toLowerCase().startsWith('https://') || main.toLowerCase().startsWith('http://') ) {
+            fs = make(HTTPFilesystem)
+        }
+        main = new UniversalPath(main, fs)
+    }
+    return main.concat(...concats)
+}

src/util/support/path.ts

@@ -12,18 +12,6 @@ import {Pipeline} from './Pipe'
  */
 export type PathLike = string | UniversalPath
 
-/**
- * Create a new UniversalPath from the given path-like segments.
- * @param parts
- */
-export function universalPath(...parts: PathLike[]): UniversalPath {
-    let [main, ...concats] = parts  // eslint-disable-line prefer-const
-    if ( !(main instanceof UniversalPath) ) {
-        main = new UniversalPath(main)
-    }
-    return main.concat(...concats)
-}
-
 /**
  * Format bytes as human-readable text.
  *
@@ -144,7 +132,7 @@ export class UniversalPath {
      * Return a new copy of this UniversalPath instance.
      */
     clone(): UniversalPath {
-        return new UniversalPath(this.initial)
+        return new UniversalPath(this.initial, this.filesystem)
     }
 
     /**
@@ -224,7 +212,7 @@ export class UniversalPath {
      */
     public concat(...paths: PathLike[]): UniversalPath {
         const resolved = nodePath.join(this.unqualified, ...(paths.map(p => typeof p === 'string' ? p : p.unqualified)))
-        return new UniversalPath(`${this.prefix}${resolved}`)
+        return new UniversalPath(`${this.prefix}${resolved}`, this.filesystem)
     }
 
     /**

src/util/support/path/HTTPFilesystem.ts

@@ -0,0 +1,86 @@
+import * as fs from 'fs'
+import * as mime from 'mime-types'
+import * as path from 'path'
+import {FilesystemOperationNotSupported, ReadOnlyFilesystem} from './ReadOnlyFilesystem'
+import {UniversalPath} from '../path'
+import {Awaitable, Maybe} from '../types'
+import {Readable} from 'stream'
+import {FileMetadata, Stat} from './Filesystem'
+import {Collection} from '../../collection/Collection'
+import {RequestInfo, RequestInit, Response} from 'node-fetch'
+import {unsafeESMImport} from '../../unsafe'
+
+const fetch = (url: RequestInfo, init?: RequestInit): Promise<Response> => unsafeESMImport('node-fetch').then(({default: nodeFetch}) => nodeFetch(url, init))
+
+/**
+ * A filesystem implementation that reads files over HTTPS.
+ */
+export class HTTPFilesystem extends ReadOnlyFilesystem {
+    getPrefix(): string {
+        return 'https://'
+    }
+
+    async getStoreFileAsTemp(args: { storePath: string }): Promise<UniversalPath> {
+        const temp = this.tempName()
+        const write = fs.createWriteStream(temp)
+        const read = await this.getStoreFileAsStream(args)
+
+        return new Promise<UniversalPath>((res, rej) => {
+            write.on('finish', () => {
+                res(new UniversalPath(temp))
+            })
+
+            write.on('error', rej)
+            read.on('error', rej)
+            read.pipe(write)
+        })
+    }
+
+    async getStoreFileAsStream(args: { storePath: string }): Promise<Readable> {
+        if ( args.storePath.startsWith('/') ) {
+            args.storePath = args.storePath.slice(1)
+        }
+
+        const result = await fetch(`${this.getPrefix()}${args.storePath}`)
+        if ( result.body ) {
+            return (new Readable()).wrap(result.body)
+        }
+
+        return Readable.from([])
+    }
+
+    async stat(args: { storePath: string }): Promise<Stat> {
+        if ( args.storePath.startsWith('/') ) {
+            args.storePath = args.storePath.slice(1)
+        }
+
+        let response: Maybe<Response> = undefined
+        try {
+            response = await fetch(`${this.getPrefix()}${args.storePath}`)
+        } catch (e) {}  // eslint-disable-line no-empty
+
+        return {
+            path: new UniversalPath(args.storePath, this),
+            exists: Boolean(response?.ok),
+            sizeInBytes: response?.size || 0,
+            mimeType: mime.lookup(path.basename(args.storePath)) || undefined,
+            tags: [],
+            accessed: undefined,
+            modified: undefined,
+            created: undefined,
+            isFile: true,
+            isDirectory: false,
+        }
+    }
+
+    getMetadata(storePath: string): Awaitable<FileMetadata> {
+        const mimeType = mime.lookup(path.basename(storePath))
+        return {
+            ...(mimeType ? { mimeType } : {}),
+        }
+    }
+
+    list(): Awaitable<Collection<string>> {
+        throw new FilesystemOperationNotSupported(this, 'list')
+    }
+}

src/util/support/path/ReadOnlyFilesystem.ts

@@ -0,0 +1,42 @@
+import {ErrorWithContext} from '../../error/ErrorWithContext'
+import {Filesystem} from './Filesystem'
+import {Awaitable} from '../types'
+import {Writable} from 'stream'
+
+/**
+ * Error thrown when attempting to perform an operation on a filesystem that doesn't support it.
+ */
+export class FilesystemOperationNotSupported extends ErrorWithContext {
+    constructor(fs: Filesystem, operation: string, context: any = {}) {
+        super(`The filesystem ${fs.constructor.name} does not support the ${operation} operation.`, context)
+    }
+}
+
+/**
+ * Abstract base class for filesystems that are read-only (e.g. HTTP/S).
+ */
+export abstract class ReadOnlyFilesystem extends Filesystem {
+    putLocalFile(): Awaitable<void> {
+        throw new FilesystemOperationNotSupported(this, 'putLocalFile')
+    }
+
+    putStoreFileAsStream(): Awaitable<Writable> {
+        throw new FilesystemOperationNotSupported(this, 'putStoreFileAsStream')
+    }
+
+    touch(): Awaitable<void> {
+        throw new FilesystemOperationNotSupported(this, 'touch')
+    }
+
+    remove(): Awaitable<void> {
+        throw new FilesystemOperationNotSupported(this, 'remove')
+    }
+
+    mkdir(): Awaitable<void> {
+        throw new FilesystemOperationNotSupported(this, 'mkdir')
+    }
+
+    setMetadata(): Awaitable<void> {
+        throw new FilesystemOperationNotSupported(this, 'setMetadata')
+    }
+}

src/util/support/types.ts

@@ -45,7 +45,7 @@ export type ParameterizedCallback<T> = ((arg: T) => any)
 export type KeyValue<T> = {key: string, value: T}
 
 /** Simple helper method to verify that a key is a keyof some object. */
-export function isKeyof<T>(key: unknown, obj: T): key is keyof T {
+export function isKeyof<T extends object>(key: unknown, obj: T): key is keyof T {
     if ( typeof key !== 'string' && typeof key !== 'symbol' ) {
         return false
     }