import * as nodePath from 'path' import * as fs from 'fs' import * as mkdirp from 'mkdirp' import * as mime from 'mime-types' import {FileNotFoundError, Filesystem} from './path/Filesystem' import {Collection} from '../collection/Collection' import {Readable, Writable} from 'stream' import {Pipeline} from './Pipe' /** * An item that could represent a path. */ export type PathLike = string | UniversalPath /** * Format bytes as human-readable text. * * @param bytes Number of bytes. * @param si True to use metric (SI) units, aka powers of 1000. False to use * binary (IEC), aka powers of 1024. * @param dp Number of decimal places to display. * @see https://stackoverflow.com/a/14919494/4971138 */ export function bytesToHumanFileSize(bytes: number, si = false, dp = 1): string { const thresh = si ? 1000 : 1024 if (Math.abs(bytes) < thresh) { return bytes + ' B' } const units = si ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'] let u = -1 const r = 10 ** dp do { bytes /= thresh ++u } while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1) return bytes.toFixed(dp) + ' ' + units[u] } /** * Walk recursively over entries in a directory. * * Right now the types are kinda weird for async iterables. This is like an async * IterableIterable that resolves a string or another IterableIterable of the same type. * * Hence why it's separate from the UniversalPath class. * * @param dir */ export async function* walk(dir: string): any { for await (const sub of await fs.promises.opendir(dir) ) { const entry = nodePath.join(dir, sub.name) if ( sub.isDirectory() ) { yield* walk(entry) } else if ( sub.isFile() ) { yield entry } } } /** * Class representing some kind of filesystem resource. */ export class UniversalPath { protected resourcePrefix!: string protected resourceLocalPath!: string protected resourceQuery: URLSearchParams = new URLSearchParams() constructor( /** * The path string this path refers to. * * @example /home/user/file.txt * @example https://site.com/file.txt */ protected readonly initial: string, protected readonly filesystem?: Filesystem, ) { this.setPrefix() this.setLocal() if ( this.isRemote ) { this.resourceQuery = (new URL(this.toRemote)).searchParams } } /** * Determine the correct prefix for this path. * @protected */ protected setPrefix(): void { if ( this.initial.toLowerCase().startsWith('http://') ) { this.resourcePrefix = 'http://' } else if ( this.initial.toLowerCase().startsWith('https://') ) { this.resourcePrefix = 'https://' } else if ( this.filesystem ) { this.resourcePrefix = this.filesystem.getPrefix() } else { this.resourcePrefix = 'file://' } } /** * Determine the "localized" string of this path. * * This is the normalized path WITHOUT the prefix. * * @example * The normalized path of "file:///home/user/file.txt" is "/home/user/file.txt". * * @protected */ protected setLocal(): void { this.resourceLocalPath = this.initial if ( this.initial.toLowerCase().startsWith(this.resourcePrefix) ) { this.resourceLocalPath = this.resourceLocalPath.slice(this.resourcePrefix.length) } if ( this.resourcePrefix === 'file://' && !this.resourceLocalPath.startsWith('/') && !this.filesystem ) { this.resourceLocalPath = nodePath.resolve(this.resourceLocalPath) } } /** * Return a new copy of this UniversalPath instance. */ clone(): UniversalPath { return new UniversalPath(this.initial, this.filesystem) } /** * Get the URLSearchParams for this resource. */ get query(): URLSearchParams { return this.resourceQuery } /** * Get the string of this resource. */ get prefix(): string { return this.resourcePrefix } /** * Returns true if this resource refers to a file on the local filesystem. */ get isLocal(): boolean { return this.resourcePrefix === 'file://' && !this.filesystem } /** * Returns true if this resource refers to a file on a remote filesystem. */ get isRemote(): boolean { return Boolean(this.resourcePrefix !== 'file://' || this.filesystem) } /** * Get the non-prefixed path to this resource. */ get unqualified(): string { return this.resourceLocalPath } /** * Get the path to this resource as it would be accessed from the current filesystem. */ get toLocal(): string { if ( this.isLocal ) { return this.resourceLocalPath } else { return `${this.prefix}${this.resourceLocalPath}` } } /** * Get the fully-prefixed path to this resource. */ get toRemote(): string { const query = this.query.toString() return `${this.prefix}${this.resourceLocalPath}${query ? '?' + query : ''}` } /** * Get the basename of the path. */ get toBase(): string { return nodePath.basename(this.resourceLocalPath) } /** * Append and resolve the given paths to this resource and return a new UniversalPath. * * @example * ```typescript * const homeDir = universalPath('home', 'user') * * homeDir.concat('file.txt').toLocal // => /home/user/file.txt * * homeDir.concat('..', 'other_user').toLocal // => /home/other_user * ``` * * @param paths */ public concat(...paths: PathLike[]): UniversalPath { const resolved = nodePath.join(this.unqualified, ...(paths.map(p => typeof p === 'string' ? p : p.unqualified))) return new UniversalPath(`${this.prefix}${resolved}`, this.filesystem) } /** * Append the given path-like item to this resource's path. * Unlike `concat`, this mutates the current instance. * @param path */ public append(path: PathLike): this { this.resourceLocalPath += String(path) return this } /** * Cast the path to a string (fully-prefixed). */ toString(): string { return `${this.prefix}${this.resourceLocalPath}` } /** * Get the extension of the resource referred to by this instance. * * @example * ```typescript * const myFile = universalPath('home', 'user', 'file.txt') * * myFile.ext // => 'txt' * ``` */ get ext(): string { return nodePath.extname(this.resourceLocalPath) } /** * Recursively walk all files in this directory. Must be a local resource. * * This returns an async generator function. * * @example * ```typescript * const configFiles = universalPath('home', 'user', '.config') * * for await (const configFile of configFiles.walk()) { * // configFile is a string * // ... do something ... * } * ``` */ walk(): any { return walk(this.resourceLocalPath) } /** * Resolves true if this resource is a directory. */ async isDirectory(): Promise { if ( this.filesystem ) { const stat = await this.filesystem.stat({ storePath: this.resourceLocalPath, }) return stat.isDirectory } try { return (await fs.promises.stat(this.resourceLocalPath)).isDirectory() } catch (e) { return false } } /** * Resolves true if this resource is a regular file. */ async isFile(): Promise { if ( this.filesystem ) { const stat = await this.filesystem.stat({ storePath: this.resourceLocalPath, }) return stat.isFile } try { return (await fs.promises.stat(this.resourceLocalPath)).isFile() } catch (e) { return false } } /** * Returns true if the given resource exists at the path. */ async exists(): Promise { if ( this.filesystem ) { const stat = await this.filesystem.stat({ storePath: this.resourceLocalPath, }) return stat.exists } try { await fs.promises.stat(this.resourceLocalPath) return true } catch (e) { return false } } /** * List any immediate children of this resource. */ async list(): Promise> { if ( this.filesystem ) { const files = await this.filesystem.list(this.resourceLocalPath) return files.map(x => this.concat(x)) } const paths = await fs.promises.readdir(this.resourceLocalPath) return Collection.collect(paths) .map(x => this.concat(x)) } /** * Recursively create this path as a directory. Equivalent to `mkdir -p` on Linux. */ async mkdir(): Promise { if ( this.filesystem ) { await this.filesystem.mkdir({ storePath: this.resourceLocalPath, }) } else { await mkdirp(this.resourceLocalPath) } } /** * Write the given data to this resource as a file. * @param data */ async write(data: string | Buffer): Promise { if ( typeof data === 'string' ) { data = Buffer.from(data, 'utf8') } if ( this.filesystem ) { const stream = await this.filesystem.putStoreFileAsStream({ storePath: this.resourceLocalPath, }) await new Promise((res, rej) => { stream.write(data, err => { if ( err ) { rej(err) } else { res() } }) }) } else { const fd = await fs.promises.open(this.resourceLocalPath, 'w') await fd.write(data) await fd.close() } } /** * Get a writable stream to this file's contents. */ async writeStream(): Promise { if ( this.filesystem ) { return this.filesystem.putStoreFileAsStream({ storePath: this.resourceLocalPath, }) } else { return fs.createWriteStream(this.resourceLocalPath) } } /** * Read the data from this resource's file as a string. */ async read(): Promise { let stream: Readable if ( this.filesystem ) { stream = await this.filesystem.getStoreFileAsStream({ storePath: this.resourceLocalPath, }) } else { stream = fs.createReadStream(this.resourceLocalPath) } const chunks: any[] = [] return new Promise((res, rej) => { stream.on('data', chunk => chunks.push(Buffer.from(chunk))) stream.on('error', rej) stream.on('end', () => res(Buffer.concat(chunks).toString('utf-8'))) }) } /** * Get the size of this resource in bytes. */ async sizeInBytes(): Promise { if ( this.filesystem ) { const stat = await this.filesystem.stat({ storePath: this.resourceLocalPath, }) if ( stat.exists ) { return stat.sizeInBytes } throw new FileNotFoundError(this.toString()) } const stat = await fs.promises.stat(this.resourceLocalPath) return stat.size } /** * Get the size of this resource, formatted in a human-readable string. */ async sizeForHumans(): Promise { return bytesToHumanFileSize(await this.sizeInBytes()) } /** * Get a readable stream of this file's contents. */ async readStream(): Promise { if ( this.filesystem ) { return this.filesystem.getStoreFileAsStream({ storePath: this.resourceLocalPath, }) } else { return fs.createReadStream(this.resourceLocalPath) } } /** * Returns true if this path exists in the subtree of the given path. * @param otherPath */ isChildOf(otherPath: UniversalPath): boolean { if ( (this.filesystem || otherPath.filesystem) && otherPath.filesystem !== this.filesystem ) { return false } if ( this.prefix !== otherPath.prefix ) { return false } const relative = nodePath.relative(otherPath.toLocal, this.toLocal) return Boolean(relative && !relative.startsWith('..') && !nodePath.isAbsolute(relative)) } /** * Returns true if the given path exists in the subtree of this path. * @param otherPath */ isParentOf(otherPath: UniversalPath): boolean { return otherPath.isChildOf(this) } /** * Returns true if the given path refers to the same resource as this path. * @param otherPath */ is(otherPath: UniversalPath): boolean { if ( (this.filesystem || otherPath.filesystem) && otherPath.filesystem !== this.filesystem ) { return false } if ( this.prefix !== otherPath.prefix ) { return false } const relative = nodePath.relative(otherPath.toLocal, this.toLocal) return relative === '' } /** * Get the mime-type of this resource. */ get mimeType(): string | false { return mime.lookup(this.toBase) } /** * Get the content-type header of this resource. */ get contentType(): string | false { return mime.contentType(this.toBase) } /** * Get the charset of this resource. */ get charset(): string | false { if ( this.mimeType ) { return mime.charset(this.mimeType) } return false } /** * Return a new Pipe of this collection. */ pipeTo(pipeline: Pipeline): TOut { return pipeline.apply(this) } /** Build and apply a pipeline. */ pipe(builder: (pipeline: Pipeline) => Pipeline): TOut { return builder(Pipeline.id()).apply(this) } }