Import other modules into monorepo
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
This commit is contained in:
200
src/util/support/path/Filesystem.ts
Normal file
200
src/util/support/path/Filesystem.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import {UniversalPath} from "../path"
|
||||
import * as path from "path"
|
||||
import * as os from "os"
|
||||
import {uuid_v4} from "../data"
|
||||
import ReadableStream = NodeJS.ReadableStream;
|
||||
import WritableStream = NodeJS.WritableStream;
|
||||
import {ErrorWithContext} from "../../error/ErrorWithContext";
|
||||
|
||||
/**
|
||||
* Error thrown when an operation is attempted on a non-existent file.
|
||||
*/
|
||||
export class FileNotFoundError extends ErrorWithContext {
|
||||
constructor(filename: string, context: any = {}) {
|
||||
super(`The specified file does not exist: ${filename}`, context)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface representing metadata that can be stored about a given file.
|
||||
*/
|
||||
export interface FileMetadata {
|
||||
/**
|
||||
* Tags associated with this file.
|
||||
*/
|
||||
tags?: string[],
|
||||
|
||||
/**
|
||||
* The mime-type of this file.
|
||||
*/
|
||||
mimeType?: string,
|
||||
|
||||
/**
|
||||
* Miscellaneous metadata about the file.
|
||||
*/
|
||||
misc?: {[key: string]: string},
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface defining information about a file.
|
||||
*/
|
||||
export interface Stat {
|
||||
/**
|
||||
* UniversalPath resource pointing to the file in its filesystem.
|
||||
*/
|
||||
path: UniversalPath,
|
||||
|
||||
/**
|
||||
* True if the file exists. False otherwise.
|
||||
*/
|
||||
exists: boolean,
|
||||
|
||||
/**
|
||||
* The size, in bytes, of the file on the remote filesystem.
|
||||
* If `exists` is false, this number is undefined.
|
||||
*/
|
||||
sizeInBytes: number,
|
||||
|
||||
/**
|
||||
* If specified, the mime-type of the remote file.
|
||||
*/
|
||||
mimeType?: string,
|
||||
|
||||
/**
|
||||
* Tags associated with the remote file.
|
||||
*/
|
||||
tags: string[],
|
||||
|
||||
accessed?: Date,
|
||||
modified?: Date,
|
||||
created?: Date,
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract base-class for remote filesystem implementations.
|
||||
*/
|
||||
export abstract class Filesystem {
|
||||
/**
|
||||
* Called when the Filesystem driver is initialized. Do any standup here.
|
||||
*/
|
||||
public open(): void | Promise<void> {}
|
||||
|
||||
/**
|
||||
* Called when the Filesystem driver is destroyed. Do any cleanup here.
|
||||
*/
|
||||
public close(): void | Promise<void> {}
|
||||
|
||||
/**
|
||||
* Get the URI prefix for this filesystem.
|
||||
* @example `file://`
|
||||
* @example `s3://`
|
||||
*/
|
||||
public abstract getPrefix(): string
|
||||
|
||||
/**
|
||||
* Get a UniversalPath that refers to a file on this filesystem.
|
||||
* @param storePath
|
||||
*/
|
||||
public getPath(storePath: string): UniversalPath {
|
||||
return new UniversalPath(storePath, this)
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a file from the local filesystem into the remote filesystem.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* await store.putLocalFile({
|
||||
* localPath: '/tmp/temp.file',
|
||||
* storePath: 'my/upload-key/temp.file',
|
||||
* mimeType: 'application/json',
|
||||
* tags: ['json', 'user-data'],
|
||||
* })
|
||||
* ```
|
||||
*
|
||||
* @param args
|
||||
*/
|
||||
public abstract putLocalFile(args: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}): void | Promise<void>
|
||||
|
||||
/**
|
||||
* Download a file in the remote filesystem to the local filesystem and return it as a UniversalPath.
|
||||
* @param args
|
||||
*/
|
||||
public abstract getStoreFileAsTemp(args: {storePath: string}): UniversalPath | Promise<UniversalPath>
|
||||
|
||||
/**
|
||||
* Open a readable stream for a file in the remote filesystem.
|
||||
* @param args
|
||||
*/
|
||||
public abstract getStoreFileAsStream(args: {storePath: string}): ReadableStream | Promise<ReadableStream>
|
||||
|
||||
/**
|
||||
* Open a writable stream for a file in the remote filesystem.
|
||||
* @param args
|
||||
*/
|
||||
public abstract putStoreFileAsStream(args: {storePath: string}): WritableStream | Promise<WritableStream>
|
||||
|
||||
/**
|
||||
* Fetch some information about a file that may or may not be in the remote filesystem without fetching the entire file.
|
||||
* @param args
|
||||
*/
|
||||
public abstract stat(args: {storePath: string}): Stat | Promise<Stat>
|
||||
|
||||
/**
|
||||
* If the file does not exist in the remote filesystem, create it. If it does exist, update the modify timestamps.
|
||||
* @param args
|
||||
*/
|
||||
public abstract touch(args: {storePath: string}): void | Promise<void>
|
||||
|
||||
/**
|
||||
* Remove the given resource(s) from the remote filesystem.
|
||||
* @param args
|
||||
*/
|
||||
public abstract remove(args: {storePath: string, recursive?: boolean }): void | Promise<void>
|
||||
|
||||
/**
|
||||
* Create the given path on the store as a directory, recursively.
|
||||
* @param args
|
||||
*/
|
||||
public abstract mkdir(args: {storePath: string}): void | Promise<void>
|
||||
|
||||
/**
|
||||
* Get the metadata object for the given file, if it exists.
|
||||
* @param storePath
|
||||
*/
|
||||
public abstract getMetadata(storePath: string): FileMetadata | Promise<FileMetadata>
|
||||
|
||||
/**
|
||||
* Set the metadata object for the given file, if the file exists.
|
||||
* @param storePath
|
||||
* @param meta
|
||||
*/
|
||||
public abstract setMetadata(storePath: string, meta: FileMetadata): void | Promise<void>
|
||||
|
||||
/**
|
||||
* Normalize the input tags into a single array of strings. This is useful for implementing the fluent
|
||||
* interface for `putLocalFile()`.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const tags: string[] = this._normalizeTags(args.tag, args.tags)
|
||||
* ```
|
||||
*
|
||||
* @param tag
|
||||
* @param tags
|
||||
* @protected
|
||||
*/
|
||||
protected _normalizeTags(tag?: string, tags?: string[]): string[] {
|
||||
if ( !tags ) tags = []
|
||||
if ( tag ) tags.push(tag)
|
||||
return tags
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the name of a temp-file on the LOCAL filesystem.
|
||||
* @protected
|
||||
*/
|
||||
protected tempName(): string {
|
||||
return path.resolve(os.tmpdir(), uuid_v4())
|
||||
}
|
||||
}
|
||||
161
src/util/support/path/LocalFilesystem.ts
Normal file
161
src/util/support/path/LocalFilesystem.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
import {FileMetadata, FileNotFoundError, Filesystem, Stat} from "./Filesystem"
|
||||
import * as fs from "fs"
|
||||
import * as path from "path"
|
||||
import {UniversalPath} from "../path"
|
||||
import * as rimraf from "rimraf"
|
||||
import * as mkdirp from "mkdirp"
|
||||
|
||||
export interface LocalFilesystemConfig {
|
||||
baseDir: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A Filesystem implementation that stores files on the local disk.
|
||||
* @todo walk
|
||||
*/
|
||||
export class LocalFilesystem extends Filesystem {
|
||||
constructor(
|
||||
protected readonly baseConfig: LocalFilesystemConfig
|
||||
) { super() }
|
||||
|
||||
async open(): Promise<void> {
|
||||
// Make sure the base directory exists
|
||||
await mkdirp(this.baseConfig.baseDir)
|
||||
await mkdirp(this.storePath(''))
|
||||
await mkdirp(this.metadataPath(''))
|
||||
}
|
||||
|
||||
public getPrefix(): string {
|
||||
return 'local://'
|
||||
}
|
||||
|
||||
public async putLocalFile({localPath, storePath, ...args}: {localPath: string, storePath: string, mimeType?: string, tags?: string[], tag?: string}) {
|
||||
await fs.promises.copyFile(localPath, this.storePath(storePath))
|
||||
await fs.promises.writeFile(this.metadataPath(storePath), JSON.stringify({
|
||||
mimeType: args.mimeType,
|
||||
tags: this._normalizeTags(args.tag, args.tags),
|
||||
}))
|
||||
}
|
||||
|
||||
public async getStoreFileAsTemp({ storePath }: {storePath: string}): Promise<UniversalPath> {
|
||||
const tempName = this.tempName()
|
||||
await fs.promises.copyFile(this.storePath(storePath), tempName)
|
||||
return new UniversalPath(tempName)
|
||||
}
|
||||
|
||||
public getStoreFileAsStream(args: { storePath: string }): fs.ReadStream | Promise<fs.ReadStream> {
|
||||
return fs.createReadStream(this.storePath(args.storePath))
|
||||
}
|
||||
|
||||
public putStoreFileAsStream(args: { storePath: string }): fs.WriteStream | Promise<fs.WriteStream> {
|
||||
return fs.createWriteStream(this.storePath(args.storePath))
|
||||
}
|
||||
|
||||
public async getMetadata(storePath: string) {
|
||||
try {
|
||||
const json = (await fs.promises.readFile(this.metadataPath(storePath))).toString('utf-8')
|
||||
return JSON.parse(json)
|
||||
} catch (e) {
|
||||
return {
|
||||
tags: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async setMetadata(storePath: string, meta: FileMetadata): Promise<void> {
|
||||
if ( !(await this.stat({storePath})).exists ) {
|
||||
throw new FileNotFoundError(storePath)
|
||||
}
|
||||
|
||||
const metaPath = this.metadataPath(storePath)
|
||||
await fs.promises.writeFile(metaPath, JSON.stringify(meta))
|
||||
}
|
||||
|
||||
public async stat(args: { storePath: string }): Promise<Stat> {
|
||||
try {
|
||||
const stat = await fs.promises.stat(this.storePath(args.storePath))
|
||||
const meta = await this.getMetadata(args.storePath)
|
||||
|
||||
return {
|
||||
path: new UniversalPath(args.storePath, this),
|
||||
exists: true,
|
||||
sizeInBytes: stat.size,
|
||||
mimeType: meta.mimeType,
|
||||
tags: meta.tags,
|
||||
accessed: stat.atime,
|
||||
modified: stat.mtime,
|
||||
created: stat.ctime,
|
||||
}
|
||||
} catch (e) {
|
||||
if ( e?.code === 'ENOENT' ) {
|
||||
return {
|
||||
path: new UniversalPath(args.storePath, this),
|
||||
exists: false,
|
||||
sizeInBytes: 0,
|
||||
tags: []
|
||||
}
|
||||
}
|
||||
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
public async touch(args: {storePath: string}): Promise<void> {
|
||||
return new Promise<void>((res, rej) => {
|
||||
const storePath = this.storePath(args.storePath)
|
||||
const time = new Date()
|
||||
fs.utimes(storePath, time, time, err => {
|
||||
if ( err ) {
|
||||
fs.open(storePath, 'w', (err2, fd) => {
|
||||
if ( err2 ) return rej(err2)
|
||||
fs.close(fd, err3 => {
|
||||
if ( err3 ) return rej(err3)
|
||||
res()
|
||||
})
|
||||
})
|
||||
} else {
|
||||
res()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
public async remove(args: { storePath: string; recursive?: boolean }): Promise<void> {
|
||||
if ( !args.recursive ) {
|
||||
await fs.promises.unlink(this.storePath(args.storePath))
|
||||
await fs.promises.unlink(this.metadataPath(args.storePath))
|
||||
} else {
|
||||
await new Promise<void>((res, rej) => {
|
||||
rimraf(this.storePath(args.storePath), err => {
|
||||
if ( err ) return rej(err)
|
||||
else {
|
||||
fs.promises.unlink(this.metadataPath(args.storePath)).then(() => res()).catch(rej)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
public async mkdir(args: {storePath: string}): Promise<void> {
|
||||
await mkdirp(this.storePath(args.storePath))
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a relative path in the store, resolve it to an absolute path.
|
||||
* @param storePath
|
||||
* @protected
|
||||
*/
|
||||
protected storePath(storePath: string): string {
|
||||
return path.resolve(this.baseConfig.baseDir, 'data', storePath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a relative path in the store, resolve it to an absolute path to the metadata JSON
|
||||
* file for that path.
|
||||
* @param storePath
|
||||
* @protected
|
||||
*/
|
||||
protected metadataPath(storePath: string): string {
|
||||
return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
|
||||
}
|
||||
}
|
||||
246
src/util/support/path/SSHFilesystem.ts
Normal file
246
src/util/support/path/SSHFilesystem.ts
Normal file
@@ -0,0 +1,246 @@
|
||||
import {FileMetadata, Filesystem, Stat} from "./Filesystem"
|
||||
import * as ssh2 from "ssh2"
|
||||
import * as path from "path"
|
||||
import * as fs from "fs"
|
||||
import ReadableStream = NodeJS.ReadableStream
|
||||
import {UniversalPath} from "../path"
|
||||
|
||||
/**
|
||||
* A Filesystem implementation that stores files on remote hosts via SFTP/SSH.
|
||||
*/
|
||||
export class SSHFilesystem extends Filesystem {
|
||||
private _ssh?: ssh2.Client
|
||||
|
||||
constructor(
|
||||
protected readonly baseConfig: { ssh: ssh2.ConnectConfig, baseDir: string },
|
||||
) { super() }
|
||||
|
||||
getPrefix(): string {
|
||||
return `sftp+${this.baseConfig.ssh.host}://`
|
||||
}
|
||||
|
||||
async getStoreFileAsTemp(args: { storePath: string }): Promise<UniversalPath> {
|
||||
const temp = this.tempName()
|
||||
const write = fs.createWriteStream(temp)
|
||||
const read = await this.getStoreFileAsStream(args)
|
||||
|
||||
return new Promise<UniversalPath>((res, rej) => {
|
||||
write.on('finish', () => {
|
||||
res(new UniversalPath(temp))
|
||||
})
|
||||
|
||||
write.on('error', rej)
|
||||
read.on('error', rej)
|
||||
read.pipe(write)
|
||||
})
|
||||
}
|
||||
|
||||
async getStoreFileAsStream(args: { storePath: string }): Promise<ReadableStream> {
|
||||
const sftp = await this.getSFTP()
|
||||
return sftp.createReadStream(this.storePath(args.storePath))
|
||||
}
|
||||
|
||||
async putLocalFile(args: { localPath: string; storePath: string; mimeType?: string; tags?: string[]; tag?: string }): Promise<void> {
|
||||
const read = fs.createReadStream(args.localPath)
|
||||
const write = await this.putStoreFileAsStream({storePath: args.storePath})
|
||||
|
||||
// write the metadata first
|
||||
const sftp = await this.getSFTP()
|
||||
await sftp.writeFile(this.metadataPath(args.storePath), JSON.stringify({
|
||||
mimeType: args.mimeType,
|
||||
tags: this._normalizeTags(args.tag, args.tags)
|
||||
}))
|
||||
|
||||
// pipe the local file to the store
|
||||
await new Promise<void>((res, rej) => {
|
||||
write.on('finish', () => res())
|
||||
write.on('error', rej)
|
||||
read.on('error', rej)
|
||||
read.pipe(write)
|
||||
})
|
||||
}
|
||||
|
||||
async putStoreFileAsStream(args: { storePath: string }): Promise<NodeJS.WritableStream> {
|
||||
const sftp = await this.getSFTP()
|
||||
return sftp.createWriteStream(this.storePath(args.storePath))
|
||||
}
|
||||
|
||||
async mkdir(args: { storePath: string }): Promise<void> {
|
||||
const sftp = await this.getSFTP()
|
||||
await new Promise<void>((res, rej) => {
|
||||
sftp.mkdir(this.storePath(args.storePath), err => {
|
||||
if ( err ) rej(err)
|
||||
else res()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async remove(args: { storePath: string; recursive?: boolean }): Promise<void> {
|
||||
const sftp = await this.getSFTP()
|
||||
|
||||
await new Promise<void>((res, rej) => {
|
||||
sftp.unlink(this.storePath(args.storePath), err => {
|
||||
if ( err ) return rej(err)
|
||||
else {
|
||||
sftp.unlink(this.metadataPath(args.storePath), err2 => {
|
||||
if ( err2 ) rej(err2)
|
||||
else res()
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async stat(args: { storePath: string }): Promise<Stat> {
|
||||
const sftp = await this.getSFTP()
|
||||
|
||||
try {
|
||||
const stat = await new Promise<any>((res, rej) => {
|
||||
sftp.stat(this.storePath(args.storePath), (err, stat) => {
|
||||
if ( err ) return rej(err)
|
||||
res(stat)
|
||||
})
|
||||
})
|
||||
|
||||
const jsonStream = sftp.createReadStream(this.metadataPath(args.storePath))
|
||||
const json = await this.streamToString(jsonStream)
|
||||
const meta = JSON.parse(json)
|
||||
|
||||
return {
|
||||
path: new UniversalPath(args.storePath, this),
|
||||
exists: true,
|
||||
sizeInBytes: stat.size,
|
||||
mimeType: meta.mimeType,
|
||||
tags: meta.tags,
|
||||
accessed: stat.atime,
|
||||
modified: stat.mtime,
|
||||
created: stat.ctime,
|
||||
}
|
||||
} catch (e) {
|
||||
return {
|
||||
path: new UniversalPath(args.storePath, this),
|
||||
exists: false,
|
||||
sizeInBytes: 0,
|
||||
tags: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async touch(args: { storePath: string }): Promise<void> {
|
||||
const sftp = await this.getSFTP()
|
||||
|
||||
return new Promise<void>((res, rej) => {
|
||||
const storePath = this.storePath(args.storePath)
|
||||
const time = new Date()
|
||||
sftp.utimes(storePath, time, time, err => {
|
||||
if ( err ) {
|
||||
sftp.open(storePath, 'w', (err2, fd) => {
|
||||
if ( err2 ) return rej(err2)
|
||||
sftp.close(fd, err3 => {
|
||||
if ( err3 ) return rej(err3)
|
||||
res()
|
||||
})
|
||||
})
|
||||
} else {
|
||||
res()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async getMetadata(storePath: string): Promise<FileMetadata> {
|
||||
try {
|
||||
const sftp = await this.getSFTP()
|
||||
return new Promise((res, rej) => {
|
||||
sftp.readFile(this.metadataPath(storePath), (err, buffer) => {
|
||||
if ( err ) rej(err)
|
||||
res(JSON.parse(buffer.toString('utf-8')))
|
||||
})
|
||||
})
|
||||
} catch (e) {
|
||||
return {
|
||||
tags: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async setMetadata(storePath: string, meta: FileMetadata): Promise<void> {
|
||||
const sftp = await this.getSFTP()
|
||||
const metaPath = this.metadataPath(storePath)
|
||||
await new Promise<void>((res, rej) => {
|
||||
sftp.writeFile(metaPath, JSON.stringify(meta), err => {
|
||||
if ( err ) rej(err)
|
||||
else res()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this._ssh?.end()
|
||||
}
|
||||
|
||||
/**
|
||||
* Using the provided `hostConfig`, create and cache the SSH connection to the host.
|
||||
* If a connection already exists, re-use it.
|
||||
*/
|
||||
async getSSH(): Promise<ssh2.Client> {
|
||||
if ( this._ssh ) return this._ssh
|
||||
|
||||
return new Promise((res, rej) => {
|
||||
const client = new ssh2.Client()
|
||||
|
||||
client.on('ready', () => {
|
||||
this._ssh = client
|
||||
res(client)
|
||||
}).connect(this.baseConfig.ssh)
|
||||
|
||||
client.on('error', rej)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Using `getSSH()`, create a new SFTP helper based on that connection.
|
||||
*/
|
||||
async getSFTP(): Promise<ssh2.SFTPWrapper> {
|
||||
const ssh = await this.getSSH()
|
||||
|
||||
return new Promise((res, rej) => {
|
||||
ssh.sftp((err, sftp) => {
|
||||
if ( err ) rej(err)
|
||||
else res(sftp)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the given store path to an absolute path on the remote filesystem.
|
||||
* @param storePath
|
||||
* @protected
|
||||
*/
|
||||
protected storePath(storePath: string): string {
|
||||
return path.resolve(this.baseConfig.baseDir, 'data', storePath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the given store path to an absolute path of a metadata file on the remote filesystem.
|
||||
* @param storePath
|
||||
* @protected
|
||||
*/
|
||||
protected metadataPath(storePath: string): string {
|
||||
return path.resolve(this.baseConfig.baseDir, 'meta', storePath + '.json')
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a readable stream, cast it to a string.
|
||||
* @param stream
|
||||
* @protected
|
||||
*/
|
||||
protected streamToString(stream: NodeJS.ReadableStream): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)))
|
||||
stream.on('error', (err) => reject(err))
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user