Initial import

This commit is contained in:
2020-11-26 19:57:37 -06:00
commit 81032e3b0b
33 changed files with 5986 additions and 0 deletions

147
app/ServerUnit.js Normal file
View File

@@ -0,0 +1,147 @@
const WebSocket = require('ws')
const http = require('http')
const url = require('url')
const fs = require('fs')
const tmp = require('tmp-promise')
const websocketStream = require('websocket-stream')
const StreamSkip = require('stream-skip')
const StreamTake = require('./ws/StreamTake')
const Socket = require('./ws/Socket')
const { Unit } = require('libflitter')
const { NodeDescriptorType } = require('./enum')
const { Buffer } = require('buffer')
const { Readable } = require('stream')
class ServerUnit extends Unit {
static get services() {
return [...super.services, 'configs', 'app', 'models', 'upload']
}
sockets = []
static name() {
return 'server'
}
name() {
return 'server'
}
async go(app) {
this.server = new WebSocket.Server({
port: this.configs.get('server.port')
})
this.server.on('connection', socket => {
socket = this.app.di().make(Socket, socket)
this.sockets.push(socket)
socket.ping()
})
this.stream_server = http.createServer().listen(this.configs.get('server.stream_port'))
this.stream_socket = websocketStream.createServer({
server: this.stream_server,
perMessageDeflate: false,
binary: true,
}, (stream, request) => {
const query = url.parse(request.url, true).query
if ( !query.writing_file ) this.on_file_stream(stream, request)
else this.on_file_write_stream(stream, request)
})
await new Promise(res => {
process.on('SIGINT', res)
})
}
async on_file_write_stream(stream, request) {
let { socket_uuid, node_uuid, length = 4096, position = 0 } = url.parse(request.url, true).query
if ( typeof position === 'string' ) position = parseInt(position)
if ( typeof length === 'string' ) length = parseInt(length)
const socket = this.sockets.find(x => x.uuid === socket_uuid)
const Node = this.models.get('fs:Node')
const node = await Node.findOne({
uuid: node_uuid,
deleted: false,
descriptor_type: NodeDescriptorType.File,
})
if ( !socket.session.temp_write_files ) socket.session.temp_write_files = {}
const placeholder = socket.session.temp_write_files?.[node.uuid] || await tmp.file()
socket.session.temp_write_files[node.uuid] = placeholder
console.log('Upload placeholder!', placeholder)
const old_file = await node.uploaded_file()
if ( old_file ) {
if ( position === 0 ) {
// This is a new write, so delete the old file
await old_file.delete()
delete node.uploaded_file_id
} else {
await this.upload.provider().download_file(old_file, placeholder.path)
}
}
console.log('write stream', stream)
console.log('write data', { placeholder, position, length })
stream.pipe(fs.createWriteStream(placeholder.path, { start: position }))
}
_bufferStream(stream) {
const chunks = []
return new Promise((resolve, reject) => {
stream.on('data', chunk => {
console.log('stream data', chunk)
chunks.push(chunk)
})
stream.on('error', reject)
stream.on('end', () => {
console.log('stream end!')
resolve(Buffer.concat(chunks))
})
})
}
async on_file_stream(stream, request) {
let { socket_uuid, node_uuid, length = 4096, position = 0 } = url.parse(request.url, true).query
if ( typeof position === 'string' ) position = parseInt(position)
if ( typeof length === 'string' ) length = parseInt(length)
// const socket = this.sockets.find(x => x.uuid === socket_uuid)
const Node = this.models.get('fs:Node')
const node = await Node.findOne({
uuid: node_uuid,
deleted: false,
descriptor_type: NodeDescriptorType.File,
})
const file = await node.uploaded_file()
if ( file ) {
const readable = this.upload.provider().read_stream(file)
const slicer = new StreamSkip({ skip: position })
const taker = new StreamTake({ take: length })
readable.pipe(slicer).pipe(taker).pipe(stream)
} else {
// If no data was written, just return an empty file
const empty = new Readable()
empty.push('')
empty.push(null)
empty.pipe(stream)
}
}
async cleanup(app) {
this.server.close()
this.stream_server.close()
this.stream_socket.close()
}
}
module.exports = exports = ServerUnit

6
app/enum.js Normal file
View File

@@ -0,0 +1,6 @@
const NodeDescriptorType = {
File: 'file',
Directory: 'directory',
}
module.exports = exports = { NodeDescriptorType }

18
app/models/Token.model.js Normal file
View File

@@ -0,0 +1,18 @@
const { Model } = require('flitter-orm')
const uuid = require('uuid').v4
const gen_token = () => {
return `${uuid()}${uuid()}${uuid()}${uuid()}`.replace(/-/g, '')
}
class Token extends Model {
static get schema() {
return {
user_uuid: String,
token_value: { type: String, default: gen_token },
active: { type: Boolean, default: true },
}
}
}
module.exports = exports = Token

31
app/models/User.model.js Normal file
View File

@@ -0,0 +1,31 @@
const { Model } = require('flitter-orm')
const uuid = require('uuid').v4
class User extends Model {
static get services() {
return [...super.services, 'models']
}
static get schema() {
return {
uuid: { type: String, default: uuid },
username: String,
}
}
async get_token() {
const Token = this.models.get('Token')
const existing = await Token.findOne({
active: true,
user_uuid: this.uuid,
})
if ( existing ) return existing
const generated = new Token({ user_uuid: this.uuid })
await generated.save()
return generated
}
}
module.exports = exports = User

160
app/models/fs/Node.model.js Normal file
View File

@@ -0,0 +1,160 @@
const uuid = require('uuid').v4
const { Model } = require('flitter-orm')
const { NodeDescriptorType } = require('../../enum')
class Node extends Model {
static get services() {
return [...super.services, 'models']
}
static get schema() {
return {
uuid: { type: String, default: uuid },
pied_name: String,
pied_parent_path: { type: String, default: '/' },
overlay_name: { type: String, default: 'mainline' },
mtime: { type: Date, default: () => new Date },
atime: { type: Date, default: () => new Date },
ctime: { type: Date, default: () => new Date },
mode: { type: Number, default: 33188 },
size: { type: Number, default: 0 },
descriptor_type: { type: String, default: NodeDescriptorType.File },
uploaded_file_id: String,
deleted: { type: Boolean, default: false },
root: { type: Boolean, default: false },
}
}
static path_parts(path) {
const path_parts = path.split('/')
const pied_name = path_parts.pop()
let pied_parent_path = path_parts.join('/')
if ( !pied_parent_path.startsWith('/') ) pied_parent_path = `/${pied_parent_path}`
return [pied_parent_path, pied_name]
}
static async get_root() {
let root = await this.findOne({
deleted: false, root: true, pied_parent_path: '/', pied_name: '/',
})
if ( !root ) {
root = new this({
pied_name: '/',
mode: 16877,
root: true,
descriptor_type: NodeDescriptorType.Directory,
})
await root.save()
}
return root
}
static async get_path(path, workspace = 'mainline') {
const [pied_parent_path, pied_name] = this.path_parts(path)
const nodes = await this.find({
pied_name,
pied_parent_path,
overlay_name: {
$in: [workspace, 'mainline'],
},
deleted: false,
root: false,
})
if ( nodes.length === 1 ) {
return nodes[0]
}
if ( nodes.length === 2 ) {
return nodes.find(x => x.overlay_name === workspace)
}
}
static async list_path(path, workspace = 'mainline') {
const nodes = await this.find({
pied_parent_path: path,
overlay_name: workspace,
// deleted: false,
root: false,
})
const mainline_nodes = await this.find({
pied_name: {
$nin: nodes.map(x => x.pied_name),
},
pied_parent_path: path,
overlay_name: 'mainline',
deleted: false,
root: false,
})
return [...nodes, ...mainline_nodes].filter(x => !x.deleted)
}
async uploaded_file() {
if ( !this.uploaded_file_id ) return;
const File = this.models.get('upload::File')
return File.findById(this.uploaded_file_id)
}
async all_descendants(workspace = 'mainline') {
const formatted_parent = `${this.pied_parent_path === '/' ? '/' : this.pied_parent_path + '/'}${this.pied_name}`
const nodes = await this.constructor.find({
pied_parent_path: {
$regex: this.root ? `/.*` : `(?:${formatted_parent}/.*)|(?:${formatted_parent}$)`,
},
overlay_name: workspace,
root: false,
// deleted: false,
})
const mainline_nodes = await this.constructor.find({
$and: [
{
pied_parent_path: {
$regex: this.root ? `/.*` : `(?:${formatted_parent}/.*)|(?:${formatted_parent}$)`,
},
overlay_name: 'mainline',
root: false,
deleted: false,
},
...nodes.map(node => {
return {
pied_parent_path: {
$ne: node.pied_parent_path,
},
pied_name: {
$ne: node.pied_name,
},
}
})
],
})
return [...nodes, ...mainline_nodes].filter(x => !x.deleted)
}
to_api() {
return {
pied_name: this.pied_name,
mtime: this.mtime,
atime: this.atime,
ctime: this.ctime,
mode: this.mode,
nlink: 1,
uid: 0, // TODO
gid: 0, // TODO
size: this.size,
}
}
}
module.exports = exports = Node

4
app/shared.js Normal file
View File

@@ -0,0 +1,4 @@
module.exports = exports = {
Errors: require('../../shared/Errors'),
Message: require('../../shared/Message'),
}

76
app/ws/Socket.js Normal file
View File

@@ -0,0 +1,76 @@
const uuid = require('uuid').v4
const { Message } = require('../shared')
const { Injectable } = require('flitter-di')
class Socket extends Injectable {
static get services() {
return [...super.services, 'app', 'output']
}
messages = []
constructor(socket) {
super()
this.socket = socket
this.uuid = uuid()
this.session = {}
this.socket.on('message', msg => this.on_message(msg))
}
ping() {
this.send(Message.route('meta.ping').expect_response())
}
send(message) {
this.output.debug(message)
if ( typeof message === 'string' ) {
this.socket.send(message)
return
}
if ( message.needs_response ) {
this.messages.push(message)
}
const serial = message.serialize()
this.socket.send(serial)
}
async on_message(msg) {
this.output.info(msg)
const message = new Message(msg)
const response = new Message()
message.socket = this
if ( message.is_response() ) {
// Try to find the message that sent the request
const request = this.messages.find(x => x.uuid() === message.response_to())
if ( request ) {
await request._response_callback(message)
request.has_response = true;
} else {
this.send(
response.response_to(message.uuid())
.error(Errors.InvalidReplyUUID)
)
}
this.messages = this.messages.filter(x => !x.has_response)
} else {
let handler;
try {
handler = require(`./routes/${message.route()}`)
} catch (e) {}
if ( !handler ) {
return this.send(socket, response.error(Errors.InvalidMessageRoute))
}
await handler(message, this.app.di().container.proxy())
}
}
}
module.exports = exports = Socket

35
app/ws/StreamTake.js Normal file
View File

@@ -0,0 +1,35 @@
const stream = require('stream')
const util = require('util')
const Transform = stream.Transform
function Take(options) {
// allow use without new
if (!(this instanceof Take)) {
return new Take(options);
}
this._toTake = options.take || undefined
// init Transform
Transform.call(this, options);
}
util.inherits(Take, Transform);
Take.prototype._transform = function (chunk, enc, cb) {
if ( typeof this._toTake == 'undefined' ) {
this.push(chunk)
}
else if (this._toTake > chunk.length) {
this._toTake -= chunk.length;
this.push(chunk)
} else {
if (this._toTake !== chunk.length) this.push(chunk.slice(0, this._toTake))
this._toTake = 0;
}
cb();
};
module.exports = Take;

View File

@@ -0,0 +1,52 @@
const { Errors } = require('../../shared')
const { NodeDescriptorType } = require('../../enum')
module.exports = exports = async (message, di) => {
const new_fd = (node_uuid) => {
if ( !message.socket.session.last_file_descriptor ) {
message.socket.session.last_file_descriptor = 0
message.socket.session.file_descriptors = {}
}
message.socket.session.last_file_descriptor += 1
message.socket.session.file_descriptors[message.socket.session.last_file_descriptor] = node_uuid
return message.socket.session.last_file_descriptor
}
const Node = di.models.get('fs:Node')
const { path, mode } = message.data()
if ( !path ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( path === '/' ) {
return message.send_response(
message.fresh.error(Errors.NodeAlreadyExists)
)
}
const existing_node = await Node.get_path(path, message.socket.session.overlay_name || 'mainline')
if ( existing_node ) {
return message.send_response(
message.fresh().error(Errors.NodeAlreadyExists)
)
}
const [pied_parent_path, pied_name] = Node.path_parts(path)
const node = new Node({
pied_name,
pied_parent_path,
overlay_name: message.socket.session.overlay_name || 'mainline',
mode: 33188, // TODO account for the mode from the client!
descriptor_type: NodeDescriptorType.File,
})
await node.save()
message.send_response(
message.fresh().data({ node: node.to_api(), descriptor: new_fd(node.uuid) })
)
}

View File

@@ -0,0 +1,26 @@
const { Errors } = require('../../shared')
module.exports = exports = async (message, di) => {
const Node = di.models.get('fs:Node')
const { path } = message.data()
let data
if ( path === '/' ) {
const root = await Node.get_root()
data = root.to_api()
} else {
const node = await Node.get_path(path, message.socket.session.overlay_name || 'mainline')
if ( !node ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
data = node.to_api()
}
message.send_response(
message.fresh().data({ node: data })
)
}

42
app/ws/routes/fs.mkdir.js Normal file
View File

@@ -0,0 +1,42 @@
const { Errors } = require('../../shared')
const { NodeDescriptorType } = require('../../enum')
module.exports = exports = async (message, di) => {
const Node = di.models.get('fs:Node')
const { path, mode } = message.data()
if ( !path ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( path === '/' ) {
return message.send_response(
message.fresh.error(Errors.NodeAlreadyExists)
)
}
const existing_node = await Node.get_path(path, message.socket.session.overlay_name || 'mainline')
if ( existing_node ) {
return message.send_response(
message.fresh().error(Errors.NodeAlreadyExists)
)
}
const [pied_parent_path, pied_name] = Node.path_parts(path)
const node = new Node({
pied_name,
pied_parent_path,
overlay_name: message.socket.session.overlay_name || 'mainline',
mode: 16877, // TODO account for the mode from the client!
descriptor_type: NodeDescriptorType.Directory,
size: 100,
})
await node.save()
message.send_response(
message.fresh().data({ node: node.to_api() })
)
}

33
app/ws/routes/fs.open.js Normal file
View File

@@ -0,0 +1,33 @@
const { Errors } = require('../../shared')
const { NodeDescriptorType } = require('../../enum')
module.exports = exports = async (message, di) => {
const send_error = (err) => message.send_response(
message.fresh().error(err)
)
const new_fd = (node_uuid) => {
if ( !message.socket.session.last_file_descriptor ) {
message.socket.session.last_file_descriptor = 0
message.socket.session.file_descriptors = {}
}
message.socket.session.last_file_descriptor += 1
message.socket.session.file_descriptors[message.socket.session.last_file_descriptor] = node_uuid
return message.socket.session.last_file_descriptor
}
const Node = di.models.get('fs:Node')
const { path, flags } = message.data()
if ( !path ) return send_error(Errors.NodeDoesNotExist)
if ( path === '/' ) return send_error(Errors.IsDirectoryDescriptor)
const node = await Node.get_path(path, message.socket.session.overlay_name || 'mainline')
if ( !node ) send_error(Errors.NodeDoesNotExist)
if ( node.descriptor_type === NodeDescriptorType.Directory ) send_error(Errors.IsDirectoryDescriptor)
message.send_response(
message.fresh().data({ node: node.to_api(), descriptor: new_fd(node.uuid) })
)
}

View File

@@ -0,0 +1,23 @@
const { Errors } = require('../../shared')
module.exports = exports = async (message, di) => {
const Node = di.models.get('fs:Node')
const { path } = message.data()
if ( path !== '/' ) {
// If the path isn't the root of the workspace, then make
// sure that the parent node actually exists
const node = await Node.get_path(path, message.socket.session.overlay_name || 'mainline')
if ( !node ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
}
const nodes = await Node.list_path(path, message.socket.session.overlay_name || 'mainline')
message.send_response(
message.fresh().data({ nodes })
)
}

View File

@@ -0,0 +1,11 @@
module.exports = exports = async (message, di) => {
const { descriptor } = message.data()
if ( message.socket.session.file_descriptors ) {
delete message.socket.session.file_descriptors[descriptor]
}
message.send_response(
message.fresh()
)
}

View File

@@ -0,0 +1,69 @@
const { Errors } = require('../../shared')
const { NodeDescriptorType } = require('../../enum')
module.exports = exports = async (message, di) => {
const Node = di.models.get('fs:Node')
const { source, destination } = message.data()
if ( !source ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( source === '/' ) {
return message.send_response(
message.fresh.error(Errors.NodePermissionFail)
)
}
const node = await Node.get_path(source, message.socket.session.overlay_name || 'mainline')
if ( !node ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
// make sure the move-to destination is valid
const [pied_parent_path, pied_name] = Node.path_parts(destination)
if ( pied_parent_path !== '/' ) {
const parent_node = await Node.get_path(pied_parent_path, message.socket.session.overlay_name || 'mainline')
if ( !parent_node ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( parent_node.descriptor_type !== NodeDescriptorType.Directory ) {
return message.send_response(
message.fresh().error(Errors.NotDirectoryDescriptor)
)
}
}
const existing_target = await Node.get_path(destination, message.socket.session.overlay_name || 'mainline')
if ( existing_target ) {
// If we're moving to an existing target, overwrite that target
const all_desc = await existing_target.all_descendants(message.socket.session.overlay_name || 'mainline')
await Promise.all(all_desc.map(desc => {
desc.deleted = true
return desc.save()
}))
existing_target.deleted = true
await existing_target.save()
}
const moved_desc = await node.all_descendants(message.socket.session.overlay_name || 'mainline')
await Promise.all(moved_desc.map(moved_node => {
moved_node.pied_parent_path = moved_node.pied_parent_path.replace(source, destination)
return moved_node.save()
}))
node.pied_parent_path = pied_parent_path
node.pied_name = pied_name
await node.save()
message.send_response(
message.fresh()
)
}

51
app/ws/routes/fs.rmdir.js Normal file
View File

@@ -0,0 +1,51 @@
const { Errors } = require('../../shared')
const { NodeDescriptorType } = require('../../enum')
module.exports = exports = async (message, di) => {
const Node = di.models.get('fs:Node')
const { path } = message.data()
if ( !path ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( path === '/' ) {
return message.send_response(
message.fresh.error(Errors.NodePermissionFail)
)
}
const node = await Node.get_path(path, message.socket.session.overlay_name || 'mainline')
if ( !node ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( node.descriptor_type !== NodeDescriptorType.Directory ) {
return message.send_response(
message.fresh().error(Errors.NotDirectoryDescriptor)
)
}
const child = await Node.findOne({
pied_parent_path: path,
overlay_name: message.socket.session.overlay_name || 'mainline',
deleted: false,
})
if ( child ) {
return message.send_response(
message.fresh().error(Errors.NodeNotEmpty)
)
}
node.deleted = true
await node.save()
message.send_response(
message.fresh()
)
}

View File

@@ -0,0 +1,39 @@
const { Errors } = require('../../shared')
const { NodeDescriptorType } = require('../../enum')
module.exports = exports = async (message, di) => {
const Node = di.models.get('fs:Node')
const { path } = message.data()
if ( !path ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( path === '/' ) {
return message.send_response(
message.fresh.error(Errors.NodePermissionFail)
)
}
const node = await Node.get_path(path, message.socket.session.overlay_name || 'mainline')
if ( !node ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
if ( node.descriptor_type !== NodeDescriptorType.File ) {
return message.send_response(
message.fresh().error(Errors.IsDirectoryDescriptor)
)
}
node.deleted = true
await node.save()
message.send_response(
message.fresh()
)
}

View File

@@ -0,0 +1,19 @@
module.exports = exports = async (message, di) => {
const Token = di.models.get('Token')
const { token_value } = message.data()
const token = await Token.findOne({ active: true, token_value })
if ( token ) {
message.socket.session.is_auth = true
message.socket.session.token_value = token_value
message.socket.session.user_uuid = token.user_uuid
return message.send_response(
message.fresh().data({ is_auth: true })
)
}
return message.send_response(
message.fresh().data({ is_auth: false })
)
}

View File

@@ -0,0 +1,4 @@
module.exports = exports = async (message, di) => {
console.log('Received ping!');
message.send_response(message.fresh())
}

View File

@@ -0,0 +1,34 @@
const { NodeDescriptorType } = require('../../enum')
const { Errors } = require('../../shared')
module.exports = exports = async (message, di) => {
const Node = di.models.get('fs:Node')
const { descriptor } = message.data()
if ( !descriptor || !message.socket.session.file_descriptors[descriptor] ) {
return message.send_response(
message.fresh().error(Errors.NoSuchDescriptor)
)
}
const node = await Node.findOne({
deleted: false,
descriptor_type: NodeDescriptorType.File,
uuid: message.socket.session.file_descriptors[descriptor],
})
if ( !node ) {
return message.send_response(
message.fresh().error(Errors.NodeDoesNotExist)
)
}
const data = {
node_uuid: node.uuid,
socket_uuid: message.socket.uuid,
}
message.send_response(
message.fresh().data(data)
)
}