Re-implement compression workers and abstraction

Implement DefaultCompression class along with a generic interface to
facilitate easy to use compression in a background thread, and make use
of this class in Storage implementation by default.

package-lock.json

@@ -9,6 +9,7 @@
             "version": "1.6.0",
             "license": "GPL-3.0-or-later",
             "dependencies": {
+                "@msgpack/msgpack": "^3.1.2",
                 "ajv": "^6.10.2",
                 "circular-json": "^0.5.9",
                 "clipboard-copy": "^3.1.0",
@@ -658,6 +659,15 @@
                 "node": ">= 8"
             }
         },
+        "node_modules/@msgpack/msgpack": {
+            "version": "3.1.2",
+            "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.1.2.tgz",
+            "integrity": "sha512-JEW4DEtBzfe8HvUYecLU9e6+XJnKDlUAIve8FvPzF3Kzs6Xo/KuZkZJsDH0wJXl/qEZbeeE7edxDNY3kMs39hQ==",
+            "license": "ISC",
+            "engines": {
+                "node": ">= 18"
+            }
+        },
         "node_modules/@nodelib/fs.scandir": {
             "version": "2.1.5",
             "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

package.json

@@ -21,6 +21,7 @@
         "package-all": "gulp --cwd gulp package.standalone.all"
     },
     "dependencies": {
+        "@msgpack/msgpack": "^3.1.2",
         "ajv": "^6.10.2",
         "circular-json": "^0.5.9",
         "clipboard-copy": "^3.1.0",

src/js/core/compression.ts

@@ -0,0 +1,49 @@
+export interface Compression {
+    compress(data: unknown): Promise<Uint8Array>;
+    decompress(data: Uint8Array): Promise<unknown>;
+}
+
+type CompressionWorkerResponse<T> = { result: T; error: null } | { result: null; error: Error };
+
+export class DefaultCompression implements Compression {
+    compress(data: unknown): Promise<Uint8Array> {
+        const { promise, reject, resolve } = Promise.withResolvers<Uint8Array>();
+
+        // NOTE: new URL(...) has to be inlined for webpack to process it correctly
+        const worker = new Worker(new URL("../webworkers/compression", import.meta.url));
+        worker.addEventListener("error", ev => reject(ev.message));
+
+        worker.addEventListener("message", ev => {
+            const response = ev.data as CompressionWorkerResponse<Uint8Array>;
+            if (response.error !== null) {
+                reject(response.error);
+                return;
+            }
+
+            resolve(response.result);
+        });
+
+        worker.postMessage(data);
+        return promise;
+    }
+
+    decompress(data: Uint8Array): Promise<unknown> {
+        const { promise, reject, resolve } = Promise.withResolvers<unknown>();
+
+        const worker = new Worker(new URL("../webworkers/decompression", import.meta.url));
+        worker.addEventListener("error", ev => reject(new Error(ev.message)));
+
+        worker.addEventListener("message", ev => {
+            const response = ev.data as CompressionWorkerResponse<unknown>;
+            if (response.error !== null) {
+                reject(response.error);
+                return;
+            }
+
+            resolve(response.result);
+        });
+
+        worker.postMessage(data, [data.buffer]);
+        return promise;
+    }
+}

src/js/mods/modloader.ts

@@ -108,7 +108,9 @@ export class ModLoader {
         const modules = import.meta.webpackContext("../", {
             recursive: true,
             regExp: /\.[jt]sx?$/,
-            exclude: /\.d\.ts$/,
+            // NOTE: Worker scripts are executed if not explicitly excluded, which causes
+            // infinite recursion!
+            exclude: /\/webworkers\/|\.d\.ts$/,
         });
 
         Array.from(modules.keys()).forEach(key => {

src/js/platform/storage.ts

@@ -1,16 +1,26 @@
 import { Application } from "@/application";
+import { Compression, DefaultCompression } from "@/core/compression";
 import { FsError } from "./fs_error";
 
 export const STORAGE_SAVES = "saves";
 export const STORAGE_MOD_PREFIX = "mod/";
 
+interface FsJob {
+    type: string;
+    filename?: string;
+    contents?: Uint8Array;
+    extension?: string;
+}
+
 export class Storage {
     readonly app: Application;
     readonly id: string;
+    readonly compression: Compression;
 
-    constructor(app: Application, id: string) {
+    constructor(app: Application, id: string, compression?: Compression) {
         this.app = app;
         this.id = id;
+        this.compression = compression ?? new DefaultCompression();
     }
 
     /**
@@ -20,18 +30,22 @@ export class Storage {
         return this.invokeFsJob({ type: "initialize" });
     }
 
-    /**
-     * Writes a string to a file asynchronously
-     */
-    writeFileAsync(filename: string, contents: unknown): Promise<void> {
-        return this.invokeFsJob({ type: "write", filename, contents });
-    }
-
     /**
      * Reads a string asynchronously
      */
     readFileAsync(filename: string): Promise<unknown> {
-        return this.invokeFsJob({ type: "read", filename });
+        return this.invokeFsJob({ type: "read", filename }).then(contents =>
+            this.compression.decompress(contents)
+        );
+    }
+
+    /**
+     * Writes a string to a file asynchronously
+     */
+    writeFileAsync(filename: string, contents: unknown): Promise<void> {
+        return this.compression
+            .compress(contents)
+            .then(contents => this.invokeFsJob({ type: "write", filename, contents }));
     }
 
     /**
@@ -46,7 +60,9 @@ export class Storage {
      * decompressed file contents, or undefined if the operation was canceled
      */
     requestOpenFile(extension: string): Promise<unknown> {
-        return this.invokeFsJob({ type: "open-external", extension });
+        return this.invokeFsJob({ type: "open-external", extension }).then(contents =>
+            contents ? this.compression.decompress(contents) : undefined
+        );
     }
 
     /**
@@ -55,10 +71,12 @@ export class Storage {
      * that file.
      */
     requestSaveFile(filename: string, contents: unknown): Promise<unknown> {
-        return this.invokeFsJob({ type: "save-external", filename, contents });
+        return this.compression
+            .compress(contents)
+            .then(contents => this.invokeFsJob({ type: "save-external", filename, contents }));
     }
 
-    private invokeFsJob(data: object) {
+    private invokeFsJob(data: FsJob) {
         return ipcRenderer
             .invoke("fs-job", {
                 id: this.id,

src/js/webworkers/compression.ts

@@ -0,0 +1,20 @@
+/// <reference lib="WebWorker" />
+
+import { encode } from "@msgpack/msgpack";
+
+async function compress(data: unknown): Promise<Uint8Array> {
+    const input = new Blob([encode(data)]).stream();
+    const gzip = new CompressionStream("gzip");
+    const response = new Response(input.pipeThrough(gzip));
+
+    return new Uint8Array(await response.arrayBuffer());
+}
+
+self.addEventListener("message", async ev => {
+    try {
+        const result = await compress(ev.data);
+        self.postMessage({ result, error: null }, [result.buffer]);
+    } catch (err) {
+        self.postMessage({ result: null, error: err });
+    }
+});

src/js/webworkers/decompression.ts

@@ -0,0 +1,24 @@
+/// <reference lib="WebWorker" />
+
+import { decodeAsync } from "@msgpack/msgpack";
+
+async function decompress(data: Uint8Array): Promise<unknown> {
+    const input = new Blob([data]).stream();
+    const gunzip = new DecompressionStream("gzip");
+
+    return await decodeAsync(input.pipeThrough(gunzip));
+}
+
+self.addEventListener("message", async ev => {
+    if (!(ev.data instanceof Uint8Array)) {
+        self.postMessage({ result: null, error: new Error("Incoming data must be of type Uint8Array") });
+        return;
+    }
+
+    try {
+        const result = await decompress(ev.data);
+        self.postMessage({ result, error: null });
+    } catch (err) {
+        self.postMessage({ result: null, error: err });
+    }
+});

src/js/webworkers/tsconfig.json

@@ -1,6 +0,0 @@
-{
-    "extends": ["../../tsconfig"],
-    "compilerOptions": {
-        "lib": ["WebWorker"]
-    }
-}

src/tsconfig.json

@@ -6,7 +6,7 @@
         "module": "es2022",
         "moduleResolution": "bundler",
         "noEmit": true,
-        "target": "ES2022",
+        "target": "ES2024",
         /* JSX Compilation */
         "paths": {
             "@/*": ["./js/*"]