Merge pull request #69 from tobspr-games/dengr1065/new-savegame-storage

Implement gzipped MessagePack savegame storage

electron/package-lock.json

@@ -9,6 +9,7 @@
             "version": "1.0.0",
             "license": "MIT",
             "dependencies": {
+                "@msgpack/msgpack": "^3.1.1",
                 "semver": "^7.7.1",
                 "zod": "^3.24.2"
             },
@@ -50,6 +51,15 @@
                 "semver": "bin/semver.js"
             }
         },
+        "node_modules/@msgpack/msgpack": {
+            "version": "3.1.1",
+            "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.1.1.tgz",
+            "integrity": "sha512-DnBpqkMOUGayNVKyTLlkM6ILmU/m/+VUxGkuQlPQVAcvreLz5jn1OlQnWd8uHKL/ZSiljpM12rjRhr51VtvJUQ==",
+            "license": "ISC",
+            "engines": {
+                "node": ">= 18"
+            }
+        },
         "node_modules/@sindresorhus/is": {
             "version": "4.6.0",
             "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz",

electron/package.json

@@ -9,6 +9,7 @@
         "start": "tsc && electron ."
     },
     "dependencies": {
+        "@msgpack/msgpack": "^3.1.1",
         "semver": "^7.7.1",
         "zod": "^3.24.2"
     },

electron/src/fsjob.ts

@@ -1,68 +1,129 @@
+import { BrowserWindow, dialog, FileFilter } from "electron";
 import fs from "fs/promises";
 import path from "path";
 import { userData } from "./config.js";
+import { StorageInterface } from "./storage/interface.js";
 
 interface GenericFsJob {
-    filename: string;
+    id: string;
 }
 
-type ListFsJob = GenericFsJob & { type: "list" };
+export type InitializeFsJob = GenericFsJob & { type: "initialize" };
-type ReadFsJob = GenericFsJob & { type: "read" };
+type ListFsJob = GenericFsJob & { type: "list"; filename: string };
-type WriteFsJob = GenericFsJob & { type: "write"; contents: string };
+type ReadFsJob = GenericFsJob & { type: "read"; filename: string };
-type DeleteFsJob = GenericFsJob & { type: "delete" };
+type WriteFsJob<T> = GenericFsJob & { type: "write"; filename: string; contents: T };
+type DeleteFsJob = GenericFsJob & { type: "delete"; filename: string };
 
-export type FsJob = ListFsJob | ReadFsJob | WriteFsJob | DeleteFsJob;
+type OpenExternalFsJob = GenericFsJob & { type: "open-external"; extension: string };
-type FsJobResult = string | string[] | void;
+type SaveExternalFsJob<T> = GenericFsJob & { type: "save-external"; filename: string; contents: T };
 
-export class FsJobHandler {
+export type FsJob<T> =
+    | InitializeFsJob
+    | ListFsJob
+    | ReadFsJob
+    | WriteFsJob<T>
+    | DeleteFsJob
+    | OpenExternalFsJob
+    | SaveExternalFsJob<T>;
+type FsJobResult<T> = T | string[] | void;
+
+export class FsJobHandler<T> {
     readonly rootDir: string;
+    private readonly storage: StorageInterface<T>;
+    private initialized = false;
 
-    constructor(subDir: string) {
+    constructor(subDir: string, storage: StorageInterface<T>) {
         this.rootDir = path.join(userData, subDir);
+        this.storage = storage;
     }
 
-    handleJob(job: FsJob): Promise<FsJobResult> {
+    async initialize(): Promise<void> {
+        if (this.initialized) {
+            return;
+        }
+
+        // Create the directory so that users know where to put files
+        await fs.mkdir(this.rootDir, { recursive: true });
+        this.initialized = true;
+    }
+
+    handleJob(job: FsJob<T>): Promise<FsJobResult<T>> {
+        switch (job.type) {
+            case "initialize":
+                return this.initialize();
+            case "open-external":
+                return this.openExternal(job.extension);
+            case "save-external":
+                return this.saveExternal(job.filename, job.contents);
+        }
+
         const filename = this.safeFileName(job.filename);
 
         switch (job.type) {
             case "list":
                 return this.list(filename);
             case "read":
-                return this.read(filename);
+                return this.storage.read(filename);
             case "write":
                 return this.write(filename, job.contents);
             case "delete":
-                return this.delete(filename);
+                return this.storage.delete(filename);
         }
 
         // @ts-expect-error this method can actually receive garbage
         throw new Error(`Unknown FS job type: ${job.type}`);
     }
 
+    private async openExternal(extension: string): Promise<T | undefined> {
+        const filters = this.getFileDialogFilters(extension === "*" ? undefined : extension);
+        const window = BrowserWindow.getAllWindows()[0]!;
+
+        const result = await dialog.showOpenDialog(window, { filters, properties: ["openFile"] });
+        if (result.canceled) {
+            return undefined;
+        }
+
+        return await this.storage.read(result.filePaths[0]);
+    }
+
+    private async saveExternal(filename: string, contents: T): Promise<void> {
+        // Try to guess extension
+        const ext = filename.indexOf(".") < 1 ? filename.split(".").at(-1)! : undefined;
+        const filters = this.getFileDialogFilters(ext);
+        const window = BrowserWindow.getAllWindows()[0]!;
+
+        const result = await dialog.showSaveDialog(window, { defaultPath: filename, filters });
+        if (result.canceled) {
+            return;
+        }
+
+        return await this.storage.write(result.filePath, contents);
+    }
+
+    private getFileDialogFilters(extension?: string): FileFilter[] {
+        const filters: FileFilter[] = [{ name: "All files", extensions: ["*"] }];
+
+        if (extension !== undefined) {
+            filters.unshift({
+                name: `${extension.toUpperCase()} files`,
+                extensions: [extension],
+            });
+        }
+
+        return filters;
+    }
+
     private list(subdir: string): Promise<string[]> {
         // Bare-bones implementation
         return fs.readdir(subdir);
     }
 
-    private read(file: string): Promise<string> {
+    private async write(file: string, contents: T): Promise<void> {
-        return fs.readFile(file, "utf-8");
-    }
-
-    private async write(file: string, contents: string): Promise<string> {
         // The target directory might not exist, ensure it does
         const parentDir = path.dirname(file);
         await fs.mkdir(parentDir, { recursive: true });
 
-        // Backups not implemented yet.
+        await this.storage.write(file, contents);
-        await fs.writeFile(file, contents, {
-            encoding: "utf-8",
-            flush: true,
-        });
-        return contents;
-    }
-
-    private delete(file: string): Promise<void> {
-        return fs.unlink(file);
     }
 
     private safeFileName(name: string) {

electron/src/index.ts

@@ -1,7 +1,6 @@
 import { BrowserWindow, app, shell } from "electron";
 import path from "path";
 import { defaultWindowTitle, pageUrl, switches } from "./config.js";
-import { FsJobHandler } from "./fsjob.js";
 import { IpcHandler } from "./ipc.js";
 import { ModLoader } from "./mods/loader.js";
 import { ModProtocolHandler } from "./mods/protocol_handler.js";
@@ -20,15 +19,9 @@ if (!app.requestSingleInstanceLock()) {
     });
 }
 
-// TODO: Implement a redirector/advanced storage system
-// Let mods have own data directories with easy access and
-// split savegames/configs - only implement backups and gzip
-// files if requested. Perhaps, use streaming to make large
-// transfers less "blocking"
-const fsJob = new FsJobHandler("saves");
 const modLoader = new ModLoader();
 const modProtocol = new ModProtocolHandler(modLoader);
-const ipc = new IpcHandler(fsJob, modLoader);
+const ipc = new IpcHandler(modLoader);
 
 function createWindow() {
     // The protocol can only be handled after "ready" event

electron/src/ipc.ts

@@ -1,13 +1,13 @@
 import { BrowserWindow, IpcMainInvokeEvent, ipcMain } from "electron";
 import { FsJob, FsJobHandler } from "./fsjob.js";
 import { ModLoader } from "./mods/loader.js";
+import { SavesStorage } from "./storage/saves.js";
 
 export class IpcHandler {
-    private readonly fsJob: FsJobHandler;
+    private readonly savesHandler = new FsJobHandler("saves", new SavesStorage());
     private readonly modLoader: ModLoader;
 
-    constructor(fsJob: FsJobHandler, modLoader: ModLoader) {
+    constructor(modLoader: ModLoader) {
-        this.fsJob = fsJob;
         this.modLoader = modLoader;
     }
 
@@ -20,8 +20,12 @@ export class IpcHandler {
         // ipcMain.handle("open-mods-folder", ...)
     }
 
-    private handleFsJob(_event: IpcMainInvokeEvent, job: FsJob) {
+    private handleFsJob(_event: IpcMainInvokeEvent, job: FsJob<unknown>) {
-        return this.fsJob.handleJob(job);
+        if (job.id !== "saves") {
+            throw new Error("Storages other than saves/ are not implemented yet");
+        }
+
+        return this.savesHandler.handleJob(job);
     }
 
     private async getMods() {

electron/src/storage/interface.ts

@@ -0,0 +1,5 @@
+export interface StorageInterface<T> {
+    read(file: string): Promise<T>;
+    write(file: string, contents: T): Promise<void>;
+    delete(file: string): Promise<void>;
+}

electron/src/storage/raw.ts

@@ -0,0 +1,16 @@
+import fs from "node:fs/promises";
+import { StorageInterface } from "./interface.js";
+
+export class RawStorage implements StorageInterface<string> {
+    read(file: string): Promise<string> {
+        return fs.readFile(file, "utf-8");
+    }
+
+    write(file: string, contents: string): Promise<void> {
+        return fs.writeFile(file, contents, "utf-8");
+    }
+
+    delete(file: string): Promise<void> {
+        return fs.unlink(file);
+    }
+}

electron/src/storage/saves.ts

@@ -0,0 +1,54 @@
+import { decodeAsync, encode } from "@msgpack/msgpack";
+import fs from "node:fs";
+import { pipeline } from "node:stream/promises";
+import { createGunzip, createGzip } from "node:zlib";
+import { StorageInterface } from "./interface.js";
+
+/**
+ * This storage implementation is used for savegame files and other
+ * ReadWriteProxy objects. It uses gzipped MessagePack as the file format.
+ */
+export class SavesStorage implements StorageInterface<unknown> {
+    async read(file: string): Promise<unknown> {
+        const stream = fs.createReadStream(file);
+        const gunzip = createGunzip();
+
+        try {
+            // Any filesystem errors will be uncovered here. This code ensures we return the most
+            // relevant rejection, or resolve with the decoded data
+            const [readResult, decodeResult] = await Promise.allSettled([
+                pipeline(stream, gunzip),
+                decodeAsync(gunzip),
+            ]);
+
+            if (decodeResult.status === "fulfilled") {
+                return decodeResult.value;
+            }
+
+            // Return the most relevant error
+            throw readResult.status === "rejected" ? readResult.reason : decodeResult.reason;
+        } finally {
+            stream.close();
+            gunzip.close();
+        }
+    }
+
+    async write(file: string, contents: unknown): Promise<void> {
+        const stream = fs.createWriteStream(file);
+        const gzip = createGzip();
+
+        try {
+            const encoded = encode(contents);
+            const blob = new Blob([encoded]);
+
+            return await pipeline(blob.stream(), gzip, stream);
+        } finally {
+            gzip.close();
+            stream.close();
+        }
+    }
+
+    delete(file: string): Promise<void> {
+        return fs.promises.unlink(file);
+    }
+}

package-lock.json

@@ -12,7 +12,6 @@
                 "ajv": "^6.10.2",
                 "circular-json": "^0.5.9",
                 "clipboard-copy": "^3.1.0",
-                "crc": "^3.8.0",
                 "debounce-promise": "^3.1.2",
                 "howler": "^2.1.2",
                 "lz-string": "^1.4.4"
@@ -2434,6 +2433,7 @@
             "version": "1.5.1",
             "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
             "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+            "dev": true,
             "funding": [
                 {
                     "type": "github",
@@ -3253,6 +3253,7 @@
             "version": "5.7.1",
             "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
             "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+            "dev": true,
             "funding": [
                 {
                     "type": "github",
@@ -4363,15 +4364,6 @@
                 "node": ">=4"
             }
         },
-        "node_modules/crc": {
-            "version": "3.8.0",
-            "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz",
-            "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==",
-            "license": "MIT",
-            "dependencies": {
-                "buffer": "^5.1.0"
-            }
-        },
         "node_modules/cross-spawn": {
             "version": "6.0.5",
             "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
@@ -9937,6 +9929,7 @@
             "version": "1.2.1",
             "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
             "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+            "dev": true,
             "funding": [
                 {
                     "type": "github",

package.json

@@ -24,7 +24,6 @@
         "ajv": "^6.10.2",
         "circular-json": "^0.5.9",
         "clipboard-copy": "^3.1.0",
-        "crc": "^3.8.0",
         "debounce-promise": "^3.1.2",
         "howler": "^2.1.2",
         "lz-string": "^1.4.4"

src/js/application.js

@@ -13,7 +13,7 @@ import { MOD_SIGNALS } from "./mods/mod_signals";
 import { MODS } from "./mods/modloader";
 import { ClientAPI } from "./platform/api";
 import { Sound } from "./platform/sound";
-import { Storage } from "./platform/storage";
+import { Storage, STORAGE_SAVES } from "./platform/storage";
 import { PlatformWrapperImplElectron } from "./platform/wrapper";
 import { ApplicationSettings } from "./profile/application_settings";
 import { SavegameManager } from "./savegame/savegame_manager";
@@ -54,21 +54,23 @@ export class Application {
 
         this.unloaded = false;
 
+        // Platform stuff
+        this.storage = new Storage(this, STORAGE_SAVES);
+        await this.storage.initialize();
+
+        this.platformWrapper = new PlatformWrapperImplElectron(this);
+
         // Global stuff
-        this.settings = new ApplicationSettings(this);
+        this.settings = new ApplicationSettings(this, this.storage);
         this.ticker = new AnimationFrame();
         this.stateMgr = new StateManager(this);
-        this.savegameMgr = new SavegameManager(this);
+        // NOTE: SavegameManager uses the passed storage, but savegames always
+        // use Application#storage
+        this.savegameMgr = new SavegameManager(this, this.storage);
         this.inputMgr = new InputDistributor(this);
         this.backgroundResourceLoader = new BackgroundResourcesLoader(this);
         this.clientApi = new ClientAPI(this);
 
-        // Platform dependent stuff
-
-        this.storage = new Storage(this);
-
-        this.platformWrapper = new PlatformWrapperImplElectron(this);
-
         this.sound = new Sound(this);
 
         // Track if the window is focused (only relevant for browser)

src/js/core/async_compression.js

@@ -1,101 +0,0 @@
-// @ts-expect-error FIXME: missing typings
-import CompressionWorker from "../webworkers/compression.worker";
-
-import { createLogger } from "./logging";
-import { round2Digits } from "./utils";
-
-const logger = createLogger("async_compression");
-
-export const compressionPrefix = String.fromCodePoint(1);
-
-/**
- * @typedef {{
- *   errorHandler: function(any) : void,
- *   resolver: function(any) : void,
- *   startTime: number
- * }} JobEntry
- */
-
-class AsynCompression {
-    constructor() {
-        this.worker = new CompressionWorker();
-
-        this.currentJobId = 1000;
-
-        /** @type {Object.<number, JobEntry>} */
-        this.currentJobs = {};
-
-        this.worker.addEventListener("message", event => {
-            const { jobId, result } = event.data;
-            const jobData = this.currentJobs[jobId];
-            if (!jobData) {
-                logger.error("Failed to resolve job result, job id", jobId, "is not known");
-                return;
-            }
-
-            const duration = performance.now() - jobData.startTime;
-            logger.log(
-                "Got job",
-                jobId,
-                "response within",
-                round2Digits(duration),
-                "ms: ",
-                result.length,
-                "bytes"
-            );
-            const resolver = jobData.resolver;
-            delete this.currentJobs[jobId];
-            resolver(result);
-        });
-
-        this.worker.addEventListener("error", err => {
-            logger.error("Got error from webworker:", err, "aborting all jobs");
-            const failureCalls = [];
-            for (const jobId in this.currentJobs) {
-                failureCalls.push(this.currentJobs[jobId].errorHandler);
-            }
-            this.currentJobs = {};
-            for (let i = 0; i < failureCalls.length; ++i) {
-                failureCalls[i](err);
-            }
-        });
-    }
-
-    /**
-     * Compresses any object
-     * @param {any} obj
-     */
-    compressObjectAsync(obj) {
-        logger.log("Compressing object async (optimized)");
-        return this.internalQueueJob("compressObject", {
-            obj,
-            compressionPrefix,
-        });
-    }
-
-    /**
-     * Queues a new job
-     * @param {string} job
-     * @param {any} data
-     * @returns {Promise<any>}
-     */
-    internalQueueJob(job, data) {
-        const jobId = ++this.currentJobId;
-        return new Promise((resolve, reject) => {
-            const errorHandler = err => {
-                logger.error("Failed to compress job", jobId, ":", err);
-                reject(err);
-            };
-            this.currentJobs[jobId] = {
-                errorHandler,
-                resolver: resolve,
-                startTime: performance.now(),
-            };
-
-            logger.log("Posting job", job, "/", jobId);
-            this.worker.postMessage({ jobId, job, data });
-        });
-    }
-}
-
-export const asyncCompressor = new AsynCompression();

src/js/core/config.ts

@@ -110,12 +110,6 @@ export const globalConfig = {
 
     rendering: {},
     debug,
-
-    // Secret vars
-    info: {
-        // Binary file salt
-        file: "Ec'])@^+*9zMevK3uMV4432x9%iK'=",
-    },
 };
 
 export const IS_MOBILE = /iPhone|iPad|iPod|Android/i.test(navigator.userAgent);

src/js/core/read_write_proxy.js

@@ -1,27 +1,21 @@
 /* typehints:start */
-import { Application } from "../application";
+import { Storage } from "@/platform/storage";
 /* typehints:end */
 
 import { FsError } from "@/platform/fs_error";
-import { compressObject, decompressObject } from "../savegame/savegame_compressor";
+import { IS_DEBUG } from "./config";
-import { asyncCompressor, compressionPrefix } from "./async_compression";
-import { IS_DEBUG, globalConfig } from "./config";
 import { ExplainedResult } from "./explained_result";
 import { createLogger } from "./logging";
-import { compressX64, decompressX64 } from "./lzstring";
-import { computeCrc } from "./sensitive_utils.encrypt";
 
 import debounce from "debounce-promise";
 
 const logger = createLogger("read_write_proxy");
 
-const salt = globalConfig.info.file;
-
 // Helper which only writes / reads if verify() works. Also performs migration
 export class ReadWriteProxy {
-    constructor(app, filename) {
+    constructor(storage, filename) {
-        /** @type {Application} */
+        /** @type {Storage} */
-        this.app = app;
+        this.storage = storage;
 
         this.filename = filename;
 
@@ -85,9 +79,8 @@ export class ReadWriteProxy {
      * @param {object} obj
      */
     static serializeObject(obj) {
-        const jsonString = JSON.stringify(compressObject(obj));
+        // TODO: Remove redundant method
-        const checksum = computeCrc(jsonString + salt);
+        return obj;
-        return compressionPrefix + compressX64(checksum + jsonString);
     }
 
     /**
@@ -95,30 +88,8 @@ export class ReadWriteProxy {
      * @param {object} text
      */
     static deserializeObject(text) {
-        const decompressed = decompressX64(text.substr(compressionPrefix.length));
+        // TODO: Remove redundant method
-        if (!decompressed) {
+        return text;
-            // LZ string decompression failure
-            throw new Error("bad-content / decompression-failed");
-        }
-        if (decompressed.length < 40) {
-            // String too short
-            throw new Error("bad-content / payload-too-small");
-        }
-
-        // Compare stored checksum with actual checksum
-        const checksum = decompressed.substring(0, 40);
-        const jsonString = decompressed.substr(40);
-
-        const desiredChecksum = computeCrc(jsonString + salt);
-
-        if (desiredChecksum !== checksum) {
-            // Checksum mismatch
-            throw new Error("bad-content / checksum-mismatch");
-        }
-
-        const parsed = JSON.parse(jsonString);
-        const decoded = decompressObject(parsed);
-        return decoded;
     }
 
     /**
@@ -142,11 +113,8 @@ export class ReadWriteProxy {
      * @returns {Promise<void>}
      */
     doWriteAsync() {
-        return asyncCompressor
+        return this.storage
-            .compressObjectAsync(this.currentData)
+            .writeFileAsync(this.filename, this.currentData)
-            .then(compressed => {
-                return this.app.storage.writeFileAsync(this.filename, compressed);
-            })
             .then(() => {
                 logger.log("📄 Wrote", this.filename);
             })
@@ -160,7 +128,7 @@ export class ReadWriteProxy {
     readAsync() {
         // Start read request
         return (
-            this.app.storage
+            this.storage
                 .readFileAsync(this.filename)
 
                 // Check for errors during read
@@ -169,73 +137,12 @@ export class ReadWriteProxy {
                         logger.log("File not found, using default data");
 
                         // File not found or unreadable, assume default file
-                        return Promise.resolve(null);
+                        return Promise.resolve(this.getDefaultData());
                     }
 
                     return Promise.reject("file-error: " + err);
                 })
 
-                // Decrypt data (if its encrypted)
-                // @ts-ignore
-                .then(rawData => {
-                    if (rawData == null) {
-                        // So, the file has not been found, use default data
-                        return JSON.stringify(compressObject(this.getDefaultData()));
-                    }
-
-                    if (rawData.startsWith(compressionPrefix)) {
-                        const decompressed = decompressX64(rawData.substr(compressionPrefix.length));
-                        if (!decompressed) {
-                            // LZ string decompression failure
-                            return Promise.reject("bad-content / decompression-failed");
-                        }
-                        if (decompressed.length < 40) {
-                            // String too short
-                            return Promise.reject("bad-content / payload-too-small");
-                        }
-
-                        // Compare stored checksum with actual checksum
-                        const checksum = decompressed.substring(0, 40);
-                        const jsonString = decompressed.slice(40);
-
-                        const desiredChecksum = computeCrc(jsonString + salt);
-
-                        if (desiredChecksum !== checksum) {
-                            // Checksum mismatch
-                            return Promise.reject(
-                                "bad-content / checksum-mismatch: " + desiredChecksum + " vs " + checksum
-                            );
-                        }
-                        return jsonString;
-                    } else {
-                        if (!G_IS_DEV) {
-                            return Promise.reject("bad-content / missing-compression");
-                        }
-                    }
-                    return rawData;
-                })
-
-                // Parse JSON, this could throw but that's fine
-                .then(res => {
-                    try {
-                        return JSON.parse(res);
-                    } catch (ex) {
-                        logger.error(
-                            "Failed to parse file content of",
-                            this.filename,
-                            ":",
-                            ex,
-                            "(content was:",
-                            res,
-                            ")"
-                        );
-                        throw new Error("invalid-serialized-data");
-                    }
-                })
-
-                // Decompress
-                .then(compressed => decompressObject(compressed))
-
                 // Verify basic structure
                 .then(contents => {
                     const result = this.internalVerifyBasicStructure(contents);
@@ -302,7 +209,7 @@ export class ReadWriteProxy {
      * @returns {Promise<void>}
      */
     deleteAsync() {
-        return this.app.storage.deleteFileAsync(this.filename);
+        return this.storage.deleteFileAsync(this.filename);
     }
 
     // Internal

src/js/core/sensitive_utils.encrypt.js

@@ -1,12 +0,0 @@
-import crc32 from "crc/crc32";
-
-// Distinguish legacy crc prefixes
-export const CRC_PREFIX = "crc32".padEnd(32, "-");
-
-/**
- * Computes the crc for a given string
- * @param {string} str
- */
-export function computeCrc(str) {
-    return CRC_PREFIX + crc32(str).toString(16).padStart(8, "0");
-}

src/js/core/utils.js

@@ -599,38 +599,6 @@ export function fillInLinkIntoTranslation(translation, link) {
         .replace("</link>", "</a>");
 }
 
-/**
- * Generates a file download
- * @param {string} filename
- * @param {string} text
- */
-export function generateFileDownload(filename, text) {
-    const element = document.createElement("a");
-    element.setAttribute("href", "data:text/plain;charset=utf-8," + encodeURIComponent(text));
-    element.setAttribute("download", filename);
-
-    element.style.display = "none";
-    document.body.appendChild(element);
-
-    element.click();
-    document.body.removeChild(element);
-}
-
-/**
- * Starts a file chooser
- * @param {string} acceptedType
- */
-export function startFileChoose(acceptedType = ".bin") {
-    const input = document.createElement("input");
-    input.type = "file";
-    input.accept = acceptedType;
-
-    return new Promise(resolve => {
-        input.onchange = _ => resolve(input.files[0]);
-        input.click();
-    });
-}
-
 const MAX_ROMAN_NUMBER = 49;
 const romanLiteralsCache = ["0"];
 

src/js/platform/storage.ts

@@ -1,53 +1,68 @@
 import { Application } from "@/application";
 import { FsError } from "./fs_error";
 
+export const STORAGE_SAVES = "saves";
+export const STORAGE_MOD_PREFIX = "mod/";
+
 export class Storage {
     readonly app: Application;
+    readonly id: string;
 
-    constructor(app: Application) {
+    constructor(app: Application, id: string) {
         this.app = app;
+        this.id = id;
     }
 
     /**
      * Initializes the storage
      */
     initialize(): Promise<void> {
-        return Promise.resolve();
+        return this.invokeFsJob({ type: "initialize" });
     }
 
     /**
      * Writes a string to a file asynchronously
      */
-    writeFileAsync(filename: string, contents: string): Promise<void> {
+    writeFileAsync(filename: string, contents: unknown): Promise<void> {
-        return ipcRenderer
+        return this.invokeFsJob({ type: "write", filename, contents });
-            .invoke("fs-job", {
-                type: "write",
-                filename,
-                contents,
-            })
-            .catch(e => this.wrapError(e));
     }
 
     /**
      * Reads a string asynchronously
      */
-    readFileAsync(filename: string): Promise<string> {
+    readFileAsync(filename: string): Promise<unknown> {
-        return ipcRenderer
+        return this.invokeFsJob({ type: "read", filename });
-            .invoke("fs-job", {
-                type: "read",
-                filename,
-            })
-            .catch(e => this.wrapError(e));
     }
 
     /**
      * Tries to delete a file
      */
     deleteFileAsync(filename: string): Promise<void> {
+        return this.invokeFsJob({ type: "delete", filename });
+    }
+
+    /**
+     * Displays the "Open File" dialog to let user pick a file. Returns the
+     * decompressed file contents, or undefined if the operation was canceled
+     */
+    requestOpenFile(extension: string): Promise<unknown> {
+        return this.invokeFsJob({ type: "open-external", extension });
+    }
+
+    /**
+     * Displays the "Save File" dialog to let user pick a file. If the user
+     * picks a file, the passed contents will be compressed and written to
+     * that file.
+     */
+    requestSaveFile(filename: string, contents: unknown): Promise<unknown> {
+        return this.invokeFsJob({ type: "save-external", filename, contents });
+    }
+
+    private invokeFsJob(data: object) {
         return ipcRenderer
             .invoke("fs-job", {
-                type: "delete",
+                id: this.id,
-                filename,
+                ...data,
             })
             .catch(e => this.wrapError(e));
     }

src/js/profile/application_settings.js

@@ -2,13 +2,13 @@
 import { Application } from "../application";
 /* typehints:end */
 
-import { ReadWriteProxy } from "../core/read_write_proxy";
-import { BoolSetting, EnumSetting, RangeSetting, BaseSetting } from "./setting_types";
-import { createLogger } from "../core/logging";
 import { ExplainedResult } from "../core/explained_result";
+import { createLogger } from "../core/logging";
+import { ReadWriteProxy } from "../core/read_write_proxy";
 import { THEMES, applyGameTheme } from "../game/theme";
-import { T } from "../translations";
 import { LANGUAGES } from "../languages";
+import { T } from "../translations";
+import { BaseSetting, BoolSetting, EnumSetting, RangeSetting } from "./setting_types";
 
 const logger = createLogger("application_settings");
 
@@ -330,8 +330,11 @@ class SettingsStorage {
 }
 
 export class ApplicationSettings extends ReadWriteProxy {
-    constructor(app) {
+    constructor(app, storage) {
-        super(app, "app_settings.bin");
+        super(storage, "app_settings.bin");
+
+        /** @type {Application} */
+        this.app = app;
 
         this.settingHandles = initializeSettings();
     }

src/js/savegame/savegame.js

@@ -6,16 +6,6 @@ import { MODS } from "../mods/modloader";
 import { BaseSavegameInterface } from "./savegame_interface";
 import { getSavegameInterface, savegameInterfaces } from "./savegame_interface_registry";
 import { SavegameSerializer } from "./savegame_serializer";
-import { SavegameInterface_V1001 } from "./schemas/1001";
-import { SavegameInterface_V1002 } from "./schemas/1002";
-import { SavegameInterface_V1003 } from "./schemas/1003";
-import { SavegameInterface_V1004 } from "./schemas/1004";
-import { SavegameInterface_V1005 } from "./schemas/1005";
-import { SavegameInterface_V1006 } from "./schemas/1006";
-import { SavegameInterface_V1007 } from "./schemas/1007";
-import { SavegameInterface_V1008 } from "./schemas/1008";
-import { SavegameInterface_V1009 } from "./schemas/1009";
-import { SavegameInterface_V1010 } from "./schemas/1010";
 
 const logger = createLogger("savegame");
 
@@ -36,7 +26,11 @@ export class Savegame extends ReadWriteProxy {
      * @param {SavegameMetadata} param0.metaDataRef Handle to the meta data
      */
     constructor(app, { internalId, metaDataRef }) {
-        super(app, "savegame-" + internalId + ".bin");
+        super(app.storage, "savegame-" + internalId + ".bin");
+
+        /** @type {Application} */
+        this.app = app;
+
         this.internalId = internalId;
         this.metaDataRef = metaDataRef;
 
@@ -108,58 +102,8 @@ export class Savegame extends ReadWriteProxy {
      * @param {SavegameData} data
      */
     migrate(data) {
-        if (data.version < 1000) {
+        if (data.version !== this.getCurrentVersion()) {
-            return ExplainedResult.bad("Can not migrate savegame, too old");
+            return ExplainedResult.bad("Savegame upgrade is not supported");
-        }
-
-        if (data.version === 1000) {
-            SavegameInterface_V1001.migrate1000to1001(data);
-            data.version = 1001;
-        }
-
-        if (data.version === 1001) {
-            SavegameInterface_V1002.migrate1001to1002(data);
-            data.version = 1002;
-        }
-
-        if (data.version === 1002) {
-            SavegameInterface_V1003.migrate1002to1003(data);
-            data.version = 1003;
-        }
-
-        if (data.version === 1003) {
-            SavegameInterface_V1004.migrate1003to1004(data);
-            data.version = 1004;
-        }
-
-        if (data.version === 1004) {
-            SavegameInterface_V1005.migrate1004to1005(data);
-            data.version = 1005;
-        }
-
-        if (data.version === 1005) {
-            SavegameInterface_V1006.migrate1005to1006(data);
-            data.version = 1006;
-        }
-
-        if (data.version === 1006) {
-            SavegameInterface_V1007.migrate1006to1007(data);
-            data.version = 1007;
-        }
-
-        if (data.version === 1007) {
-            SavegameInterface_V1008.migrate1007to1008(data);
-            data.version = 1008;
-        }
-
-        if (data.version === 1008) {
-            SavegameInterface_V1009.migrate1008to1009(data);
-            data.version = 1009;
-        }
-
-        if (data.version === 1009) {
-            SavegameInterface_V1010.migrate1009to1010(data);
-            data.version = 1010;
         }
 
         return ExplainedResult.good();

src/js/savegame/savegame_compressor.js

@@ -1,168 +0,0 @@
-const charmap =
-    "!#%&'()*+,-./:;<=>?@[]^_`{|}~¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
-let compressionCache = {};
-let decompressionCache = {};
-
-/**
- * Compresses an integer into a tight string representation
- * @param {number} i
- * @returns {string}
- */
-function compressInt(i) {
-    // Zero value breaks
-    i += 1;
-
-    // save `i` as the cache key
-    // to avoid it being modified by the
-    // rest of the function.
-    const cache_key = i;
-
-    if (compressionCache[cache_key]) {
-        return compressionCache[cache_key];
-    }
-    let result = "";
-    do {
-        result += charmap[i % charmap.length];
-        i = Math.floor(i / charmap.length);
-    } while (i > 0);
-    return (compressionCache[cache_key] = result);
-}
-
-/**
- * Decompresses an integer from its tight string representation
- * @param {string} s
- * @returns {number}
- */
-function decompressInt(s) {
-    if (decompressionCache[s]) {
-        return decompressionCache[s];
-    }
-    s = "" + s;
-    let result = 0;
-    for (let i = s.length - 1; i >= 0; --i) {
-        result = result * charmap.length + charmap.indexOf(s.charAt(i));
-    }
-    // Fixes zero value break fix from above
-    result -= 1;
-    return (decompressionCache[s] = result);
-}
-
-// Sanity
-if (G_IS_DEV) {
-    for (let i = 0; i < 10000; ++i) {
-        if (decompressInt(compressInt(i)) !== i) {
-            throw new Error(
-                "Bad compression for: " +
-                    i +
-                    " compressed: " +
-                    compressInt(i) +
-                    " decompressed: " +
-                    decompressInt(compressInt(i))
-            );
-        }
-    }
-}
-
-/**
- * @param {any} obj
- * @param {Map} keys
- * @param {Map} values
- * @returns {any[]|object|number|string}
- */
-function compressObjectInternal(obj, keys, values) {
-    if (Array.isArray(obj)) {
-        let result = [];
-        for (let i = 0; i < obj.length; ++i) {
-            result.push(compressObjectInternal(obj[i], keys, values));
-        }
-        return result;
-    } else if (typeof obj === "object" && obj !== null) {
-        let result = {};
-        for (const key in obj) {
-            let index = keys.get(key);
-            if (index === undefined) {
-                index = keys.size;
-                keys.set(key, index);
-            }
-            const value = obj[key];
-            result[compressInt(index)] = compressObjectInternal(value, keys, values);
-        }
-        return result;
-    } else if (typeof obj === "string") {
-        let index = values.get(obj);
-        if (index === undefined) {
-            index = values.size;
-            values.set(obj, index);
-        }
-        return compressInt(index);
-    }
-    return obj;
-}
-
-/**
- * @param {Map} hashMap
- * @returns {Array}
- */
-function indexMapToArray(hashMap) {
-    const result = new Array(hashMap.size);
-    hashMap.forEach((index, key) => {
-        result[index] = key;
-    });
-    return result;
-}
-
-/**
- * @param {object} obj
- */
-export function compressObject(obj) {
-    const keys = new Map();
-    const values = new Map();
-    const data = compressObjectInternal(obj, keys, values);
-    return {
-        keys: indexMapToArray(keys),
-        values: indexMapToArray(values),
-        data,
-    };
-}
-
-/**
- * @param {object} obj
- * @param {string[]} keys
- * @param {any[]} values
- * @returns {object}
- */
-function decompressObjectInternal(obj, keys = [], values = []) {
-    if (Array.isArray(obj)) {
-        let result = [];
-        for (let i = 0; i < obj.length; ++i) {
-            result.push(decompressObjectInternal(obj[i], keys, values));
-        }
-        return result;
-    } else if (typeof obj === "object" && obj !== null) {
-        let result = {};
-        for (const key in obj) {
-            const realIndex = decompressInt(key);
-            const value = obj[key];
-            result[keys[realIndex]] = decompressObjectInternal(value, keys, values);
-        }
-        return result;
-    } else if (typeof obj === "string") {
-        const realIndex = decompressInt(obj);
-        return values[realIndex];
-    }
-    return obj;
-}
-
-/**
- * @param {object} obj
- */
-export function decompressObject(obj) {
-    if (obj.keys && obj.values && obj.data) {
-        const keys = obj.keys;
-        const values = obj.values;
-        const result = decompressObjectInternal(obj.data, keys, values);
-        return result;
-    }
-    return obj;
-}

src/js/savegame/savegame_manager.js

@@ -1,3 +1,7 @@
+/* typehints:start */
+import { Application } from "@/application";
+/* typehints:end */
+
 import { globalConfig } from "../core/config";
 import { ExplainedResult } from "../core/explained_result";
 import { createLogger } from "../core/logging";
@@ -17,8 +21,11 @@ export const enumLocalSavegameStatus = {
 };
 
 export class SavegameManager extends ReadWriteProxy {
-    constructor(app) {
+    constructor(app, storage) {
-        super(app, "savegames.bin");
+        super(storage, "savegames.bin");
+
+        /** @type {Application} */
+        this.app = app;
 
         this.currentData = this.getDefaultData();
     }

src/js/states/main_menu.js

@@ -2,16 +2,13 @@ import { globalConfig, THIRDPARTY_URLS } from "../core/config";
 import { GameState } from "../core/game_state";
 import { DialogWithForm } from "../core/modal_dialog_elements";
 import { FormElementInput } from "../core/modal_dialog_forms";
-import { ReadWriteProxy } from "../core/read_write_proxy";
 import {
     formatSecondsToTimeAgo,
-    generateFileDownload,
     getLogoSprite,
     makeButton,
     makeDiv,
     makeDivElement,
     removeAllChildren,
-    startFileChoose,
     waitNextFrame,
 } from "../core/utils";
 import { HUDModalDialogs } from "../game/hud/parts/modal_dialogs";
@@ -137,58 +134,34 @@ export class MainMenuState extends GameState {
     /**
      * Asks the user to import a savegame
      */
-    requestImportSavegame() {
+    async requestImportSavegame() {
-        // Create a 'fake' file-input to accept savegames
+        const closeLoader = this.dialogs.showLoadingDialog();
-        startFileChoose(".bin").then(file => {
+        await waitNextFrame();
-            if (file) {
-                const closeLoader = this.dialogs.showLoadingDialog();
-                waitNextFrame().then(() => {
-                    const reader = new FileReader();
-                    reader.addEventListener("load", event => {
-                        const contents = event.target.result;
-                        let realContent;
 
-                        try {
+        const data = await this.app.storage.requestOpenFile("bin");
-                            realContent = ReadWriteProxy.deserializeObject(contents);
+        if (data === undefined) {
-                        } catch (err) {
+            // User canceled the request
-                            closeLoader();
+            closeLoader();
-                            this.dialogs.showWarning(
+            return;
-                                T.dialogs.importSavegameError.title,
+        }
-                                T.dialogs.importSavegameError.text + "<br><br>" + err
-                            );
-                            return;
-                        }
 
-                        this.app.savegameMgr.importSavegame(realContent).then(
+        try {
-                            () => {
+            this.app.savegameMgr.importSavegame(data);
-                                closeLoader();
+            closeLoader();
-                                this.dialogs.showWarning(
+            this.dialogs.showWarning(
-                                    T.dialogs.importSavegameSuccess.title,
+                T.dialogs.importSavegameSuccess.title,
-                                    T.dialogs.importSavegameSuccess.text
+                T.dialogs.importSavegameSuccess.text
-                                );
+            );
 
-                                this.renderMainMenu();
+            this.renderMainMenu();
-                                this.renderSavegames();
+            this.renderSavegames();
-                            },
+        } catch (err) {
-                            err => {
+            closeLoader();
-                                closeLoader();
+            this.dialogs.showWarning(
-                                this.dialogs.showWarning(
+                T.dialogs.importSavegameError.title,
-                                    T.dialogs.importSavegameError.title,
+                T.dialogs.importSavegameError.text + ":<br><br>" + err
-                                    T.dialogs.importSavegameError.text + ":<br><br>" + err
+            );
-                                );
+        }
-                            }
-                        );
-                    });
-                    reader.addEventListener("error", error => {
-                        this.dialogs.showWarning(
-                            T.dialogs.importSavegameError.title,
-                            T.dialogs.importSavegameError.text + ":<br><br>" + error
-                        );
-                    });
-                    reader.readAsText(file, "utf-8");
-                });
-            }
-        });
     }
 
     onBackButton() {
@@ -602,9 +575,8 @@ export class MainMenuState extends GameState {
     downloadGame(game) {
         const savegame = this.app.savegameMgr.getSavegameById(game.internalId);
         savegame.readAsync().then(() => {
-            const data = ReadWriteProxy.serializeObject(savegame.currentData);
             const filename = (game.name || "unnamed") + ".bin";
-            generateFileDownload(filename, data);
+            savegame.storage.requestSaveFile(filename, savegame.currentData);
         });
     }
 

src/js/webworkers/compression.worker.js

@@ -1,31 +0,0 @@
-import { globalConfig } from "../core/config";
-import { compressX64 } from "../core/lzstring";
-import { computeCrc } from "../core/sensitive_utils.encrypt";
-import { compressObject } from "../savegame/savegame_compressor";
-
-self.addEventListener("message", event => {
-    // @ts-ignore
-    const { jobId, job, data } = event.data;
-    const result = performJob(job, data);
-
-    // @ts-ignore
-    self.postMessage({ jobId, result });
-});
-
-function performJob(job, data) {
-    switch (job) {
-        case "compressX64": {
-            return compressX64(data);
-        }
-
-        case "compressObject": {
-            const optimized = compressObject(data.obj);
-            const stringified = JSON.stringify(optimized);
-
-            const checksum = computeCrc(stringified + globalConfig.info.file);
-            return data.compressionPrefix + compressX64(checksum + stringified);
-        }
-        default:
-            throw new Error("Webworker: Unknown job: " + job);
-    }
-}