Improve saving speed by using crc32 instead of sha1 for checksum

package.json

@@ -34,6 +34,7 @@
         "clipboard-copy": "^3.1.0",
         "colors": "^1.3.3",
         "core-js": "3",
+        "crc": "^3.8.0",
         "cssnano-preset-advanced": "^4.0.7",
         "email-validator": "^2.0.4",
         "eslint": "7.1.0",

src/js/core/async_compression.js

@@ -53,7 +53,15 @@ class AsynCompression {
             }
 
             const duration = performance.now() - jobData.startTime;
-            logger.log("Got job", jobId, "response within", round2Digits(duration), "ms");
+            logger.log(
+                "Got job",
+                jobId,
+                "response within",
+                round2Digits(duration),
+                "ms: ",
+                result.length,
+                "bytes"
+            );
             const resolver = jobData.resolver;
             delete this.currentJobs[jobId];
             resolver(result);
@@ -73,13 +81,13 @@ class AsynCompression {
     }
 
     /**
-     * Compresses file
+     * Compresses any object
-     * @param {string} text
+     * @param {any} obj
      */
-    compressFileAsync(text) {
+    compressObjectAsync(obj) {
-        logger.log("Compressing", text.length, "bytes async");
+        logger.log("Compressing object async (optimized)");
-        return this.internalQueueJob("compressFile", {
+        return this.internalQueueJob("compressObject", {
-            text,
+            obj,
             compressionPrefix,
         });
     }

src/js/core/read_write_proxy.js

@@ -2,7 +2,7 @@
 import { Application } from "../application";
 /* typehints:end */
 
-import { sha1 } from "./sensitive_utils.encrypt";
+import { sha1, CRC_PREFIX } from "./sensitive_utils.encrypt";
 import { createLogger } from "./logging";
 import { FILE_NOT_FOUND } from "../platform/storage";
 import { accessNestedPropertyReverse } from "./utils";
@@ -11,6 +11,7 @@ import { ExplainedResult } from "./explained_result";
 import { decompressX64, compressX64 } from "./lzstring";
 import { asyncCompressor, compressionPrefix } from "./async_compression";
 import { compressObject, decompressObject } from "../savegame/savegame_compressor";
+import crc32 from "crc/crc32";
 
 const logger = createLogger("read_write_proxy");
 
@@ -84,7 +85,7 @@ export class ReadWriteProxy {
      */
     static serializeObject(obj) {
         const jsonString = JSON.stringify(compressObject(obj));
-        const checksum = sha1(jsonString + salt);
+        const checksum = CRC_PREFIX + crc32(jsonString + salt).toString(16);
         return compressionPrefix + compressX64(checksum + jsonString);
     }
 
@@ -106,7 +107,11 @@ export class ReadWriteProxy {
         // Compare stored checksum with actual checksum
         const checksum = decompressed.substring(0, 40);
         const jsonString = decompressed.substr(40);
-        const desiredChecksum = sha1(jsonString + salt);
+
+        const desiredChecksum = checksum.startsWith(CRC_PREFIX)
+            ? CRC_PREFIX + crc32(jsonString + salt).toString(16)
+            : sha1(jsonString + salt);
+
         if (desiredChecksum !== checksum) {
             // Checksum mismatch
             throw new Error("bad-content / checksum-mismatch");
@@ -119,7 +124,7 @@ export class ReadWriteProxy {
 
     /**
      * Writes the data asychronously, fails if verify() fails
-     * @returns {Promise<string>}
+     * @returns {Promise<void>}
      */
     writeAsync() {
         const verifyResult = this.internalVerifyEntry(this.currentData);
@@ -128,35 +133,14 @@ export class ReadWriteProxy {
             logger.error("Tried to write invalid data to", this.filename, "reason:", verifyResult.reason);
             return Promise.reject(verifyResult.reason);
         }
-        const jsonString = JSON.stringify(compressObject(this.currentData));
-
-        // if (!this.app.pageVisible || this.app.unloaded) {
-        //     logger.log("Saving file sync because in unload handler");
-        //     const checksum = sha1(jsonString + salt);
-        //     let compressed = compressionPrefix + compressX64(checksum + jsonString);
-        //     if (G_IS_DEV && IS_DEBUG) {
-        //         compressed = jsonString;
-        //     }
-
-        //     if (!this.app.storage.writeFileSyncIfSupported(this.filename, compressed)) {
-        //         return Promise.reject("Failed to write " + this.filename + " sync!");
-        //     } else {
-        //         logger.log("📄 Wrote (sync!)", this.filename);
-        //         return Promise.resolve(compressed);
-        //     }
-        // }
 
         return asyncCompressor
-            .compressFileAsync(jsonString)
+            .compressObjectAsync(this.currentData)
             .then(compressed => {
-                if (G_IS_DEV && IS_DEBUG) {
-                    compressed = jsonString;
-                }
                 return this.app.storage.writeFileAsync(this.filename, compressed);
             })
             .then(() => {
                 logger.log("📄 Wrote", this.filename);
-                return jsonString;
             })
             .catch(err => {
                 logger.error("Failed to write", this.filename, ":", err);
@@ -205,7 +189,11 @@ export class ReadWriteProxy {
                         // Compare stored checksum with actual checksum
                         const checksum = decompressed.substring(0, 40);
                         const jsonString = decompressed.substr(40);
-                        const desiredChecksum = sha1(jsonString + salt);
+
+                        const desiredChecksum = checksum.startsWith(CRC_PREFIX)
+                            ? CRC_PREFIX + crc32(jsonString + salt).toString(16)
+                            : sha1(jsonString + salt);
+
                         if (desiredChecksum !== checksum) {
                             // Checksum mismatch
                             return Promise.reject("bad-content / checksum-mismatch");

src/js/core/sensitive_utils.encrypt.js

@@ -10,3 +10,6 @@ export function sha1(str) {
 export function getNameOfProvider() {
     return window[decompressX64("DYewxghgLgliB2Q")][decompressX64("BYewzgLgdghgtgUyA")];
 }
+
+// Distinguish legacy crc prefixes
+export const CRC_PREFIX = "crc32".padEnd(32, "-");

src/js/savegame/savegame_compressor.js

@@ -90,9 +90,6 @@ function compressObjectInternal(obj, keys = [], values = []) {
 }
 
 export function compressObject(obj) {
-    if (G_IS_DEV) {
-        return obj;
-    }
     const keys = [];
     const values = [];
     const data = compressObjectInternal(obj, keys, values);

src/js/webworkers/compression.worker.js

@@ -1,6 +1,8 @@
 import { compressX64 } from "../core/lzstring";
 import { globalConfig } from "../core/config";
-import { sha1 } from "../core/sensitive_utils.encrypt";
+import { compressObject } from "../savegame/savegame_compressor";
+import { CRC_PREFIX } from "../core/sensitive_utils.encrypt";
+import crc32 from "crc/crc32";
 
 function accessNestedPropertyReverse(obj, keys) {
     let result = obj;
@@ -24,9 +26,13 @@ function performJob(job, data) {
         case "compressX64": {
             return compressX64(data);
         }
-        case "compressFile": {
+
-            const checksum = sha1(data.text + salt);
+        case "compressObject": {
-            return data.compressionPrefix + compressX64(checksum + data.text);
+            const optimized = compressObject(data.obj);
+            const stringified = JSON.stringify(optimized);
+
+            const checksum = CRC_PREFIX + crc32(stringified + salt).toString(16);
+            return data.compressionPrefix + compressX64(checksum + stringified);
         }
         default:
             throw new Error("Webworker: Unknown job: " + job);

yarn.lock

@@ -1877,6 +1877,14 @@ buffer@^4.3.0:
     ieee754 "^1.1.4"
     isarray "^1.0.0"
 
+buffer@^5.1.0:
+  version "5.6.0"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
+  integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==
+  dependencies:
+    base64-js "^1.0.2"
+    ieee754 "^1.1.4"
+
 buffer@^5.2.0, buffer@^5.2.1:
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.5.0.tgz#9c3caa3d623c33dd1c7ef584b89b88bf9c9bc1ce"
@@ -2441,6 +2449,13 @@ cosmiconfig@^5.0.0:
     js-yaml "^3.13.1"
     parse-json "^4.0.0"
 
+crc@^3.8.0:
+  version "3.8.0"
+  resolved "https://registry.yarnpkg.com/crc/-/crc-3.8.0.tgz#ad60269c2c856f8c299e2c4cc0de4556914056c6"
+  integrity sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==
+  dependencies:
+    buffer "^5.1.0"
+
 create-ecdh@^4.0.0:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff"