Archives/tobspr_shapez.io
1
0
Fork 0
mirror of https://github.com/tobspr/shapez.io.git synced 2024-10-27 20:34:29 +00:00
Code Issues Projects Releases Wiki Activity

Improve saving speed by using crc32 instead of sha1 for checksum

Browse Source
This commit is contained in:
tobspr 2020-08-10 23:31:02 +02:00
parent 9701a143ec
commit 2203d5fc6a
7 changed files with 59 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -34,6 +34,7 @@
"clipboard-copy": "^3.1.0",
"colors": "^1.3.3",
"core-js": "3",
"crc": "^3.8.0",
"cssnano-preset-advanced": "^4.0.7",
"email-validator": "^2.0.4",
"eslint": "7.1.0",

22
src/js/core/async_compression.js
View File

@ -53,7 +53,15 @@ class AsynCompression {
}
const duration = performance.now() - jobData.startTime;
logger.log("Got job", jobId, "response within", round2Digits(duration), "ms");
logger.log(
"Got job",
jobId,
"response within",
round2Digits(duration),
"ms: ",
result.length,
"bytes"
);
const resolver = jobData.resolver;
delete this.currentJobs[jobId];
resolver(result);
@ -73,13 +81,13 @@ class AsynCompression {
}
/**
* Compresses file
* @param {string} text
* Compresses any object
* @param {any} obj
*/
compressFileAsync(text) {
logger.log("Compressing", text.length, "bytes async");
return this.internalQueueJob("compressFile", {
text,
compressObjectAsync(obj) {
logger.log("Compressing object async (optimized)");
return this.internalQueueJob("compressObject", {
obj,
compressionPrefix,
});
}

42
src/js/core/read_write_proxy.js
View File

@ -2,7 +2,7 @@
import { Application } from "../application";
/* typehints:end */
import { sha1 } from "./sensitive_utils.encrypt";
import { sha1, CRC_PREFIX } from "./sensitive_utils.encrypt";
import { createLogger } from "./logging";
import { FILE_NOT_FOUND } from "../platform/storage";
import { accessNestedPropertyReverse } from "./utils";
@ -11,6 +11,7 @@ import { ExplainedResult } from "./explained_result";
import { decompressX64, compressX64 } from "./lzstring";
import { asyncCompressor, compressionPrefix } from "./async_compression";
import { compressObject, decompressObject } from "../savegame/savegame_compressor";
import crc32 from "crc/crc32";
const logger = createLogger("read_write_proxy");
@ -84,7 +85,7 @@ export class ReadWriteProxy {
*/
static serializeObject(obj) {
const jsonString = JSON.stringify(compressObject(obj));
const checksum = sha1(jsonString + salt);
const checksum = CRC_PREFIX + crc32(jsonString + salt).toString(16);
return compressionPrefix + compressX64(checksum + jsonString);
}
@ -106,7 +107,11 @@ export class ReadWriteProxy {
// Compare stored checksum with actual checksum
const checksum = decompressed.substring(0, 40);
const jsonString = decompressed.substr(40);
const desiredChecksum = sha1(jsonString + salt);
const desiredChecksum = checksum.startsWith(CRC_PREFIX)
? CRC_PREFIX + crc32(jsonString + salt).toString(16)
: sha1(jsonString + salt);
if (desiredChecksum !== checksum) {
// Checksum mismatch
throw new Error("bad-content / checksum-mismatch");
@ -119,7 +124,7 @@ export class ReadWriteProxy {
/**
* Writes the data asychronously, fails if verify() fails
* @returns {Promise<string>}
* @returns {Promise<void>}
*/
writeAsync() {
const verifyResult = this.internalVerifyEntry(this.currentData);
@ -128,35 +133,14 @@ export class ReadWriteProxy {
logger.error("Tried to write invalid data to", this.filename, "reason:", verifyResult.reason);
return Promise.reject(verifyResult.reason);
}
const jsonString = JSON.stringify(compressObject(this.currentData));
// if (!this.app.pageVisible || this.app.unloaded) {
// logger.log("Saving file sync because in unload handler");
// const checksum = sha1(jsonString + salt);
// let compressed = compressionPrefix + compressX64(checksum + jsonString);
// if (G_IS_DEV && IS_DEBUG) {
// compressed = jsonString;
// }
// if (!this.app.storage.writeFileSyncIfSupported(this.filename, compressed)) {
// return Promise.reject("Failed to write " + this.filename + " sync!");
// } else {
// logger.log("📄 Wrote (sync!)", this.filename);
// return Promise.resolve(compressed);
// }
// }
return asyncCompressor
.compressFileAsync(jsonString)
.compressObjectAsync(this.currentData)
.then(compressed => {
if (G_IS_DEV && IS_DEBUG) {
compressed = jsonString;
}
return this.app.storage.writeFileAsync(this.filename, compressed);
})
.then(() => {
logger.log("📄 Wrote", this.filename);
return jsonString;
})
.catch(err => {
logger.error("Failed to write", this.filename, ":", err);
@ -205,7 +189,11 @@ export class ReadWriteProxy {
// Compare stored checksum with actual checksum
const checksum = decompressed.substring(0, 40);
const jsonString = decompressed.substr(40);
const desiredChecksum = sha1(jsonString + salt);
const desiredChecksum = checksum.startsWith(CRC_PREFIX)
? CRC_PREFIX + crc32(jsonString + salt).toString(16)
: sha1(jsonString + salt);
if (desiredChecksum !== checksum) {
// Checksum mismatch
return Promise.reject("bad-content / checksum-mismatch");

3
src/js/core/sensitive_utils.encrypt.js
View File

@ -10,3 +10,6 @@ export function sha1(str) {
export function getNameOfProvider() {
return window[decompressX64("DYewxghgLgliB2Q")][decompressX64("BYewzgLgdghgtgUyA")];
}
// Distinguish legacy crc prefixes
export const CRC_PREFIX = "crc32".padEnd(32, "-");

3
src/js/savegame/savegame_compressor.js
View File

@ -90,9 +90,6 @@ function compressObjectInternal(obj, keys = [], values = []) {
}
export function compressObject(obj) {
if (G_IS_DEV) {
return obj;
}
const keys = [];
const values = [];
const data = compressObjectInternal(obj, keys, values);

14
src/js/webworkers/compression.worker.js
View File

@ -1,6 +1,8 @@
import { compressX64 } from "../core/lzstring";
import { globalConfig } from "../core/config";
import { sha1 } from "../core/sensitive_utils.encrypt";
import { compressObject } from "../savegame/savegame_compressor";
import { CRC_PREFIX } from "../core/sensitive_utils.encrypt";
import crc32 from "crc/crc32";
function accessNestedPropertyReverse(obj, keys) {
let result = obj;
@ -24,9 +26,13 @@ function performJob(job, data) {
case "compressX64": {
return compressX64(data);
}
case "compressFile": {
const checksum = sha1(data.text + salt);
return data.compressionPrefix + compressX64(checksum + data.text);
case "compressObject": {
const optimized = compressObject(data.obj);
const stringified = JSON.stringify(optimized);
const checksum = CRC_PREFIX + crc32(stringified + salt).toString(16);
return data.compressionPrefix + compressX64(checksum + stringified);
}
default:
throw new Error("Webworker: Unknown job: " + job);

15
yarn.lock
View File

@ -1877,6 +1877,14 @@ buffer@^4.3.0:
ieee754 "^1.1.4"
isarray "^1.0.0"
buffer@^5.1.0:
version "5.6.0"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==
dependencies:
base64-js "^1.0.2"
ieee754 "^1.1.4"
buffer@^5.2.0, buffer@^5.2.1:
version "5.5.0"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.5.0.tgz#9c3caa3d623c33dd1c7ef584b89b88bf9c9bc1ce"
@ -2441,6 +2449,13 @@ cosmiconfig@^5.0.0:
js-yaml "^3.13.1"
parse-json "^4.0.0"
crc@^3.8.0:
version "3.8.0"
resolved "https://registry.yarnpkg.com/crc/-/crc-3.8.0.tgz#ad60269c2c856f8c299e2c4cc0de4556914056c6"
integrity sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==
dependencies:
buffer "^5.1.0"
create-ecdh@^4.0.0:
version "4.0.3"
resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff"