From ff2274f671c7c58845efc7724dc38ec5a96a70e8 Mon Sep 17 00:00:00 2001
From: LeopoldTal <leopold.tal.dg@gmail.com>
Date: Mon, 21 Sep 2020 08:37:50 +0200
Subject: [PATCH] fix size overflow when building sounds (#676)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Building the standalone with V8 v8 (Node v14) fails at step `sounds.musicHQ`, either by running out of memory or with the error "Invalid string length". Building with V8 v7 (Node v12) succeeds.

This is because `gulp-cache` builds a 994-MB string when [stringifying file contents](https://github.com/jgable/gulp-cache/blob/master/src/task-proxy.js#L262) to cache them. V8 v8 [limits string length to around 537 MB](https://github.com/v8/v8/blob/master/src/objects/string.h#L384), and thus cannot represent this string. (V8 v7 allows around 1074 MB, which is why the build passes on Node v12.)

But [`theme-full.mp3`](https://github.com/tobspr/shapez.io/blob/master/res_raw/sounds/music/theme-full.mp3) is only 79 MB: how did we get 1250% overhead?

Unlike plaintext files, binary files are read as buffers, but [by default](https://github.com/jgable/gulp-cache/blob/master/src/index.js#L46) `gulp-cache` stringifies them naively, producing the extremely inefficient representation:

````
[
  {
    "cwd": "/Users/tobspr/shapez.io/gulp",
    "base": "/Users/tobspr/shapez.io/res_raw/sounds/music",
    "contents": {
      "type": "Buffer",
      "data": [
        73,
        68,
        51,
        4,
        0,
        0,
…etc.
````

Fortunately, `gulp-cache` [can read base64-encoded cache files](https://github.com/jgable/gulp-cache/blob/master/src/index.js#L26).

Instead of using the default file transform function, **pass a `value` option to base64-encode the file contents**. This results in only 33% overhead on cache file size (106 MB for the largest file).

This has multiple benefits:

- Fixes the build failure
- Requires less memory (from 6 GB down to < 1 GB on my machine)
- When cache files are found, the `sounds.musicHQ` is much faster (from ~30 s down to ~4 s on my machine)
- Smaller cache files on disk
---
 gulp/sounds.js | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/gulp/sounds.js b/gulp/sounds.js
index c38e7a6e..aaae606f 100644
--- a/gulp/sounds.js
+++ b/gulp/sounds.js
@@ -16,6 +16,12 @@ function gulptasksSounds($, gulp, buildFolder) {
         cacheDirName: "shapezio-precompiled-sounds",
     });
 
+    function getFileCacheValue(file) {
+        const { _isVinyl, base, cwd, contents, history, stat, path } = file;
+        const encodedContents = Buffer.from(contents).toString('base64');
+        return { _isVinyl, base, cwd, contents: encodedContents, history, stat, path };
+    }
+
     // Encodes the game music
     gulp.task("sounds.music", () => {
         return gulp
@@ -34,6 +40,7 @@ function gulptasksSounds($, gulp, buildFolder) {
                     {
                         name: "music",
                         fileCache,
+                        value: getFileCacheValue,
                     }
                 )
             )
@@ -58,6 +65,7 @@ function gulptasksSounds($, gulp, buildFolder) {
                     {
                         name: "music-high-quality",
                         fileCache,
+                        value: getFileCacheValue,
                     }
                 )
             )