1
0
mirror of https://github.com/tobspr/shapez.io.git synced 2024-10-27 20:34:29 +00:00
tobspr_shapez.io/gulp/sounds.js
LeopoldTal ff2274f671
fix size overflow when building sounds (#676)
Building the standalone with V8 v8 (Node v14) fails at step `sounds.musicHQ`, either by running out of memory or with the error "Invalid string length". Building with V8 v7 (Node v12) succeeds.

This is because `gulp-cache` builds a 994-MB string when [stringifying file contents](https://github.com/jgable/gulp-cache/blob/master/src/task-proxy.js#L262) to cache them. V8 v8 [limits string length to around 537 MB](https://github.com/v8/v8/blob/master/src/objects/string.h#L384), and thus cannot represent this string. (V8 v7 allows around 1074 MB, which is why the build passes on Node v12.)

But [`theme-full.mp3`](https://github.com/tobspr/shapez.io/blob/master/res_raw/sounds/music/theme-full.mp3) is only 79 MB: how did we get 1250% overhead?

Unlike plaintext files, binary files are read as buffers, but [by default](https://github.com/jgable/gulp-cache/blob/master/src/index.js#L46) `gulp-cache` stringifies them naively, producing the extremely inefficient representation:

````
[
  {
    "cwd": "/Users/tobspr/shapez.io/gulp",
    "base": "/Users/tobspr/shapez.io/res_raw/sounds/music",
    "contents": {
      "type": "Buffer",
      "data": [
        73,
        68,
        51,
        4,
        0,
        0,
…etc.
````

Fortunately, `gulp-cache` [can read base64-encoded cache files](https://github.com/jgable/gulp-cache/blob/master/src/index.js#L26).

Instead of using the default file transform function, **pass a `value` option to base64-encode the file contents**. This results in only 33% overhead on cache file size (106 MB for the largest file).

This has multiple benefits:

- Fixes the build failure
- Requires less memory (from 6 GB down to < 1 GB on my machine)
- When cache files are found, the `sounds.musicHQ` is much faster (from ~30 s down to ~4 s on my machine)
- Smaller cache files on disk
2020-09-21 08:37:50 +02:00

136 lines
4.9 KiB
JavaScript

const path = require("path");
const audiosprite = require("gulp-audiosprite");
function gulptasksSounds($, gulp, buildFolder) {
// Gather some basic infos
const soundsDir = path.join(__dirname, "..", "res_raw", "sounds");
const builtSoundsDir = path.join(__dirname, "..", "res_built", "sounds");
gulp.task("sounds.clear", () => {
return gulp.src(builtSoundsDir, { read: false, allowEmpty: true }).pipe($.clean({ force: true }));
});
const filters = ["volume=0.2"];
const fileCache = new $.cache.Cache({
cacheDirName: "shapezio-precompiled-sounds",
});
function getFileCacheValue(file) {
const { _isVinyl, base, cwd, contents, history, stat, path } = file;
const encodedContents = Buffer.from(contents).toString('base64');
return { _isVinyl, base, cwd, contents: encodedContents, history, stat, path };
}
// Encodes the game music
gulp.task("sounds.music", () => {
return gulp
.src([path.join(soundsDir, "music", "**", "*.wav"), path.join(soundsDir, "music", "**", "*.mp3")])
.pipe($.plumber())
.pipe(
$.cache(
$.fluentFfmpeg("mp3", function (cmd) {
return cmd
.audioBitrate(48)
.audioChannels(1)
.audioFrequency(22050)
.audioCodec("libmp3lame")
.audioFilters(["volume=0.15"]);
}),
{
name: "music",
fileCache,
value: getFileCacheValue,
}
)
)
.pipe(gulp.dest(path.join(builtSoundsDir, "music")));
});
// Encodes the game music in high quality for the standalone
gulp.task("sounds.musicHQ", () => {
return gulp
.src([path.join(soundsDir, "music", "**", "*.wav"), path.join(soundsDir, "music", "**", "*.mp3")])
.pipe($.plumber())
.pipe(
$.cache(
$.fluentFfmpeg("mp3", function (cmd) {
return cmd
.audioBitrate(256)
.audioChannels(2)
.audioFrequency(44100)
.audioCodec("libmp3lame")
.audioFilters(["volume=0.15"]);
}),
{
name: "music-high-quality",
fileCache,
value: getFileCacheValue,
}
)
)
.pipe(gulp.dest(path.join(builtSoundsDir, "music")));
});
// Encodes the ui sounds
gulp.task("sounds.sfxGenerateSprites", () => {
return gulp
.src([path.join(soundsDir, "sfx", "**", "*.wav"), path.join(soundsDir, "sfx", "**", "*.mp3")])
.pipe($.plumber())
.pipe(
audiosprite({
format: "howler",
output: "sfx",
gap: 0.1,
export: "mp3",
})
)
.pipe(gulp.dest(path.join(builtSoundsDir)));
});
gulp.task("sounds.sfxOptimize", () => {
return gulp
.src([path.join(builtSoundsDir, "sfx.mp3")])
.pipe($.plumber())
.pipe(
$.fluentFfmpeg("mp3", function (cmd) {
return cmd
.audioBitrate(128)
.audioChannels(1)
.audioFrequency(22050)
.audioCodec("libmp3lame")
.audioFilters(filters);
})
)
.pipe(gulp.dest(path.join(builtSoundsDir)));
});
gulp.task("sounds.sfxCopyAtlas", () => {
return gulp
.src([path.join(builtSoundsDir, "sfx.json")])
.pipe(gulp.dest(path.join(__dirname, "..", "src", "js", "built-temp")));
});
gulp.task(
"sounds.sfx",
gulp.series("sounds.sfxGenerateSprites", "sounds.sfxOptimize", "sounds.sfxCopyAtlas")
);
gulp.task("sounds.copy", () => {
return gulp
.src(path.join(builtSoundsDir, "**", "*.mp3"))
.pipe($.plumber())
.pipe($.cached("sounds.copy"))
.pipe(gulp.dest(path.join(buildFolder, "res", "sounds")));
});
gulp.task("sounds.buildall", gulp.parallel("sounds.music", "sounds.sfx"));
gulp.task("sounds.buildallHQ", gulp.parallel("sounds.musicHQ", "sounds.sfx"));
gulp.task("sounds.fullbuild", gulp.series("sounds.clear", "sounds.buildall", "sounds.copy"));
gulp.task("sounds.fullbuildHQ", gulp.series("sounds.clear", "sounds.buildallHQ", "sounds.copy"));
gulp.task("sounds.dev", gulp.series("sounds.buildall", "sounds.copy"));
}
module.exports = {
gulptasksSounds,
};