From fee481f9e970a980c4b567016940cbafc905ea54 Mon Sep 17 00:00:00 2001
From: tmpolaczyk <44604217+tmpolaczyk@users.noreply.github.com>
Date: Wed, 24 Jul 2024 11:04:52 +0200
Subject: [PATCH] Use jobserver in wasm-builder to limit concurrency of spawned
cargo processes (#4946)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
When building multiple runtimes in parallel, each of them will try to
use the concurrency set by the parent cargo process. For example, in a
system with 8 cpu cores, building 3 runtimes in parallel creates 8 * 3
tasks. This results in the system hanging because of the high cpu and
memory usage.
This PR allows the substrate_wasm_builder to use the same [jobserver][1]
as the parent cargo process, making all invocations of cargo share the
same concurrency pool. So in a system with 8 cores, there will never be
more than 8 tasks running at the same time.
Implementation roughly based on [cargo][2] but with less unsafe.
This can be tested by telling cargo to use half the cpu cores, like
`cargo build -j4` in an 8 core machine. Before this PR it will use 100%
cpu when building 2 runtimes in parallel, after this PR it will always
use 50%.
[1]:
https://doc.rust-lang.org/cargo/reference/build-scripts.html#jobserver
[2]:
https://github.com/rust-lang/cargo/blob/d1b5f0759eedf5f1126c781c64232856956069ad/src/cargo/util/context/mod.rs#L271
---------
Co-authored-by: Bastian Köcher <git@kchr.de>
---
Cargo.lock | 1 +
Cargo.toml | 1 +
substrate/utils/wasm-builder/Cargo.toml | 1 +
substrate/utils/wasm-builder/src/builder.rs | 2 ++
.../utils/wasm-builder/src/wasm_project.rs | 17 +++++++++++++++++
5 files changed, 22 insertions(+)
diff --git a/Cargo.lock b/Cargo.lock
index 13e6b716e66..90415606bd3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -21520,6 +21520,7 @@ dependencies = [
"console",
"filetime",
"frame-metadata",
+ "jobserver",
"merkleized-metadata",
"parity-scale-codec",
"parity-wasm",
diff --git a/Cargo.toml b/Cargo.toml
index 1699bd97a8e..9b14dfd3cdf 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -803,6 +803,7 @@ is-terminal = { version = "0.4.9" }
is_executable = { version = "1.0.1" }
isahc = { version = "1.2" }
itertools = { version = "0.11" }
+jobserver = { version = "0.1.26" }
jsonpath_lib = { version = "0.3" }
jsonrpsee = { version = "0.23.2" }
jsonrpsee-core = { version = "0.23.2" }
diff --git a/substrate/utils/wasm-builder/Cargo.toml b/substrate/utils/wasm-builder/Cargo.toml
index f084400c12e..28fdebf209b 100644
--- a/substrate/utils/wasm-builder/Cargo.toml
+++ b/substrate/utils/wasm-builder/Cargo.toml
@@ -27,6 +27,7 @@ filetime = { workspace = true }
wasm-opt = { workspace = true }
parity-wasm = { workspace = true }
polkavm-linker = { workspace = true }
+jobserver = { workspace = true }
# Dependencies required for the `metadata-hash` feature.
merkleized-metadata = { optional = true, workspace = true }
diff --git a/substrate/utils/wasm-builder/src/builder.rs b/substrate/utils/wasm-builder/src/builder.rs
index 37c6c4aa743..eb761a103d6 100644
--- a/substrate/utils/wasm-builder/src/builder.rs
+++ b/substrate/utils/wasm-builder/src/builder.rs
@@ -346,6 +346,8 @@ fn build_project(
check_for_runtime_version_section: bool,
#[cfg(feature = "metadata-hash")] enable_metadata_hash: Option<MetadataExtraInfo>,
) {
+ // Init jobserver as soon as possible
+ crate::wasm_project::get_jobserver();
let cargo_cmd = match crate::prerequisites::check(target) {
Ok(cmd) => cmd,
Err(err_msg) => {
diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs
index 20fa9fc1aa3..63887389fb1 100644
--- a/substrate/utils/wasm-builder/src/wasm_project.rs
+++ b/substrate/utils/wasm-builder/src/wasm_project.rs
@@ -31,6 +31,7 @@ use std::{
ops::Deref,
path::{Path, PathBuf},
process,
+ sync::OnceLock,
};
use strum::{EnumIter, IntoEnumIterator};
use toml::value::Table;
@@ -899,6 +900,12 @@ fn build_bloaty_blob(
}
}
+ // Inherit jobserver in child cargo command to ensure we don't try to use more concurrency than
+ // available
+ if let Some(c) = get_jobserver() {
+ c.configure(&mut build_cmd);
+ }
+
println!("{}", colorize_info_message("Information that should be included in a bug report."));
println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd);
println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version());
@@ -1178,3 +1185,13 @@ fn copy_blob_to_target_directory(cargo_manifest: &Path, blob_binary: &WasmBinary
)
.expect("Copies blob binary to `WASM_TARGET_DIRECTORY`.");
}
+
+// Get jobserver from parent cargo command
+pub fn get_jobserver() -> &'static Option<jobserver::Client> {
+ static JOBSERVER: OnceLock<Option<jobserver::Client>> = OnceLock::new();
+
+ JOBSERVER.get_or_init(|| {
+ // Unsafe because it deals with raw fds
+ unsafe { jobserver::Client::from_env() }
+ })
+}
--
GitLab