Newer
Older
// Copyright (C) Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! PVF artifacts (final compiled code blobs).
//!
//! # Lifecycle of an artifact
//! 1. During node start-up, we will check the cached artifacts, if any. The stale and corrupted
//! ones are pruned. The valid ones are registered in the [`Artifacts`] table.
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
//!
//! 2. In order to be executed, a PVF should be prepared first. This means that artifacts should
//! have an [`ArtifactState::Prepared`] entry for that artifact in the table. If not, the
//! preparation process kicks in. The execution request is stashed until after the preparation is
//! done, and the artifact state in the host is set to [`ArtifactState::Preparing`]. Preparation
//! goes through the preparation queue and the pool.
//!
//! 1. If the artifact is already being processed, we add another execution request to the
//! existing preparation job, without starting a new one.
//!
//! 2. Note that if the state is [`ArtifactState::FailedToProcess`], we usually do not retry
//! preparation, though we may under certain conditions.
//!
//! 3. The pool gets an available worker and instructs it to work on the given PVF. The worker
//! starts compilation. When the worker finishes successfully, it writes the serialized artifact
//! into a temporary file and notifies the host that it's done. The host atomically moves
//! (renames) the temporary file to the destination filename of the artifact.
//!
//! 4. If the worker concluded successfully or returned an error, then the pool notifies the queue.
//! In both cases, the queue reports to the host that the result is ready.
//!
//! 5. The host will react by changing the artifact state to either [`ArtifactState::Prepared`] or
//! [`ArtifactState::FailedToProcess`] for the PVF in question. On success, the
//! `last_time_needed` will be set to the current time. It will also dispatch the pending
//! execution requests.
//!
//! 6. On success, the execution request will come through the execution queue and ultimately be
//! processed by an execution worker. When this worker receives the request, it will read the
//! requested artifact. If it doesn't exist it reports an internal error. A request for execution
//! will bump the `last_time_needed` to the current time.
//!
//! 7. There is a separate process for pruning the prepared artifacts whose `last_time_needed` is
//! older by a predefined parameter. This process is run very rarely (say, once a day). Once the
//! artifact is expired it is removed from disk eagerly atomically.
use crate::{host::PrecheckResultSender, LOG_TARGET};
use polkadot_core_primitives::Hash;
use polkadot_node_core_pvf_common::{
error::PrepareError, prepare::PrepareStats, pvf::PvfPrepData, RUNTIME_VERSION,
};
use polkadot_node_primitives::NODE_VERSION;
use polkadot_parachain_primitives::primitives::ValidationCodeHash;
use polkadot_primitives::ExecutorParamsHash;
const RUNTIME_PREFIX: &str = "wasmtime_v";
const NODE_PREFIX: &str = "polkadot_v";
fn artifact_prefix() -> String {
format!("{}{}_{}{}", RUNTIME_PREFIX, RUNTIME_VERSION, NODE_PREFIX, NODE_VERSION)
}
/// Identifier of an artifact. Encodes a code hash of the PVF and a hash of executor parameter set.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ArtifactId {
pub(crate) code_hash: ValidationCodeHash,
pub(crate) executor_params_hash: ExecutorParamsHash,
}
impl ArtifactId {
/// Creates a new artifact ID with the given hash.
pub fn new(code_hash: ValidationCodeHash, executor_params_hash: ExecutorParamsHash) -> Self {
Self { code_hash, executor_params_hash }
/// Returns an artifact ID that corresponds to the PVF with given executor params.
pub fn from_pvf_prep_data(pvf: &PvfPrepData) -> Self {
Self::new(pvf.code_hash(), pvf.executor_params().hash())
}
/// Returns the canonical path to the concluded artifact.
pub(crate) fn path(&self, cache_path: &Path, checksum: &str) -> PathBuf {
let file_name = format!(
"{}_{:#x}_{:#x}_0x{}",
artifact_prefix(),
self.executor_params_hash,
checksum
/// Tries to recover the artifact id from the given file name.
/// Return `None` if the given file name is invalid.
/// VALID_NAME := <PREFIX> _ <CODE_HASH> _ <PARAM_HASH> _ <CHECKSUM>
fn from_file_name(file_name: &str) -> Option<Self> {
let file_name = file_name.strip_prefix(&artifact_prefix())?.strip_prefix('_')?;
let parts: Vec<&str> = file_name.split('_').collect();
if let [code_hash, param_hash, _checksum] = parts[..] {
let code_hash = Hash::from_str(code_hash).ok()?.into();
let executor_params_hash =
ExecutorParamsHash::from_hash(Hash::from_str(param_hash).ok()?);
return Some(Self { code_hash, executor_params_hash })
}
None
}
/// A bundle of the artifact ID and the path.
///
/// Rationale for having this is two-fold:
///
/// - While we can derive the artifact path from the artifact id, it makes sense to carry it around
/// sometimes to avoid extra work.
/// - At the same time, carrying only path limiting the ability for logging.
#[derive(Debug, Clone)]
pub struct ArtifactPathId {
pub(crate) id: ArtifactId,
pub(crate) path: PathBuf,
}
impl ArtifactPathId {
pub(crate) fn new(artifact_id: ArtifactId, path: &Path) -> Self {
Self { id: artifact_id, path: path.to_owned() }
pub enum ArtifactState {
/// The artifact is ready to be used by the executor.
///
/// That means that the artifact should be accessible through the path obtained by the artifact
/// id (unless, it was removed externally).
Prepared {
/// The path of the compiled artifact.
path: PathBuf,
/// The time when the artifact was last needed.
///
/// This is updated when we get the heads up for this artifact or when we just discover
/// this file.
last_time_needed: SystemTime,
/// Stats produced by successful preparation.
prepare_stats: PrepareStats,
},
/// A task to prepare this artifact is scheduled.
Preparing {
/// List of result senders that are waiting for a response.
waiting_for_response: Vec<PrecheckResultSender>,
/// The number of times this artifact has failed to prepare.
num_failures: u32,
},
/// The code couldn't be compiled due to an error. Such artifacts
/// never reach the executor and stay in the host's memory.
FailedToProcess {
/// Keep track of the last time that processing this artifact failed.
last_time_failed: SystemTime,
/// The number of times this artifact has failed to prepare.
num_failures: u32,
/// The last error encountered for preparation.
error: PrepareError,
},
}
/// A container of all known artifact ids and their states.
pub struct Artifacts {
inner: HashMap<ArtifactId, ArtifactState>,
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
#[cfg(test)]
pub(crate) fn empty() -> Self {
Self { inner: HashMap::new() }
}
#[cfg(test)]
pub(crate) fn len(&self) -> usize {
self.inner.len()
}
/// Create an empty table and populate it with valid artifacts as [`ArtifactState::Prepared`],
/// if any. The existing caches will be checked by their file name to determine whether they are
/// valid, e.g., matching the current node version. The ones deemed invalid will be pruned.
pub async fn new_and_prune(cache_path: &Path) -> Self {
let mut artifacts = Self { inner: HashMap::new() };
artifacts.insert_and_prune(cache_path).await;
artifacts
}
async fn insert_and_prune(&mut self, cache_path: &Path) {
async fn is_corrupted(path: &Path) -> bool {
let checksum = match tokio::fs::read(path).await {
Ok(bytes) => blake3::hash(&bytes),
Err(err) => {
// just remove the file if we cannot read it
gum::warn!(
target: LOG_TARGET,
?err,
"unable to read artifact {:?} when checking integrity, removing...",
path,
);
return true
},
};
if let Some(file_name) = path.file_name() {
if let Some(file_name) = file_name.to_str() {
return !file_name.ends_with(checksum.to_hex().as_str())
}
}
true
}
// Insert the entry into the artifacts table if it is valid.
// Otherwise, prune it.
async fn insert_or_prune(
artifacts: &mut Artifacts,
entry: &tokio::fs::DirEntry,
cache_path: &Path,
) {
let file_type = entry.file_type().await;
let file_name = entry.file_name();
match file_type {
Ok(file_type) =>
if !file_type.is_file() {
return
},
Err(err) => {
gum::warn!(
target: LOG_TARGET,
?err,
"unable to get file type for {:?}",
file_name,
);
return
},
}
if let Some(file_name) = file_name.to_str() {
let id = ArtifactId::from_file_name(file_name);
let path = cache_path.join(file_name);
if id.is_none() || is_corrupted(&path).await {
gum::warn!(
target: LOG_TARGET,
"discarding invalid artifact {:?}",
&path,
);
let _ = tokio::fs::remove_file(&path).await;
return
}
if let Some(id) = id {
gum::debug!(
target: LOG_TARGET,
"reusing existing {:?} for node version v{}",
&path,
NODE_VERSION,
);
artifacts.insert_prepared(id, path, SystemTime::now(), Default::default());
}
} else {
gum::warn!(
target: LOG_TARGET,
"non-Unicode file name {:?} found in {:?}",
file_name,
cache_path,
);
}
}
// Make sure that the cache path directory and all its parents are created.
let _ = tokio::fs::create_dir_all(cache_path).await;
let mut dir = match tokio::fs::read_dir(cache_path).await {
Ok(dir) => dir,
Err(err) => {
gum::error!(
target: LOG_TARGET,
?err,
"failed to read dir {:?}",
cache_path,
);
return
},
};
loop {
match dir.next_entry().await {
Ok(Some(entry)) => insert_or_prune(self, &entry, cache_path).await,
Ok(None) => break,
Err(err) => {
gum::warn!(
target: LOG_TARGET,
?err,
"error processing artifacts in {:?}",
cache_path,
);
break
},
}
}
}
/// Returns the state of the given artifact by its ID.
pub fn artifact_state_mut(&mut self, artifact_id: &ArtifactId) -> Option<&mut ArtifactState> {
self.inner.get_mut(artifact_id)
}
/// Inform the table about the artifact with the given ID. The state will be set to "preparing".
///
/// This function must be used only for brand-new artifacts and should never be used for
pub fn insert_preparing(
&mut self,
artifact_id: ArtifactId,
waiting_for_response: Vec<PrecheckResultSender>,
.insert(artifact_id, ArtifactState::Preparing { waiting_for_response, num_failures: 0 })
}
/// Insert an artifact with the given ID as "prepared".
///
/// This function should only be used to build the artifact table at startup with valid
/// artifact caches.
pub(crate) fn insert_prepared(
&mut self,
artifact_id: ArtifactId,
// See the precondition.
always!(self
.inner
.insert(artifact_id, ArtifactState::Prepared { path, last_time_needed, prepare_stats })
/// Remove artifacts older than the given TTL and return id and path of the removed ones.
pub fn prune(&mut self, artifact_ttl: Duration) -> Vec<(ArtifactId, PathBuf)> {
let now = SystemTime::now();
let mut to_remove = vec![];
for (k, v) in self.inner.iter() {
if let ArtifactState::Prepared { last_time_needed, ref path, .. } = *v {
if now
.duration_since(last_time_needed)
.map(|age| age > artifact_ttl)
.unwrap_or(false)
{
to_remove.push((k.clone(), path.clone()));
}
}
}
for artifact in &to_remove {
self.inner.remove(&artifact.0);
}
to_remove
}
}
#[cfg(test)]
mod tests {
use super::{artifact_prefix as prefix, ArtifactId, Artifacts, NODE_VERSION, RUNTIME_VERSION};
use polkadot_primitives::ExecutorParamsHash;
use std::{
fs,
io::Write,
path::{Path, PathBuf},
str::FromStr,
};
fn rand_hash(len: usize) -> String {
let mut rng = rand::thread_rng();
let hex: Vec<_> = "0123456789abcdef".chars().collect();
(0..len).map(|_| hex[rng.gen_range(0..hex.len())]).collect()
}
fn file_name(code_hash: &str, param_hash: &str, checksum: &str) -> String {
format!("{}_0x{}_0x{}_0x{}", prefix(), code_hash, param_hash, checksum)
}
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
fn create_artifact(
dir: impl AsRef<Path>,
prefix: &str,
code_hash: impl AsRef<str>,
params_hash: impl AsRef<str>,
) -> (PathBuf, String) {
fn artifact_path_without_checksum(
dir: impl AsRef<Path>,
prefix: &str,
code_hash: impl AsRef<str>,
params_hash: impl AsRef<str>,
) -> PathBuf {
let mut path = dir.as_ref().to_path_buf();
let file_name =
format!("{}_0x{}_0x{}", prefix, code_hash.as_ref(), params_hash.as_ref(),);
path.push(file_name);
path
}
let (code_hash, params_hash) = (code_hash.as_ref(), params_hash.as_ref());
let path = artifact_path_without_checksum(dir, prefix, code_hash, params_hash);
let mut file = fs::File::create(&path).unwrap();
let content = format!("{}{}", code_hash, params_hash).into_bytes();
file.write_all(&content).unwrap();
let checksum = blake3::hash(&content).to_hex().to_string();
(path, checksum)
}
fn create_rand_artifact(dir: impl AsRef<Path>, prefix: &str) -> (PathBuf, String) {
create_artifact(dir, prefix, rand_hash(64), rand_hash(64))
}
fn concluded_path(path: impl AsRef<Path>, checksum: &str) -> PathBuf {
let path = path.as_ref();
let mut file_name = path.file_name().unwrap().to_os_string();
file_name.push("_0x");
file_name.push(checksum);
path.with_file_name(file_name)
}
#[test]
fn artifact_prefix() {
assert_eq!(prefix(), format!("wasmtime_v{}_polkadot_v{}", RUNTIME_VERSION, NODE_VERSION));
#[test]
fn from_file_name() {
assert!(ArtifactId::from_file_name("").is_none());
assert!(ArtifactId::from_file_name("junk").is_none());
let file_name = file_name(
"0022800000000000000000000000000000000000000000000000000000000000",
"0033900000000000000000000000000000000000000000000000000000000000",
"00000000000000000000000000000000",
ArtifactId::from_file_name(&file_name),
Some(ArtifactId::new(
hex_literal::hex![
"0022800000000000000000000000000000000000000000000000000000000000"
]
.into(),
ExecutorParamsHash::from_hash(sp_core::H256(hex_literal::hex![
"0033900000000000000000000000000000000000000000000000000000000000"
])),
let dir = Path::new("/test");
let code_hash = "1234567890123456789012345678901234567890123456789012345678901234";
let params_hash = "4321098765432109876543210987654321098765432109876543210987654321";
let checksum = "34567890123456789012345678901234";
let file_name = file_name(code_hash, params_hash, checksum);
let code_hash = H256::from_str(code_hash).unwrap();
let params_hash = H256::from_str(params_hash).unwrap();
let path = ArtifactId::new(code_hash.into(), ExecutorParamsHash::from_hash(params_hash))
.path(dir, checksum);
assert_eq!(path.to_str().unwrap(), format!("/test/{}", file_name));
async fn remove_stale_cache_on_startup() {
let cache_dir = tempfile::Builder::new().prefix("test-cache-").tempdir().unwrap();
// invalid prefix
create_rand_artifact(&cache_dir, "");
create_rand_artifact(&cache_dir, "wasmtime_polkadot_v");
create_rand_artifact(&cache_dir, "wasmtime_v8.0.0_polkadot_v1.0.0");
let prefix = prefix();
// no checksum
create_rand_artifact(&cache_dir, &prefix);
// invalid hashes
let (path, checksum) = create_artifact(&cache_dir, &prefix, "000", "000001");
let new_path = concluded_path(&path, &checksum);
fs::rename(&path, &new_path).unwrap();
// checksum tampered
let (path, checksum) = create_rand_artifact(&cache_dir, &prefix);
let new_path = concluded_path(&path, checksum.chars().rev().collect::<String>().as_str());
fs::rename(&path, &new_path).unwrap();
// valid
let (path, checksum) = create_rand_artifact(&cache_dir, &prefix);
let new_path = concluded_path(&path, &checksum);
fs::rename(&path, &new_path).unwrap();
assert_eq!(fs::read_dir(&cache_dir).unwrap().count(), 7);
let artifacts = Artifacts::new_and_prune(cache_dir.path()).await;
assert_eq!(fs::read_dir(&cache_dir).unwrap().count(), 1);
assert_eq!(artifacts.len(), 1);
fs::remove_dir_all(cache_dir).unwrap();