Unverified Commit 0609dc74 authored by Bernhard Schuster's avatar Bernhard Schuster Committed by GitHub
Browse files

add additional assurances to `create_inherent` (#4349)



* minor: move checks into separate fn

* add additional validity checks

* simplify shuffling

* Closes potential OOB weight

* improve docs

* fooo

* remove obsolete comment

* move filtering into the rollback-transaction

Technically this is not necessary but avoids future footguns.

* move check up and avoid duplicate checks

* refactor: make sure backed candidates are sane, even more

* doc wording
Co-authored-by: default avatarZeke Mostov <z.mostov@gmail.com>

* refactor: avoid const generics for sake of wasm size

`true` -> `FullCheck::Skip`, `false` -> `FullCheck::Yes`.

* chore: unify `CandidateCheckContext` instance names

* refactor: introduce `IndexedRetain` for `Vec<T>`

* chore: make tests prefix free

* doc: re-introduce removed comment

* refactor: remove another const generic to save some wasm size
Co-authored-by: default avatarZeke Mostov <z.mostov@gmail.com>
parent 8cd96d9f
Pipeline #167795 canceled with stages
in 1 minute and 43 seconds
......@@ -51,13 +51,14 @@ All failed checks should lead to an unrecoverable error making the block invalid
1. For each applied bit of each availability-bitfield, set the bit for the validator in the `CandidatePendingAvailability`'s `availability_votes` bitfield. Track all candidates that now have >2/3 of bits set in their `availability_votes`. These candidates are now available and can be enacted.
1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number.
1. Return a list of `(CoreIndex, CandidateHash)` from freed cores consisting of the cores where candidates have become available.
* `sanitize_bitfields<T: crate::inclusion::Config, const CHECK_SIGS: bool>(
* `sanitize_bitfields<T: crate::inclusion::Config>(
unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
disputed_bitfield: DisputedBitfield,
expected_bits: usize,
parent_hash: T::Hash,
session_index: SessionIndex,
validators: &[ValidatorId],
full_check: FullCheck,
)`:
1. check that `disputed_bitfield` has the same number of bits as the `expected_bits`, iff not return early with an empty vec.
1. each of the below checks is for each bitfield. If a check does not pass the bitfield will be skipped.
......@@ -65,7 +66,7 @@ All failed checks should lead to an unrecoverable error making the block invalid
1. check that the number of bits is equal to `expected_bits`.
1. check that the validator index is strictly increasing (and thus also unique).
1. check that the validator bit index is not out of bounds.
1. check the validators signature, iff `CHECK_SIGS=true`.
1. check the validators signature, iff `full_check=FullCheck::Yes`.
* `sanitize_backed_candidates<T: crate::inclusion::Config, F: Fn(CandidateHash) -> bool>(
relay_parent: T::Hash,
......
......@@ -22,7 +22,7 @@
use crate::{
configuration, disputes, dmp, hrmp, paras,
paras_inherent::{sanitize_bitfields, DisputedBitfield, VERIFY_SIGS},
paras_inherent::{sanitize_bitfields, DisputedBitfield},
scheduler::CoreAssignment,
shared, ump,
};
......@@ -56,6 +56,19 @@ pub struct AvailabilityBitfieldRecord<N> {
submitted_at: N, // for accounting, as meaning of bits may change over time.
}
/// Determines if all checks should be applied or if a subset was already completed
/// in a code path that will be executed afterwards or was already executed before.
#[derive(Encode, Decode, PartialEq, Eq, RuntimeDebug, TypeInfo)]
pub(crate) enum FullCheck {
/// Yes, do a full check, skip nothing.
Yes,
/// Skip a subset of checks that are already completed before.
///
/// Attention: Should only be used when absolutely sure that the required
/// checks are completed before.
Skip,
}
/// A backed candidate pending availability.
#[derive(Encode, Decode, PartialEq, TypeInfo)]
#[cfg_attr(test, derive(Debug, Default))]
......@@ -403,13 +416,14 @@ impl<T: Config> Pallet<T> {
let session_index = shared::Pallet::<T>::session_index();
let parent_hash = frame_system::Pallet::<T>::parent_hash();
let checked_bitfields = sanitize_bitfields::<T, VERIFY_SIGS>(
let checked_bitfields = sanitize_bitfields::<T>(
signed_bitfields,
disputed_bitfield,
expected_bits,
parent_hash,
session_index,
&validators[..],
FullCheck::Yes,
);
let freed_cores = Self::update_pending_availability_and_get_freed_cores::<_, true>(
......@@ -427,12 +441,16 @@ impl<T: Config> Pallet<T> {
///
/// Both should be sorted ascending by core index, and the candidates should be a subset of
/// scheduled cores. If these conditions are not met, the execution of the function fails.
pub(crate) fn process_candidates(
pub(crate) fn process_candidates<GV>(
parent_storage_root: T::Hash,
candidates: Vec<BackedCandidate<T::Hash>>,
scheduled: Vec<CoreAssignment>,
group_validators: impl Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>,
) -> Result<ProcessedCandidates<T::Hash>, DispatchError> {
group_validators: GV,
full_check: FullCheck,
) -> Result<ProcessedCandidates<T::Hash>, DispatchError>
where
GV: Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>,
{
ensure!(candidates.len() <= scheduled.len(), Error::<T>::UnscheduledCandidate);
if scheduled.is_empty() {
......@@ -446,7 +464,7 @@ impl<T: Config> Pallet<T> {
// before of the block where we include a candidate (i.e. this code path).
let now = <frame_system::Pallet<T>>::block_number();
let relay_parent_number = now - One::one();
let check_cx = CandidateCheckContext::<T>::new(now, relay_parent_number);
let check_ctx = CandidateCheckContext::<T>::new(now, relay_parent_number);
// Collect candidate receipts with backers.
let mut candidate_receipt_with_backing_validator_indices =
......@@ -481,54 +499,20 @@ impl<T: Config> Pallet<T> {
//
// In the meantime, we do certain sanity checks on the candidates and on the scheduled
// list.
'a: for (candidate_idx, backed_candidate) in candidates.iter().enumerate() {
'next_backed_candidate: for (candidate_idx, backed_candidate) in
candidates.iter().enumerate()
{
if let FullCheck::Yes = full_check {
check_ctx.verify_backed_candidate(
parent_hash,
candidate_idx,
backed_candidate,
)?;
}
let para_id = backed_candidate.descriptor().para_id;
let mut backers = bitvec::bitvec![BitOrderLsb0, u8; 0; validators.len()];
// we require that the candidate is in the context of the parent block.
ensure!(
backed_candidate.descriptor().relay_parent == parent_hash,
Error::<T>::CandidateNotInParentContext,
);
ensure!(
backed_candidate.descriptor().check_collator_signature().is_ok(),
Error::<T>::NotCollatorSigned,
);
let validation_code_hash =
<paras::Pallet<T>>::validation_code_hash_at(para_id, now, None)
// A candidate for a parachain without current validation code is not scheduled.
.ok_or_else(|| Error::<T>::UnscheduledCandidate)?;
ensure!(
backed_candidate.descriptor().validation_code_hash == validation_code_hash,
Error::<T>::InvalidValidationCodeHash,
);
ensure!(
backed_candidate.descriptor().para_head ==
backed_candidate.candidate.commitments.head_data.hash(),
Error::<T>::ParaHeadMismatch,
);
if let Err(err) = check_cx.check_validation_outputs(
para_id,
&backed_candidate.candidate.commitments.head_data,
&backed_candidate.candidate.commitments.new_validation_code,
backed_candidate.candidate.commitments.processed_downward_messages,
&backed_candidate.candidate.commitments.upward_messages,
T::BlockNumber::from(backed_candidate.candidate.commitments.hrmp_watermark),
&backed_candidate.candidate.commitments.horizontal_messages,
) {
log::debug!(
target: LOG_TARGET,
"Validation outputs checking during inclusion of a candidate {} for parachain `{}` failed: {:?}",
candidate_idx,
u32::from(para_id),
err,
);
Err(err.strip_into_dispatch_err::<T>())?;
};
for (i, assignment) in scheduled[skip..].iter().enumerate() {
check_assignment_in_order(assignment)?;
......@@ -631,7 +615,7 @@ impl<T: Config> Pallet<T> {
backers,
assignment.group_idx,
));
continue 'a
continue 'next_backed_candidate
}
}
......@@ -682,7 +666,7 @@ impl<T: Config> Pallet<T> {
availability_votes,
relay_parent_number,
backers: backers.to_bitvec(),
backed_in_number: check_cx.now,
backed_in_number: check_ctx.now,
backing_group: group,
},
);
......@@ -704,9 +688,9 @@ impl<T: Config> Pallet<T> {
// `relay_parent_number` is equal to `now`.
let now = <frame_system::Pallet<T>>::block_number();
let relay_parent_number = now;
let check_cx = CandidateCheckContext::<T>::new(now, relay_parent_number);
let check_ctx = CandidateCheckContext::<T>::new(now, relay_parent_number);
if let Err(err) = check_cx.check_validation_outputs(
if let Err(err) = check_ctx.check_validation_outputs(
para_id,
&validation_outputs.head_data,
&validation_outputs.new_validation_code,
......@@ -941,17 +925,78 @@ impl<BlockNumber> AcceptanceCheckErr<BlockNumber> {
}
/// A collection of data required for checking a candidate.
struct CandidateCheckContext<T: Config> {
pub(crate) struct CandidateCheckContext<T: Config> {
config: configuration::HostConfiguration<T::BlockNumber>,
now: T::BlockNumber,
relay_parent_number: T::BlockNumber,
}
impl<T: Config> CandidateCheckContext<T> {
fn new(now: T::BlockNumber, relay_parent_number: T::BlockNumber) -> Self {
pub(crate) fn new(now: T::BlockNumber, relay_parent_number: T::BlockNumber) -> Self {
Self { config: <configuration::Pallet<T>>::config(), now, relay_parent_number }
}
/// Execute verification of the candidate.
///
/// Assures:
/// * correct expected relay parent reference
/// * collator signature check passes
/// * code hash of commitments matches current code hash
/// * para head in the descriptor and commitments match
pub(crate) fn verify_backed_candidate(
&self,
parent_hash: <T as frame_system::Config>::Hash,
candidate_idx: usize,
backed_candidate: &BackedCandidate<<T as frame_system::Config>::Hash>,
) -> Result<(), Error<T>> {
let para_id = backed_candidate.descriptor().para_id;
let now = self.now;
// we require that the candidate is in the context of the parent block.
ensure!(
backed_candidate.descriptor().relay_parent == parent_hash,
Error::<T>::CandidateNotInParentContext,
);
ensure!(
backed_candidate.descriptor().check_collator_signature().is_ok(),
Error::<T>::NotCollatorSigned,
);
let validation_code_hash = <paras::Pallet<T>>::validation_code_hash_at(para_id, now, None)
// A candidate for a parachain without current validation code is not scheduled.
.ok_or_else(|| Error::<T>::UnscheduledCandidate)?;
ensure!(
backed_candidate.descriptor().validation_code_hash == validation_code_hash,
Error::<T>::InvalidValidationCodeHash,
);
ensure!(
backed_candidate.descriptor().para_head ==
backed_candidate.candidate.commitments.head_data.hash(),
Error::<T>::ParaHeadMismatch,
);
if let Err(err) = self.check_validation_outputs(
para_id,
&backed_candidate.candidate.commitments.head_data,
&backed_candidate.candidate.commitments.new_validation_code,
backed_candidate.candidate.commitments.processed_downward_messages,
&backed_candidate.candidate.commitments.upward_messages,
T::BlockNumber::from(backed_candidate.candidate.commitments.hrmp_watermark),
&backed_candidate.candidate.commitments.horizontal_messages,
) {
log::debug!(
target: LOG_TARGET,
"Validation outputs checking during inclusion of a candidate {} for parachain `{}` failed: {:?}",
candidate_idx,
u32::from(para_id),
err,
);
Err(err.strip_into_dispatch_err::<T>())?;
};
Ok(())
}
/// Check the given outputs after candidate validation on whether it passes the acceptance
/// criteria.
fn check_validation_outputs(
......@@ -1935,6 +1980,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_b_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::UnscheduledCandidate
);
......@@ -1990,6 +2036,7 @@ pub(crate) mod tests {
vec![backed_b, backed_a],
vec![chain_a_assignment.clone(), chain_b_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::UnscheduledCandidate
);
......@@ -2023,6 +2070,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::InsufficientBacking
);
......@@ -2058,6 +2106,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::CandidateNotInParentContext
);
......@@ -2097,6 +2146,7 @@ pub(crate) mod tests {
thread_a_assignment.clone(),
],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::WrongCollator,
);
......@@ -2135,6 +2185,7 @@ pub(crate) mod tests {
vec![backed],
vec![thread_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::NotCollatorSigned
);
......@@ -2185,6 +2236,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::CandidateScheduledBeforeParaFree
);
......@@ -2228,6 +2280,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::CandidateScheduledBeforeParaFree
);
......@@ -2279,6 +2332,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::PrematureCodeUpgrade
);
......@@ -2313,6 +2367,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Err(Error::<Test>::ValidationDataHashMismatch.into()),
);
......@@ -2348,6 +2403,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::InvalidValidationCodeHash
);
......@@ -2383,6 +2439,7 @@ pub(crate) mod tests {
vec![backed],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
),
Error::<Test>::ParaHeadMismatch
);
......@@ -2552,6 +2609,7 @@ pub(crate) mod tests {
thread_a_assignment.clone(),
],
&group_validators,
FullCheck::Yes,
)
.expect("candidates scheduled, in order, and backed");
......@@ -2742,6 +2800,7 @@ pub(crate) mod tests {
vec![backed_a],
vec![chain_a_assignment.clone()],
&group_validators,
FullCheck::Yes,
)
.expect("candidates scheduled, in order, and backed");
......
......@@ -23,7 +23,9 @@
use crate::{
disputes::DisputesHandler,
inclusion, initializer,
inclusion,
inclusion::{CandidateCheckContext, FullCheck},
initializer,
scheduler::{self, CoreAssignment, FreedReason},
shared, ump,
};
......@@ -42,21 +44,21 @@ use primitives::v1::{
UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex,
PARACHAINS_INHERENT_IDENTIFIER,
};
use rand::{Rng, SeedableRng};
use rand::{seq::SliceRandom, SeedableRng};
use scale_info::TypeInfo;
use sp_runtime::traits::Header as HeaderT;
use sp_runtime::traits::{Header as HeaderT, One};
use sp_std::{
cmp::Ordering,
collections::{btree_map::BTreeMap, btree_set::BTreeSet},
prelude::*,
vec::Vec,
};
#[cfg(feature = "runtime-benchmarks")]
mod benchmarking;
const LOG_TARGET: &str = "runtime::inclusion-inherent";
const SKIP_SIG_VERIFY: bool = false;
pub(crate) const VERIFY_SIGS: bool = true;
pub trait WeightInfo {
/// Variant over `v`, the count of dispute statements in a dispute statement set. This gives the
......@@ -158,6 +160,29 @@ fn backed_candidates_weight<T: frame_system::Config + Config>(
.fold(0, |acc, x| acc.saturating_add(x))
}
/// A helper trait to allow calling retain while getting access
/// to the index of the item in the `vec`.
trait IndexedRetain<T> {
/// Retains only the elements specified by the predicate.
///
/// In other words, remove all elements `e` residing at
/// index `i` such that `f(i, &e)` returns `false`. This method
/// operates in place, visiting each element exactly once in the
/// original order, and preserves the order of the retained elements.
fn indexed_retain(&mut self, f: impl FnMut(usize, &T) -> bool);
}
impl<T> IndexedRetain<T> for Vec<T> {
fn indexed_retain(&mut self, mut f: impl FnMut(usize, &T) -> bool) {
let mut idx = 0_usize;
self.retain(move |item| {
let ret = f(idx, item);
idx += 1_usize;
ret
})
}
}
/// A bitfield concerning concluded disputes for candidates
/// associated to the core index equivalent to the bit position.
#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
......@@ -252,25 +277,24 @@ pub mod pallet {
// (`enter`) and the off-chain checks by the block author (this function). Once we are confident
// in all the logic in this module this check should be removed to optimize performance.
let inherent_data =
match Self::enter(frame_system::RawOrigin::None.into(), inherent_data.clone()) {
Ok(_) => inherent_data,
Err(err) => {
log::error!(
target: LOG_TARGET,
"dropping paras inherent data because they produced \
let inherent_data = match Self::enter_inner(inherent_data.clone(), FullCheck::Skip) {
Ok(_) => inherent_data,
Err(err) => {
log::error!(
target: LOG_TARGET,
"dropping paras inherent data because they produced \
an invalid paras inherent: {:?}",
err.error,
);
err.error,
);
ParachainsInherentData {
bitfields: Vec::new(),
backed_candidates: Vec::new(),
disputes: Vec::new(),
parent_header: inherent_data.parent_header,
}
},
};
ParachainsInherentData {
bitfields: Vec::new(),
backed_candidates: Vec::new(),
disputes: Vec::new(),
parent_header: inherent_data.parent_header,
}
},
};
Some(Call::enter { data: inherent_data })
}
......@@ -329,180 +353,192 @@ pub mod pallet {
ensure!(!Included::<T>::exists(), Error::<T>::TooManyInclusionInherents);
Included::<T>::set(Some(()));
let ParachainsInherentData {
bitfields: mut signed_bitfields,
mut backed_candidates,
parent_header,
mut disputes,
} = data;
Self::enter_inner(data, FullCheck::Yes)
}
}
}
log::debug!(
target: LOG_TARGET,
"[enter] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}",
signed_bitfields.len(),
backed_candidates.len(),
disputes.len()
);
impl<T: Config> Pallet<T> {
pub(crate) fn enter_inner(
data: ParachainsInherentData<T::Header>,
full_check: FullCheck,
) -> DispatchResultWithPostInfo {
let ParachainsInherentData {
bitfields: mut signed_bitfields,
mut backed_candidates,
parent_header,
mut disputes,
} = data;
// Check that the submitted parent header indeed corresponds to the previous block hash.
let parent_hash = <frame_system::Pallet<T>>::parent_hash();
ensure!(
parent_header.hash().as_ref() == parent_hash.as_ref(),
Error::<T>::InvalidParentHeader,
);
log::debug!(
target: LOG_TARGET,
"[enter] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}",
signed_bitfields.len(),
backed_candidates.len(),
disputes.len()
);
let mut candidate_weight = backed_candidates_weight::<T>(&backed_candidates);
let mut bitfields_weight = signed_bitfields_weight::<T>(signed_bitfields.len());
let disputes_weight = dispute_statements_weight::<T>(&disputes);
// Check that the submitted parent header indeed corresponds to the previous block hash.
let parent_hash = <frame_system::Pallet<T>>::parent_hash();
ensure!(
parent_header.hash().as_ref() == parent_hash.as_ref(),
Error::<T>::InvalidParentHeader,
);
let max_block_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
let now = <frame_system::Pallet<T>>::block_number();
// Potentially trim inherent data to ensure processing will be within weight limits
let total_weight = {
if candidate_weight
.saturating_add(bitfields_weight)
.saturating_add(disputes_weight) >
max_block_weight
{
// if the total weight is over the max block weight, first try clearing backed
// candidates and bitfields.
backed_candidates.clear();
candidate_weight = 0;
signed_bitfields.clear();
bitfields_weight = 0;
}
let mut candidate_weight = backed_candidates_weight::<T>(&backed_candidates);
let mut bitfields_weight = signed_bitfields_weight::<T>(signed_bitfields.len());
let disputes_weight = dispute_statements_weight::<T>(&disputes);
if disputes_weight > max_block_weight {
// if disputes are by themselves overweight already, trim the disputes.
debug_assert!(candidate_weight == 0 && bitfields_weight == 0);
let max_block_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
let entropy = compute_entropy::<T>(parent_hash);
let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
// Potentially trim inherent data to ensure processing will be within weight limits
let total_weight = {
if candidate_weight
.saturating_add(bitfields_weight)
.saturating_add(disputes_weight) >
max_block_weight
{
// if the total weight is over the max block weight, first try clearing backed
// candidates and bitfields.
backed_candidates.clear();
candidate_weight = 0;
signed_bitfields.clear();
bitfields_weight = 0;
}
let remaining_weight =
limit_disputes::<T>(&mut disputes, max_block_weight, &mut rng);
max_block_weight.saturating_sub(remaining_weight)
} else {
candidate_weight
.saturating_add(bitfields_weight)
.saturating_add(disputes_weight)
}
};
if disputes_weight > max_block_weight {
// if disputes are by themselves overweight already, trim the disputes.
debug_assert!(candidate_weight == 0 && bitfields_weight == 0);
let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
let entropy = compute_entropy::<T>(parent_hash);
let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
// Handle disputes logic.
let current_session = <shared::Pallet<T>>::session_index();
let disputed_bitfield = {
let new_current_dispute_sets: Vec<_> = disputes
.iter()
.filter(|s| s.session == current_session)
.map(|s| (s.session, s.candidate_hash))
.collect();
<