From 3069b0af398e5a0374802c31201637e97f4a530a Mon Sep 17 00:00:00 2001
From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com>
Date: Fri, 27 Oct 2023 14:50:30 +0300
Subject: [PATCH] make polkadot die graciously (#2056)
While investigating some db migrations that make the node startup fail,
I noticed that the node wasn't exiting and that the log file were
growing exponentially, until my whole system was freezing and that makes
it really hard to actually find why it was failing in the first place.
E.g:
```
ls -lh /tmp/zombie-01a04c2a2c0265d85f6440cf01c0f44a_-51319-uyggzuD4wEpV/bob.log
32,6G oct 27 11:16 /tmp/zombie-01a04c2a2c0265d85f6440cf01c0f44a_-51319-uyggzuD4wEpV/bob.log
```
This was happening because the following errors were being printed
continously without the subsystem main loop exiting:
From dispute-coordinator:
```
WARN tokio-runtime-worker parachain::dispute-coordinator: error=Subsystem(Generated(Context("Signal channel is terminated and empty.")))
```
From availability recovery:
```
Erasure task channel closed. Node shutting down ?
```
Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>
---
polkadot/node/core/dispute-coordinator/src/lib.rs | 3 ++-
polkadot/node/network/availability-recovery/src/lib.rs | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/polkadot/node/core/dispute-coordinator/src/lib.rs b/polkadot/node/core/dispute-coordinator/src/lib.rs
index 3f0fd401314..e96fee81240 100644
--- a/polkadot/node/core/dispute-coordinator/src/lib.rs
+++ b/polkadot/node/core/dispute-coordinator/src/lib.rs
@@ -27,6 +27,7 @@
use std::sync::Arc;
+use error::FatalError;
use futures::FutureExt;
use gum::CandidateHash;
@@ -431,7 +432,7 @@ impl DisputeCoordinatorSubsystem {
#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
async fn wait_for_first_leaf<Context>(ctx: &mut Context) -> Result<Option<ActivatedLeaf>> {
loop {
- match ctx.recv().await? {
+ match ctx.recv().await.map_err(FatalError::SubsystemReceive)? {
FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(None),
FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
if let Some(activated) = update.activated {
diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs
index e2146981da9..9acc48ea92e 100644
--- a/polkadot/node/network/availability-recovery/src/lib.rs
+++ b/polkadot/node/network/availability-recovery/src/lib.rs
@@ -822,6 +822,7 @@ async fn erasure_task_thread(
target: LOG_TARGET,
"Erasure task channel closed. Node shutting down ?",
);
+ break
},
}
}
--
GitLab