From 49c8942f8a7541e1d596e6ca9ed27ae16dd76de3 Mon Sep 17 00:00:00 2001 From: Gav Date: Tue, 11 Sep 2018 20:31:19 +0200 Subject: [PATCH] Other bits of big refactor --- core/bft/Cargo.toml | 22 + core/bft/README.adoc | 13 + core/bft/src/error.rs | 95 + core/bft/src/lib.rs | 1124 +++++++++++ core/cli/Cargo.toml | 35 + core/cli/README.adoc | 13 + core/cli/build.rs | 59 + core/cli/doc/shell-completion.adoc | 41 + core/cli/src/cli.yml | 229 +++ core/cli/src/error.rs | 37 + core/cli/src/informant.rs | 124 ++ core/cli/src/lib.rs | 571 ++++++ core/cli/src/panic_hook.rs | 69 + core/client/Cargo.toml | 32 + core/client/README.adoc | 13 + core/client/db/Cargo.toml | 24 + core/client/db/src/cache.rs | 432 +++++ core/client/db/src/lib.rs | 715 +++++++ core/client/db/src/light.rs | 392 ++++ core/client/db/src/utils.rs | 174 ++ core/client/src/backend.rs | 107 ++ core/client/src/block_builder.rs | 140 ++ core/client/src/blockchain.rs | 92 + core/client/src/call_executor.rs | 199 ++ core/client/src/cht.rs | 274 +++ core/client/src/client.rs | 810 ++++++++ core/client/src/error.rs | 154 ++ core/client/src/genesis.rs | 202 ++ core/client/src/in_mem.rs | 439 +++++ core/client/src/lib.rs | 69 + core/client/src/light/backend.rs | 229 +++ core/client/src/light/blockchain.rs | 142 ++ core/client/src/light/call_executor.rs | 191 ++ core/client/src/light/fetcher.rs | 308 +++ core/client/src/light/mod.rs | 77 + core/client/src/notifications.rs | 289 +++ core/codec/Cargo.toml | 12 + core/codec/README.adoc | 13 + core/codec/derive/Cargo.toml | 20 + core/codec/derive/src/decode.rs | 111 ++ core/codec/derive/src/encode.rs | 153 ++ core/codec/derive/src/lib.rs | 126 ++ core/codec/derive/tests/mod.rs | 151 ++ core/codec/src/codec.rs | 540 ++++++ core/codec/src/joiner.rs | 33 + core/codec/src/keyedvec.rs | 36 + core/codec/src/lib.rs | 45 + core/environmental/Cargo.toml | 8 + core/environmental/README.adoc | 13 + core/environmental/src/lib.rs | 383 ++++ core/environmental/with_std.rs | 32 + core/environmental/without_std.rs | 70 + core/executor/Cargo.toml | 33 + core/executor/README.adoc | 13 + core/executor/src/error.rs | 81 + core/executor/src/lib.rs | 93 + core/executor/src/native_executor.rs | 224 +++ core/executor/src/sandbox.rs | 676 +++++++ core/executor/src/wasm_executor.rs | 718 +++++++ core/executor/src/wasm_utils.rs | 199 ++ core/executor/wasm/Cargo.lock | 216 +++ core/executor/wasm/Cargo.toml | 19 + core/executor/wasm/build.sh | 8 + core/executor/wasm/src/lib.rs | 122 ++ core/extrinsic-pool/Cargo.toml | 19 + core/extrinsic-pool/README.adoc | 13 + core/extrinsic-pool/src/error.rs | 33 + core/extrinsic-pool/src/lib.rs | 49 + core/extrinsic-pool/src/listener.rs | 95 + core/extrinsic-pool/src/pool.rs | 606 ++++++ core/extrinsic-pool/src/rotator.rs | 209 +++ core/extrinsic-pool/src/watcher.rs | 103 + core/keyring/Cargo.toml | 9 + core/keyring/README.adoc | 13 + core/keyring/src/lib.rs | 176 ++ core/keystore/Cargo.toml | 19 + core/keystore/README.adoc | 13 + core/keystore/src/lib.rs | 293 +++ core/misbehavior-check/Cargo.toml | 19 + core/misbehavior-check/README.adoc | 13 + core/misbehavior-check/src/lib.rs | 206 ++ core/network-libp2p/Cargo.toml | 34 + core/network-libp2p/README.adoc | 13 + core/network-libp2p/src/connection_filter.rs | 31 + core/network-libp2p/src/custom_proto.rs | 290 +++ core/network-libp2p/src/error.rs | 221 +++ core/network-libp2p/src/lib.rs | 75 + core/network-libp2p/src/network_state.rs | 953 ++++++++++ core/network-libp2p/src/service.rs | 1437 ++++++++++++++ core/network-libp2p/src/timeouts.rs | 115 ++ core/network-libp2p/src/topology.rs | 655 +++++++ core/network-libp2p/src/traits.rs | 299 +++ core/network-libp2p/src/transport.rs | 50 + core/network-libp2p/tests/tests.rs | 130 ++ core/network/Cargo.toml | 30 + core/network/README.adoc | 13 + core/network/src/blocks.rs | 282 +++ core/network/src/chain.rs | 105 ++ core/network/src/config.rs | 32 + core/network/src/consensus_gossip.rs | 382 ++++ core/network/src/error.rs | 35 + core/network/src/import_queue.rs | 593 ++++++ core/network/src/io.rs | 72 + core/network/src/lib.rs | 69 + core/network/src/message.rs | 375 ++++ core/network/src/on_demand.rs | 736 ++++++++ core/network/src/protocol.rs | 679 +++++++ core/network/src/service.rs | 340 ++++ core/network/src/specialization.rs | 48 + core/network/src/sync.rs | 428 +++++ core/network/src/test/mod.rs | 330 ++++ core/network/src/test/sync.rs | 121 ++ core/primitives/Cargo.toml | 58 + core/primitives/README.adoc | 13 + core/primitives/src/authority_id.rs | 106 ++ core/primitives/src/bytes.rs | 158 ++ core/primitives/src/ed25519.rs | 357 ++++ core/primitives/src/hash.rs | 166 ++ core/primitives/src/hasher.rs | 53 + core/primitives/src/hashing.rs | 106 ++ core/primitives/src/hexdisplay.rs | 94 + core/primitives/src/lib.rs | 142 ++ core/primitives/src/rlp_codec.rs | 123 ++ core/primitives/src/sandbox.rs | 221 +++ core/primitives/src/storage.rs | 44 + core/primitives/src/tests.rs | 17 + core/primitives/src/u32_trait.rs | 89 + core/primitives/src/uint.rs | 99 + core/pwasm-alloc/Cargo.lock | 37 + core/pwasm-alloc/Cargo.toml | 23 + core/pwasm-alloc/README.adoc | 25 + core/pwasm-alloc/build.rs | 14 + core/pwasm-alloc/src/lib.rs | 34 + core/pwasm-libc/Cargo.lock | 4 + core/pwasm-libc/Cargo.toml | 15 + core/pwasm-libc/README.adoc | 24 + core/pwasm-libc/src/lib.rs | 55 + core/rpc-servers/Cargo.toml | 14 + core/rpc-servers/README.adoc | 14 + core/rpc-servers/src/lib.rs | 93 + core/rpc/Cargo.toml | 27 + core/rpc/README.adoc | 13 + core/rpc/src/author/error.rs | 68 + core/rpc/src/author/mod.rs | 162 ++ core/rpc/src/author/tests.rs | 178 ++ core/rpc/src/chain/error.rs | 42 + core/rpc/src/chain/mod.rs | 165 ++ core/rpc/src/chain/tests.rs | 207 ++ core/rpc/src/errors.rs | 34 + core/rpc/src/helpers.rs | 25 + core/rpc/src/lib.rs | 59 + core/rpc/src/metadata.rs | 54 + core/rpc/src/state/error.rs | 54 + core/rpc/src/state/mod.rs | 277 +++ core/rpc/src/state/tests.rs | 186 ++ core/rpc/src/subscriptions.rs | 85 + core/rpc/src/system/error.rs | 40 + core/rpc/src/system/mod.rs | 41 + core/rpc/src/system/tests.rs | 54 + core/runtime-io/Cargo.toml | 32 + core/runtime-io/README.adoc | 13 + core/runtime-io/build.rs | 14 + core/runtime-io/src/lib.rs | 35 + core/runtime-io/with_std.rs | 266 +++ core/runtime-io/without_std.rs | 329 ++++ core/runtime-sandbox/Cargo.toml | 31 + core/runtime-sandbox/README.adoc | 13 + core/runtime-sandbox/build.rs | 14 + core/runtime-sandbox/src/lib.rs | 221 +++ core/runtime-sandbox/with_std.rs | 482 +++++ core/runtime-sandbox/without_std.rs | 303 +++ core/runtime-std/Cargo.toml | 18 + core/runtime-std/README.adoc | 14 + core/runtime-std/build.rs | 14 + core/runtime-std/src/lib.rs | 51 + core/runtime-std/with_std.rs | 38 + core/runtime-std/without_std.rs | 46 + core/serializer/Cargo.toml | 8 + core/serializer/README.adoc | 14 + core/serializer/src/lib.rs | 46 + core/service/Cargo.toml | 31 + core/service/README.adoc | 14 + core/service/src/chain_ops.rs | 139 ++ core/service/src/chain_spec.rs | 170 ++ core/service/src/components.rs | 256 +++ core/service/src/config.rs | 121 ++ core/service/src/error.rs | 35 + core/service/src/lib.rs | 424 +++++ core/state-db/Cargo.toml | 14 + core/state-db/README.adoc | 13 + core/state-db/src/lib.rs | 407 ++++ core/state-db/src/pruning.rs | 267 +++ core/state-db/src/test.rs | 83 + core/state-db/src/unfinalized.rs | 533 ++++++ core/state-machine/Cargo.toml | 21 + core/state-machine/README.adoc | 13 + core/state-machine/src/backend.rs | 193 ++ core/state-machine/src/ext.rs | 171 ++ core/state-machine/src/lib.rs | 691 +++++++ core/state-machine/src/proving_backend.rs | 183 ++ core/state-machine/src/testing.rs | 118 ++ core/state-machine/src/trie_backend.rs | 362 ++++ core/telemetry/Cargo.toml | 15 + core/telemetry/README.adoc | 13 + core/telemetry/src/lib.rs | 160 ++ core/test-client/Cargo.toml | 18 + core/test-client/README.adoc | 13 + core/test-client/src/block_builder_ext.rs | 42 + core/test-client/src/client_ext.rs | 103 + core/test-client/src/lib.rs | 66 + core/test-runtime/Cargo.toml | 36 + core/test-runtime/README.adoc | 13 + core/test-runtime/src/genesismap.rs | 67 + core/test-runtime/src/lib.rs | 156 ++ core/test-runtime/src/system.rs | 281 +++ core/test-runtime/wasm/Cargo.lock | 709 +++++++ core/test-runtime/wasm/Cargo.toml | 26 + core/test-runtime/wasm/build.sh | 8 + core/test-runtime/wasm/src | 1 + framework/client/Cargo.toml | 32 + framework/executor/Cargo.toml | 33 + framework/network/Cargo.toml | 30 + framework/runtime-support/Cargo.toml | 32 + framework/test-runtime/Cargo.toml | 36 + node/Cargo.toml | 18 + node/api/Cargo.toml | 13 + node/api/src/lib.rs | 155 ++ node/build.rs | 24 + node/cli/Cargo.toml | 12 + node/cli/src/cli.yml | 12 + node/cli/src/error.rs | 29 + node/cli/src/lib.rs | 122 ++ node/consensus/Cargo.toml | 26 + node/consensus/README.adoc | 5 + node/consensus/src/error.rs | 51 + node/consensus/src/evaluation.rs | 96 + node/consensus/src/lib.rs | 445 +++++ node/consensus/src/offline_tracker.rs | 137 ++ node/consensus/src/service.rs | 172 ++ node/executor/Cargo.toml | 28 + node/executor/src/lib.rs | 523 ++++++ node/network/Cargo.toml | 17 + node/network/src/consensus.rs | 297 +++ node/network/src/lib.rs | 117 ++ node/primitives/Cargo.toml | 29 + node/primitives/src/lib.rs | 94 + node/runtime/Cargo.toml | 61 + node/runtime/src/checked_block.rs | 94 + node/runtime/src/lib.rs | 386 ++++ node/runtime/wasm/Cargo.lock | 568 ++++++ node/runtime/wasm/Cargo.toml | 38 + node/runtime/wasm/build.sh | 8 + node/runtime/wasm/src | 1 + node/service/Cargo.toml | 26 + node/service/src/chain_spec.rs | 209 +++ node/service/src/lib.rs | 213 +++ node/src/main.rs | 69 + node/transaction-pool/Cargo.toml | 18 + node/transaction-pool/src/error.rs | 73 + node/transaction-pool/src/lib.rs | 236 +++ runtime/README.adoc | 6 + runtime/balances/Cargo.toml | 36 + runtime/balances/src/address.rs | 111 ++ runtime/balances/src/genesis_config.rs | 84 + runtime/balances/src/lib.rs | 660 +++++++ runtime/balances/src/mock.rs | 77 + runtime/balances/src/tests.rs | 324 ++++ runtime/consensus/Cargo.toml | 32 + runtime/consensus/src/lib.rs | 267 +++ runtime/contract/Cargo.toml | 41 + runtime/contract/src/account_db.rs | 180 ++ runtime/contract/src/double_map.rs | 90 + runtime/contract/src/exec.rs | 266 +++ runtime/contract/src/gas.rs | 178 ++ runtime/contract/src/genesis_config.rs | 54 + runtime/contract/src/lib.rs | 278 +++ runtime/contract/src/tests.rs | 670 +++++++ runtime/contract/src/vm/env_def/macros.rs | 285 +++ runtime/contract/src/vm/env_def/mod.rs | 315 ++++ runtime/contract/src/vm/mod.rs | 553 ++++++ runtime/contract/src/vm/prepare.rs | 286 +++ runtime/council/Cargo.toml | 43 + runtime/council/src/lib.rs | 238 +++ runtime/council/src/motions.rs | 372 ++++ runtime/council/src/seats.rs | 1355 ++++++++++++++ runtime/council/src/voting.rs | 505 +++++ runtime/democracy/Cargo.toml | 37 + runtime/democracy/src/lib.rs | 681 +++++++ runtime/democracy/src/vote_threshold.rs | 97 + runtime/example/Cargo.toml | 34 + runtime/example/src/lib.rs | 415 +++++ runtime/executive/Cargo.toml | 33 + runtime/executive/src/lib.rs | 359 ++++ runtime/primitives/Cargo.toml | 34 + runtime/primitives/src/bft.rs | 195 ++ runtime/primitives/src/generic/block.rs | 105 ++ .../src/generic/checked_extrinsic.rs | 59 + runtime/primitives/src/generic/digest.rs | 150 ++ runtime/primitives/src/generic/header.rs | 167 ++ runtime/primitives/src/generic/mod.rs | 33 + runtime/primitives/src/generic/tests.rs | 105 ++ .../src/generic/unchecked_extrinsic.rs | 159 ++ runtime/primitives/src/lib.rs | 434 +++++ runtime/primitives/src/testing.rs | 138 ++ runtime/primitives/src/traits.rs | 462 +++++ runtime/session/Cargo.toml | 40 + runtime/session/src/lib.rs | 467 +++++ runtime/staking/Cargo.toml | 45 + runtime/staking/src/genesis_config.rs | 77 + runtime/staking/src/lib.rs | 576 ++++++ runtime/staking/src/mock.rs | 126 ++ runtime/staking/src/tests.rs | 502 +++++ runtime/support/Cargo.toml | 32 + runtime/support/README.adoc | 14 + runtime/support/src/dispatch.rs | 584 ++++++ runtime/support/src/event.rs | 298 +++ runtime/support/src/hashable.rs | 38 + runtime/support/src/lib.rs | 191 ++ runtime/support/src/metadata.rs | 459 +++++ runtime/support/src/storage/generator.rs | 1659 +++++++++++++++++ runtime/support/src/storage/mod.rs | 609 ++++++ runtime/system/Cargo.toml | 32 + runtime/system/src/lib.rs | 426 +++++ runtime/timestamp/Cargo.toml | 32 + runtime/timestamp/src/lib.rs | 210 +++ runtime/treasury/Cargo.toml | 34 + runtime/treasury/src/lib.rs | 546 ++++++ runtime/version/Cargo.toml | 22 + runtime/version/src/lib.rs | 130 ++ 329 files changed, 57639 insertions(+) create mode 100644 core/bft/Cargo.toml create mode 100644 core/bft/README.adoc create mode 100644 core/bft/src/error.rs create mode 100644 core/bft/src/lib.rs create mode 100644 core/cli/Cargo.toml create mode 100644 core/cli/README.adoc create mode 100644 core/cli/build.rs create mode 100644 core/cli/doc/shell-completion.adoc create mode 100644 core/cli/src/cli.yml create mode 100644 core/cli/src/error.rs create mode 100644 core/cli/src/informant.rs create mode 100644 core/cli/src/lib.rs create mode 100644 core/cli/src/panic_hook.rs create mode 100644 core/client/Cargo.toml create mode 100644 core/client/README.adoc create mode 100644 core/client/db/Cargo.toml create mode 100644 core/client/db/src/cache.rs create mode 100644 core/client/db/src/lib.rs create mode 100644 core/client/db/src/light.rs create mode 100644 core/client/db/src/utils.rs create mode 100644 core/client/src/backend.rs create mode 100644 core/client/src/block_builder.rs create mode 100644 core/client/src/blockchain.rs create mode 100644 core/client/src/call_executor.rs create mode 100644 core/client/src/cht.rs create mode 100644 core/client/src/client.rs create mode 100644 core/client/src/error.rs create mode 100644 core/client/src/genesis.rs create mode 100644 core/client/src/in_mem.rs create mode 100644 core/client/src/lib.rs create mode 100644 core/client/src/light/backend.rs create mode 100644 core/client/src/light/blockchain.rs create mode 100644 core/client/src/light/call_executor.rs create mode 100644 core/client/src/light/fetcher.rs create mode 100644 core/client/src/light/mod.rs create mode 100644 core/client/src/notifications.rs create mode 100644 core/codec/Cargo.toml create mode 100644 core/codec/README.adoc create mode 100644 core/codec/derive/Cargo.toml create mode 100644 core/codec/derive/src/decode.rs create mode 100644 core/codec/derive/src/encode.rs create mode 100644 core/codec/derive/src/lib.rs create mode 100644 core/codec/derive/tests/mod.rs create mode 100644 core/codec/src/codec.rs create mode 100644 core/codec/src/joiner.rs create mode 100644 core/codec/src/keyedvec.rs create mode 100644 core/codec/src/lib.rs create mode 100644 core/environmental/Cargo.toml create mode 100644 core/environmental/README.adoc create mode 100644 core/environmental/src/lib.rs create mode 100644 core/environmental/with_std.rs create mode 100644 core/environmental/without_std.rs create mode 100644 core/executor/Cargo.toml create mode 100644 core/executor/README.adoc create mode 100644 core/executor/src/error.rs create mode 100644 core/executor/src/lib.rs create mode 100644 core/executor/src/native_executor.rs create mode 100644 core/executor/src/sandbox.rs create mode 100644 core/executor/src/wasm_executor.rs create mode 100644 core/executor/src/wasm_utils.rs create mode 100644 core/executor/wasm/Cargo.lock create mode 100644 core/executor/wasm/Cargo.toml create mode 100755 core/executor/wasm/build.sh create mode 100644 core/executor/wasm/src/lib.rs create mode 100644 core/extrinsic-pool/Cargo.toml create mode 100644 core/extrinsic-pool/README.adoc create mode 100644 core/extrinsic-pool/src/error.rs create mode 100644 core/extrinsic-pool/src/lib.rs create mode 100644 core/extrinsic-pool/src/listener.rs create mode 100644 core/extrinsic-pool/src/pool.rs create mode 100644 core/extrinsic-pool/src/rotator.rs create mode 100644 core/extrinsic-pool/src/watcher.rs create mode 100644 core/keyring/Cargo.toml create mode 100644 core/keyring/README.adoc create mode 100644 core/keyring/src/lib.rs create mode 100644 core/keystore/Cargo.toml create mode 100644 core/keystore/README.adoc create mode 100644 core/keystore/src/lib.rs create mode 100644 core/misbehavior-check/Cargo.toml create mode 100644 core/misbehavior-check/README.adoc create mode 100644 core/misbehavior-check/src/lib.rs create mode 100644 core/network-libp2p/Cargo.toml create mode 100644 core/network-libp2p/README.adoc create mode 100644 core/network-libp2p/src/connection_filter.rs create mode 100644 core/network-libp2p/src/custom_proto.rs create mode 100644 core/network-libp2p/src/error.rs create mode 100644 core/network-libp2p/src/lib.rs create mode 100644 core/network-libp2p/src/network_state.rs create mode 100644 core/network-libp2p/src/service.rs create mode 100644 core/network-libp2p/src/timeouts.rs create mode 100644 core/network-libp2p/src/topology.rs create mode 100644 core/network-libp2p/src/traits.rs create mode 100644 core/network-libp2p/src/transport.rs create mode 100644 core/network-libp2p/tests/tests.rs create mode 100644 core/network/Cargo.toml create mode 100644 core/network/README.adoc create mode 100644 core/network/src/blocks.rs create mode 100644 core/network/src/chain.rs create mode 100644 core/network/src/config.rs create mode 100644 core/network/src/consensus_gossip.rs create mode 100644 core/network/src/error.rs create mode 100644 core/network/src/import_queue.rs create mode 100644 core/network/src/io.rs create mode 100644 core/network/src/lib.rs create mode 100644 core/network/src/message.rs create mode 100644 core/network/src/on_demand.rs create mode 100644 core/network/src/protocol.rs create mode 100644 core/network/src/service.rs create mode 100644 core/network/src/specialization.rs create mode 100644 core/network/src/sync.rs create mode 100644 core/network/src/test/mod.rs create mode 100644 core/network/src/test/sync.rs create mode 100644 core/primitives/Cargo.toml create mode 100644 core/primitives/README.adoc create mode 100644 core/primitives/src/authority_id.rs create mode 100644 core/primitives/src/bytes.rs create mode 100644 core/primitives/src/ed25519.rs create mode 100644 core/primitives/src/hash.rs create mode 100644 core/primitives/src/hasher.rs create mode 100644 core/primitives/src/hashing.rs create mode 100644 core/primitives/src/hexdisplay.rs create mode 100644 core/primitives/src/lib.rs create mode 100644 core/primitives/src/rlp_codec.rs create mode 100644 core/primitives/src/sandbox.rs create mode 100644 core/primitives/src/storage.rs create mode 100644 core/primitives/src/tests.rs create mode 100644 core/primitives/src/u32_trait.rs create mode 100644 core/primitives/src/uint.rs create mode 100644 core/pwasm-alloc/Cargo.lock create mode 100644 core/pwasm-alloc/Cargo.toml create mode 100644 core/pwasm-alloc/README.adoc create mode 100644 core/pwasm-alloc/build.rs create mode 100644 core/pwasm-alloc/src/lib.rs create mode 100644 core/pwasm-libc/Cargo.lock create mode 100644 core/pwasm-libc/Cargo.toml create mode 100644 core/pwasm-libc/README.adoc create mode 100644 core/pwasm-libc/src/lib.rs create mode 100644 core/rpc-servers/Cargo.toml create mode 100644 core/rpc-servers/README.adoc create mode 100644 core/rpc-servers/src/lib.rs create mode 100644 core/rpc/Cargo.toml create mode 100644 core/rpc/README.adoc create mode 100644 core/rpc/src/author/error.rs create mode 100644 core/rpc/src/author/mod.rs create mode 100644 core/rpc/src/author/tests.rs create mode 100644 core/rpc/src/chain/error.rs create mode 100644 core/rpc/src/chain/mod.rs create mode 100644 core/rpc/src/chain/tests.rs create mode 100644 core/rpc/src/errors.rs create mode 100644 core/rpc/src/helpers.rs create mode 100644 core/rpc/src/lib.rs create mode 100644 core/rpc/src/metadata.rs create mode 100644 core/rpc/src/state/error.rs create mode 100644 core/rpc/src/state/mod.rs create mode 100644 core/rpc/src/state/tests.rs create mode 100644 core/rpc/src/subscriptions.rs create mode 100644 core/rpc/src/system/error.rs create mode 100644 core/rpc/src/system/mod.rs create mode 100644 core/rpc/src/system/tests.rs create mode 100644 core/runtime-io/Cargo.toml create mode 100644 core/runtime-io/README.adoc create mode 100644 core/runtime-io/build.rs create mode 100644 core/runtime-io/src/lib.rs create mode 100644 core/runtime-io/with_std.rs create mode 100644 core/runtime-io/without_std.rs create mode 100755 core/runtime-sandbox/Cargo.toml create mode 100644 core/runtime-sandbox/README.adoc create mode 100755 core/runtime-sandbox/build.rs create mode 100755 core/runtime-sandbox/src/lib.rs create mode 100755 core/runtime-sandbox/with_std.rs create mode 100755 core/runtime-sandbox/without_std.rs create mode 100644 core/runtime-std/Cargo.toml create mode 100644 core/runtime-std/README.adoc create mode 100644 core/runtime-std/build.rs create mode 100644 core/runtime-std/src/lib.rs create mode 100644 core/runtime-std/with_std.rs create mode 100644 core/runtime-std/without_std.rs create mode 100644 core/serializer/Cargo.toml create mode 100644 core/serializer/README.adoc create mode 100644 core/serializer/src/lib.rs create mode 100644 core/service/Cargo.toml create mode 100644 core/service/README.adoc create mode 100644 core/service/src/chain_ops.rs create mode 100644 core/service/src/chain_spec.rs create mode 100644 core/service/src/components.rs create mode 100644 core/service/src/config.rs create mode 100644 core/service/src/error.rs create mode 100644 core/service/src/lib.rs create mode 100644 core/state-db/Cargo.toml create mode 100644 core/state-db/README.adoc create mode 100644 core/state-db/src/lib.rs create mode 100644 core/state-db/src/pruning.rs create mode 100644 core/state-db/src/test.rs create mode 100644 core/state-db/src/unfinalized.rs create mode 100644 core/state-machine/Cargo.toml create mode 100644 core/state-machine/README.adoc create mode 100644 core/state-machine/src/backend.rs create mode 100644 core/state-machine/src/ext.rs create mode 100644 core/state-machine/src/lib.rs create mode 100644 core/state-machine/src/proving_backend.rs create mode 100644 core/state-machine/src/testing.rs create mode 100644 core/state-machine/src/trie_backend.rs create mode 100644 core/telemetry/Cargo.toml create mode 100644 core/telemetry/README.adoc create mode 100644 core/telemetry/src/lib.rs create mode 100644 core/test-client/Cargo.toml create mode 100644 core/test-client/README.adoc create mode 100644 core/test-client/src/block_builder_ext.rs create mode 100644 core/test-client/src/client_ext.rs create mode 100644 core/test-client/src/lib.rs create mode 100644 core/test-runtime/Cargo.toml create mode 100644 core/test-runtime/README.adoc create mode 100644 core/test-runtime/src/genesismap.rs create mode 100644 core/test-runtime/src/lib.rs create mode 100644 core/test-runtime/src/system.rs create mode 100644 core/test-runtime/wasm/Cargo.lock create mode 100644 core/test-runtime/wasm/Cargo.toml create mode 100755 core/test-runtime/wasm/build.sh create mode 120000 core/test-runtime/wasm/src create mode 100644 framework/client/Cargo.toml create mode 100644 framework/executor/Cargo.toml create mode 100644 framework/network/Cargo.toml create mode 100644 framework/runtime-support/Cargo.toml create mode 100644 framework/test-runtime/Cargo.toml create mode 100644 node/Cargo.toml create mode 100644 node/api/Cargo.toml create mode 100644 node/api/src/lib.rs create mode 100644 node/build.rs create mode 100644 node/cli/Cargo.toml create mode 100644 node/cli/src/cli.yml create mode 100644 node/cli/src/error.rs create mode 100644 node/cli/src/lib.rs create mode 100644 node/consensus/Cargo.toml create mode 100644 node/consensus/README.adoc create mode 100644 node/consensus/src/error.rs create mode 100644 node/consensus/src/evaluation.rs create mode 100644 node/consensus/src/lib.rs create mode 100644 node/consensus/src/offline_tracker.rs create mode 100644 node/consensus/src/service.rs create mode 100644 node/executor/Cargo.toml create mode 100644 node/executor/src/lib.rs create mode 100644 node/network/Cargo.toml create mode 100644 node/network/src/consensus.rs create mode 100644 node/network/src/lib.rs create mode 100644 node/primitives/Cargo.toml create mode 100644 node/primitives/src/lib.rs create mode 100644 node/runtime/Cargo.toml create mode 100644 node/runtime/src/checked_block.rs create mode 100644 node/runtime/src/lib.rs create mode 100644 node/runtime/wasm/Cargo.lock create mode 100644 node/runtime/wasm/Cargo.toml create mode 100755 node/runtime/wasm/build.sh create mode 120000 node/runtime/wasm/src create mode 100644 node/service/Cargo.toml create mode 100644 node/service/src/chain_spec.rs create mode 100644 node/service/src/lib.rs create mode 100644 node/src/main.rs create mode 100644 node/transaction-pool/Cargo.toml create mode 100644 node/transaction-pool/src/error.rs create mode 100644 node/transaction-pool/src/lib.rs create mode 100644 runtime/README.adoc create mode 100644 runtime/balances/Cargo.toml create mode 100644 runtime/balances/src/address.rs create mode 100644 runtime/balances/src/genesis_config.rs create mode 100644 runtime/balances/src/lib.rs create mode 100644 runtime/balances/src/mock.rs create mode 100644 runtime/balances/src/tests.rs create mode 100644 runtime/consensus/Cargo.toml create mode 100644 runtime/consensus/src/lib.rs create mode 100644 runtime/contract/Cargo.toml create mode 100644 runtime/contract/src/account_db.rs create mode 100644 runtime/contract/src/double_map.rs create mode 100644 runtime/contract/src/exec.rs create mode 100644 runtime/contract/src/gas.rs create mode 100644 runtime/contract/src/genesis_config.rs create mode 100644 runtime/contract/src/lib.rs create mode 100644 runtime/contract/src/tests.rs create mode 100644 runtime/contract/src/vm/env_def/macros.rs create mode 100644 runtime/contract/src/vm/env_def/mod.rs create mode 100644 runtime/contract/src/vm/mod.rs create mode 100644 runtime/contract/src/vm/prepare.rs create mode 100644 runtime/council/Cargo.toml create mode 100644 runtime/council/src/lib.rs create mode 100644 runtime/council/src/motions.rs create mode 100644 runtime/council/src/seats.rs create mode 100644 runtime/council/src/voting.rs create mode 100644 runtime/democracy/Cargo.toml create mode 100644 runtime/democracy/src/lib.rs create mode 100644 runtime/democracy/src/vote_threshold.rs create mode 100644 runtime/example/Cargo.toml create mode 100644 runtime/example/src/lib.rs create mode 100644 runtime/executive/Cargo.toml create mode 100644 runtime/executive/src/lib.rs create mode 100644 runtime/primitives/Cargo.toml create mode 100644 runtime/primitives/src/bft.rs create mode 100644 runtime/primitives/src/generic/block.rs create mode 100644 runtime/primitives/src/generic/checked_extrinsic.rs create mode 100644 runtime/primitives/src/generic/digest.rs create mode 100644 runtime/primitives/src/generic/header.rs create mode 100644 runtime/primitives/src/generic/mod.rs create mode 100644 runtime/primitives/src/generic/tests.rs create mode 100644 runtime/primitives/src/generic/unchecked_extrinsic.rs create mode 100644 runtime/primitives/src/lib.rs create mode 100644 runtime/primitives/src/testing.rs create mode 100644 runtime/primitives/src/traits.rs create mode 100644 runtime/session/Cargo.toml create mode 100644 runtime/session/src/lib.rs create mode 100644 runtime/staking/Cargo.toml create mode 100644 runtime/staking/src/genesis_config.rs create mode 100644 runtime/staking/src/lib.rs create mode 100644 runtime/staking/src/mock.rs create mode 100644 runtime/staking/src/tests.rs create mode 100644 runtime/support/Cargo.toml create mode 100644 runtime/support/README.adoc create mode 100644 runtime/support/src/dispatch.rs create mode 100644 runtime/support/src/event.rs create mode 100644 runtime/support/src/hashable.rs create mode 100644 runtime/support/src/lib.rs create mode 100644 runtime/support/src/metadata.rs create mode 100644 runtime/support/src/storage/generator.rs create mode 100644 runtime/support/src/storage/mod.rs create mode 100644 runtime/system/Cargo.toml create mode 100644 runtime/system/src/lib.rs create mode 100644 runtime/timestamp/Cargo.toml create mode 100644 runtime/timestamp/src/lib.rs create mode 100644 runtime/treasury/Cargo.toml create mode 100644 runtime/treasury/src/lib.rs create mode 100644 runtime/version/Cargo.toml create mode 100644 runtime/version/src/lib.rs diff --git a/core/bft/Cargo.toml b/core/bft/Cargo.toml new file mode 100644 index 00000000000..609a27088b4 --- /dev/null +++ b/core/bft/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "substrate-bft" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +futures = "0.1.17" +substrate-codec = { path = "../codec" } +substrate-primitives = { path = "../primitives" } +substrate-runtime-support = { path = "../../runtime/support" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-runtime-version = { path = "../../runtime/version" } + +tokio = "0.1.7" +parking_lot = "0.4" +error-chain = "0.12" +log = "0.3" +rhododendron = "0.3" + +[dev-dependencies] +substrate-keyring = { path = "../keyring" } +substrate-executor = { path = "../executor" } diff --git a/core/bft/README.adoc b/core/bft/README.adoc new file mode 100644 index 00000000000..8f4939087ee --- /dev/null +++ b/core/bft/README.adoc @@ -0,0 +1,13 @@ + += Substrate BFT + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/bft/src/error.rs b/core/bft/src/error.rs new file mode 100644 index 00000000000..fb41e7efa90 --- /dev/null +++ b/core/bft/src/error.rs @@ -0,0 +1,95 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Error types in the BFT service. +use runtime_version::RuntimeVersion; +use primitives::ed25519; + +error_chain! { + errors { + /// Missing state at block with given descriptor. + StateUnavailable(b: String) { + description("State missing at given block."), + display("State unavailable at block {}", b), + } + + /// I/O terminated unexpectedly + IoTerminated { + description("I/O terminated unexpectedly."), + display("I/O terminated unexpectedly."), + } + + /// Unable to schedule wakeup. + FaultyTimer(e: ::tokio::timer::Error) { + description("Timer error"), + display("Timer error: {}", e), + } + + /// Unable to propose a block. + CannotPropose { + description("Unable to create block proposal."), + display("Unable to create block proposal."), + } + + /// Error checking signature + InvalidSignature(s: ed25519::Signature, a: ::primitives::AuthorityId) { + description("Message signature is invalid"), + display("Message signature {:?} by {:?} is invalid.", s, a), + } + + /// Account is not an authority. + InvalidAuthority(a: ::primitives::AuthorityId) { + description("Message sender is not a valid authority"), + display("Message sender {:?} is not a valid authority.", a), + } + + /// Authoring interface does not match the runtime. + IncompatibleAuthoringRuntime(native: RuntimeVersion, on_chain: RuntimeVersion) { + description("Authoring for current runtime is not supported"), + display("Authoring for current runtime is not supported. Native ({}) cannot author for on-chain ({}).", native, on_chain), + } + + /// Authoring interface does not match the runtime. + RuntimeVersionMissing { + description("Current runtime has no version"), + display("Authoring for current runtime is not supported since it has no version."), + } + + /// Authoring interface does not match the runtime. + NativeRuntimeMissing { + description("This build has no native runtime"), + display("Authoring in current build is not supported since it has no runtime."), + } + + /// Justification requirements not met. + InvalidJustification { + description("Invalid justification"), + display("Invalid justification."), + } + + /// Some other error. + Other(e: Box<::std::error::Error + Send>) { + description("Other error") + display("Other error: {}", e.description()) + } + } +} + +impl From<::rhododendron::InputStreamConcluded> for Error { + fn from(_: ::rhododendron::InputStreamConcluded) -> Error { + ErrorKind::IoTerminated.into() + } +} diff --git a/core/bft/src/lib.rs b/core/bft/src/lib.rs new file mode 100644 index 00000000000..e6f2615f0fd --- /dev/null +++ b/core/bft/src/lib.rs @@ -0,0 +1,1124 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! BFT Agreement based on a rotating proposer in different rounds. +//! +//! Where this crate refers to input stream, should never logically conclude. +//! The logic in this crate assumes that messages flushed to the output stream +//! will eventually reach other nodes and that our own messages are not included +//! in the input stream. +//! +//! Note that it is possible to witness agreement being reached without ever +//! seeing the candidate. Any candidates seen will be checked for validity. +//! +//! Although technically the agreement will always complete (given the eventual +//! delivery of messages), in practice it is possible for this future to +//! conclude without having witnessed the conclusion. +//! In general, this future should be pre-empted by the import of a justification +//! set for this block height. +// end::description[] + +#![recursion_limit="128"] + +pub mod error; + +extern crate substrate_codec as codec; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_support as runtime_support; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_runtime_version as runtime_version; +extern crate tokio; +extern crate parking_lot; +extern crate rhododendron; + +#[macro_use] +extern crate log; + +extern crate futures; + +#[macro_use] +extern crate error_chain; + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Instant, Duration}; + +use codec::Encode; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{Block, Header}; +use runtime_primitives::bft::{Message as PrimitiveMessage, Action as PrimitiveAction, Justification as PrimitiveJustification}; +use primitives::{AuthorityId, ed25519, ed25519::LocalizedSignature}; + +use futures::{Async, Stream, Sink, Future, IntoFuture}; +use futures::sync::oneshot; +use tokio::timer::Delay; +use parking_lot::Mutex; + +pub use rhododendron::{InputStreamConcluded, AdvanceRoundReason}; +pub use error::{Error, ErrorKind}; + +// statuses for an agreement +mod status { + pub const LIVE: usize = 0; + pub const BAD: usize = 1; + pub const GOOD: usize = 2; +} + +/// Messages over the proposal. +/// Each message carries an associated round number. +pub type Message = rhododendron::Message::Hash>; + +/// Localized message type. +pub type LocalizedMessage = rhododendron::LocalizedMessage< + B, + ::Hash, + AuthorityId, + LocalizedSignature +>; + +/// Justification of some hash. +pub type Justification = rhododendron::Justification; + +/// Justification of a prepare message. +pub type PrepareJustification = rhododendron::PrepareJustification; + +/// Unchecked justification. +pub struct UncheckedJustification(rhododendron::UncheckedJustification); + +impl UncheckedJustification { + /// Create a new, unchecked justification. + pub fn new(digest: H, signatures: Vec, round_number: usize) -> Self { + UncheckedJustification(rhododendron::UncheckedJustification { + digest, + signatures, + round_number, + }) + } +} + +impl From> for UncheckedJustification { + fn from(inner: rhododendron::UncheckedJustification) -> Self { + UncheckedJustification(inner) + } +} + +impl From> for UncheckedJustification { + fn from(just: PrimitiveJustification) -> Self { + UncheckedJustification(rhododendron::UncheckedJustification { + round_number: just.round_number as usize, + digest: just.hash, + signatures: just.signatures.into_iter().map(|(from, sig)| LocalizedSignature { + signer: from.into(), + signature: sig, + }).collect(), + }) + } +} + +impl Into> for UncheckedJustification { + fn into(self) -> PrimitiveJustification { + PrimitiveJustification { + round_number: self.0.round_number as u32, + hash: self.0.digest, + signatures: self.0.signatures.into_iter().map(|s| (s.signer.into(), s.signature)).collect(), + } + } +} + +/// Result of a committed round of BFT +pub type Committed = rhododendron::Committed::Hash, LocalizedSignature>; + +/// Communication between BFT participants. +pub type Communication = rhododendron::Communication::Hash, AuthorityId, LocalizedSignature>; + +/// Misbehavior observed from BFT participants. +pub type Misbehavior = rhododendron::Misbehavior; + +/// Environment producer for a BFT instance. Creates proposer instance and communication streams. +pub trait Environment { + /// The proposer type this creates. + type Proposer: Proposer; + /// The input stream type. + type Input: Stream, Error=>::Error>; + /// The output stream type. + type Output: Sink, SinkError=>::Error>; + /// Error which can occur upon creation. + type Error: From; + + /// Initialize the proposal logic on top of a specific header. + /// Produces the proposer and message streams for this instance of BFT agreement. + // TODO: provide state context explicitly? + fn init(&self, parent_header: &B::Header, authorities: &[AuthorityId], sign_with: Arc) + -> Result<(Self::Proposer, Self::Input, Self::Output), Self::Error>; +} + +/// Logic for a proposer. +/// +/// This will encapsulate creation and evaluation of proposals at a specific +/// block. +pub trait Proposer { + /// Error type which can occur when proposing or evaluating. + type Error: From + From + ::std::fmt::Debug + 'static; + /// Future that resolves to a committed proposal. + type Create: IntoFuture; + /// Future that resolves when a proposal is evaluated. + type Evaluate: IntoFuture; + + /// Create a proposal. + fn propose(&self) -> Self::Create; + + /// Evaluate proposal. True means valid. + fn evaluate(&self, proposal: &B) -> Self::Evaluate; + + /// Import witnessed misbehavior. + fn import_misbehavior(&self, misbehavior: Vec<(AuthorityId, Misbehavior)>); + + /// Determine the proposer for a given round. This should be a deterministic function + /// with consistent results across all authorities. + fn round_proposer(&self, round_number: usize, authorities: &[AuthorityId]) -> AuthorityId; + + /// Hook called when a BFT round advances without a proposal. + fn on_round_end(&self, _round_number: usize, _proposed: bool) { } +} + +/// Block import trait. +pub trait BlockImport { + /// Import a block alongside its corresponding justification. + fn import_block(&self, block: B, justification: Justification, authorities: &[AuthorityId]) -> bool; +} + +/// Trait for getting the authorities at a given block. +pub trait Authorities { + /// Get the authorities at the given block. + fn authorities(&self, at: &BlockId) -> Result, Error>; +} + +// caches the round number to start at if we end up with BFT consensus on the same +// parent hash more than once (happens if block is bad). +// +// this will force a committed but locally-bad block to be considered analogous to +// a round advancement vote. +#[derive(Debug)] +struct RoundCache { + hash: Option, + start_round: usize, +} + +/// Instance of BFT agreement. +struct BftInstance { + key: Arc, + authorities: Vec, + parent_hash: B::Hash, + round_timeout_multiplier: u64, + cache: Arc>>, + proposer: P, +} + +impl> BftInstance + where + B: Clone + Eq, + B::Hash: ::std::hash::Hash + +{ + fn round_timeout_duration(&self, round: usize) -> Duration { + // 2^(min(6, x/8)) * 10 + // Grows exponentially starting from 10 seconds, capped at 640 seconds. + const ROUND_INCREMENT_STEP: usize = 8; + + let round = round / ROUND_INCREMENT_STEP; + let round = ::std::cmp::min(6, round) as u32; + + let timeout = 1u64.checked_shl(round) + .unwrap_or_else(u64::max_value) + .saturating_mul(self.round_timeout_multiplier); + + Duration::from_secs(timeout) + } + + fn update_round_cache(&self, current_round: usize) { + let mut cache = self.cache.lock(); + if cache.hash.as_ref() == Some(&self.parent_hash) { + cache.start_round = current_round + 1; + } + } +} + +impl> rhododendron::Context for BftInstance + where + B: Clone + Eq, + B::Hash: ::std::hash::Hash, +{ + type Error = P::Error; + type AuthorityId = AuthorityId; + type Digest = B::Hash; + type Signature = LocalizedSignature; + type Candidate = B; + type RoundTimeout = Box>; + type CreateProposal = ::Future; + type EvaluateProposal = ::Future; + + fn local_id(&self) -> AuthorityId { + self.key.public().into() + } + + fn proposal(&self) -> Self::CreateProposal { + self.proposer.propose().into_future() + } + + fn candidate_digest(&self, proposal: &B) -> B::Hash { + proposal.hash() + } + + fn sign_local(&self, message: Message) -> LocalizedMessage { + sign_message(message, &*self.key, self.parent_hash.clone()) + } + + fn round_proposer(&self, round: usize) -> AuthorityId { + self.proposer.round_proposer(round, &self.authorities[..]) + } + + fn proposal_valid(&self, proposal: &B) -> Self::EvaluateProposal { + self.proposer.evaluate(proposal).into_future() + } + + fn begin_round_timeout(&self, round: usize) -> Self::RoundTimeout { + let timeout = self.round_timeout_duration(round); + let fut = Delay::new(Instant::now() + timeout) + .map_err(|e| Error::from(ErrorKind::FaultyTimer(e))) + .map_err(Into::into); + + Box::new(fut) + } + + fn on_advance_round( + &self, + accumulator: &::rhododendron::Accumulator, + round: usize, + next_round: usize, + reason: AdvanceRoundReason, + ) { + use std::collections::HashSet; + + let collect_pubkeys = |participants: HashSet<&Self::AuthorityId>| participants.into_iter() + .map(|p| ed25519::Public::from_raw(p.0)) + .collect::>(); + + let round_timeout = self.round_timeout_duration(next_round); + debug!(target: "bft", "Advancing to round {} from {}", next_round, round); + debug!(target: "bft", "Participating authorities: {:?}", + collect_pubkeys(accumulator.participants())); + debug!(target: "bft", "Voting authorities: {:?}", + collect_pubkeys(accumulator.voters())); + debug!(target: "bft", "Round {} should end in at most {} seconds from now", next_round, round_timeout.as_secs()); + + self.update_round_cache(next_round); + + if let AdvanceRoundReason::Timeout = reason { + self.proposer.on_round_end(round, accumulator.proposal().is_some()); + } + } +} + +/// A future that resolves either when canceled (witnessing a block from the network at same height) +/// or when agreement completes. +pub struct BftFuture where + B: Block + Clone + Eq, + B::Hash: ::std::hash::Hash, + P: Proposer, + InStream: Stream, Error=P::Error>, + OutSink: Sink, SinkError=P::Error>, +{ + inner: rhododendron::Agreement, InStream, OutSink>, + status: Arc, + cancel: oneshot::Receiver<()>, + import: Arc, +} + +impl Future for BftFuture where + B: Block + Clone + Eq, + B::Hash: ::std::hash::Hash, + P: Proposer, + P::Error: ::std::fmt::Display, + I: BlockImport, + InStream: Stream, Error=P::Error>, + OutSink: Sink, SinkError=P::Error>, +{ + type Item = (); + type Error = (); + + fn poll(&mut self) -> ::futures::Poll<(), ()> { + // service has canceled the future. bail + let cancel = match self.cancel.poll() { + Ok(Async::Ready(())) | Err(_) => true, + Ok(Async::NotReady) => false, + }; + + // TODO: handle and log this error in a way which isn't noisy on exit. + let committed = match self.inner.poll().map_err(|_| ()) { + Ok(Async::Ready(x)) => x, + Ok(Async::NotReady) => + return Ok(if cancel { Async::Ready(()) } else { Async::NotReady }), + Err(()) => return Err(()), + }; + + // if something was committed, the round leader must have proposed. + self.inner.context().proposer.on_round_end(committed.round_number, true); + + // If we didn't see the proposal (very unlikely), + // we will get the block from the network later. + if let Some(justified_block) = committed.candidate { + let hash = justified_block.hash(); + info!(target: "bft", "Importing block #{} ({}) directly from BFT consensus", + justified_block.header().number(), hash); + + let import_ok = self.import.import_block( + justified_block, + committed.justification, + &self.inner.context().authorities + ); + + if !import_ok { + warn!(target: "bft", "{:?} was bad block agreed on in round #{}", + hash, committed.round_number); + self.status.store(status::BAD, Ordering::Release); + } else { + self.status.store(status::GOOD, Ordering::Release); + } + } else { + // assume good unless we received the proposal. + self.status.store(status::GOOD, Ordering::Release); + } + + self.inner.context().update_round_cache(committed.round_number); + + Ok(Async::Ready(())) + } +} + +impl Drop for BftFuture where + B: Block + Clone + Eq, + B::Hash: ::std::hash::Hash, + P: Proposer, + InStream: Stream, Error=P::Error>, + OutSink: Sink, SinkError=P::Error>, +{ + fn drop(&mut self) { + // TODO: have a trait member to pass misbehavior reports into. + let misbehavior = self.inner.drain_misbehavior().collect::>(); + self.inner.context().proposer.import_misbehavior(misbehavior); + } +} + +struct AgreementHandle { + status: Arc, + send_cancel: Option>, +} + +impl AgreementHandle { + fn status(&self) -> usize { + self.status.load(Ordering::Acquire) + } +} + +impl Drop for AgreementHandle { + fn drop(&mut self) { + if let Some(sender) = self.send_cancel.take() { + let _ = sender.send(()); + } + } +} + +/// The BftService kicks off the agreement process on top of any blocks it +/// is notified of. +/// +/// This assumes that it is being run in the context of a tokio runtime. +pub struct BftService { + client: Arc, + live_agreement: Mutex>, + round_cache: Arc>>, + round_timeout_multiplier: u64, + key: Arc, // TODO: key changing over time. + factory: P, +} + +impl BftService + where + B: Block + Clone + Eq, + P: Environment, + >::Error: ::std::fmt::Display, + I: BlockImport + Authorities, +{ + + /// Create a new service instance. + pub fn new(client: Arc, key: Arc, factory: P) -> BftService { + BftService { + client: client, + live_agreement: Mutex::new(None), + round_cache: Arc::new(Mutex::new(RoundCache { + hash: None, + start_round: 0, + })), + round_timeout_multiplier: 10, + key: key, // TODO: key changing over time. + factory, + } + } + + /// Get the local Authority ID. + pub fn local_id(&self) -> AuthorityId { + // TODO: based on a header and some keystore. + self.key.public().into() + } + + /// Signal that a valid block with the given header has been imported. + /// + /// If the local signing key is an authority, this will begin the consensus process to build a + /// block on top of it. If the executor fails to run the future, an error will be returned. + /// Returns `None` if the agreement on the block with given parent is already in progress. + pub fn build_upon(&self, header: &B::Header) + -> Result>::Proposer, + I, +

>::Input, +

>::Output, + >>, P::Error> + where + { + let hash = header.hash(); + + let mut live_agreement = self.live_agreement.lock(); + let can_build = live_agreement.as_ref() + .map_or(true, |x| self.can_build_on_inner(header, x)); + + if !can_build { + return Ok(None) + } + + let authorities = self.client.authorities(&BlockId::Hash(hash.clone()))?; + + let n = authorities.len(); + let max_faulty = max_faulty_of(n); + trace!(target: "bft", "Initiating agreement on top of #{}, {:?}", header.number(), hash); + trace!(target: "bft", "max_faulty_of({})={}", n, max_faulty); + + let local_id = self.local_id(); + + if !authorities.contains(&local_id) { + // cancel current agreement + live_agreement.take(); + Err(ErrorKind::InvalidAuthority(local_id).into())?; + } + + let (proposer, input, output) = self.factory.init(header, &authorities, self.key.clone())?; + + let bft_instance = BftInstance { + proposer, + parent_hash: hash.clone(), + cache: self.round_cache.clone(), + round_timeout_multiplier: self.round_timeout_multiplier, + key: self.key.clone(), + authorities: authorities, + }; + + let mut agreement = rhododendron::agree( + bft_instance, + n, + max_faulty, + input, + output, + ); + + // fast forward round number if necessary. + { + let mut cache = self.round_cache.lock(); + trace!(target: "bft", "Round cache: {:?}", &*cache); + if cache.hash.as_ref() == Some(&hash) { + trace!(target: "bft", "Fast-forwarding to round {}", cache.start_round); + let start_round = cache.start_round; + cache.start_round += 1; + + drop(cache); + agreement.fast_forward(start_round); + } else { + *cache = RoundCache { + hash: Some(hash.clone()), + start_round: 1, + }; + } + } + + let status = Arc::new(AtomicUsize::new(status::LIVE)); + let (tx, rx) = oneshot::channel(); + + // cancel current agreement. + *live_agreement = Some((header.clone(), AgreementHandle { + send_cancel: Some(tx), + status: status.clone(), + })); + + Ok(Some(BftFuture { + inner: agreement, + status: status, + cancel: rx, + import: self.client.clone(), + })) + } + + /// Cancel current agreement if any. + pub fn cancel_agreement(&self) { + self.live_agreement.lock().take(); + } + + /// Whether we can build using the given header. + pub fn can_build_on(&self, header: &B::Header) -> bool { + self.live_agreement.lock().as_ref() + .map_or(true, |x| self.can_build_on_inner(header, x)) + } + + /// Get a reference to the underyling client. + pub fn client(&self) -> &I { &*self.client } + + fn can_build_on_inner(&self, header: &B::Header, live: &(B::Header, AgreementHandle)) -> bool { + let hash = header.hash(); + let &(ref live_header, ref handle) = live; + match handle.status() { + _ if *header != *live_header && *live_header.parent_hash() != hash => true, // can always follow with next block. + status::BAD => hash == live_header.hash(), // bad block can be re-agreed on. + _ => false, // canceled won't appear since we overwrite the handle before returning. + } + } +} + +/// Given a total number of authorities, yield the maximum faulty that would be allowed. +/// This will always be under 1/3. +pub fn max_faulty_of(n: usize) -> usize { + n.saturating_sub(1) / 3 +} + +/// Given a total number of authorities, yield the minimum required signatures. +/// This will always be over 2/3. +pub fn bft_threshold(n: usize) -> usize { + n - max_faulty_of(n) +} + +fn check_justification_signed_message(authorities: &[AuthorityId], message: &[u8], just: UncheckedJustification) + -> Result, UncheckedJustification> +{ + // TODO: return additional error information. + just.0.check(authorities.len() - max_faulty_of(authorities.len()), |_, _, sig| { + let auth_id = sig.signer.clone().into(); + if !authorities.contains(&auth_id) { return None } + + if ed25519::verify_strong(&sig.signature, message, &sig.signer) { + Some(sig.signer.0) + } else { + None + } + }).map_err(UncheckedJustification) +} + +/// Check a full justification for a header hash. +/// Provide all valid authorities. +/// +/// On failure, returns the justification back. +pub fn check_justification(authorities: &[AuthorityId], parent: B::Hash, just: UncheckedJustification) + -> Result, UncheckedJustification> +{ + let message = Encode::encode(&PrimitiveMessage:: { + parent, + action: PrimitiveAction::Commit(just.0.round_number as u32, just.0.digest.clone()), + }); + + check_justification_signed_message(authorities, &message[..], just) +} + +/// Check a prepare justification for a header hash. +/// Provide all valid authorities. +/// +/// On failure, returns the justification back. +pub fn check_prepare_justification(authorities: &[AuthorityId], parent: B::Hash, just: UncheckedJustification) + -> Result, UncheckedJustification> +{ + let message = Encode::encode(&PrimitiveMessage:: { + parent, + action: PrimitiveAction::Prepare(just.0.round_number as u32, just.0.digest.clone()), + }); + + check_justification_signed_message(authorities, &message[..], just) +} + +/// Check proposal message signatures and authority. +/// Provide all valid authorities. +pub fn check_proposal( + authorities: &[AuthorityId], + parent_hash: &B::Hash, + propose: &::rhododendron::LocalizedProposal) + -> Result<(), Error> +{ + if !authorities.contains(&propose.sender) { + return Err(ErrorKind::InvalidAuthority(propose.sender.into()).into()); + } + + let action_header = PrimitiveAction::ProposeHeader(propose.round_number as u32, propose.digest.clone()); + let action_propose = PrimitiveAction::Propose(propose.round_number as u32, propose.proposal.clone()); + check_action::(action_header, parent_hash, &propose.digest_signature)?; + check_action::(action_propose, parent_hash, &propose.full_signature) +} + +/// Check vote message signatures and authority. +/// Provide all valid authorities. +pub fn check_vote( + authorities: &[AuthorityId], + parent_hash: &B::Hash, + vote: &::rhododendron::LocalizedVote) + -> Result<(), Error> +{ + if !authorities.contains(&vote.sender) { + return Err(ErrorKind::InvalidAuthority(vote.sender.into()).into()); + } + + let action = match vote.vote { + ::rhododendron::Vote::Prepare(r, ref h) => PrimitiveAction::Prepare(r as u32, h.clone()), + ::rhododendron::Vote::Commit(r, ref h) => PrimitiveAction::Commit(r as u32, h.clone()), + ::rhododendron::Vote::AdvanceRound(r) => PrimitiveAction::AdvanceRound(r as u32), + }; + check_action::(action, parent_hash, &vote.signature) +} + +fn check_action(action: PrimitiveAction, parent_hash: &B::Hash, sig: &LocalizedSignature) -> Result<(), Error> { + let primitive = PrimitiveMessage { + parent: parent_hash.clone(), + action, + }; + + let message = Encode::encode(&primitive); + if ed25519::verify_strong(&sig.signature, &message, &sig.signer) { + Ok(()) + } else { + Err(ErrorKind::InvalidSignature(sig.signature.into(), sig.signer.clone().into()).into()) + } +} + +/// Sign a BFT message with the given key. +pub fn sign_message(message: Message, key: &ed25519::Pair, parent_hash: B::Hash) -> LocalizedMessage { + let signer = key.public(); + + let sign_action = |action: PrimitiveAction| { + let primitive = PrimitiveMessage { + parent: parent_hash.clone(), + action, + }; + + let to_sign = Encode::encode(&primitive); + LocalizedSignature { + signer: signer.clone(), + signature: key.sign(&to_sign), + } + }; + + match message { + ::rhododendron::Message::Propose(r, proposal) => { + let header_hash = proposal.hash(); + let action_header = PrimitiveAction::ProposeHeader(r as u32, header_hash.clone()); + let action_propose = PrimitiveAction::Propose(r as u32, proposal.clone()); + + ::rhododendron::LocalizedMessage::Propose(::rhododendron::LocalizedProposal { + round_number: r, + proposal, + digest: header_hash, + sender: signer.clone().into(), + digest_signature: sign_action(action_header), + full_signature: sign_action(action_propose), + }) + } + ::rhododendron::Message::Vote(vote) => { + let action = match vote { + ::rhododendron::Vote::Prepare(r, ref h) => PrimitiveAction::Prepare(r as u32, h.clone()), + ::rhododendron::Vote::Commit(r, ref h) => PrimitiveAction::Commit(r as u32, h.clone()), + ::rhododendron::Vote::AdvanceRound(r) => PrimitiveAction::AdvanceRound(r as u32), + }; + + ::rhododendron::LocalizedMessage::Vote(::rhododendron::LocalizedVote { + vote: vote, + sender: signer.clone().into(), + signature: sign_action(action), + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + use runtime_primitives::testing::{Block as GenericTestBlock, Header as TestHeader}; + use primitives::H256; + use self::keyring::Keyring; + + extern crate substrate_keyring as keyring; + + type TestBlock = GenericTestBlock<()>; + + struct FakeClient { + authorities: Vec, + imported_heights: Mutex> + } + + impl BlockImport for FakeClient { + fn import_block(&self, block: TestBlock, _justification: Justification, _authorities: &[AuthorityId]) -> bool { + assert!(self.imported_heights.lock().insert(block.header.number)); + true + } + } + + impl Authorities for FakeClient { + fn authorities(&self, _at: &BlockId) -> Result, Error> { + Ok(self.authorities.clone()) + } + } + + // "black hole" output sink. + struct Comms(::std::marker::PhantomData); + + impl Sink for Comms { + type SinkItem = Communication; + type SinkError = E; + + fn start_send(&mut self, _item: Communication) -> ::futures::StartSend, E> { + Ok(::futures::AsyncSink::Ready) + } + + fn poll_complete(&mut self) -> ::futures::Poll<(), E> { + Ok(Async::Ready(())) + } + } + + impl Stream for Comms { + type Item = Communication; + type Error = E; + + fn poll(&mut self) -> ::futures::Poll, Self::Error> { + Ok(::futures::Async::NotReady) + } + } + + struct DummyFactory; + struct DummyProposer(u64); + + impl Environment for DummyFactory { + type Proposer = DummyProposer; + type Input = Comms; + type Output = Comms; + type Error = Error; + + fn init(&self, parent_header: &TestHeader, _authorities: &[AuthorityId], _sign_with: Arc) + -> Result<(DummyProposer, Self::Input, Self::Output), Error> + { + Ok((DummyProposer(parent_header.number + 1), Comms(::std::marker::PhantomData), Comms(::std::marker::PhantomData))) + } + } + + impl Proposer for DummyProposer { + type Error = Error; + type Create = Result; + type Evaluate = Result; + + fn propose(&self) -> Result { + + Ok(TestBlock { + header: from_block_number(self.0), + extrinsics: Default::default() + }) + } + + fn evaluate(&self, proposal: &TestBlock) -> Result { + Ok(proposal.header.number == self.0) + } + + fn import_misbehavior(&self, _misbehavior: Vec<(AuthorityId, Misbehavior)>) {} + + fn round_proposer(&self, round_number: usize, authorities: &[AuthorityId]) -> AuthorityId { + authorities[round_number % authorities.len()].clone() + } + } + + fn make_service(client: FakeClient) + -> BftService + { + BftService { + client: Arc::new(client), + live_agreement: Mutex::new(None), + round_cache: Arc::new(Mutex::new(RoundCache { + hash: None, + start_round: 0, + })), + round_timeout_multiplier: 10, + key: Arc::new(Keyring::One.into()), + factory: DummyFactory + } + } + + fn sign_vote(vote: ::rhododendron::Vote, key: &ed25519::Pair, parent_hash: H256) -> LocalizedSignature { + match sign_message::(vote.into(), key, parent_hash) { + ::rhododendron::LocalizedMessage::Vote(vote) => vote.signature, + _ => panic!("signing vote leads to signed vote"), + } + } + + fn from_block_number(num: u64) -> TestHeader { + TestHeader::new( + num, + Default::default(), + Default::default(), + Default::default(), + Default::default(), + ) + } + + #[test] + fn future_gets_preempted() { + let client = FakeClient { + authorities: vec![ + Keyring::One.to_raw_public().into(), + Keyring::Two.to_raw_public().into(), + Keyring::Alice.to_raw_public().into(), + Keyring::Eve.to_raw_public().into(), + ], + imported_heights: Mutex::new(HashSet::new()), + }; + + let service = make_service(client); + + let first = from_block_number(2); + let first_hash = first.hash(); + + let mut second = from_block_number(3); + second.parent_hash = first_hash; + let _second_hash = second.hash(); + + let mut first_bft = service.build_upon(&first).unwrap().unwrap(); + assert!(service.live_agreement.lock().as_ref().unwrap().0 == first); + + let _second_bft = service.build_upon(&second).unwrap(); + assert!(service.live_agreement.lock().as_ref().unwrap().0 != first); + assert!(service.live_agreement.lock().as_ref().unwrap().0 == second); + + // first_bft has been cancelled. need to swap out so we can check it. + let (_tx, mut rx) = oneshot::channel(); + ::std::mem::swap(&mut rx, &mut first_bft.cancel); + + assert!(rx.wait().is_ok()); + } + + #[test] + fn max_faulty() { + assert_eq!(max_faulty_of(3), 0); + assert_eq!(max_faulty_of(4), 1); + assert_eq!(max_faulty_of(100), 33); + assert_eq!(max_faulty_of(0), 0); + assert_eq!(max_faulty_of(11), 3); + assert_eq!(max_faulty_of(99), 32); + } + + #[test] + fn justification_check_works() { + let parent_hash = Default::default(); + let hash = [0xff; 32].into(); + + let authorities = vec![ + Keyring::One.to_raw_public().into(), + Keyring::Two.to_raw_public().into(), + Keyring::Alice.to_raw_public().into(), + Keyring::Eve.to_raw_public().into(), + ]; + + let authorities_keys = vec![ + Keyring::One.into(), + Keyring::Two.into(), + Keyring::Alice.into(), + Keyring::Eve.into(), + ]; + + let unchecked = UncheckedJustification(rhododendron::UncheckedJustification { + digest: hash, + round_number: 1, + signatures: authorities_keys.iter().take(3).map(|key| { + sign_vote(rhododendron::Vote::Commit(1, hash).into(), key, parent_hash) + }).collect(), + }); + + assert!(check_justification::(&authorities, parent_hash, unchecked).is_ok()); + + let unchecked = UncheckedJustification(rhododendron::UncheckedJustification { + digest: hash, + round_number: 0, // wrong round number (vs. the signatures) + signatures: authorities_keys.iter().take(3).map(|key| { + sign_vote(rhododendron::Vote::Commit(1, hash).into(), key, parent_hash) + }).collect(), + }); + + assert!(check_justification::(&authorities, parent_hash, unchecked).is_err()); + + // not enough signatures. + let unchecked = UncheckedJustification(rhododendron::UncheckedJustification { + digest: hash, + round_number: 1, + signatures: authorities_keys.iter().take(2).map(|key| { + sign_vote(rhododendron::Vote::Commit(1, hash).into(), key, parent_hash) + }).collect(), + }); + + assert!(check_justification::(&authorities, parent_hash, unchecked).is_err()); + + // wrong hash. + let unchecked = UncheckedJustification(rhododendron::UncheckedJustification { + digest: [0xfe; 32].into(), + round_number: 1, + signatures: authorities_keys.iter().take(3).map(|key| { + sign_vote(rhododendron::Vote::Commit(1, hash).into(), key, parent_hash) + }).collect(), + }); + + assert!(check_justification::(&authorities, parent_hash, unchecked).is_err()); + } + + #[test] + fn propose_check_works() { + let parent_hash = Default::default(); + + let authorities = vec![ + Keyring::Alice.to_raw_public().into(), + Keyring::Eve.to_raw_public().into(), + ]; + + let block = TestBlock { + header: from_block_number(1), + extrinsics: Default::default() + }; + + let proposal = sign_message(::rhododendron::Message::Propose(1, block.clone()), &Keyring::Alice.pair(), parent_hash);; + if let ::rhododendron::LocalizedMessage::Propose(proposal) = proposal { + assert!(check_proposal(&authorities, &parent_hash, &proposal).is_ok()); + let mut invalid_round = proposal.clone(); + invalid_round.round_number = 0; + assert!(check_proposal(&authorities, &parent_hash, &invalid_round).is_err()); + let mut invalid_digest = proposal.clone(); + invalid_digest.digest = [0xfe; 32].into(); + assert!(check_proposal(&authorities, &parent_hash, &invalid_digest).is_err()); + } else { + assert!(false); + } + + // Not an authority + let proposal = sign_message::(::rhododendron::Message::Propose(1, block), &Keyring::Bob.pair(), parent_hash);; + if let ::rhododendron::LocalizedMessage::Propose(proposal) = proposal { + assert!(check_proposal(&authorities, &parent_hash, &proposal).is_err()); + } else { + assert!(false); + } + } + + #[test] + fn vote_check_works() { + let parent_hash: H256 = Default::default(); + let hash: H256 = [0xff; 32].into(); + + let authorities = vec![ + Keyring::Alice.to_raw_public().into(), + Keyring::Eve.to_raw_public().into(), + ]; + + let vote = sign_message::(::rhododendron::Message::Vote(::rhododendron::Vote::Prepare(1, hash)), &Keyring::Alice.pair(), parent_hash);; + if let ::rhododendron::LocalizedMessage::Vote(vote) = vote { + assert!(check_vote::(&authorities, &parent_hash, &vote).is_ok()); + let mut invalid_sender = vote.clone(); + invalid_sender.signature.signer = Keyring::Eve.into(); + assert!(check_vote::(&authorities, &parent_hash, &invalid_sender).is_err()); + } else { + assert!(false); + } + + // Not an authority + let vote = sign_message::(::rhododendron::Message::Vote(::rhododendron::Vote::Prepare(1, hash)), &Keyring::Bob.pair(), parent_hash);; + if let ::rhododendron::LocalizedMessage::Vote(vote) = vote { + assert!(check_vote::(&authorities, &parent_hash, &vote).is_err()); + } else { + assert!(false); + } + } + + #[test] + fn drop_bft_future_does_not_deadlock() { + let client = FakeClient { + authorities: vec![ + Keyring::One.to_raw_public().into(), + Keyring::Two.to_raw_public().into(), + Keyring::Alice.to_raw_public().into(), + Keyring::Eve.to_raw_public().into(), + ], + imported_heights: Mutex::new(HashSet::new()), + }; + + let service = make_service(client); + + let first = from_block_number(2); + let first_hash = first.hash(); + + let mut second = from_block_number(3); + second.parent_hash = first_hash; + + let _ = service.build_upon(&first).unwrap(); + assert!(service.live_agreement.lock().as_ref().unwrap().0 == first); + service.live_agreement.lock().take(); + } + + #[test] + fn bft_can_build_though_skipped() { + let client = FakeClient { + authorities: vec![ + Keyring::One.to_raw_public().into(), + Keyring::Two.to_raw_public().into(), + Keyring::Alice.to_raw_public().into(), + Keyring::Eve.to_raw_public().into(), + ], + imported_heights: Mutex::new(HashSet::new()), + }; + + let service = make_service(client); + + let first = from_block_number(2); + let first_hash = first.hash(); + + let mut second = from_block_number(3); + second.parent_hash = first_hash; + + let mut third = from_block_number(4); + third.parent_hash = second.hash(); + + let _ = service.build_upon(&first).unwrap(); + assert!(service.live_agreement.lock().as_ref().unwrap().0 == first); + // BFT has not seen second, but will move forward on third + service.build_upon(&third).unwrap(); + assert!(service.live_agreement.lock().as_ref().unwrap().0 == third); + + // but we are not walking backwards + service.build_upon(&second).unwrap(); + assert!(service.live_agreement.lock().as_ref().unwrap().0 == third); + } +} diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml new file mode 100644 index 00000000000..ef24f7cf2b4 --- /dev/null +++ b/core/cli/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "substrate-cli" +version = "0.3.0" +authors = ["Parity Technologies "] +description = "Substrate CLI interface." +build = "build.rs" + +[dependencies] +clap = { version = "~2.32", features = ["yaml"] } +backtrace = "0.3" +env_logger = "0.4" +error-chain = "0.12" +log = "0.3" +atty = "0.2" +regex = "1" +time = "0.1" +slog = "^2" +ansi_term = "0.10" +lazy_static = "1.0" +app_dirs = "1.2" +tokio = "0.1.7" +futures = "0.1.17" +fdlimit = "0.1" +exit-future = "0.1" +sysinfo = "0.5.7" +substrate-client = { path = "../client" } +substrate-network = { path = "../network" } +substrate-network-libp2p = { path = "../network-libp2p" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-service = { path = "../service" } +substrate-telemetry = { path = "../telemetry" } +names = "0.11.0" + +[build-dependencies] +clap = "~2.32" diff --git a/core/cli/README.adoc b/core/cli/README.adoc new file mode 100644 index 00000000000..2b9b74362d3 --- /dev/null +++ b/core/cli/README.adoc @@ -0,0 +1,13 @@ + += Substrate CLI + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +Polkadot CLI library + +include::doc/shell-completion.adoc[] diff --git a/core/cli/build.rs b/core/cli/build.rs new file mode 100644 index 00000000000..645e98d5e8a --- /dev/null +++ b/core/cli/build.rs @@ -0,0 +1,59 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[macro_use] +extern crate clap; + +use std::fs; +use std::env; +use clap::Shell; +use std::path::Path; + +fn main() { + build_shell_completion(); +} + +/// Build shell completion scripts for all known shells +/// Full list in https://github.com/kbknapp/clap-rs/blob/e9d0562a1dc5dfe731ed7c767e6cee0af08f0cf9/src/app/parser.rs#L123 +fn build_shell_completion() { + let shells = [Shell::Bash, Shell::Fish, Shell::Zsh, Shell::Elvish, Shell::PowerShell]; + for shell in shells.iter() { + build_completion(shell); + } +} + +/// Build the shell auto-completion for a given Shell +fn build_completion(shell: &Shell) { + let yml = load_yaml!("src/cli.yml"); + + let outdir = match env::var_os("OUT_DIR") { + None => return, + Some(dir) => dir, + }; + let path = Path::new(&outdir) + .parent().unwrap() + .parent().unwrap() + .parent().unwrap() + .join("completion-scripts"); + + fs::create_dir(&path).ok(); + + let mut app = clap::App::from_yaml(&yml); + app.gen_completions( + "polkadot", + *shell, + &path); +} diff --git a/core/cli/doc/shell-completion.adoc b/core/cli/doc/shell-completion.adoc new file mode 100644 index 00000000000..8afbd37adb9 --- /dev/null +++ b/core/cli/doc/shell-completion.adoc @@ -0,0 +1,41 @@ + +== Shell completion + +The Substrate cli command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system. + +Assuming you built a release version using `cargo build --release` and use `bash` run the following: + +[source, shell] +source target/release/completion-scripts/substrate.bash + +You can find completion scripts for: +- bash +- fish +- zsh +- elvish +- powershell + +To make this change persistent, you can proceed as follow: + +.First install + +[source, shell] +---- +COMPL_DIR=$HOME/.completion +mkdir -p $COMPL_DIR +cp -f target/release/completion-scripts/substrate.bash $COMPL_DIR/ +echo "source $COMPL_DIR/substrate.bash" >> $HOME/.bash_profile +source $HOME/.bash_profile +---- + +.Update + +When you build a new version of Substrate, the following will ensure you auto-completion script matches the current binary: + +[source, shell] +---- +COMPL_DIR=$HOME/.completion +mkdir -p $COMPL_DIR +cp -f target/release/completion-scripts/substrate.bash $COMPL_DIR/ +source $HOME/.bash_profile +---- diff --git a/core/cli/src/cli.yml b/core/cli/src/cli.yml new file mode 100644 index 00000000000..96c80e111c5 --- /dev/null +++ b/core/cli/src/cli.yml @@ -0,0 +1,229 @@ +name: {name} +author: {author} +about: {description} +args: + - log: + short: l + long: log + value_name: LOG_PATTERN + help: Sets a custom logging filter + takes_value: true + - base-path: + long: base-path + short: d + value_name: PATH + help: Specify custom base path + takes_value: true + - keystore-path: + long: keystore-path + value_name: PATH + help: Specify custom keystore path + takes_value: true + - key: + long: key + value_name: STRING + help: Specify additional key seed + takes_value: true + - node-key: + long: node-key + value_name: KEY + help: Specify node secret key (64-character hex string) + takes_value: true + - validator: + long: validator + help: Enable validator mode + takes_value: false + - light: + long: light + help: Run in light client mode + takes_value: false + - dev: + long: dev + help: Run in development mode; implies --chain=dev --validator --key Alice + takes_value: false + - listen-addr: + long: listen-addr + value_name: LISTEN_ADDR + help: Listen on this multiaddress + takes_value: true + multiple: true + - port: + long: port + value_name: PORT + help: Specify p2p protocol TCP port. Only used if --listen-addr is not specified. + takes_value: true + - rpc-external: + long: rpc-external + help: Listen to all RPC interfaces (default is local) + takes_value: false + - ws-external: + long: ws-external + help: Listen to all Websocket interfaces (default is local) + takes_value: false + - rpc-port: + long: rpc-port + value_name: PORT + help: Specify HTTP RPC server TCP port + takes_value: true + - ws-port: + long: ws-port + value_name: PORT + help: Specify WebSockets RPC server TCP port + takes_value: true + - bootnodes: + long: bootnodes + value_name: URL + help: Specify a list of bootnodes + takes_value: true + multiple: true + - reserved-nodes: + long: reserved-nodes + value_name: URL + help: Specify a list of reserved node addresses + takes_value: true + multiple: true + - min-peers: + long: min-peers + value_name: MIN_PEERS + help: Specify the minimum number of peers + takes_value: true + - max-peers: + long: max-peers + value_name: MAX_PEERS + help: Specify the maximum number of peers + takes_value: true + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification (one of krummelanke, dev, local or staging) + takes_value: true + - pruning: + long: pruning + value_name: PRUNING_MODE + help: Specify the pruning mode, a number of blocks to keep or "archive". Default is 256. + takes_value: true + - name: + long: name + value_name: NAME + help: The human-readable name for this node, as reported to the telemetry server, if enabled + takes_value: true + - telemetry: + short: t + long: telemetry + help: Should connect to the Polkadot telemetry server (telemetry is off by default on local chains) + takes_value: false + - no-telemetry: + long: no-telemetry + help: Should not connect to the Polkadot telemetry server (telemetry is on by default on global chains) + takes_value: false + - telemetry-url: + long: telemetry-url + value_name: TELEMETRY_URL + help: The URL of the telemetry server. Implies --telemetry + takes_value: true + - execution: + long: execution + value_name: STRATEGY + help: The means of execution used when calling into the runtime. Can be either wasm, native or both. +subcommands: + - build-spec: + about: Build a spec.json file, outputing to stdout + args: + - raw: + long: raw + help: Force raw genesis storage output. + takes_value: false + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification (one of krummelanke, dev, local or staging) + takes_value: true + - export-blocks: + about: Export blocks to a file + args: + - OUTPUT: + index: 1 + help: Output file name or stdout if unspecified. + required: false + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification. + takes_value: true + - base-path: + long: base-path + short: d + value_name: PATH + help: Specify custom base path. + takes_value: true + - from: + long: from + value_name: BLOCK + help: Specify starting block number. 1 by default. + takes_value: true + - to: + long: to + value_name: BLOCK + help: Specify last block number. Best block by default. + takes_value: true + - json: + long: json + help: Use JSON output rather than binary. + takes_value: false + - import-blocks: + about: Import blocks from file. + args: + - INPUT: + index: 1 + help: Input file or stdin if unspecified. + required: false + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification. + takes_value: true + - base-path: + long: base-path + short: d + value_name: PATH + help: Specify custom base path. + takes_value: true + - execution: + long: execution + value_name: STRATEGY + help: The means of execution used when calling into the runtime. Can be either wasm, native or both. + - max-heap-pages: + long: max-heap-pages + value_name: COUNT + help: The maximum number of 64KB pages to ever allocate for Wasm execution. Don't alter this unless you know what you're doing. + - revert: + about: Revert chain to the previous state + args: + - NUM: + index: 1 + help: Number of blocks to revert. Default is 256. + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification. + takes_value: true + - base-path: + long: base-path + short: d + value_name: PATH + help: Specify custom base path. + takes_value: true + - purge-chain: + about: Remove the whole chain data. + args: + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification. + takes_value: true + - base-path: + long: base-path + short: d + value_name: PATH + help: Specify custom base path. + takes_value: true diff --git a/core/cli/src/error.rs b/core/cli/src/error.rs new file mode 100644 index 00000000000..ec70a5b70b7 --- /dev/null +++ b/core/cli/src/error.rs @@ -0,0 +1,37 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Initialization errors. + +use client; + +error_chain! { + foreign_links { + Io(::std::io::Error) #[doc="IO error"]; + Cli(::clap::Error) #[doc="CLI error"]; + Service(::service::Error) #[doc="Substrate service error"]; + } + links { + Client(client::error::Error, client::error::ErrorKind) #[doc="Client error"]; + } + errors { + /// Input error. + Input(m: String) { + description("Invalid input"), + display("{}", m), + } + } +} diff --git a/core/cli/src/informant.rs b/core/cli/src/informant.rs new file mode 100644 index 00000000000..eacc95c50db --- /dev/null +++ b/core/cli/src/informant.rs @@ -0,0 +1,124 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Console informant. Prints sync progress and block events. Runs on the calling thread. + +use ansi_term::Colour; +use std::time::{Duration, Instant}; +use futures::{Future, Stream}; +use service::{Service, Components}; +use tokio::runtime::TaskExecutor; +use tokio::timer::Interval; +use sysinfo::{get_current_pid, ProcessExt, System, SystemExt}; +use network::{SyncState, SyncProvider}; +use client::BlockchainEvents; +use runtime_primitives::traits::{Header, As}; + +const TIMER_INTERVAL_MS: u64 = 5000; + +/// Spawn informant on the event loop +pub fn start(service: &Service, exit: ::exit_future::Exit, handle: TaskExecutor) + where + C: Components, +{ + let interval = Interval::new(Instant::now(), Duration::from_millis(TIMER_INTERVAL_MS)); + + let network = service.network(); + let client = service.client(); + let txpool = service.extrinsic_pool(); + let mut last_number = None; + + let mut sys = System::new(); + let self_pid = get_current_pid(); + + let display_notifications = interval.map_err(|e| debug!("Timer error: {:?}", e)).for_each(move |_| { + let sync_status = network.status(); + + if let Ok(best_block) = client.best_block_header() { + let hash = best_block.hash(); + let num_peers = sync_status.num_peers; + let best_number: u64 = best_block.number().as_(); + let speed = move || speed(best_number, last_number); + let (status, target) = match (sync_status.sync.state, sync_status.sync.best_seen_block) { + (SyncState::Idle, _) => ("Idle".into(), "".into()), + (SyncState::Downloading, None) => (format!("Syncing{}", speed()), "".into()), + (SyncState::Downloading, Some(n)) => (format!("Syncing{}", speed()), format!(", target=#{}", n)), + }; + last_number = Some(best_number); + let txpool_status = txpool.light_status(); + info!( + target: "substrate", + "{}{} ({} peers), best: #{} ({})", + Colour::White.bold().paint(&status), + target, + Colour::White.bold().paint(format!("{}", sync_status.num_peers)), + Colour::White.paint(format!("{}", best_number)), + hash + ); + + // get cpu usage and memory usage of this process + let (cpu_usage, memory) = if sys.refresh_process(self_pid) { + let proc = sys.get_process(self_pid).expect("Above refresh_process succeeds, this should be Some(), qed"); + (proc.cpu_usage(), proc.memory()) + } else { (0.0, 0) }; + + telemetry!( + "system.interval"; + "status" => format!("{}{}", status, target), + "peers" => num_peers, + "height" => best_number, + "best" => ?hash, + "txcount" => txpool_status.transaction_count, + "cpu" => cpu_usage, + "memory" => memory + ); + } else { + warn!("Error getting best block information"); + } + + Ok(()) + }); + + let client = service.client(); + let display_block_import = client.import_notification_stream().for_each(|n| { + info!(target: "substrate", "Imported #{} ({})", n.header.number(), n.hash); + Ok(()) + }); + + let txpool = service.extrinsic_pool(); + let display_txpool_import = txpool.import_notification_stream().for_each(move |_| { + let status = txpool.light_status(); + telemetry!("txpool.import"; "mem_usage" => status.mem_usage, "count" => status.transaction_count, "sender" => status.senders); + Ok(()) + }); + + let informant_work = display_notifications.join3(display_block_import, display_txpool_import); + handle.spawn(exit.until(informant_work).map(|_| ())); +} + +fn speed(best_number: u64, last_number: Option) -> String { + let speed = match last_number { + Some(num) => (best_number.saturating_sub(num) * 10_000 / TIMER_INTERVAL_MS) as f64, + None => 0.0 + }; + + if speed < 1.0 { + "".into() + } else { + format!(" {:4.1} bps", speed / 10.0) + } +} + diff --git a/core/cli/src/lib.rs b/core/cli/src/lib.rs new file mode 100644 index 00000000000..94d8849b05b --- /dev/null +++ b/core/cli/src/lib.rs @@ -0,0 +1,571 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Substrate CLI library. +// end::description[] + +#![warn(missing_docs)] +#![warn(unused_extern_crates)] + +extern crate app_dirs; +extern crate env_logger; +extern crate atty; +extern crate ansi_term; +extern crate regex; +extern crate time; +extern crate fdlimit; +extern crate futures; +extern crate tokio; +extern crate names; +extern crate backtrace; +extern crate sysinfo; + +extern crate substrate_client as client; +extern crate substrate_network as network; +extern crate substrate_network_libp2p as network_libp2p; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_service as service; +#[macro_use] +extern crate slog; // needed until we can reexport `slog_info` from `substrate_telemetry` +#[macro_use] +extern crate substrate_telemetry; +extern crate exit_future; + +#[macro_use] +extern crate lazy_static; +extern crate clap; +#[macro_use] +extern crate error_chain; +#[macro_use] +extern crate log; + +pub mod error; +pub mod informant; +mod panic_hook; + +use network_libp2p::AddrComponent; +use runtime_primitives::traits::As; +use service::{ + ServiceFactory, FactoryFullConfiguration, RuntimeGenesis, + FactoryGenesis, PruningMode, ChainSpec, +}; +use network::NonReservedPeerMode; + +use std::io::{Write, Read, stdin, stdout}; +use std::iter; +use std::fs; +use std::fs::File; +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use names::{Generator, Name}; +use regex::Regex; + +use futures::Future; + +/// Executable version. Used to pass version information from the root crate. +pub struct VersionInfo { + /// Implementation version. + pub version: &'static str, + /// SCM Commit hash. + pub commit: &'static str, + /// Executable file name. + pub executable_name: &'static str, + /// Executable file description. + pub description: &'static str, + /// Executable file author. + pub author: &'static str, +} + +/// CLI Action +pub enum Action { + /// Substrate handled the command. No need to do anything. + ExecutedInternally, + /// Service mode requested. Caller should start the service. + RunService((FactoryFullConfiguration, E)), +} + +/// Something that can be converted into an exit signal. +pub trait IntoExit { + /// Exit signal type. + type Exit: Future + Send + 'static; + /// Convert into exit signal. + fn into_exit(self) -> Self::Exit; +} + +fn load_spec(matches: &clap::ArgMatches, factory: F) -> Result, String> + where G: RuntimeGenesis, F: FnOnce(&str) -> Result>, String>, +{ + let chain_key = matches.value_of("chain").unwrap_or_else(|| if matches.is_present("dev") { "dev" } else { "" }); + let spec = match factory(chain_key)? { + Some(spec) => spec, + None => ChainSpec::from_json_file(PathBuf::from(chain_key))? + }; + Ok(spec) +} + +fn base_path(matches: &clap::ArgMatches) -> PathBuf { + matches.value_of("base-path") + .map(|x| Path::new(x).to_owned()) + .unwrap_or_else(default_base_path) +} + +/// Check whether a node name is considered as valid +fn is_node_name_valid(_name: &str) -> Result<(), &str> { + const MAX_NODE_NAME_LENGTH: usize = 32; + let name = _name.to_string(); + if name.chars().count() >= MAX_NODE_NAME_LENGTH { + return Err("Node name too long"); + } + + let invalid_chars = r"[\\.@]"; + let re = Regex::new(invalid_chars).unwrap(); + if re.is_match(&name) { + return Err("Node name should not contain invalid chars such as '.' and '@'"); + } + + let invalid_patterns = r"(https?:\\/+)?(www)+"; + let re = Regex::new(invalid_patterns).unwrap(); + if re.is_match(&name) { + return Err("Node name should not contain urls"); + } + + Ok(()) +} + +/// Parse command line arguments and execute commands or return service configuration. +/// +/// IANA unassigned port ranges that we could use: +/// 6717-6766 Unassigned +/// 8504-8553 Unassigned +/// 9556-9591 Unassigned +/// 9803-9874 Unassigned +/// 9926-9949 Unassigned +pub fn prepare_execution( + args: I, + exit: E, + version: VersionInfo, + spec_factory: S, + impl_name: &'static str, +) -> error::Result> +where + I: IntoIterator, + T: Into + Clone, + E: IntoExit, + F: ServiceFactory, + S: FnOnce(&str) -> Result>>, String>, +{ + panic_hook::set(); + + let full_version = service::config::full_version_from_strs( + version.version, + version.commit + ); + let yaml = format!(include_str!("./cli.yml"), + name = version.executable_name, + description = version.description, + author = version.author, + ); + let yaml = &clap::YamlLoader::load_from_str(&yaml).expect("Invalid yml file")[0]; + let matches = match clap::App::from_yaml(yaml) + .version(&(full_version + "\n")[..]) + .get_matches_from_safe(args) { + Ok(m) => m, + Err(e) => e.exit(), + }; + + // TODO [ToDr] Split parameters parsing from actual execution. + let log_pattern = matches.value_of("log").unwrap_or(""); + init_logger(log_pattern); + fdlimit::raise_fd_limit(); + + if let Some(matches) = matches.subcommand_matches("build-spec") { + let spec = load_spec(&matches, spec_factory)?; + build_spec::(matches, spec)?; + return Ok(Action::ExecutedInternally); + } + + if let Some(matches) = matches.subcommand_matches("export-blocks") { + let spec = load_spec(&matches, spec_factory)?; + export_blocks::(matches, spec, exit.into_exit())?; + return Ok(Action::ExecutedInternally); + } + + if let Some(matches) = matches.subcommand_matches("import-blocks") { + let spec = load_spec(&matches, spec_factory)?; + import_blocks::(matches, spec, exit.into_exit())?; + return Ok(Action::ExecutedInternally); + } + + if let Some(matches) = matches.subcommand_matches("revert") { + let spec = load_spec(&matches, spec_factory)?; + revert_chain::(matches, spec)?; + return Ok(Action::ExecutedInternally); + } + + if let Some(matches) = matches.subcommand_matches("purge-chain") { + let spec = load_spec(&matches, spec_factory)?; + purge_chain::(matches, spec)?; + return Ok(Action::ExecutedInternally); + } + + let spec = load_spec(&matches, spec_factory)?; + let mut config = service::Configuration::default_with_spec(spec); + + config.impl_name = impl_name; + config.impl_commit = version.commit; + config.impl_version = version.version; + + config.name = match matches.value_of("name") { + None => Generator::with_naming(Name::Numbered).next().unwrap(), + Some(name) => name.into(), + }; + match is_node_name_valid(&config.name) { + Ok(_) => (), + Err(msg) => return Err(error::ErrorKind::Input( + format!("Invalid node name '{}'. Reason: {}. If unsure, use none.", config.name, msg)).into()) + } + + let base_path = base_path(&matches); + + config.keystore_path = matches.value_of("keystore") + .map(|x| Path::new(x).to_owned()) + .unwrap_or_else(|| keystore_path(&base_path, config.chain_spec.id())) + .to_string_lossy() + .into(); + + config.database_path = db_path(&base_path, config.chain_spec.id()).to_string_lossy().into(); + + config.pruning = match matches.value_of("pruning") { + Some("archive") => PruningMode::ArchiveAll, + None => PruningMode::default(), + Some(s) => PruningMode::keep_blocks(s.parse() + .map_err(|_| error::ErrorKind::Input("Invalid pruning mode specified".to_owned()))?), + }; + + let role = + if matches.is_present("light") { + config.execution_strategy = service::ExecutionStrategy::NativeWhenPossible; + service::Roles::LIGHT + } else if matches.is_present("validator") || matches.is_present("dev") { + config.execution_strategy = service::ExecutionStrategy::Both; + service::Roles::AUTHORITY + } else { + config.execution_strategy = service::ExecutionStrategy::NativeWhenPossible; + service::Roles::FULL + }; + + if let Some(s) = matches.value_of("execution") { + config.execution_strategy = match s { + "both" => service::ExecutionStrategy::Both, + "native" => service::ExecutionStrategy::NativeWhenPossible, + "wasm" => service::ExecutionStrategy::AlwaysWasm, + _ => return Err(error::ErrorKind::Input("Invalid execution mode specified".to_owned()).into()), + }; + } + + config.roles = role; + { + config.network.boot_nodes.extend(matches + .values_of("bootnodes") + .map_or(Default::default(), |v| v.map(|n| n.to_owned()).collect::>())); + config.network.config_path = Some(network_path(&base_path, config.chain_spec.id()).to_string_lossy().into()); + config.network.net_config_path = config.network.config_path.clone(); + config.network.reserved_nodes.extend(matches + .values_of("reserved-nodes") + .map_or(Default::default(), |v| v.map(|n| n.to_owned()).collect::>())); + if !config.network.reserved_nodes.is_empty() { + config.network.non_reserved_mode = NonReservedPeerMode::Deny; + } + + config.network.listen_addresses = Vec::new(); + for addr in matches.values_of("listen-addr").unwrap_or_default() { + let addr = addr.parse().map_err(|_| "Invalid listen multiaddress")?; + config.network.listen_addresses.push(addr); + } + if config.network.listen_addresses.is_empty() { + let port = match matches.value_of("port") { + Some(port) => port.parse().map_err(|_| "Invalid p2p port value specified.")?, + None => 30333, + }; + config.network.listen_addresses = vec![ + iter::once(AddrComponent::IP4(Ipv4Addr::new(0, 0, 0, 0))) + .chain(iter::once(AddrComponent::TCP(port))) + .collect() + ]; + } + + config.network.public_addresses = Vec::new(); + + config.network.client_version = config.client_id(); + config.network.use_secret = match matches.value_of("node-key").map(|s| s.parse()) { + Some(Ok(secret)) => Some(secret), + Some(Err(err)) => return Err(format!("Error parsing node key: {}", err).into()), + None => None, + }; + + let min_peers = match matches.value_of("min-peers") { + Some(min_peers) => min_peers.parse().map_err(|_| "Invalid min-peers value specified.")?, + None => 25, + }; + let max_peers = match matches.value_of("max-peers") { + Some(max_peers) => max_peers.parse().map_err(|_| "Invalid max-peers value specified.")?, + None => 50, + }; + if min_peers > max_peers { + return Err(error::ErrorKind::Input("Min-peers mustn't be larger than max-peers.".to_owned()).into()); + } + config.network.min_peers = min_peers; + config.network.max_peers = max_peers; + } + + config.keys = matches.values_of("key").unwrap_or_default().map(str::to_owned).collect(); + if matches.is_present("dev") { + config.keys.push("Alice".into()); + } + + let rpc_interface: &str = if matches.is_present("rpc-external") { "0.0.0.0" } else { "127.0.0.1" }; + let ws_interface: &str = if matches.is_present("ws-external") { "0.0.0.0" } else { "127.0.0.1" }; + + config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), "rpc-port", &matches)?); + config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), "ws-port", &matches)?); + + // Override telemetry + if matches.is_present("no-telemetry") { + config.telemetry_url = None; + } else if let Some(url) = matches.value_of("telemetry-url") { + config.telemetry_url = Some(url.to_owned()); + } + + Ok(Action::RunService((config, exit))) +} + +fn build_spec(matches: &clap::ArgMatches, spec: ChainSpec>) -> error::Result<()> + where F: ServiceFactory, +{ + info!("Building chain spec"); + let raw = matches.is_present("raw"); + let json = service::chain_ops::build_spec::>(spec, raw)?; + print!("{}", json); + Ok(()) +} + +fn export_blocks(matches: &clap::ArgMatches, spec: ChainSpec>, exit: E) -> error::Result<()> + where F: ServiceFactory, E: Future + Send + 'static, +{ + let base_path = base_path(matches); + let mut config = service::Configuration::default_with_spec(spec); + config.database_path = db_path(&base_path, config.chain_spec.id()).to_string_lossy().into(); + info!("DB path: {}", config.database_path); + let from: u64 = match matches.value_of("from") { + Some(v) => v.parse().map_err(|_| "Invalid --from argument")?, + None => 1, + }; + + let to: Option = match matches.value_of("to") { + Some(v) => Some(v.parse().map_err(|_| "Invalid --to argument")?), + None => None, + }; + let json = matches.is_present("json"); + + let file: Box = match matches.value_of("OUTPUT") { + Some(filename) => Box::new(File::create(filename)?), + None => Box::new(stdout()), + }; + + Ok(service::chain_ops::export_blocks::(config, exit, file, As::sa(from), to.map(As::sa), json)?) +} + +fn import_blocks(matches: &clap::ArgMatches, spec: ChainSpec>, exit: E) -> error::Result<()> + where F: ServiceFactory, E: Future + Send + 'static, +{ + let base_path = base_path(matches); + let mut config = service::Configuration::default_with_spec(spec); + config.database_path = db_path(&base_path, config.chain_spec.id()).to_string_lossy().into(); + + if let Some(s) = matches.value_of("execution") { + config.execution_strategy = match s { + "both" => service::ExecutionStrategy::Both, + "native" => service::ExecutionStrategy::NativeWhenPossible, + "wasm" => service::ExecutionStrategy::AlwaysWasm, + _ => return Err(error::ErrorKind::Input("Invalid execution mode specified".to_owned()).into()), + }; + } + + let file: Box = match matches.value_of("INPUT") { + Some(filename) => Box::new(File::open(filename)?), + None => Box::new(stdin()), + }; + + Ok(service::chain_ops::import_blocks::(config, exit, file)?) +} + +fn revert_chain(matches: &clap::ArgMatches, spec: ChainSpec>) -> error::Result<()> + where F: ServiceFactory, +{ + let base_path = base_path(matches); + let mut config = service::Configuration::default_with_spec(spec); + config.database_path = db_path(&base_path, config.chain_spec.id()).to_string_lossy().into(); + + let blocks = match matches.value_of("NUM") { + Some(v) => v.parse().map_err(|_| "Invalid block count specified")?, + None => 256, + }; + + Ok(service::chain_ops::revert_chain::(config, As::sa(blocks))?) +} + +fn purge_chain(matches: &clap::ArgMatches, spec: ChainSpec>) -> error::Result<()> + where F: ServiceFactory, +{ + let base_path = base_path(matches); + let database_path = db_path(&base_path, spec.id()); + + print!("Are you sure to remove {:?}? (y/n)", &database_path); + stdout().flush().expect("failed to flush stdout"); + + let mut input = String::new(); + stdin().read_line(&mut input)?; + let input = input.trim(); + + match input.chars().nth(0) { + Some('y') | Some('Y') => { + fs::remove_dir_all(&database_path)?; + println!("{:?} removed.", &database_path); + }, + _ => println!("Aborted"), + } + + Ok(()) +} + +fn parse_address(default: &str, port_param: &str, matches: &clap::ArgMatches) -> Result { + let mut address: SocketAddr = default.parse().ok().ok_or_else(|| format!("Invalid address specified for --{}.", port_param))?; + if let Some(port) = matches.value_of(port_param) { + let port: u16 = port.parse().ok().ok_or_else(|| format!("Invalid port for --{} specified.", port_param))?; + address.set_port(port); + } + + Ok(address) +} + +fn keystore_path(base_path: &Path, chain_id: &str) -> PathBuf { + let mut path = base_path.to_owned(); + path.push("chains"); + path.push(chain_id); + path.push("keystore"); + path +} + +fn db_path(base_path: &Path, chain_id: &str) -> PathBuf { + let mut path = base_path.to_owned(); + path.push("chains"); + path.push(chain_id); + path.push("db"); + path +} + +fn network_path(base_path: &Path, chain_id: &str) -> PathBuf { + let mut path = base_path.to_owned(); + path.push("chains"); + path.push(chain_id); + path.push("network"); + path +} + +fn default_base_path() -> PathBuf { + use app_dirs::{AppInfo, AppDataType}; + + let app_info = AppInfo { + name: "Polkadot", + author: "Parity Technologies", + }; + + app_dirs::get_app_root( + AppDataType::UserData, + &app_info, + ).expect("app directories exist on all supported platforms; qed") +} + +fn init_logger(pattern: &str) { + use ansi_term::Colour; + + let mut builder = env_logger::LogBuilder::new(); + // Disable info logging by default for some modules: + builder.filter(Some("ws"), log::LogLevelFilter::Warn); + builder.filter(Some("hyper"), log::LogLevelFilter::Warn); + // Enable info for others. + builder.filter(None, log::LogLevelFilter::Info); + + if let Ok(lvl) = std::env::var("RUST_LOG") { + builder.parse(&lvl); + } + + builder.parse(pattern); + let isatty = atty::is(atty::Stream::Stderr); + let enable_color = isatty; + + let format = move |record: &log::LogRecord| { + let timestamp = time::strftime("%Y-%m-%d %H:%M:%S", &time::now()).expect("Error formatting log timestamp"); + + let mut output = if log::max_log_level() <= log::LogLevelFilter::Info { + format!("{} {}", Colour::Black.bold().paint(timestamp), record.args()) + } else { + let name = ::std::thread::current().name().map_or_else(Default::default, |x| format!("{}", Colour::Blue.bold().paint(x))); + format!("{} {} {} {} {}", Colour::Black.bold().paint(timestamp), name, record.level(), record.target(), record.args()) + }; + + if !enable_color { + output = kill_color(output.as_ref()); + } + + if !isatty && record.level() <= log::LogLevel::Info && atty::is(atty::Stream::Stdout) { + // duplicate INFO/WARN output to console + println!("{}", output); + } + output + }; + builder.format(format); + + builder.init().expect("Logger initialized only once."); +} + +fn kill_color(s: &str) -> String { + lazy_static! { + static ref RE: Regex = Regex::new("\x1b\\[[^m]+m").expect("Error initializing color regex"); + } + RE.replace_all(s, "").to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tests_node_name_good() { + assert!(is_node_name_valid("short name").is_ok()); + } + + #[test] + fn tests_node_name_bad() { + assert!(is_node_name_valid("long names are not very cool for the ui").is_err()); + assert!(is_node_name_valid("Dots.not.Ok").is_err()); + assert!(is_node_name_valid("http://visit.me").is_err()); + assert!(is_node_name_valid("https://visit.me").is_err()); + assert!(is_node_name_valid("www.visit.me").is_err()); + assert!(is_node_name_valid("email@domain").is_err()); + } +} diff --git a/core/cli/src/panic_hook.rs b/core/cli/src/panic_hook.rs new file mode 100644 index 00000000000..c7cea5c8537 --- /dev/null +++ b/core/cli/src/panic_hook.rs @@ -0,0 +1,69 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Custom panic hook with bug report link + +use backtrace::Backtrace; +use std::io::{self, Write}; +use std::panic::{self, PanicInfo}; +use std::thread; + +/// Set the panic hook +pub fn set() { + panic::set_hook(Box::new(panic_hook)); +} + +static ABOUT_PANIC: &str = " +This is a bug. Please report it at: + + https://github.com/paritytech/polkadot/issues/new +"; + +fn panic_hook(info: &PanicInfo) { + let location = info.location(); + let file = location.as_ref().map(|l| l.file()).unwrap_or(""); + let line = location.as_ref().map(|l| l.line()).unwrap_or(0); + + let msg = match info.payload().downcast_ref::<&'static str>() { + Some(s) => *s, + None => match info.payload().downcast_ref::() { + Some(s) => &s[..], + None => "Box", + } + }; + + let thread = thread::current(); + let name = thread.name().unwrap_or(""); + + let backtrace = Backtrace::new(); + + let mut stderr = io::stderr(); + + let _ = writeln!(stderr, ""); + let _ = writeln!(stderr, "===================="); + let _ = writeln!(stderr, ""); + let _ = writeln!(stderr, "{:?}", backtrace); + let _ = writeln!(stderr, ""); + let _ = writeln!( + stderr, + "Thread '{}' panicked at '{}', {}:{}", + name, msg, file, line + ); + + let _ = writeln!(stderr, "{}", ABOUT_PANIC); + ::std::process::exit(1); +} + diff --git a/core/client/Cargo.toml b/core/client/Cargo.toml new file mode 100644 index 00000000000..886410f74ee --- /dev/null +++ b/core/client/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-client" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +error-chain = "0.12" +fnv = "1.0" +log = "0.3" +parking_lot = "0.4" +triehash = "0.2" +hex-literal = "0.1" +futures = "0.1.17" + +slog = "^2" +heapsize = "0.4" +substrate-bft = { path = "../bft" } +substrate-codec = { path = "../codec" } +substrate-executor = { path = "../executor" } +substrate-primitives = { path = "../primitives" } +substrate-runtime-io = { path = "../runtime-io" } +substrate-runtime-support = { path = "../../runtime/support" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-state-machine = { path = "../state-machine" } +substrate-keyring = { path = "../keyring" } +substrate-telemetry = { path = "../telemetry" } +hashdb = "0.2.1" +patricia-trie = "0.2.1" +rlp = "0.2.4" + +[dev-dependencies] +substrate-test-client = { path = "../test-client" } diff --git a/core/client/README.adoc b/core/client/README.adoc new file mode 100644 index 00000000000..d644b1d039b --- /dev/null +++ b/core/client/README.adoc @@ -0,0 +1,13 @@ + += Client + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/client/db/Cargo.toml b/core/client/db/Cargo.toml new file mode 100644 index 00000000000..b0a6071d210 --- /dev/null +++ b/core/client/db/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "substrate-client-db" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +parking_lot = "0.4" +log = "0.3" +kvdb = "0.1" +kvdb-rocksdb = "0.1.3" +hashdb = "0.2.1" +memorydb = "0.2.1" +substrate-primitives = { path = "../../primitives" } +substrate-client = { path = "../../client" } +substrate-state-machine = { path = "../../state-machine" } +substrate-codec = { path = "../../codec" } +substrate-codec-derive = { path = "../../codec/derive" } +substrate-executor = { path = "../../executor" } +substrate-state-db = { path = "../../state-db" } +substrate-runtime-support = { path = "../../../runtime/support" } +substrate-runtime-primitives = { path = "../../../runtime/primitives" } + +[dev-dependencies] +kvdb-memorydb = "0.1" diff --git a/core/client/db/src/cache.rs b/core/client/db/src/cache.rs new file mode 100644 index 00000000000..194e307e069 --- /dev/null +++ b/core/client/db/src/cache.rs @@ -0,0 +1,432 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! DB-backed cache of blockchain data. + +use std::sync::Arc; +use parking_lot::RwLock; + +use kvdb::{KeyValueDB, DBTransaction}; + +use client::blockchain::Cache as BlockchainCache; +use client::error::Result as ClientResult; +use codec::{Codec, Encode, Decode}; +use primitives::AuthorityId; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{Block as BlockT, As, NumberFor}; +use utils::{COLUMN_META, BlockKey, db_err, meta_keys, read_id, db_key_to_number, number_to_db_key}; + +/// Database-backed cache of blockchain data. +pub struct DbCache { + db: Arc, + block_index_column: Option, + authorities_at: DbCacheList>, +} + +impl DbCache + where + Block: BlockT, + NumberFor: As, +{ + /// Create new cache. + pub fn new( + db: Arc, + block_index_column: Option, + authorities_column: Option + ) -> ClientResult { + Ok(DbCache { + db: db.clone(), + block_index_column, + authorities_at: DbCacheList::new(db, meta_keys::BEST_AUTHORITIES, authorities_column)?, + }) + } + + /// Get authorities_cache. + pub fn authorities_at_cache(&self) -> &DbCacheList> { + &self.authorities_at + } +} + +impl BlockchainCache for DbCache + where + Block: BlockT, + NumberFor: As, +{ + fn authorities_at(&self, at: BlockId) -> Option> { + let authorities_at = read_id(&*self.db, self.block_index_column, at).and_then(|at| match at { + Some(at) => self.authorities_at.value_at_key(at), + None => Ok(None), + }); + + match authorities_at { + Ok(authorities) => authorities, + Err(error) => { + warn!("Trying to read authorities from db cache has failed with: {}", error); + None + }, + } + } +} + +/// Database-backed blockchain cache which holds its entries as a list. +/// The meta column holds the pointer to the best known cache entry and +/// every entry points to the previous entry. +/// New entry appears when the set of authorities changes in block, so the +/// best entry here means the entry that is valid for the best block (and +/// probably for its ascendants). +pub struct DbCacheList { + db: Arc, + meta_key: &'static [u8], + column: Option, + /// Best entry at the moment. None means that cache has no entries at all. + best_entry: RwLock, T>>>, +} + +/// Single cache entry. +#[derive(Clone)] +#[cfg_attr(test, derive(Debug, PartialEq))] +pub struct Entry { + /// first block, when this value became actual + valid_from: N, + /// None means that we do not know the value starting from `valid_from` block + value: Option, +} + +/// Internal representation of the single cache entry. The entry points to the +/// previous entry in the cache, allowing us to traverse back in time in list-style. +#[derive(Encode, Decode)] +#[cfg_attr(test, derive(Debug, PartialEq))] +struct StorageEntry { + /// None if valid from the beginning + prev_valid_from: Option, + /// None means that we do not know the value starting from `valid_from` block + value: Option, +} + +impl DbCacheList + where + Block: BlockT, + NumberFor: As, + T: Clone + PartialEq + Codec, +{ + /// Creates new cache list. + fn new(db: Arc, meta_key: &'static [u8], column: Option) -> ClientResult { + let best_entry = RwLock::new(db.get(COLUMN_META, meta_key) + .map_err(db_err) + .and_then(|block| match block { + Some(block) => { + let valid_from = db_key_to_number(&block)?; + read_storage_entry::(&*db, column, valid_from) + .map(|entry| Some(Entry { + valid_from, + value: entry + .expect("meta entry references the entry at the block; storage entry at block exists when referenced; qed") + .value, + })) + }, + None => Ok(None), + })?); + + Ok(DbCacheList { + db, + column, + meta_key, + best_entry, + }) + } + + /// Gets the best known entry. + pub fn best_entry(&self) -> Option, T>> { + self.best_entry.read().clone() + } + + /// Commits the new best pending value to the database. Returns Some if best entry must + /// be updated after transaction is committed. + pub fn commit_best_entry( + &self, + transaction: &mut DBTransaction, + valid_from: NumberFor, + pending_value: Option + ) -> Option, T>> { + let best_entry = self.best_entry(); + let update_best_entry = match ( + best_entry.as_ref().and_then(|a| a.value.as_ref()), + pending_value.as_ref() + ) { + (Some(best_value), Some(pending_value)) => best_value != pending_value, + (None, Some(_)) | (Some(_), None) => true, + (None, None) => false, + }; + if !update_best_entry { + return None; + } + + let valid_from_key = number_to_db_key(valid_from); + transaction.put(COLUMN_META, self.meta_key, &valid_from_key); + transaction.put(self.column, &valid_from_key, &StorageEntry { + prev_valid_from: best_entry.map(|b| b.valid_from), + value: pending_value.clone(), + }.encode()); + + Some(Entry { + valid_from, + value: pending_value, + }) + } + + /// Updates the best in-memory cache entry. Must be called after transaction with changes + /// from commit_best_entry has been committed. + pub fn update_best_entry(&self, best_entry: Option, T>>) { + *self.best_entry.write() = best_entry; + } + + /// Prune all entries from the beginning up to the block (including entry at the number). Returns + /// the number of pruned entries. Pruning never deletes the latest entry in the cache. + pub fn prune_entries( + &self, + transaction: &mut DBTransaction, + last_to_prune: NumberFor + ) -> ClientResult { + // find the last entry we want to keep + let mut last_entry_to_keep = match self.best_entry() { + Some(best_entry) => best_entry.valid_from, + None => return Ok(0), + }; + let mut first_entry_to_remove = last_entry_to_keep; + while first_entry_to_remove > last_to_prune { + last_entry_to_keep = first_entry_to_remove; + + let entry = read_storage_entry::(&*self.db, self.column, first_entry_to_remove)? + .expect("entry referenced from the next entry; entry exists when referenced; qed"); + // if we have reached the first list entry + // AND all list entries are for blocks that are later than last_to_prune + // => nothing to prune + first_entry_to_remove = match entry.prev_valid_from { + Some(prev_valid_from) => prev_valid_from, + None => return Ok(0), + } + } + + // remove all entries, starting from entry_to_remove + let mut pruned = 0; + let mut entry_to_remove = Some(first_entry_to_remove); + while let Some(current_entry) = entry_to_remove { + let entry = read_storage_entry::(&*self.db, self.column, current_entry)? + .expect("referenced entry exists; entry_to_remove is a reference to the entry; qed"); + + if current_entry != last_entry_to_keep { + transaction.delete(self.column, &number_to_db_key(current_entry)); + pruned += 1; + } + entry_to_remove = entry.prev_valid_from; + } + + let mut entry = read_storage_entry::(&*self.db, self.column, last_entry_to_keep)? + .expect("last_entry_to_keep >= first_entry_to_remove; that means that we're leaving this entry in the db; qed"); + entry.prev_valid_from = None; + transaction.put(self.column, &number_to_db_key(last_entry_to_keep), &entry.encode()); + + Ok(pruned) + } + + /// Reads the cached value, actual at given block. Returns None if the value was not cached + /// or if it has been pruned. + fn value_at_key(&self, key: BlockKey) -> ClientResult> { + let at = db_key_to_number::>(&key)?; + let best_valid_from = match self.best_entry() { + // there are entries in cache + Some(best_entry) => { + // we're looking for the best value + if at >= best_entry.valid_from { + return Ok(best_entry.value); + } + + // we're looking for the value of older blocks + best_entry.valid_from + }, + // there are no entries in the cache + None => return Ok(None), + }; + + let mut entry = read_storage_entry::(&*self.db, self.column, best_valid_from)? + .expect("self.best_entry().is_some() if there's entry for best_valid_from; qed"); + loop { + let prev_valid_from = match entry.prev_valid_from { + Some(prev_valid_from) => prev_valid_from, + None => return Ok(None), + }; + + let prev_entry = read_storage_entry::(&*self.db, self.column, prev_valid_from)? + .expect("entry referenced from the next entry; entry exists when referenced; qed"); + if at >= prev_valid_from { + return Ok(prev_entry.value); + } + + entry = prev_entry; + } + } +} + +/// Reads the entry at the block with given number. +fn read_storage_entry( + db: &KeyValueDB, + column: Option, + number: NumberFor +) -> ClientResult, T>>> + where + Block: BlockT, + NumberFor: As, + T: Codec, +{ + db.get(column, &number_to_db_key(number)) + .and_then(|entry| match entry { + Some(entry) => Ok(StorageEntry::, T>::decode(&mut &entry[..])), + None => Ok(None), + }) + .map_err(db_err) +} + +#[cfg(test)] +mod tests { + use runtime_primitives::testing::Block as RawBlock; + use light::{AUTHORITIES_ENTRIES_TO_KEEP, columns, LightStorage}; + use light::tests::insert_block; + use super::*; + + type Block = RawBlock; + + #[test] + fn authorities_storage_entry_serialized() { + let test_cases: Vec>> = vec![ + StorageEntry { prev_valid_from: Some(42), value: Some(vec![[1u8; 32].into()]) }, + StorageEntry { prev_valid_from: None, value: Some(vec![[1u8; 32].into(), [2u8; 32].into()]) }, + StorageEntry { prev_valid_from: None, value: None }, + ]; + + for expected in test_cases { + let serialized = expected.encode(); + let deserialized = StorageEntry::decode(&mut &serialized[..]).unwrap(); + assert_eq!(expected, deserialized); + } + } + + #[test] + fn best_authorities_are_updated() { + let db = LightStorage::new_test(); + let authorities_at: Vec<(usize, Option>>)> = vec![ + (0, None), + (0, None), + (1, Some(Entry { valid_from: 1, value: Some(vec![[2u8; 32].into()]) })), + (1, Some(Entry { valid_from: 1, value: Some(vec![[2u8; 32].into()]) })), + (2, Some(Entry { valid_from: 3, value: Some(vec![[4u8; 32].into()]) })), + (2, Some(Entry { valid_from: 3, value: Some(vec![[4u8; 32].into()]) })), + (3, Some(Entry { valid_from: 5, value: None })), + (3, Some(Entry { valid_from: 5, value: None })), + ]; + + // before any block, there are no entries in cache + assert!(db.cache().authorities_at_cache().best_entry().is_none()); + assert_eq!(db.db().iter(columns::AUTHORITIES).count(), 0); + + // insert blocks and check that best_authorities() returns correct result + let mut prev_hash = Default::default(); + for number in 0..authorities_at.len() { + let authorities_at_number = authorities_at[number].1.clone().and_then(|e| e.value); + prev_hash = insert_block(&db, &prev_hash, number as u64, authorities_at_number); + assert_eq!(db.cache().authorities_at_cache().best_entry(), authorities_at[number].1); + assert_eq!(db.db().iter(columns::AUTHORITIES).count(), authorities_at[number].0); + } + + // check that authorities_at() returns correct results for all retrospective blocks + for number in 1..authorities_at.len() + 1 { + assert_eq!(db.cache().authorities_at(BlockId::Number(number as u64)), + authorities_at.get(number + 1) + .or_else(|| authorities_at.last()) + .unwrap().1.clone().and_then(|e| e.value)); + } + + // now check that cache entries are pruned when new blocks are inserted + let mut current_entries_count = authorities_at.last().unwrap().0; + let pruning_starts_at = AUTHORITIES_ENTRIES_TO_KEEP as usize; + for number in authorities_at.len()..authorities_at.len() + pruning_starts_at { + prev_hash = insert_block(&db, &prev_hash, number as u64, None); + if number > pruning_starts_at { + let prev_entries_count = authorities_at[number - pruning_starts_at].0; + let entries_count = authorities_at.get(number - pruning_starts_at + 1).map(|e| e.0) + .unwrap_or_else(|| authorities_at.last().unwrap().0); + current_entries_count -= entries_count - prev_entries_count; + } + + // there's always at least 1 entry in the cache (after first insertion) + assert_eq!(db.db().iter(columns::AUTHORITIES).count(), ::std::cmp::max(current_entries_count, 1)); + } + } + + #[test] + fn best_authorities_are_pruned() { + let db = LightStorage::::new_test(); + let mut transaction = DBTransaction::new(); + + // insert first entry at block#100 + db.cache().authorities_at_cache().update_best_entry( + db.cache().authorities_at_cache().commit_best_entry(&mut transaction, 100, Some(vec![[1u8; 32].into()]))); + db.db().write(transaction).unwrap(); + + // no entries are pruned, since there's only one entry in the cache + let mut transaction = DBTransaction::new(); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 50).unwrap(), 0); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 100).unwrap(), 0); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 150).unwrap(), 0); + + // insert second entry at block#200 + let mut transaction = DBTransaction::new(); + db.cache().authorities_at_cache().update_best_entry( + db.cache().authorities_at_cache().commit_best_entry(&mut transaction, 200, Some(vec![[2u8; 32].into()]))); + db.db().write(transaction).unwrap(); + + let mut transaction = DBTransaction::new(); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 50).unwrap(), 0); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 100).unwrap(), 1); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 150).unwrap(), 1); + // still only 1 entry is removed since pruning never deletes the last entry + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 200).unwrap(), 1); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 250).unwrap(), 1); + + // physically remove entry for block#100 from db + let mut transaction = DBTransaction::new(); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 150).unwrap(), 1); + db.db().write(transaction).unwrap(); + + assert_eq!(db.cache().authorities_at_cache().best_entry().unwrap().value, Some(vec![[2u8; 32].into()])); + assert_eq!(db.cache().authorities_at(BlockId::Number(50)), None); + assert_eq!(db.cache().authorities_at(BlockId::Number(100)), None); + assert_eq!(db.cache().authorities_at(BlockId::Number(150)), None); + assert_eq!(db.cache().authorities_at(BlockId::Number(200)), Some(vec![[2u8; 32].into()])); + assert_eq!(db.cache().authorities_at(BlockId::Number(250)), Some(vec![[2u8; 32].into()])); + + // try to delete last entry => failure (no entries are removed) + let mut transaction = DBTransaction::new(); + assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 300).unwrap(), 0); + db.db().write(transaction).unwrap(); + + assert_eq!(db.cache().authorities_at_cache().best_entry().unwrap().value, Some(vec![[2u8; 32].into()])); + assert_eq!(db.cache().authorities_at(BlockId::Number(50)), None); + assert_eq!(db.cache().authorities_at(BlockId::Number(100)), None); + assert_eq!(db.cache().authorities_at(BlockId::Number(150)), None); + assert_eq!(db.cache().authorities_at(BlockId::Number(200)), Some(vec![[2u8; 32].into()])); + assert_eq!(db.cache().authorities_at(BlockId::Number(250)), Some(vec![[2u8; 32].into()])); + } +} diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs new file mode 100644 index 00000000000..acbd935ce8c --- /dev/null +++ b/core/client/db/src/lib.rs @@ -0,0 +1,715 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Client backend that uses RocksDB database as storage. +// end::description[] + +extern crate substrate_client as client; +extern crate kvdb_rocksdb; +extern crate kvdb; +extern crate hashdb; +extern crate memorydb; +extern crate parking_lot; +extern crate substrate_state_machine as state_machine; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_support as runtime_support; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_codec as codec; +extern crate substrate_executor as executor; +extern crate substrate_state_db as state_db; + +#[macro_use] +extern crate log; + +#[macro_use] +extern crate substrate_codec_derive; + +#[cfg(test)] +extern crate kvdb_memorydb; + +pub mod light; + +mod cache; +mod utils; + +use std::sync::Arc; +use std::path::PathBuf; +use std::io; + +use codec::{Decode, Encode}; +use hashdb::Hasher; +use kvdb::{KeyValueDB, DBTransaction}; +use memorydb::MemoryDB; +use parking_lot::RwLock; +use primitives::{H256, AuthorityId, Blake2Hasher, RlpCodec}; +use runtime_primitives::generic::BlockId; +use runtime_primitives::bft::Justification; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, Hash, HashFor, NumberFor, Zero}; +use runtime_primitives::BuildStorage; +use state_machine::backend::Backend as StateBackend; +use executor::RuntimeInfo; +use state_machine::{CodeExecutor, DBValue, ExecutionStrategy}; +use utils::{Meta, db_err, meta_keys, number_to_db_key, db_key_to_number, open_database, + read_db, read_id, read_meta}; +use state_db::StateDb; +pub use state_db::PruningMode; + +const FINALIZATION_WINDOW: u64 = 32; + +/// DB-backed patricia trie state, transaction type is an overlay of changes to commit. +pub type DbState = state_machine::TrieBackend; + +/// Database settings. +pub struct DatabaseSettings { + /// Cache size in bytes. If `None` default is used. + pub cache_size: Option, + /// Path to the database. + pub path: PathBuf, + /// Pruning mode. + pub pruning: PruningMode, +} + +/// Create an instance of db-backed client. +pub fn new_client( + settings: DatabaseSettings, + executor: E, + genesis_storage: S, + execution_strategy: ExecutionStrategy, +) -> Result, client::LocalCallExecutor, E>, Block>, client::error::Error> + where + Block: BlockT, + E: CodeExecutor + RuntimeInfo, + S: BuildStorage, +{ + let backend = Arc::new(Backend::new(settings, FINALIZATION_WINDOW)?); + let executor = client::LocalCallExecutor::new(backend.clone(), executor); + Ok(client::Client::new(backend, executor, genesis_storage, execution_strategy)?) +} + +mod columns { + pub const META: Option = Some(0); + pub const STATE: Option = Some(1); + pub const STATE_META: Option = Some(2); + pub const BLOCK_INDEX: Option = Some(3); + pub const HEADER: Option = Some(4); + pub const BODY: Option = Some(5); + pub const JUSTIFICATION: Option = Some(6); +} + +struct PendingBlock { + header: Block::Header, + justification: Option>, + body: Option>, + is_best: bool, +} + +// wrapper that implements trait required for state_db +struct StateMetaDb<'a>(&'a KeyValueDB); + +impl<'a> state_db::MetaDb for StateMetaDb<'a> { + type Error = io::Error; + + fn get_meta(&self, key: &[u8]) -> Result>, Self::Error> { + self.0.get(columns::STATE_META, key).map(|r| r.map(|v| v.to_vec())) + } +} + +/// Block database +pub struct BlockchainDb { + db: Arc, + meta: RwLock::Number, Block::Hash>>, +} + +impl BlockchainDb { + fn new(db: Arc) -> Result { + let meta = read_meta::(&*db, columns::HEADER)?; + Ok(BlockchainDb { + db, + meta: RwLock::new(meta) + }) + } + + fn update_meta(&self, hash: Block::Hash, number: ::Number, is_best: bool) { + if is_best { + let mut meta = self.meta.write(); + if number == Zero::zero() { + meta.genesis_hash = hash; + } + meta.best_number = number; + meta.best_hash = hash; + } + } +} + +impl client::blockchain::HeaderBackend for BlockchainDb { + fn header(&self, id: BlockId) -> Result, client::error::Error> { + match read_db(&*self.db, columns::BLOCK_INDEX, columns::HEADER, id)? { + Some(header) => match Block::Header::decode(&mut &header[..]) { + Some(header) => Ok(Some(header)), + None => return Err(client::error::ErrorKind::Backend("Error decoding header".into()).into()), + } + None => Ok(None), + } + } + + fn info(&self) -> Result, client::error::Error> { + let meta = self.meta.read(); + Ok(client::blockchain::Info { + best_hash: meta.best_hash, + best_number: meta.best_number, + genesis_hash: meta.genesis_hash, + }) + } + + fn status(&self, id: BlockId) -> Result { + let exists = match id { + BlockId::Hash(_) => read_id(&*self.db, columns::BLOCK_INDEX, id)?.is_some(), + BlockId::Number(n) => n <= self.meta.read().best_number, + }; + match exists { + true => Ok(client::blockchain::BlockStatus::InChain), + false => Ok(client::blockchain::BlockStatus::Unknown), + } + } + + fn number(&self, hash: Block::Hash) -> Result::Number>, client::error::Error> { + read_id::(&*self.db, columns::BLOCK_INDEX, BlockId::Hash(hash)) + .and_then(|key| match key { + Some(key) => Ok(Some(db_key_to_number(&key)?)), + None => Ok(None), + }) + } + + fn hash(&self, number: ::Number) -> Result, client::error::Error> { + read_db::(&*self.db, columns::BLOCK_INDEX, columns::HEADER, BlockId::Number(number)).map(|x| + x.map(|raw| HashFor::::hash(&raw[..])).map(Into::into) + ) + } +} + +impl client::blockchain::Backend for BlockchainDb { + fn body(&self, id: BlockId) -> Result>, client::error::Error> { + match read_db(&*self.db, columns::BLOCK_INDEX, columns::BODY, id)? { + Some(body) => match Decode::decode(&mut &body[..]) { + Some(body) => Ok(Some(body)), + None => return Err(client::error::ErrorKind::Backend("Error decoding body".into()).into()), + } + None => Ok(None), + } + } + + fn justification(&self, id: BlockId) -> Result>, client::error::Error> { + match read_db(&*self.db, columns::BLOCK_INDEX, columns::JUSTIFICATION, id)? { + Some(justification) => match Decode::decode(&mut &justification[..]) { + Some(justification) => Ok(Some(justification)), + None => return Err(client::error::ErrorKind::Backend("Error decoding justification".into()).into()), + } + None => Ok(None), + } + } + + fn cache(&self) -> Option<&client::blockchain::Cache> { + None + } +} + +/// Database transaction +pub struct BlockImportOperation { + old_state: DbState, + updates: MemoryDB, + pending_block: Option>, +} + +impl client::backend::BlockImportOperation +for BlockImportOperation +where Block: BlockT, +{ + type State = DbState; + + fn state(&self) -> Result, client::error::Error> { + Ok(Some(&self.old_state)) + } + + fn set_block_data(&mut self, header: Block::Header, body: Option>, justification: Option>, is_best: bool) -> Result<(), client::error::Error> { + assert!(self.pending_block.is_none(), "Only one block per operation is allowed"); + self.pending_block = Some(PendingBlock { + header, + body, + justification, + is_best, + }); + Ok(()) + } + + fn update_authorities(&mut self, _authorities: Vec) { + // currently authorities are not cached on full nodes + } + + fn update_storage(&mut self, update: MemoryDB) -> Result<(), client::error::Error> { + self.updates = update; + Ok(()) + } + + fn reset_storage, Vec)>>(&mut self, iter: I) -> Result<(), client::error::Error> { + // TODO: wipe out existing trie. + let (_, update) = self.old_state.storage_root(iter.into_iter().map(|(k, v)| (k, Some(v)))); + self.updates = update; + Ok(()) + } +} + +struct StorageDb { + pub db: Arc, + pub state_db: StateDb, +} + +impl state_machine::Storage for StorageDb { + fn get(&self, key: &H256) -> Result, String> { + self.state_db.get(&key.0.into(), self).map(|r| r.map(|v| DBValue::from_slice(&v))) + .map_err(|e| format!("Database backend error: {:?}", e)) + } +} + +impl state_db::HashDb for StorageDb { + type Error = io::Error; + type Hash = H256; + + fn get(&self, key: &H256) -> Result>, Self::Error> { + self.db.get(columns::STATE, &key[..]).map(|r| r.map(|v| v.to_vec())) + } +} + + +/// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks. +/// Otherwise, trie nodes are kept only from the most recent block. +pub struct Backend { + storage: Arc>, + blockchain: BlockchainDb, + finalization_window: u64, +} + +impl Backend { + /// Create a new instance of database backend. + pub fn new(config: DatabaseSettings, finalization_window: u64) -> Result { + let db = open_database(&config, "full")?; + + Backend::from_kvdb(db as Arc<_>, config.pruning, finalization_window) + } + + #[cfg(test)] + fn new_test(keep_blocks: u32) -> Self { + use utils::NUM_COLUMNS; + + let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS)); + + Backend::from_kvdb(db as Arc<_>, PruningMode::keep_blocks(keep_blocks), 0).expect("failed to create test-db") + } + + fn from_kvdb(db: Arc, pruning: PruningMode, finalization_window: u64) -> Result { + let blockchain = BlockchainDb::new(db.clone())?; + let map_e = |e: state_db::Error| ::client::error::Error::from(format!("State database error: {:?}", e)); + let state_db: StateDb = StateDb::new(pruning, &StateMetaDb(&*db)).map_err(map_e)?; + let storage_db = StorageDb { + db, + state_db, + }; + + Ok(Backend { + storage: Arc::new(storage_db), + blockchain, + finalization_window, + }) + } +} + +fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitSet) { + for (key, val) in commit.data.inserted.into_iter() { + transaction.put(columns::STATE, &key[..], &val); + } + for key in commit.data.deleted.into_iter() { + transaction.delete(columns::STATE, &key[..]); + } + for (key, val) in commit.meta.inserted.into_iter() { + transaction.put(columns::STATE_META, &key[..], &val); + } + for key in commit.meta.deleted.into_iter() { + transaction.delete(columns::STATE_META, &key[..]); + } +} + +impl client::backend::Backend for Backend where Block: BlockT { + type BlockImportOperation = BlockImportOperation; + type Blockchain = BlockchainDb; + type State = DbState; + + fn begin_operation(&self, block: BlockId) -> Result { + let state = self.state_at(block)?; + Ok(BlockImportOperation { + pending_block: None, + old_state: state, + updates: MemoryDB::default(), + }) + } + + fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> Result<(), client::error::Error> { + use client::blockchain::HeaderBackend; + let mut transaction = DBTransaction::new(); + if let Some(pending_block) = operation.pending_block { + let hash = pending_block.header.hash(); + let number = pending_block.header.number().clone(); + let key = number_to_db_key(number.clone()); + transaction.put(columns::HEADER, &key, &pending_block.header.encode()); + if let Some(body) = pending_block.body { + transaction.put(columns::BODY, &key, &body.encode()); + } + if let Some(justification) = pending_block.justification { + transaction.put(columns::JUSTIFICATION, &key, &justification.encode()); + } + transaction.put(columns::BLOCK_INDEX, hash.as_ref(), &key); + if pending_block.is_best { + transaction.put(columns::META, meta_keys::BEST_BLOCK, &key); + } + let mut changeset: state_db::ChangeSet = state_db::ChangeSet::default(); + for (key, (val, rc)) in operation.updates.drain() { + if rc > 0 { + changeset.inserted.push((key.0.into(), val.to_vec())); + } else if rc < 0 { + changeset.deleted.push(key.0.into()); + } + } + let number_u64 = number.as_().into(); + let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset); + apply_state_commit(&mut transaction, commit); + + //finalize an older block + if number_u64 > self.finalization_window { + let finalizing_hash = if self.finalization_window == 0 { + Some(hash) + } else { + let finalizing = number_u64 - self.finalization_window; + if finalizing > self.storage.state_db.best_finalized() { + self.blockchain.hash(As::sa(finalizing))? + } else { + None + } + }; + if let Some(finalizing_hash) = finalizing_hash { + trace!(target: "db", "Finalizing block #{} ({:?})", number_u64 - self.finalization_window, finalizing_hash); + let commit = self.storage.state_db.finalize_block(&finalizing_hash); + apply_state_commit(&mut transaction, commit); + } + } + + debug!(target: "db", "DB Commit {:?} ({}), best = {}", hash, number, pending_block.is_best); + self.storage.db.write(transaction).map_err(db_err)?; + self.blockchain.update_meta(hash, number, pending_block.is_best); + } + Ok(()) + } + + fn revert(&self, n: NumberFor) -> Result, client::error::Error> { + use client::blockchain::HeaderBackend; + let mut best = self.blockchain.info()?.best_number; + for c in 0 .. n.as_() { + if best == As::sa(0) { + return Ok(As::sa(c)) + } + let mut transaction = DBTransaction::new(); + match self.storage.state_db.revert_one() { + Some(commit) => { + apply_state_commit(&mut transaction, commit); + let removed = self.blockchain.hash(best)?.ok_or_else( + || client::error::ErrorKind::UnknownBlock( + format!("Error reverting to {}. Block hash not found.", best)))?; + best -= As::sa(1); + let key = number_to_db_key(best.clone()); + let hash = self.blockchain.hash(best)?.ok_or_else( + || client::error::ErrorKind::UnknownBlock( + format!("Error reverting to {}. Block hash not found.", best)))?; + transaction.put(columns::META, meta_keys::BEST_BLOCK, &key); + transaction.delete(columns::BLOCK_INDEX, removed.as_ref()); + self.storage.db.write(transaction).map_err(db_err)?; + self.blockchain.update_meta(hash, best, true); + } + None => return Ok(As::sa(c)) + } + } + Ok(n) + } + + fn blockchain(&self) -> &BlockchainDb { + &self.blockchain + } + + fn state_at(&self, block: BlockId) -> Result { + use client::blockchain::HeaderBackend as BcHeaderBackend; + + // special case for genesis initialization + match block { + BlockId::Hash(h) if h == Default::default() => + return Ok(DbState::with_storage_for_genesis(self.storage.clone())), + _ => {} + } + + match self.blockchain.header(block) { + Ok(Some(ref hdr)) if !self.storage.state_db.is_pruned(hdr.number().as_()) => { + let root = H256::from_slice(hdr.state_root().as_ref()); + Ok(DbState::with_storage(self.storage.clone(), root)) + }, + Err(e) => Err(e), + _ => Err(client::error::ErrorKind::UnknownBlock(format!("{:?}", block)).into()), + } + } +} + +impl client::backend::LocalBackend for Backend +where Block: BlockT {} + +#[cfg(test)] +mod tests { + use hashdb::HashDB; + use super::*; + use client::backend::Backend as BTrait; + use client::backend::BlockImportOperation as Op; + use client::blockchain::HeaderBackend as BlockchainHeaderBackend; + use runtime_primitives::testing::{Header, Block as RawBlock}; + + type Block = RawBlock; + + #[test] + fn block_hash_inserted_correctly() { + let db = Backend::::new_test(1); + for i in 0..10 { + assert!(db.blockchain().hash(i).unwrap().is_none()); + + { + let id = if i == 0 { + BlockId::Hash(Default::default()) + } else { + BlockId::Number(i - 1) + }; + + let mut op = db.begin_operation(id).unwrap(); + let header = Header { + number: i, + parent_hash: if i == 0 { + Default::default() + } else { + db.blockchain.hash(i - 1).unwrap().unwrap() + }, + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + op.set_block_data( + header, + Some(vec![]), + None, + true, + ).unwrap(); + db.commit_operation(op).unwrap(); + } + + assert!(db.blockchain().hash(i).unwrap().is_some()) + } + } + + #[test] + fn set_state_data() { + let db = Backend::::new_test(2); + { + let mut op = db.begin_operation(BlockId::Hash(Default::default())).unwrap(); + let mut header = Header { + number: 0, + parent_hash: Default::default(), + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let storage = vec![ + (vec![1, 3, 5], vec![2, 4, 6]), + (vec![1, 2, 3], vec![9, 9, 9]), + ]; + + header.state_root = op.old_state.storage_root(storage + .iter() + .cloned() + .map(|(x, y)| (x, Some(y))) + ).0.into(); + + op.reset_storage(storage.iter().cloned()).unwrap(); + op.set_block_data( + header, + Some(vec![]), + None, + true + ).unwrap(); + + db.commit_operation(op).unwrap(); + + let state = db.state_at(BlockId::Number(0)).unwrap(); + + assert_eq!(state.storage(&[1, 3, 5]).unwrap(), Some(vec![2, 4, 6])); + assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); + assert_eq!(state.storage(&[5, 5, 5]).unwrap(), None); + + } + + { + let mut op = db.begin_operation(BlockId::Number(0)).unwrap(); + let mut header = Header { + number: 1, + parent_hash: Default::default(), + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let storage = vec![ + (vec![1, 3, 5], None), + (vec![5, 5, 5], Some(vec![4, 5, 6])), + ]; + + let (root, overlay) = op.old_state.storage_root(storage.iter().cloned()); + op.update_storage(overlay).unwrap(); + header.state_root = root.into(); + + op.set_block_data( + header, + Some(vec![]), + None, + true + ).unwrap(); + + db.commit_operation(op).unwrap(); + + let state = db.state_at(BlockId::Number(1)).unwrap(); + + assert_eq!(state.storage(&[1, 3, 5]).unwrap(), None); + assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); + assert_eq!(state.storage(&[5, 5, 5]).unwrap(), Some(vec![4, 5, 6])); + } + } + + #[test] + fn delete_only_when_negative_rc() { + let key; + let backend = Backend::::new_test(0); + + let hash = { + let mut op = backend.begin_operation(BlockId::Hash(Default::default())).unwrap(); + let mut header = Header { + number: 0, + parent_hash: Default::default(), + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let storage: Vec<(_, _)> = vec![]; + + header.state_root = op.old_state.storage_root(storage + .iter() + .cloned() + .map(|(x, y)| (x, Some(y))) + ).0.into(); + let hash = header.hash(); + + op.reset_storage(storage.iter().cloned()).unwrap(); + + key = op.updates.insert(b"hello"); + op.set_block_data( + header, + Some(vec![]), + None, + true + ).unwrap(); + + backend.commit_operation(op).unwrap(); + + assert_eq!(backend.storage.db.get(::columns::STATE, &key.0[..]).unwrap().unwrap(), &b"hello"[..]); + hash + }; + + let hash = { + let mut op = backend.begin_operation(BlockId::Number(0)).unwrap(); + let mut header = Header { + number: 1, + parent_hash: hash, + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let storage: Vec<(_, _)> = vec![]; + + header.state_root = op.old_state.storage_root(storage + .iter() + .cloned() + .map(|(x, y)| (x, Some(y))) + ).0.into(); + let hash = header.hash(); + + op.updates.insert(b"hello"); + op.updates.remove(&key); + op.set_block_data( + header, + Some(vec![]), + None, + true + ).unwrap(); + + backend.commit_operation(op).unwrap(); + + assert_eq!(backend.storage.db.get(::columns::STATE, &key.0[..]).unwrap().unwrap(), &b"hello"[..]); + hash + }; + + { + let mut op = backend.begin_operation(BlockId::Number(1)).unwrap(); + let mut header = Header { + number: 2, + parent_hash: hash, + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let storage: Vec<(_, _)> = vec![]; + + header.state_root = op.old_state.storage_root(storage + .iter() + .cloned() + .map(|(x, y)| (x, Some(y))) + ).0.into(); + + op.updates.remove(&key); + op.set_block_data( + header, + Some(vec![]), + None, + true + ).unwrap(); + + backend.commit_operation(op).unwrap(); + + assert!(backend.storage.db.get(::columns::STATE, &key.0[..]).unwrap().is_none()); + } + } +} diff --git a/core/client/db/src/light.rs b/core/client/db/src/light.rs new file mode 100644 index 00000000000..9b82aec1f30 --- /dev/null +++ b/core/client/db/src/light.rs @@ -0,0 +1,392 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! RocksDB-based light client blockchain storage. + +use std::sync::Arc; +use parking_lot::RwLock; + +use kvdb::{KeyValueDB, DBTransaction}; + +use client::blockchain::{BlockStatus, Cache as BlockchainCache, + HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo}; +use client::cht; +use client::error::{ErrorKind as ClientErrorKind, Result as ClientResult}; +use client::light::blockchain::Storage as LightBlockchainStorage; +use codec::{Decode, Encode}; +use primitives::{AuthorityId, H256, Blake2Hasher}; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash, HashFor, + Zero, One, As, NumberFor}; +use cache::DbCache; +use utils::{meta_keys, Meta, db_err, number_to_db_key, db_key_to_number, open_database, + read_db, read_id, read_meta}; +use DatabaseSettings; + +pub(crate) mod columns { + pub const META: Option = ::utils::COLUMN_META; + pub const BLOCK_INDEX: Option = Some(1); + pub const HEADER: Option = Some(2); + pub const AUTHORITIES: Option = Some(3); + pub const CHT: Option = Some(4); +} + +/// Keep authorities for last 'AUTHORITIES_ENTRIES_TO_KEEP' blocks. +pub(crate) const AUTHORITIES_ENTRIES_TO_KEEP: u64 = cht::SIZE; + +/// Light blockchain storage. Stores most recent headers + CHTs for older headers. +pub struct LightStorage { + db: Arc, + meta: RwLock::Header as HeaderT>::Number, Block::Hash>>, + cache: DbCache, +} + +#[derive(Clone, PartialEq, Debug)] +struct BestAuthorities { + /// first block, when this set became actual + valid_from: N, + /// None means that we do not know the set starting from `valid_from` block + authorities: Option>, +} + +impl LightStorage + where + Block: BlockT, +{ + /// Create new storage with given settings. + pub fn new(config: DatabaseSettings) -> ClientResult { + let db = open_database(&config, "light")?; + + Self::from_kvdb(db as Arc<_>) + } + + #[cfg(test)] + pub(crate) fn new_test() -> Self { + use utils::NUM_COLUMNS; + + let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS)); + + Self::from_kvdb(db as Arc<_>).expect("failed to create test-db") + } + + fn from_kvdb(db: Arc) -> ClientResult { + let cache = DbCache::new(db.clone(), columns::BLOCK_INDEX, columns::AUTHORITIES)?; + let meta = RwLock::new(read_meta::(&*db, columns::HEADER)?); + + Ok(LightStorage { + db, + meta, + cache, + }) + } + + #[cfg(test)] + pub(crate) fn db(&self) -> &Arc { + &self.db + } + + #[cfg(test)] + pub(crate) fn cache(&self) -> &DbCache { + &self.cache + } + + fn update_meta(&self, hash: Block::Hash, number: <::Header as HeaderT>::Number, is_best: bool) { + if is_best { + let mut meta = self.meta.write(); + if number == <::Header as HeaderT>::Number::zero() { + meta.genesis_hash = hash; + } + + meta.best_number = number; + meta.best_hash = hash; + } + } +} + +impl BlockchainHeaderBackend for LightStorage + where + Block: BlockT, +{ + fn header(&self, id: BlockId) -> ClientResult> { + match read_db(&*self.db, columns::BLOCK_INDEX, columns::HEADER, id)? { + Some(header) => match Block::Header::decode(&mut &header[..]) { + Some(header) => Ok(Some(header)), + None => return Err(ClientErrorKind::Backend("Error decoding header".into()).into()), + } + None => Ok(None), + } + } + + fn info(&self) -> ClientResult> { + let meta = self.meta.read(); + Ok(BlockchainInfo { + best_hash: meta.best_hash, + best_number: meta.best_number, + genesis_hash: meta.genesis_hash, + }) + } + + fn status(&self, id: BlockId) -> ClientResult { + let exists = match id { + BlockId::Hash(_) => read_id(&*self.db, columns::BLOCK_INDEX, id)?.is_some(), + BlockId::Number(n) => n <= self.meta.read().best_number, + }; + match exists { + true => Ok(BlockStatus::InChain), + false => Ok(BlockStatus::Unknown), + } + } + + fn number(&self, hash: Block::Hash) -> ClientResult::Header as HeaderT>::Number>> { + read_id::(&*self.db, columns::BLOCK_INDEX, BlockId::Hash(hash)) + .and_then(|key| match key { + Some(key) => Ok(Some(db_key_to_number(&key)?)), + None => Ok(None), + }) + } + + fn hash(&self, number: <::Header as HeaderT>::Number) -> ClientResult> { + read_db::(&*self.db, columns::BLOCK_INDEX, columns::HEADER, BlockId::Number(number)).map(|x| + x.map(|raw| HashFor::::hash(&raw[..])).map(Into::into) + ) + } +} + +impl LightBlockchainStorage for LightStorage + where + Block: BlockT, + Block::Hash: From, +{ + fn import_header(&self, is_new_best: bool, header: Block::Header, authorities: Option>) -> ClientResult<()> { + let mut transaction = DBTransaction::new(); + + let hash = header.hash(); + let number = *header.number(); + let key = number_to_db_key(number); + + transaction.put(columns::HEADER, &key, &header.encode()); + transaction.put(columns::BLOCK_INDEX, hash.as_ref(), &key); + + let best_authorities = if is_new_best { + transaction.put(columns::META, meta_keys::BEST_BLOCK, &key); + + // cache authorities for previous block + let number: u64 = number.as_(); + let previous_number = number.checked_sub(1); + let best_authorities = previous_number + .and_then(|previous_number| self.cache.authorities_at_cache() + .commit_best_entry(&mut transaction, As::sa(previous_number), authorities)); + + // prune authorities from 'ancient' blocks + if let Some(ancient_number) = number.checked_sub(AUTHORITIES_ENTRIES_TO_KEEP) { + self.cache.authorities_at_cache().prune_entries(&mut transaction, As::sa(ancient_number))?; + } + + best_authorities + } else { + None + }; + + // build new CHT if required + if let Some(new_cht_number) = cht::is_build_required(cht::SIZE, *header.number()) { + let new_cht_start: NumberFor = cht::start_number(cht::SIZE, new_cht_number); + let new_cht_root: Option = cht::compute_root::( + cht::SIZE, new_cht_number, (new_cht_start.as_()..) + .map(|num| self.hash(As::sa(num)).unwrap_or_default())); + + if let Some(new_cht_root) = new_cht_root { + transaction.put(columns::CHT, &number_to_db_key(new_cht_start), new_cht_root.as_ref()); + + let mut prune_block = new_cht_start; + let new_cht_end = cht::end_number(cht::SIZE, new_cht_number); + trace!(target: "db", "Replacing blocks [{}..{}] with CHT#{}", new_cht_start, new_cht_end, new_cht_number); + + while prune_block <= new_cht_end { + transaction.delete(columns::HEADER, &number_to_db_key(prune_block)); + prune_block += <::Header as HeaderT>::Number::one(); + } + } + } + + debug!("Light DB Commit {:?} ({})", hash, number); + self.db.write(transaction).map_err(db_err)?; + self.update_meta(hash, number, is_new_best); + if let Some(best_authorities) = best_authorities { + self.cache.authorities_at_cache().update_best_entry(Some(best_authorities)); + } + + Ok(()) + } + + fn cht_root(&self, cht_size: u64, block: <::Header as HeaderT>::Number) -> ClientResult { + let no_cht_for_block = || ClientErrorKind::Backend(format!("CHT for block {} not exists", block)).into(); + + let cht_number = cht::block_to_cht_number(cht_size, block).ok_or_else(no_cht_for_block)?; + let cht_start = cht::start_number(cht_size, cht_number); + self.db.get(columns::CHT, &number_to_db_key(cht_start)).map_err(db_err)? + .ok_or_else(no_cht_for_block) + .and_then(|hash| Block::Hash::decode(&mut &*hash).ok_or_else(no_cht_for_block)) + } + + fn cache(&self) -> Option<&BlockchainCache> { + Some(&self.cache) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use client::cht; + use runtime_primitives::testing::{H256 as Hash, Header, Block as RawBlock}; + use super::*; + + type Block = RawBlock; + + pub fn insert_block( + db: &LightStorage, + parent: &Hash, + number: u64, + authorities: Option> + ) -> Hash { + let header = Header { + number: number.into(), + parent_hash: *parent, + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let hash = header.hash(); + db.import_header(true, header, authorities).unwrap(); + hash + } + + #[test] + fn returns_known_header() { + let db = LightStorage::new_test(); + let known_hash = insert_block(&db, &Default::default(), 0, None); + let header_by_hash = db.header(BlockId::Hash(known_hash)).unwrap().unwrap(); + let header_by_number = db.header(BlockId::Number(0)).unwrap().unwrap(); + assert_eq!(header_by_hash, header_by_number); + } + + #[test] + fn does_not_return_unknown_header() { + let db = LightStorage::::new_test(); + assert!(db.header(BlockId::Hash(1.into())).unwrap().is_none()); + assert!(db.header(BlockId::Number(0)).unwrap().is_none()); + } + + #[test] + fn returns_info() { + let db = LightStorage::new_test(); + let genesis_hash = insert_block(&db, &Default::default(), 0, None); + let info = db.info().unwrap(); + assert_eq!(info.best_hash, genesis_hash); + assert_eq!(info.best_number, 0); + assert_eq!(info.genesis_hash, genesis_hash); + let best_hash = insert_block(&db, &genesis_hash, 1, None); + let info = db.info().unwrap(); + assert_eq!(info.best_hash, best_hash); + assert_eq!(info.best_number, 1); + assert_eq!(info.genesis_hash, genesis_hash); + } + + #[test] + fn returns_block_status() { + let db = LightStorage::new_test(); + let genesis_hash = insert_block(&db, &Default::default(), 0, None); + assert_eq!(db.status(BlockId::Hash(genesis_hash)).unwrap(), BlockStatus::InChain); + assert_eq!(db.status(BlockId::Number(0)).unwrap(), BlockStatus::InChain); + assert_eq!(db.status(BlockId::Hash(1.into())).unwrap(), BlockStatus::Unknown); + assert_eq!(db.status(BlockId::Number(1)).unwrap(), BlockStatus::Unknown); + } + + #[test] + fn returns_block_hash() { + let db = LightStorage::new_test(); + let genesis_hash = insert_block(&db, &Default::default(), 0, None); + assert_eq!(db.hash(0).unwrap(), Some(genesis_hash)); + assert_eq!(db.hash(1).unwrap(), None); + } + + #[test] + fn import_header_works() { + let db = LightStorage::new_test(); + + let genesis_hash = insert_block(&db, &Default::default(), 0, None); + assert_eq!(db.db.iter(columns::HEADER).count(), 1); + assert_eq!(db.db.iter(columns::BLOCK_INDEX).count(), 1); + + let _ = insert_block(&db, &genesis_hash, 1, None); + assert_eq!(db.db.iter(columns::HEADER).count(), 2); + assert_eq!(db.db.iter(columns::BLOCK_INDEX).count(), 2); + } + + #[test] + fn ancient_headers_are_replaced_with_cht() { + let db = LightStorage::new_test(); + + // insert genesis block header (never pruned) + let mut prev_hash = insert_block(&db, &Default::default(), 0, None); + + // insert SIZE blocks && ensure that nothing is pruned + for number in 0..cht::SIZE { + prev_hash = insert_block(&db, &prev_hash, 1 + number, None); + } + assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE) as usize); + assert_eq!(db.db.iter(columns::CHT).count(), 0); + + // insert next SIZE blocks && ensure that nothing is pruned + for number in 0..cht::SIZE { + prev_hash = insert_block(&db, &prev_hash, 1 + cht::SIZE + number, None); + } + assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + cht::SIZE) as usize); + assert_eq!(db.db.iter(columns::CHT).count(), 0); + + // insert block #{2 * cht::SIZE + 1} && check that new CHT is created + headers of this CHT are pruned + insert_block(&db, &prev_hash, 1 + cht::SIZE + cht::SIZE, None); + assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + 1) as usize); + assert_eq!(db.db.iter(columns::CHT).count(), 1); + assert!((0..cht::SIZE).all(|i| db.db.get(columns::HEADER, &number_to_db_key(1 + i)).unwrap().is_none())); + } + + #[test] + fn get_cht_fails_for_genesis_block() { + assert!(LightStorage::::new_test().cht_root(cht::SIZE, 0).is_err()); + } + + #[test] + fn get_cht_fails_for_non_existant_cht() { + assert!(LightStorage::::new_test().cht_root(cht::SIZE, (cht::SIZE / 2) as u64).is_err()); + } + + #[test] + fn get_cht_works() { + let db = LightStorage::new_test(); + + // insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created + let mut prev_hash = Default::default(); + for i in 0..1 + cht::SIZE + cht::SIZE + 1 { + prev_hash = insert_block(&db, &prev_hash, i as u64, None); + } + + let cht_root_1 = db.cht_root(cht::SIZE, cht::start_number(cht::SIZE, 0)).unwrap(); + let cht_root_2 = db.cht_root(cht::SIZE, (cht::start_number(cht::SIZE, 0) + cht::SIZE / 2) as u64).unwrap(); + let cht_root_3 = db.cht_root(cht::SIZE, cht::end_number(cht::SIZE, 0)).unwrap(); + assert_eq!(cht_root_1, cht_root_2); + assert_eq!(cht_root_2, cht_root_3); + } +} diff --git a/core/client/db/src/utils.rs b/core/client/db/src/utils.rs new file mode 100644 index 00000000000..a2c24ddce06 --- /dev/null +++ b/core/client/db/src/utils.rs @@ -0,0 +1,174 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Db-based backend utility structures and functions, used by both +//! full and light storages. + +use std::sync::Arc; +use std::io; + +use kvdb::{KeyValueDB, DBTransaction}; +use kvdb_rocksdb::{Database, DatabaseConfig}; + +use client; +use codec::Decode; +use hashdb::DBValue; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{As, Block as BlockT, Header as HeaderT, Hash, HashFor, Zero}; +use DatabaseSettings; + +/// Number of columns in the db. Must be the same for both full && light dbs. +/// Otherwise RocksDb will fail to open database && check its type. +pub const NUM_COLUMNS: u32 = 7; +/// Meta column. The set of keys in the column is shared by full && light storages. +pub const COLUMN_META: Option = Some(0); + +/// Keys of entries in COLUMN_META. +pub mod meta_keys { + /// Type of storage (full or light). + pub const TYPE: &[u8; 4] = b"type"; + /// Best block key. + pub const BEST_BLOCK: &[u8; 4] = b"best"; + /// Best authorities block key. + pub const BEST_AUTHORITIES: &[u8; 4] = b"auth"; +} + +/// Database metadata. +pub struct Meta { + /// Hash of the best known block. + pub best_hash: H, + /// Number of the best known block. + pub best_number: N, + /// Hash of the genesis block. + pub genesis_hash: H, +} + +/// Type of block key in the database (LE block number). +pub type BlockKey = [u8; 4]; + +/// Convert block number into key (LE representation). +pub fn number_to_db_key(n: N) -> BlockKey where N: As { + let n: u64 = n.as_(); + assert!(n & 0xffffffff00000000 == 0); + + [ + (n >> 24) as u8, + ((n >> 16) & 0xff) as u8, + ((n >> 8) & 0xff) as u8, + (n & 0xff) as u8 + ] +} + +/// Convert block key into block number. +pub fn db_key_to_number(key: &[u8]) -> client::error::Result where N: As { + match key.len() { + 4 => Ok((key[0] as u64) << 24 + | (key[1] as u64) << 16 + | (key[2] as u64) << 8 + | (key[3] as u64)).map(As::sa), + _ => Err(client::error::ErrorKind::Backend("Invalid block key".into()).into()), + } +} + +/// Maps database error to client error +pub fn db_err(err: io::Error) -> client::error::Error { + use std::error::Error; + client::error::ErrorKind::Backend(err.description().into()).into() +} + +/// Open RocksDB database. +pub fn open_database(config: &DatabaseSettings, db_type: &str) -> client::error::Result> { + let mut db_config = DatabaseConfig::with_columns(Some(NUM_COLUMNS)); + db_config.memory_budget = config.cache_size; + let path = config.path.to_str().ok_or_else(|| client::error::ErrorKind::Backend("Invalid database path".into()))?; + let db = Database::open(&db_config, &path).map_err(db_err)?; + + // check database type + match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? { + Some(stored_type) => { + if db_type.as_bytes() != &*stored_type { + return Err(client::error::ErrorKind::Backend( + format!("Unexpected database type. Expected: {}", db_type)).into()); + } + }, + None => { + let mut transaction = DBTransaction::new(); + transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_bytes()); + db.write(transaction).map_err(db_err)?; + }, + } + + Ok(Arc::new(db)) +} + +/// Convert block id to block key, reading number from db if required. +pub fn read_id(db: &KeyValueDB, col_index: Option, id: BlockId) -> Result, client::error::Error> + where + Block: BlockT, +{ + match id { + BlockId::Hash(h) => db.get(col_index, h.as_ref()) + .map(|v| v.map(|v| { + let mut key: [u8; 4] = [0; 4]; + key.copy_from_slice(&v); + key + })).map_err(db_err), + BlockId::Number(n) => Ok(Some(number_to_db_key(n))), + } +} + +/// Read database column entry for the given block. +pub fn read_db(db: &KeyValueDB, col_index: Option, col: Option, id: BlockId) -> client::error::Result> + where + Block: BlockT, +{ + read_id(db, col_index, id).and_then(|key| match key { + Some(key) => db.get(col, &key).map_err(db_err), + None => Ok(None), + }) +} + +/// Read meta from the database. +pub fn read_meta(db: &KeyValueDB, col_header: Option) -> Result::Header as HeaderT>::Number, Block::Hash>, client::error::Error> + where + Block: BlockT, +{ + let genesis_number = <::Header as HeaderT>::Number::zero(); + let (best_hash, best_number) = if let Some(Some(header)) = db.get(COLUMN_META, meta_keys::BEST_BLOCK).and_then(|id| + match id { + Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]))), + None => Ok(None), + }).map_err(db_err)? + { + let hash = header.hash(); + debug!("DB Opened blockchain db, best {:?} ({})", hash, header.number()); + (hash, *header.number()) + } else { + (Default::default(), genesis_number) + }; + + let genesis_hash = db.get(col_header, &number_to_db_key(genesis_number)) + .map_err(db_err)? + .map(|raw| HashFor::::hash(&raw[..])) + .unwrap_or_default() + .into(); + + Ok(Meta { + best_hash, + best_number, + genesis_hash, + }) +} diff --git a/core/client/src/backend.rs b/core/client/src/backend.rs new file mode 100644 index 00000000000..7466c8fb332 --- /dev/null +++ b/core/client/src/backend.rs @@ -0,0 +1,107 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot Client data backend + +use error; +use primitives::AuthorityId; +use runtime_primitives::bft::Justification; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{Block as BlockT, NumberFor}; +use state_machine::backend::Backend as StateBackend; +use patricia_trie::NodeCodec; +use hashdb::Hasher; + +/// Block insertion operation. Keeps hold if the inserted block state and data. +pub trait BlockImportOperation +where + Block: BlockT, + H: Hasher, + C: NodeCodec, +{ + /// Associated state backend type. + type State: StateBackend; + + /// Returns pending state. Returns None for backends with locally-unavailable state data. + fn state(&self) -> error::Result>; + /// Append block data to the transaction. + fn set_block_data( + &mut self, + header: Block::Header, + body: Option>, + justification: Option>, + is_new_best: bool + ) -> error::Result<()>; + + /// Append authorities set to the transaction. This is a set of parent block (set which + /// has been used to check justification of this block). + fn update_authorities(&mut self, authorities: Vec); + /// Inject storage data into the database. + fn update_storage(&mut self, update: >::Transaction) -> error::Result<()>; + /// Inject storage data into the database replacing any existing data. + fn reset_storage, Vec)>>(&mut self, iter: I) -> error::Result<()>; +} + +/// Client backend. Manages the data layer. +/// +/// Note on state pruning: while an object from `state_at` is alive, the state +/// should not be pruned. The backend should internally reference-count +/// its state objects. +/// +/// The same applies for live `BlockImportOperation`s: while an import operation building on a parent `P` +/// is alive, the state for `P` should not be pruned. +pub trait Backend: Send + Sync +where + Block: BlockT, + H: Hasher, + C: NodeCodec, +{ + /// Associated block insertion operation type. + type BlockImportOperation: BlockImportOperation; + /// Associated blockchain backend type. + type Blockchain: ::blockchain::Backend; + /// Associated state backend type. + type State: StateBackend; + + /// Begin a new block insertion transaction with given parent block id. + /// When constructing the genesis, this is called with all-zero hash. + fn begin_operation(&self, block: BlockId) -> error::Result; + /// Commit block insertion. + fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>; + /// Returns reference to blockchain backend. + fn blockchain(&self) -> &Self::Blockchain; + /// Returns state backend with post-state of given block. + fn state_at(&self, block: BlockId) -> error::Result; + /// Attempts to revert the chain by `n` blocks. Returns the number of blocks that were + /// successfully reverted. + fn revert(&self, n: NumberFor) -> error::Result>; +} + +/// Mark for all Backend implementations, that are making use of state data, stored locally. +pub trait LocalBackend: Backend +where + Block: BlockT, + H: Hasher, + C: NodeCodec, +{} + +/// Mark for all Backend implementations, that are fetching required state data from remote nodes. +pub trait RemoteBackend: Backend +where + Block: BlockT, + H: Hasher, + C: NodeCodec, +{} diff --git a/core/client/src/block_builder.rs b/core/client/src/block_builder.rs new file mode 100644 index 00000000000..0d78f30cc41 --- /dev/null +++ b/core/client/src/block_builder.rs @@ -0,0 +1,140 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Utility struct to build a block. + +use std::vec::Vec; +use codec::{Decode, Encode}; +use state_machine::{self, native_when_possible}; +use runtime_primitives::traits::{Header as HeaderT, Hash, Block as BlockT, One, HashFor}; +use runtime_primitives::generic::BlockId; +use {backend, error, Client, CallExecutor}; +use runtime_primitives::{ApplyResult, ApplyOutcome}; +use patricia_trie::NodeCodec; +use primitives::{Blake2Hasher, RlpCodec}; +use hashdb::Hasher; +use rlp::Encodable; + +/// Utility for building new (valid) blocks from a stream of extrinsics. +pub struct BlockBuilder +where + B: backend::Backend, + E: CallExecutor + Clone, + Block: BlockT, + H: Hasher, + H::Out: Encodable + Ord, + C: NodeCodec, +{ + header: ::Header, + extrinsics: Vec<::Extrinsic>, + executor: E, + state: B::State, + changes: state_machine::OverlayedChanges, +} + +impl BlockBuilder +where + B: backend::Backend, + E: CallExecutor + Clone, + Block: BlockT, +{ + /// Create a new instance of builder from the given client, building on the latest block. + pub fn new(client: &Client) -> error::Result { + client.info().and_then(|i| Self::at_block(&BlockId::Hash(i.chain.best_hash), client)) + } + + /// Create a new instance of builder from the given client using a particular block's ID to + /// build upon. + pub fn at_block(block_id: &BlockId, client: &Client) -> error::Result { + let number = client.block_number_from_id(block_id)? + .ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", block_id)))? + + One::one(); + + let parent_hash = client.block_hash_from_id(block_id)? + .ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", block_id)))?; + + let executor = client.executor().clone(); + let state = client.state_at(block_id)?; + let mut changes = Default::default(); + let header = <::Header as HeaderT>::new( + number, + Default::default(), + Default::default(), + parent_hash, + Default::default() + ); + + executor.call_at_state(&state, &mut changes, "initialise_block", &header.encode(), native_when_possible())?; + changes.commit_prospective(); + + Ok(BlockBuilder { + header, + extrinsics: Vec::new(), + executor, + state, + changes, + }) + } + + /// Push onto the block's list of extrinsics. This will ensure the extrinsic + /// can be validly executed (by executing it); if it is invalid, it'll be returned along with + /// the error. Otherwise, it will return a mutable reference to self (in order to chain). + pub fn push(&mut self, xt: ::Extrinsic) -> error::Result<()> { + match self.executor.call_at_state(&self.state, &mut self.changes, "apply_extrinsic", &xt.encode(), native_when_possible()) { + Ok((result, _)) => { + match ApplyResult::decode(&mut result.as_slice()) { + Some(Ok(ApplyOutcome::Success)) | Some(Ok(ApplyOutcome::Fail)) => { + self.extrinsics.push(xt); + self.changes.commit_prospective(); + Ok(()) + } + Some(Err(e)) => { + self.changes.discard_prospective(); + Err(error::ErrorKind::ApplyExtinsicFailed(e).into()) + } + None => { + self.changes.discard_prospective(); + Err(error::ErrorKind::CallResultDecode("apply_extrinsic").into()) + } + } + } + Err(e) => { + self.changes.discard_prospective(); + Err(e) + } + } + } + + /// Consume the builder to return a valid `Block` containing all pushed extrinsics. + pub fn bake(mut self) -> error::Result { + let (output, _) = self.executor.call_at_state( + &self.state, + &mut self.changes, + "finalise_block", + &[], + native_when_possible(), + )?; + self.header = <::Header as Decode>::decode(&mut &output[..]) + .expect("Header came straight out of runtime so must be valid"); + + debug_assert_eq!( + self.header.extrinsics_root().clone(), + HashFor::::ordered_trie_root(self.extrinsics.iter().map(Encode::encode)), + ); + + Ok(::new(self.header, self.extrinsics)) + } +} diff --git a/core/client/src/blockchain.rs b/core/client/src/blockchain.rs new file mode 100644 index 00000000000..ab5dc036e96 --- /dev/null +++ b/core/client/src/blockchain.rs @@ -0,0 +1,92 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot blockchain trait + +use primitives::AuthorityId; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor}; +use runtime_primitives::generic::BlockId; +use runtime_primitives::bft::Justification; + +use error::{ErrorKind, Result}; + +/// Blockchain database header backend. Does not perform any validation. +pub trait HeaderBackend: Send + Sync { + /// Get block header. Returns `None` if block is not found. + fn header(&self, id: BlockId) -> Result>; + /// Get blockchain info. + fn info(&self) -> Result>; + /// Get block status. + fn status(&self, id: BlockId) -> Result; + /// Get block number by hash. Returns `None` if the header is not in the chain. + fn number(&self, hash: Block::Hash) -> Result::Header as HeaderT>::Number>>; + /// Get block hash by number. Returns `None` if the header is not in the chain. + fn hash(&self, number: NumberFor) -> Result>; + + /// Get block header. Returns `UnknownBlock` error if block is not found. + fn expect_header(&self, id: BlockId) -> Result { + self.header(id)?.ok_or_else(|| ErrorKind::UnknownBlock(format!("{}", id)).into()) + } +} + +/// Blockchain database backend. Does not perform any validation. +pub trait Backend: HeaderBackend { + /// Get block body. Returns `None` if block is not found. + fn body(&self, id: BlockId) -> Result::Extrinsic>>>; + /// Get block justification. Returns `None` if justification does not exist. + fn justification(&self, id: BlockId) -> Result>>; + + /// Returns data cache reference, if it is enabled on this backend. + fn cache(&self) -> Option<&Cache>; +} + +/// Blockchain optional data cache. +pub trait Cache: Send + Sync { + /// Returns the set of authorities, that was active at given block or None if there's no entry in the cache. + fn authorities_at(&self, block: BlockId) -> Option>; +} + +/// Block import outcome +pub enum ImportResult { + /// Imported successfully. + Imported, + /// Block already exists, skippped. + AlreadyInChain, + /// Unknown parent. + UnknownParent, + /// Other errror. + Err(E), +} + +/// Blockchain info +#[derive(Debug)] +pub struct Info { + /// Best block hash. + pub best_hash: <::Header as HeaderT>::Hash, + /// Best block number. + pub best_number: <::Header as HeaderT>::Number, + /// Genesis block hash. + pub genesis_hash: <::Header as HeaderT>::Hash, +} + +/// Block status. +#[derive(Debug, PartialEq, Eq)] +pub enum BlockStatus { + /// Already in the blockchain. + InChain, + /// Not in the queue or the blockchain. + Unknown, +} diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs new file mode 100644 index 00000000000..d7802557418 --- /dev/null +++ b/core/client/src/call_executor.rs @@ -0,0 +1,199 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::sync::Arc; +use std::cmp::Ord; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::Block as BlockT; +use state_machine::{self, OverlayedChanges, Ext, + CodeExecutor, ExecutionManager, native_when_possible}; +use executor::{RuntimeVersion, RuntimeInfo}; +use patricia_trie::NodeCodec; +use hashdb::Hasher; +use rlp::Encodable; +use codec::Decode; +use primitives::{Blake2Hasher, RlpCodec}; + +use backend; +use error; + +/// Information regarding the result of a call. +#[derive(Debug, Clone)] +pub struct CallResult { + /// The data that was returned from the call. + pub return_data: Vec, + /// The changes made to the state by the call. + pub changes: OverlayedChanges, +} + +/// Method call executor. +pub trait CallExecutor +where + B: BlockT, + H: Hasher, + H::Out: Ord + Encodable, + C: NodeCodec, +{ + /// Externalities error type. + type Error: state_machine::Error; + + /// Execute a call to a contract on top of state in a block of given hash. + /// + /// No changes are made. + fn call(&self, + id: &BlockId, + method: &str, + call_data: &[u8], + ) -> Result; + + /// Extract RuntimeVersion of given block + /// + /// No changes are made. + fn runtime_version(&self, id: &BlockId) -> Result; + + /// Execute a call to a contract on top of given state. + /// + /// No changes are made. + fn call_at_state< + S: state_machine::Backend, + F: FnOnce(Result, Self::Error>, Result, Self::Error>) -> Result, Self::Error>, + >(&self, + state: &S, + overlay: &mut OverlayedChanges, + method: &str, + call_data: &[u8], + manager: ExecutionManager + ) -> Result<(Vec, S::Transaction), error::Error>; + + /// Execute a call to a contract on top of given state, gathering execution proof. + /// + /// No changes are made. + fn prove_at_state>(&self, + state: S, + overlay: &mut OverlayedChanges, + method: &str, + call_data: &[u8] + ) -> Result<(Vec, Vec>), error::Error>; + + /// Get runtime version if supported. + fn native_runtime_version(&self) -> Option; +} + +/// Call executor that executes methods locally, querying all required +/// data from local backend. +pub struct LocalCallExecutor { + backend: Arc, + executor: E, +} + +impl LocalCallExecutor { + /// Creates new instance of local call executor. + pub fn new(backend: Arc, executor: E) -> Self { + LocalCallExecutor { backend, executor } + } +} + +impl Clone for LocalCallExecutor where E: Clone { + fn clone(&self) -> Self { + LocalCallExecutor { + backend: self.backend.clone(), + executor: self.executor.clone(), + } + } +} + +impl CallExecutor for LocalCallExecutor +where + B: backend::LocalBackend, + E: CodeExecutor + RuntimeInfo, + Block: BlockT, +{ + type Error = E::Error; + + fn call(&self, + id: &BlockId, + method: &str, + call_data: &[u8], + ) -> error::Result { + let mut changes = OverlayedChanges::default(); + let (return_data, _) = self.call_at_state( + &self.backend.state_at(*id)?, + &mut changes, + method, + call_data, + native_when_possible(), + )?; + Ok(CallResult { return_data, changes }) + } + + fn runtime_version(&self, id: &BlockId) -> error::Result { + let mut overlay = OverlayedChanges::default(); + let state = self.backend.state_at(*id)?; + use state_machine::Backend; + let code = state.storage(b":code") + .map_err(|e| error::ErrorKind::Execution(Box::new(e)))? + .ok_or(error::ErrorKind::VersionInvalid)? + .to_vec(); + let heap_pages = state.storage(b":heappages") + .map_err(|e| error::ErrorKind::Execution(Box::new(e)))? + .and_then(|v| u64::decode(&mut &v[..])) + .unwrap_or(8) as usize; + + self.executor.runtime_version(&mut Ext::new(&mut overlay, &state), heap_pages, &code) + .ok_or(error::ErrorKind::VersionInvalid.into()) + } + + fn call_at_state< + S: state_machine::Backend, + F: FnOnce(Result, Self::Error>, Result, Self::Error>) -> Result, Self::Error>, + >(&self, + state: &S, + changes: &mut OverlayedChanges, + method: &str, + call_data: &[u8], + manager: ExecutionManager, + ) -> error::Result<(Vec, S::Transaction)> { + state_machine::execute_using_consensus_failure_handler( + state, + changes, + &self.executor, + method, + call_data, + manager, + ).map_err(Into::into) + } + + fn prove_at_state>(&self, + state: S, + changes: &mut OverlayedChanges, + method: &str, + call_data: &[u8] + ) -> Result<(Vec, Vec>), error::Error> { + state_machine::prove_execution( + state, + changes, + &self.executor, + method, + call_data, + ) + .map(|(result, proof, _)| (result, proof)) + .map_err(Into::into) + } + + fn native_runtime_version(&self) -> Option { + ::NATIVE_VERSION + } +} diff --git a/core/client/src/cht.rs b/core/client/src/cht.rs new file mode 100644 index 00000000000..2f2f38523b1 --- /dev/null +++ b/core/client/src/cht.rs @@ -0,0 +1,274 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Canonical hash trie definitions and helper functions. +//! +//! Each CHT is a trie mapping block numbers to canonical hash. +//! One is generated for every `SIZE` blocks, allowing us to discard those blocks in +//! favor of the trie root. When the "ancient" blocks need to be accessed, we simply +//! request an inclusion proof of a specific block number against the trie with the +//! root has. A correct proof implies that the claimed block is identical to the one +//! we discarded. + +use hashdb; +use heapsize::HeapSizeOf; +use patricia_trie::NodeCodec; +use rlp::Encodable; +use triehash; + +use primitives::H256; +use runtime_primitives::traits::{As, Header as HeaderT, SimpleArithmetic, One}; +use state_machine::backend::InMemory as InMemoryState; +use state_machine::{prove_read, read_proof_check}; + +use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult}; + +/// The size of each CHT. This value is passed to every CHT-related function from +/// production code. Other values are passed from tests. +pub const SIZE: u64 = 2048; + +/// Returns Some(cht_number) if CHT is need to be built when the block with given number is canonized. +pub fn is_build_required(cht_size: u64, block_num: N) -> Option + where + N: Clone + SimpleArithmetic, +{ + let block_cht_num = block_to_cht_number(cht_size, block_num.clone())?; + let two = N::one() + N::one(); + if block_cht_num < two { + return None; + } + let cht_start = start_number(cht_size, block_cht_num.clone()); + if cht_start != block_num { + return None; + } + + Some(block_cht_num - two) +} + +/// Compute a CHT root from an iterator of block hashes. Fails if shorter than +/// SIZE items. The items are assumed to proceed sequentially from `start_number(cht_num)`. +/// Discards the trie's nodes. +pub fn compute_root( + cht_size: u64, + cht_num: Header::Number, + hashes: I, +) -> Option + where + Header: HeaderT, + Header::Hash: From, + Hasher: hashdb::Hasher, + Hasher::Out: Ord + Encodable, + I: IntoIterator>, +{ + build_pairs::(cht_size, cht_num, hashes) + .map(|pairs| triehash::trie_root::(pairs).into()) +} + +/// Build CHT-based header proof. +pub fn build_proof( + cht_size: u64, + cht_num: Header::Number, + block_num: Header::Number, + hashes: I +) -> Option>> + where + Header: HeaderT, + Hasher: hashdb::Hasher, + Hasher::Out: Ord + Encodable + HeapSizeOf, + Codec: NodeCodec, + I: IntoIterator>, +{ + let transaction = build_pairs::(cht_size, cht_num, hashes)? + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect::>(); + let storage = InMemoryState::::default().update(transaction); + let (value, proof) = prove_read(storage, &encode_cht_key(block_num)).ok()?; + if value.is_none() { + None + } else { + Some(proof) + } +} + +/// Check CHT-based header proof. +pub fn check_proof( + local_root: Header::Hash, + local_number: Header::Number, + remote_hash: Header::Hash, + remote_proof: Vec> +) -> ClientResult<()> + where + Header: HeaderT, + Header::Hash: From, + Hasher: hashdb::Hasher, + Hasher::Out: Ord + Encodable + HeapSizeOf + From, + Codec: NodeCodec, +{ + let local_cht_key = encode_cht_key(local_number); + let local_cht_value = read_proof_check::(local_root.into(), remote_proof, + &local_cht_key).map_err(|e| ClientError::from(e))?; + let local_cht_value = local_cht_value.ok_or_else(|| ClientErrorKind::InvalidHeaderProof)?; + let local_hash: Header::Hash = decode_cht_value(&local_cht_value).ok_or_else(|| ClientErrorKind::InvalidHeaderProof)?; + match local_hash == remote_hash { + true => Ok(()), + false => Err(ClientErrorKind::InvalidHeaderProof.into()), + } +} + +/// Build pairs for computing CHT. +fn build_pairs( + cht_size: u64, + cht_num: Header::Number, + hashes: I +) -> Option, Vec)>> + where + Header: HeaderT, + I: IntoIterator>, +{ + let start_num = start_number(cht_size, cht_num); + let mut pairs = Vec::new(); + let mut hash_number = start_num; + for hash in hashes.into_iter().take(cht_size as usize) { + pairs.push(hash.map(|hash| ( + encode_cht_key(hash_number).to_vec(), + encode_cht_value(hash) + ))?); + hash_number += Header::Number::one(); + } + + if pairs.len() as u64 == cht_size { + Some(pairs) + } else { + None + } +} + +/// Get the starting block of a given CHT. +/// CHT 0 includes block 1...SIZE, +/// CHT 1 includes block SIZE + 1 ... 2*SIZE +/// More generally: CHT N includes block (1 + N*SIZE)...((N+1)*SIZE). +/// This is because the genesis hash is assumed to be known +/// and including it would be redundant. +pub fn start_number(cht_size: u64, cht_num: N) -> N { + (cht_num * As::sa(cht_size)) + N::one() +} + +/// Get the ending block of a given CHT. +pub fn end_number(cht_size: u64, cht_num: N) -> N { + (cht_num + N::one()) * As::sa(cht_size) +} + +/// Convert a block number to a CHT number. +/// Returns `None` for `block_num` == 0, `Some` otherwise. +pub fn block_to_cht_number(cht_size: u64, block_num: N) -> Option { + if block_num == N::zero() { + None + } else { + Some((block_num - N::one()) / As::sa(cht_size)) + } +} + +/// Convert header number into CHT key. +pub fn encode_cht_key>(number: N) -> Vec { + let number: u64 = number.as_(); + vec![ + (number >> 56) as u8, + ((number >> 48) & 0xff) as u8, + ((number >> 40) & 0xff) as u8, + ((number >> 32) & 0xff) as u8, + ((number >> 24) & 0xff) as u8, + ((number >> 16) & 0xff) as u8, + ((number >> 8) & 0xff) as u8, + (number & 0xff) as u8 + ] +} + +/// Convert header hash into CHT value. +fn encode_cht_value>(hash: Hash) -> Vec { + hash.as_ref().to_vec() +} + +/// Convert CHT value into block header hash. +pub fn decode_cht_value>(value: &[u8]) -> Option { + match value.len() { + 32 => Some(H256::from_slice(&value[0..32]).into()), + _ => None, + } + +} + +#[cfg(test)] +mod tests { + use primitives::{Blake2Hasher, RlpCodec}; + use test_client::runtime::Header; + use super::*; + + #[test] + fn is_build_required_works() { + assert_eq!(is_build_required(SIZE, 0), None); + assert_eq!(is_build_required(SIZE, 1), None); + assert_eq!(is_build_required(SIZE, SIZE), None); + assert_eq!(is_build_required(SIZE, SIZE + 1), None); + assert_eq!(is_build_required(SIZE, 2 * SIZE), None); + assert_eq!(is_build_required(SIZE, 2 * SIZE + 1), Some(0)); + assert_eq!(is_build_required(SIZE, 3 * SIZE), None); + assert_eq!(is_build_required(SIZE, 3 * SIZE + 1), Some(1)); + } + + #[test] + fn start_number_works() { + assert_eq!(start_number(SIZE, 0), 1); + assert_eq!(start_number(SIZE, 1), SIZE + 1); + assert_eq!(start_number(SIZE, 2), SIZE + SIZE + 1); + } + + #[test] + fn end_number_works() { + assert_eq!(end_number(SIZE, 0), SIZE); + assert_eq!(end_number(SIZE, 1), SIZE + SIZE); + assert_eq!(end_number(SIZE, 2), SIZE + SIZE + SIZE); + } + + #[test] + fn build_pairs_fails_when_no_enough_blocks() { + assert!(build_pairs::(SIZE, 0, vec![Some(1.into()); SIZE as usize / 2]).is_none()); + } + + #[test] + fn build_pairs_fails_when_missing_block() { + assert!(build_pairs::(SIZE, 0, ::std::iter::repeat(Some(1.into())).take(SIZE as usize / 2) + .chain(::std::iter::once(None)) + .chain(::std::iter::repeat(Some(2.into())).take(SIZE as usize / 2 - 1))).is_none()); + } + + #[test] + fn compute_root_works() { + assert!(compute_root::(SIZE, 42, vec![Some(1.into()); SIZE as usize]).is_some()); + } + + #[test] + fn build_proof_fails_when_querying_wrong_block() { + assert!(build_proof::( + SIZE, 0, (SIZE * 1000) as u64, vec![Some(1.into()); SIZE as usize]).is_none()); + } + + #[test] + fn build_proof_works() { + assert!(build_proof::( + SIZE, 0, (SIZE / 2) as u64, vec![Some(1.into()); SIZE as usize]).is_some()); + } +} diff --git a/core/client/src/client.rs b/core/client/src/client.rs new file mode 100644 index 00000000000..9246e61ec85 --- /dev/null +++ b/core/client/src/client.rs @@ -0,0 +1,810 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate Client + +use std::sync::Arc; +use futures::sync::mpsc; +use parking_lot::{Mutex, RwLock}; +use primitives::AuthorityId; +use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Block as RuntimeBlock}}; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One, As, NumberFor}; +use runtime_primitives::BuildStorage; +use runtime_support::metadata::JSONMetadataDecodable; +use primitives::{Blake2Hasher, RlpCodec}; +use primitives::storage::{StorageKey, StorageData}; +use codec::{Encode, Decode}; +use state_machine::{ + Ext, OverlayedChanges, Backend as StateBackend, CodeExecutor, + ExecutionStrategy, ExecutionManager, prove_read +}; + +use backend::{self, BlockImportOperation}; +use blockchain::{self, Info as ChainInfo, Backend as ChainBackend, HeaderBackend as ChainHeaderBackend}; +use call_executor::{CallExecutor, LocalCallExecutor}; +use executor::{RuntimeVersion, RuntimeInfo}; +use notifications::{StorageNotifications, StorageEventStream}; +use {cht, error, in_mem, block_builder, runtime_io, bft, genesis}; + +/// Type that implements `futures::Stream` of block import events. +pub type BlockchainEventStream = mpsc::UnboundedReceiver>; + +/// Substrate Client +pub struct Client where Block: BlockT { + backend: Arc, + executor: E, + storage_notifications: Mutex>, + import_notification_sinks: Mutex>>>, + import_lock: Mutex<()>, + importing_block: RwLock>, // holds the block hash currently being imported. TODO: replace this with block queue + execution_strategy: ExecutionStrategy, +} + +/// A source of blockchain evenets. +pub trait BlockchainEvents { + /// Get block import event stream. + fn import_notification_stream(&self) -> BlockchainEventStream; + + /// Get storage changes event stream. + /// + /// Passing `None` as `filter_keys` subscribes to all storage changes. + fn storage_changes_notification_stream(&self, filter_keys: Option<&[StorageKey]>) -> error::Result>; +} + +/// Chain head information. +pub trait ChainHead { + /// Get best block header. + fn best_block_header(&self) -> Result<::Header, error::Error>; +} + +/// Fetch block body by ID. +pub trait BlockBody { + /// Get block body by ID. Returns `None` if the body is not stored. + fn block_body(&self, id: &BlockId) -> error::Result::Extrinsic>>>; +} + +/// Client info +// TODO: split queue info from chain info and amalgamate into single struct. +#[derive(Debug)] +pub struct ClientInfo { + /// Best block hash. + pub chain: ChainInfo, + /// Best block number in the queue. + pub best_queued_number: Option<<::Header as HeaderT>::Number>, + /// Best queued block hash. + pub best_queued_hash: Option, +} + +/// Block import result. +#[derive(Debug)] +pub enum ImportResult { + /// Added to the import queue. + Queued, + /// Already in the import queue. + AlreadyQueued, + /// Already in the blockchain. + AlreadyInChain, + /// Block or parent is known to be bad. + KnownBad, + /// Block parent is not in the chain. + UnknownParent, +} + +/// Block status. +#[derive(Debug, PartialEq, Eq)] +pub enum BlockStatus { + /// Added to the import queue. + Queued, + /// Already in the blockchain. + InChain, + /// Block or parent is known to be bad. + KnownBad, + /// Not in the queue or the blockchain. + Unknown, +} + +/// Block data origin. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum BlockOrigin { + /// Genesis block built into the client. + Genesis, + /// Block is part of the initial sync with the network. + NetworkInitialSync, + /// Block was broadcasted on the network. + NetworkBroadcast, + /// Block that was received from the network and validated in the consensus process. + ConsensusBroadcast, + /// Block that was collated by this node. + Own, + /// Block was imported from a file. + File, +} + +/// Summary of an imported block +#[derive(Clone, Debug)] +pub struct BlockImportNotification { + /// Imported block header hash. + pub hash: Block::Hash, + /// Imported block origin. + pub origin: BlockOrigin, + /// Imported block header. + pub header: Block::Header, + /// Is this the new best block. + pub is_new_best: bool, +} + +/// A header paired with a justification which has already been checked. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct JustifiedHeader { + header: ::Header, + justification: ::bft::Justification, + authorities: Vec, +} + +impl JustifiedHeader { + /// Deconstruct the justified header into parts. + pub fn into_inner(self) -> (::Header, ::bft::Justification, Vec) { + (self.header, self.justification, self.authorities) + } +} + +/// Create an instance of in-memory client. +pub fn new_in_mem( + executor: E, + genesis_storage: S, +) -> error::Result, LocalCallExecutor, E>, Block>> + where + E: CodeExecutor + RuntimeInfo, + S: BuildStorage, + Block: BlockT, +{ + let backend = Arc::new(in_mem::Backend::new()); + let executor = LocalCallExecutor::new(backend.clone(), executor); + Client::new(backend, executor, genesis_storage, ExecutionStrategy::NativeWhenPossible) +} + +impl Client where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + /// Creates new Substrate Client with given blockchain and code executor. + pub fn new( + backend: Arc, + executor: E, + build_genesis_storage: S, + execution_strategy: ExecutionStrategy, + ) -> error::Result { + if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() { + let genesis_storage = build_genesis_storage.build_storage()?; + let genesis_block = genesis::construct_genesis_block::(&genesis_storage); + info!("Initialising Genesis block/state (state: {}, header-hash: {})", genesis_block.header().state_root(), genesis_block.header().hash()); + let mut op = backend.begin_operation(BlockId::Hash(Default::default()))?; + op.reset_storage(genesis_storage.into_iter())?; + op.set_block_data(genesis_block.deconstruct().0, Some(vec![]), None, true)?; + backend.commit_operation(op)?; + } + Ok(Client { + backend, + executor, + storage_notifications: Default::default(), + import_notification_sinks: Default::default(), + import_lock: Default::default(), + importing_block: Default::default(), + execution_strategy, + }) + } + + /// Get a reference to the state at a given block. + pub fn state_at(&self, block: &BlockId) -> error::Result { + self.backend.state_at(*block) + } + + /// Expose backend reference. To be used in tests only + pub fn backend(&self) -> &Arc { + &self.backend + } + + /// Return single storage entry of contract under given address in state in a block of given hash. + pub fn storage(&self, id: &BlockId, key: &StorageKey) -> error::Result> { + Ok(self.state_at(id)? + .storage(&key.0).map_err(|e| error::Error::from_state(Box::new(e)))? + .map(StorageData)) + } + + /// Get the code at a given block. + pub fn code_at(&self, id: &BlockId) -> error::Result> { + Ok(self.storage(id, &StorageKey(b":code".to_vec()))? + .expect("None is returned if there's no value stored for the given key; ':code' key is always defined; qed").0) + } + + /// Get the set of authorities at a given block. + pub fn authorities_at(&self, id: &BlockId) -> error::Result> { + match self.backend.blockchain().cache().and_then(|cache| cache.authorities_at(*id)) { + Some(cached_value) => Ok(cached_value), + None => self.executor.call(id, "authorities",&[]) + .and_then(|r| Vec::::decode(&mut &r.return_data[..]) + .ok_or(error::ErrorKind::AuthLenInvalid.into())) + } + } + + /// Get the RuntimeVersion at a given block. + pub fn runtime_version_at(&self, id: &BlockId) -> error::Result { + // TODO: Post Poc-2 return an error if version is missing + self.executor.runtime_version(id) + } + + /// Get call executor reference. + pub fn executor(&self) -> &E { + &self.executor + } + + /// Returns the runtime metadata as JSON. + pub fn json_metadata(&self, id: &BlockId) -> error::Result { + self.executor.call(id, "json_metadata",&[]) + .and_then(|r| Vec::::decode(&mut &r.return_data[..]) + .ok_or("JSON Metadata decoding failed".into())) + .and_then(|metadata| { + let mut json = metadata.into_iter().enumerate().fold(String::from("{"), + |mut json, (i, m)| { + if i > 0 { + json.push_str(","); + } + let (mtype, val) = m.into_json_string(); + json.push_str(&format!(r#" "{}": {}"#, mtype, val)); + json + } + ); + json.push_str(" }"); + + Ok(json) + }) + } + + /// Reads storage value at a given block + key, returning read proof. + pub fn read_proof(&self, id: &BlockId, key: &[u8]) -> error::Result>> { + self.state_at(id) + .and_then(|state| prove_read(state, key) + .map(|(_, proof)| proof) + .map_err(Into::into)) + } + + /// Execute a call to a contract on top of state in a block of given hash + /// AND returning execution proof. + /// + /// No changes are made. + pub fn execution_proof(&self, id: &BlockId, method: &str, call_data: &[u8]) -> error::Result<(Vec, Vec>)> { + self.state_at(id).and_then(|state| self.executor.prove_at_state(state, &mut Default::default(), method, call_data)) + } + + /// Reads given header and generates CHT-based header proof. + pub fn header_proof(&self, id: &BlockId) -> error::Result<(Block::Header, Vec>)> { + self.header_proof_with_cht_size(id, cht::SIZE) + } + + /// Reads given header and generates CHT-based header proof for CHT of given size. + pub fn header_proof_with_cht_size(&self, id: &BlockId, cht_size: u64) -> error::Result<(Block::Header, Vec>)> { + let proof_error = || error::ErrorKind::Backend(format!("Failed to generate header proof for {:?}", id)); + let header = self.header(id)?.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", id)))?; + let block_num = *header.number(); + let cht_num = cht::block_to_cht_number(cht_size, block_num).ok_or_else(proof_error)?; + let cht_start = cht::start_number(cht_size, cht_num); + let headers = (cht_start.as_()..).map(|num| self.block_hash(As::sa(num)).unwrap_or_default()); + let proof = cht::build_proof::(cht_size, cht_num, block_num, headers) + .ok_or_else(proof_error)?; + Ok((header, proof)) + } + + /// Set up the native execution environment to call into a native runtime code. + pub fn using_environment T, T>( + &self, f: F + ) -> error::Result { + self.using_environment_at(&BlockId::Number(self.info()?.chain.best_number), &mut Default::default(), f) + } + + /// Set up the native execution environment to call into a native runtime code. + pub fn using_environment_at T, T>( + &self, + id: &BlockId, + overlay: &mut OverlayedChanges, + f: F + ) -> error::Result { + Ok(runtime_io::with_externalities(&mut Ext::new(overlay, &self.state_at(id)?), f)) + } + + /// Create a new block, built on the head of the chain. + pub fn new_block(&self) -> error::Result> + where E: Clone + { + block_builder::BlockBuilder::new(self) + } + + /// Create a new block, built on top of `parent`. + pub fn new_block_at(&self, parent: &BlockId) -> error::Result> + where E: Clone + { + block_builder::BlockBuilder::at_block(parent, &self) + } + + /// Call a runtime function at given block. + pub fn call_api(&self, at: &BlockId, function: &'static str, args: &A) -> error::Result + where + A: Encode, + R: Decode, + { + let parent = at; + let header = <::Header as HeaderT>::new( + self.block_number_from_id(&parent)? + .ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", parent)))? + As::sa(1), + Default::default(), + Default::default(), + self.block_hash_from_id(&parent)? + .ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", parent)))?, + Default::default() + ); + self.state_at(&parent).and_then(|state| { + let mut overlay = Default::default(); + let execution_manager = || ExecutionManager::Both(|wasm_result, native_result| { + warn!("Consensus error between wasm and native runtime execution at block {:?}", at); + warn!(" Function {:?}", function); + warn!(" Native result {:?}", native_result); + warn!(" Wasm result {:?}", wasm_result); + wasm_result + }); + self.executor().call_at_state( + &state, + &mut overlay, + "initialise_block", + &header.encode(), + execution_manager() + )?; + let (r, _) = args.using_encoded(|input| + self.executor().call_at_state( + &state, + &mut overlay, + function, + input, + execution_manager() + ))?; + Ok(R::decode(&mut &r[..]) + .ok_or_else(|| error::Error::from(error::ErrorKind::CallResultDecode(function)))?) + }) + } + + /// Check a header's justification. + pub fn check_justification( + &self, + header: ::Header, + justification: ::bft::UncheckedJustification, + ) -> error::Result> { + let parent_hash = header.parent_hash().clone(); + let authorities = self.authorities_at(&BlockId::Hash(parent_hash))?; + let just = ::bft::check_justification::(&authorities[..], parent_hash, justification) + .map_err(|_| + error::ErrorKind::BadJustification( + format!("{}", header.hash()) + ) + )?; + Ok(JustifiedHeader { + header, + justification: just, + authorities, + }) + } + + /// Queue a block for import. + pub fn import_block( + &self, + origin: BlockOrigin, + header: JustifiedHeader, + body: Option::Extrinsic>>, + ) -> error::Result { + let (header, justification, authorities) = header.into_inner(); + let parent_hash = header.parent_hash().clone(); + match self.backend.blockchain().status(BlockId::Hash(parent_hash))? { + blockchain::BlockStatus::InChain => {}, + blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent), + } + let hash = header.hash(); + let _import_lock = self.import_lock.lock(); + let height: u64 = header.number().as_(); + *self.importing_block.write() = Some(hash); + let result = self.execute_and_import_block(origin, hash, header, justification, body, authorities); + *self.importing_block.write() = None; + telemetry!("block.import"; + "height" => height, + "best" => ?hash, + "origin" => ?origin + ); + result + } + + fn execute_and_import_block( + &self, + origin: BlockOrigin, + hash: Block::Hash, + header: Block::Header, + justification: bft::Justification, + body: Option>, + authorities: Vec, + ) -> error::Result { + let parent_hash = header.parent_hash().clone(); + match self.backend.blockchain().status(BlockId::Hash(hash))? { + blockchain::BlockStatus::InChain => return Ok(ImportResult::AlreadyInChain), + blockchain::BlockStatus::Unknown => {}, + } + + let mut transaction = self.backend.begin_operation(BlockId::Hash(parent_hash))?; + let (storage_update, storage_changes) = match transaction.state()? { + Some(transaction_state) => { + let mut overlay = Default::default(); + let mut r = self.executor.call_at_state( + transaction_state, + &mut overlay, + "execute_block", + &::new(header.clone(), body.clone().unwrap_or_default()).encode(), + match (origin, self.execution_strategy) { + (BlockOrigin::NetworkInitialSync, _) | (_, ExecutionStrategy::NativeWhenPossible) => + ExecutionManager::NativeWhenPossible, + (_, ExecutionStrategy::AlwaysWasm) => ExecutionManager::AlwaysWasm, + _ => ExecutionManager::Both(|wasm_result, native_result| { + warn!("Consensus error between wasm and native block execution at block {}", hash); + warn!(" Header {:?}", header); + warn!(" Native result {:?}", native_result); + warn!(" Wasm result {:?}", wasm_result); + telemetry!("block.execute.consensus_failure"; + "hash" => ?hash, + "origin" => ?origin, + "header" => ?header + ); + wasm_result + }), + }, + ); + let (_, storage_update) = r?; + overlay.commit_prospective(); + (Some(storage_update), Some(overlay.into_committed())) + }, + None => (None, None) + }; + + let is_new_best = header.number() == &(self.backend.blockchain().info()?.best_number + One::one()); + trace!("Imported {}, (#{}), best={}, origin={:?}", hash, header.number(), is_new_best, origin); + let unchecked: bft::UncheckedJustification<_> = justification.uncheck().into(); + transaction.set_block_data(header.clone(), body, Some(unchecked.into()), is_new_best)?; + transaction.update_authorities(authorities); + if let Some(storage_update) = storage_update { + transaction.update_storage(storage_update)?; + } + self.backend.commit_operation(transaction)?; + + if origin == BlockOrigin::NetworkBroadcast || origin == BlockOrigin::Own || origin == BlockOrigin::ConsensusBroadcast { + + if let Some(storage_changes) = storage_changes { + // TODO [ToDr] How to handle re-orgs? Should we re-emit all storage changes? + self.storage_notifications.lock() + .trigger(&hash, storage_changes); + } + + let notification = BlockImportNotification:: { + hash: hash, + origin: origin, + header: header, + is_new_best: is_new_best, + }; + self.import_notification_sinks.lock() + .retain(|sink| sink.unbounded_send(notification.clone()).is_ok()); + } + Ok(ImportResult::Queued) + } + + /// Attempts to revert the chain by `n` blocks. Returns the number of blocks that were + /// successfully reverted. + pub fn revert(&self, n: NumberFor) -> error::Result> { + Ok(self.backend.revert(n)?) + } + + /// Get blockchain info. + pub fn info(&self) -> error::Result> { + let info = self.backend.blockchain().info().map_err(|e| error::Error::from_blockchain(Box::new(e)))?; + Ok(ClientInfo { + chain: info, + best_queued_hash: None, + best_queued_number: None, + }) + } + + /// Get block status. + pub fn block_status(&self, id: &BlockId) -> error::Result { + // TODO: more efficient implementation + if let BlockId::Hash(ref h) = id { + if self.importing_block.read().as_ref().map_or(false, |importing| h == importing) { + return Ok(BlockStatus::Queued); + } + } + match self.backend.blockchain().header(*id).map_err(|e| error::Error::from_blockchain(Box::new(e)))?.is_some() { + true => Ok(BlockStatus::InChain), + false => Ok(BlockStatus::Unknown), + } + } + + /// Get block hash by number. + pub fn block_hash(&self, block_number: <::Header as HeaderT>::Number) -> error::Result> { + self.backend.blockchain().hash(block_number) + } + + /// Convert an arbitrary block ID into a block hash. + pub fn block_hash_from_id(&self, id: &BlockId) -> error::Result> { + match *id { + BlockId::Hash(h) => Ok(Some(h)), + BlockId::Number(n) => self.block_hash(n), + } + } + + /// Convert an arbitrary block ID into a block hash. + pub fn block_number_from_id(&self, id: &BlockId) -> error::Result::Header as HeaderT>::Number>> { + match *id { + BlockId::Hash(_) => Ok(self.header(id)?.map(|h| h.number().clone())), + BlockId::Number(n) => Ok(Some(n)), + } + } + + /// Get block header by id. + pub fn header(&self, id: &BlockId) -> error::Result::Header>> { + self.backend.blockchain().header(*id) + } + + /// Get block body by id. + pub fn body(&self, id: &BlockId) -> error::Result::Extrinsic>>> { + self.backend.blockchain().body(*id) + } + + /// Get block justification set by id. + pub fn justification(&self, id: &BlockId) -> error::Result>> { + self.backend.blockchain().justification(*id) + } + + /// Get full block by id. + pub fn block(&self, id: &BlockId) -> error::Result>> { + Ok(match (self.header(id)?, self.body(id)?, self.justification(id)?) { + (Some(header), Some(extrinsics), Some(justification)) => + Some(SignedBlock { block: RuntimeBlock { header, extrinsics }, justification }), + _ => None, + }) + } + + /// Get best block header. + pub fn best_block_header(&self) -> error::Result<::Header> { + let info = self.backend.blockchain().info().map_err(|e| error::Error::from_blockchain(Box::new(e)))?; + Ok(self.header(&BlockId::Hash(info.best_hash))?.expect("Best block header must always exist")) + } +} + +impl bft::BlockImport for Client + where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + fn import_block( + &self, + block: Block, + justification: ::bft::Justification, + authorities: &[AuthorityId] + ) -> bool { + let (header, extrinsics) = block.deconstruct(); + let justified_header = JustifiedHeader { + header: header, + justification, + authorities: authorities.to_vec(), + }; + + self.import_block(BlockOrigin::ConsensusBroadcast, justified_header, Some(extrinsics)).is_ok() + } +} + +impl bft::Authorities for Client + where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + fn authorities(&self, at: &BlockId) -> Result, bft::Error> { + let on_chain_version: Result<_, bft::Error> = self.runtime_version_at(at) + .map_err(|e| { trace!("Error getting runtime version {:?}", e); bft::ErrorKind::RuntimeVersionMissing.into() }); + let on_chain_version = on_chain_version?; + let native_version: Result<_, bft::Error> = self.executor.native_runtime_version() + .ok_or_else(|| bft::ErrorKind::NativeRuntimeMissing.into()); + let native_version = native_version?; + if !on_chain_version.can_author_with(&native_version) { + return Err(bft::ErrorKind::IncompatibleAuthoringRuntime(on_chain_version, native_version).into()) + } + self.authorities_at(at).map_err(|_| { + let descriptor = format!("{:?}", at); + bft::ErrorKind::StateUnavailable(descriptor).into() + }) + } +} + +impl BlockchainEvents for Client +where + E: CallExecutor, + Block: BlockT, +{ + /// Get block import event stream. + fn import_notification_stream(&self) -> BlockchainEventStream { + let (sink, stream) = mpsc::unbounded(); + self.import_notification_sinks.lock().push(sink); + stream + } + + /// Get storage changes event stream. + fn storage_changes_notification_stream(&self, filter_keys: Option<&[StorageKey]>) -> error::Result> { + Ok(self.storage_notifications.lock().listen(filter_keys)) + } +} + +impl ChainHead for Client +where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + fn best_block_header(&self) -> error::Result<::Header> { + Client::best_block_header(self) + } +} + +impl BlockBody for Client + where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + fn block_body(&self, id: &BlockId) -> error::Result::Extrinsic>>> { + self.body(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use keyring::Keyring; + use test_client::{self, TestClient}; + use test_client::client::BlockOrigin; + use test_client::client::backend::Backend as TestBackend; + use test_client::{runtime as test_runtime, BlockBuilderExt}; + use test_client::runtime::Transfer; + + #[test] + fn client_initialises_from_genesis_ok() { + let client = test_client::new(); + + assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Alice.to_raw_public().into())).unwrap(), 1000); + assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Ferdie.to_raw_public().into())).unwrap(), 0); + } + + #[test] + fn authorities_call_works() { + let client = test_client::new(); + + assert_eq!(client.info().unwrap().chain.best_number, 0); + assert_eq!(client.authorities_at(&BlockId::Number(0)).unwrap(), vec![ + Keyring::Alice.to_raw_public().into(), + Keyring::Bob.to_raw_public().into(), + Keyring::Charlie.to_raw_public().into() + ]); + } + + #[test] + fn block_builder_works_with_no_transactions() { + let client = test_client::new(); + + let builder = client.new_block().unwrap(); + + client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + + assert_eq!(client.info().unwrap().chain.best_number, 1); + } + + #[test] + fn block_builder_works_with_transactions() { + let client = test_client::new(); + + let mut builder = client.new_block().unwrap(); + + builder.push_transfer(Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Ferdie.to_raw_public().into(), + amount: 42, + nonce: 0, + }).unwrap(); + + client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + + assert_eq!(client.info().unwrap().chain.best_number, 1); + assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap()); + assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Alice.to_raw_public().into())).unwrap(), 958); + assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Ferdie.to_raw_public().into())).unwrap(), 42); + } + + #[test] + fn client_uses_authorities_from_blockchain_cache() { + let client = test_client::new(); + test_client::client::in_mem::cache_authorities_at( + client.backend().blockchain(), + Default::default(), + Some(vec![[1u8; 32].into()])); + assert_eq!(client.authorities_at( + &BlockId::Hash(Default::default())).unwrap(), + vec![[1u8; 32].into()]); + } + + #[test] + fn block_builder_does_not_include_invalid() { + let client = test_client::new(); + + let mut builder = client.new_block().unwrap(); + + builder.push_transfer(Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Ferdie.to_raw_public().into(), + amount: 42, + nonce: 0, + }).unwrap(); + + assert!(builder.push_transfer(Transfer { + from: Keyring::Eve.to_raw_public().into(), + to: Keyring::Alice.to_raw_public().into(), + amount: 42, + nonce: 0, + }).is_err()); + + client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + + assert_eq!(client.info().unwrap().chain.best_number, 1); + assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap()); + assert_eq!(client.body(&BlockId::Number(1)).unwrap().unwrap().len(), 1) + } + + #[test] + fn json_metadata() { + let client = test_client::new(); + + let mut builder = client.new_block().unwrap(); + + builder.push_transfer(Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Ferdie.to_raw_public().into(), + amount: 42, + nonce: 0, + }).unwrap(); + + assert!(builder.push_transfer(Transfer { + from: Keyring::Eve.to_raw_public().into(), + to: Keyring::Alice.to_raw_public().into(), + amount: 42, + nonce: 0, + }).is_err()); + + client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + + assert_eq!( + client.json_metadata(&BlockId::Number(1)).unwrap(), + r#"{ "events": { "name": "Test", "events": { "event": hallo } } }"# + ); + } +} diff --git a/core/client/src/error.rs b/core/client/src/error.rs new file mode 100644 index 00000000000..b1eb27385ba --- /dev/null +++ b/core/client/src/error.rs @@ -0,0 +1,154 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot client possible errors. + +use std; +use state_machine; +use runtime_primitives::ApplyError; + +error_chain! { + errors { + /// Backend error. + Backend(s: String) { + description("Unrecoverable backend error"), + display("Backend error: {}", s), + } + + /// Unknown block. + UnknownBlock(h: String) { + description("unknown block"), + display("UnknownBlock: {}", &*h), + } + + /// Applying extrinsic error. + ApplyExtinsicFailed(e: ApplyError) { + description("Extrinsic error"), + display("Extrinsic error: {:?}", e), + } + + /// Execution error. + Execution(e: Box) { + description("execution error"), + display("Execution: {}", e), + } + + /// Blockchain error. + Blockchain(e: Box) { + description("Blockchain error"), + display("Blockchain: {}", e), + } + + /// Invalid state data. + AuthLenEmpty { + description("authority count state error"), + display("Current state of blockchain has no authority count value"), + } + + /// Invalid state data. + AuthEmpty(i: u32) { + description("authority value state error"), + display("Current state of blockchain has no authority value for index {}", i), + } + + /// Invalid state data. + AuthLenInvalid { + description("authority count state error"), + display("Current state of blockchain has invalid authority count value"), + } + + /// Cound not get runtime version. + VersionInvalid { + description("Runtime version error"), + display("On-chain runtime does not specify version"), + } + + /// Invalid state data. + AuthInvalid(i: u32) { + description("authority value state error"), + display("Current state of blockchain has invalid authority value for index {}", i), + } + + /// Bad justification for header. + BadJustification(h: String) { + description("bad justification for header"), + display("bad justification for header: {}", &*h), + } + + /// Not available on light client. + NotAvailableOnLightClient { + description("not available on light client"), + display("This method is not currently available when running in light client mode"), + } + + /// Invalid remote header proof. + InvalidHeaderProof { + description("invalid header proof"), + display("Remote node has responded with invalid header proof"), + } + + /// Invalid remote execution proof. + InvalidExecutionProof { + description("invalid execution proof"), + display("Remote node has responded with invalid execution proof"), + } + + /// Remote fetch has been cancelled. + RemoteFetchCancelled { + description("remote fetch cancelled"), + display("Remote data fetch has been cancelled"), + } + + /// Remote fetch has been failed. + RemoteFetchFailed { + description("remote fetch failed"), + display("Remote data fetch has been failed"), + } + + /// Error decoding call result. + CallResultDecode(method: &'static str) { + description("Error decoding call result") + display("Error decoding call result of {}", method) + } + } +} + +// TODO [ToDr] Temporary, state_machine::Error should be a regular error not Box. +impl From> for Error { + fn from(e: Box) -> Self { + ErrorKind::Execution(e).into() + } +} + +impl From for Error { + fn from(e: state_machine::backend::Void) -> Self { + match e {} + } +} + +impl Error { + /// Chain a blockchain error. + pub fn from_blockchain(e: Box) -> Self { + ErrorKind::Blockchain(e).into() + } + + /// Chain a state error. + pub fn from_state(e: Box) -> Self { + ErrorKind::Execution(e).into() + } +} + +impl state_machine::Error for Error {} diff --git a/core/client/src/genesis.rs b/core/client/src/genesis.rs new file mode 100644 index 00000000000..9c7810aedca --- /dev/null +++ b/core/client/src/genesis.rs @@ -0,0 +1,202 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tool for creating the genesis block. + +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, Zero}; +use runtime_primitives::StorageMap; + +/// Create a genesis block, given the initial storage. +pub fn construct_genesis_block< + Block: BlockT +> ( + storage: &StorageMap +) -> Block { + let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root(storage.clone().into_iter()); + let extrinsics_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root(::std::iter::empty::<(&[u8], &[u8])>()); + Block::new( + <::Header as HeaderT>::new( + Zero::zero(), + extrinsics_root, + state_root, + Default::default(), + Default::default() + ), + Default::default() + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use codec::{Encode, Decode, Joiner}; + use keyring::Keyring; + use executor::NativeExecutionDispatch; + use state_machine::{execute, OverlayedChanges, ExecutionStrategy}; + use state_machine::backend::InMemory; + use test_client; + use test_client::runtime::genesismap::{GenesisConfig, additional_storage_with_genesis}; + use test_client::runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest, Extrinsic}; + use primitives::{Blake2Hasher, RlpCodec, ed25519::{Public, Pair}}; + + native_executor_instance!(Executor, test_client::runtime::api::dispatch, test_client::runtime::VERSION, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm")); + + fn executor() -> ::executor::NativeExecutor { + NativeExecutionDispatch::new() + } + + fn construct_block(backend: &InMemory, number: BlockNumber, parent_hash: Hash, state_root: Hash, txs: Vec) -> (Vec, Hash) { + use triehash::ordered_trie_root; + + let transactions = txs.into_iter().map(|tx| { + let signature = Pair::from(Keyring::from_public(Public::from_raw(tx.from.0)).unwrap()) + .sign(&tx.encode()).into(); + + Extrinsic { transfer: tx, signature } + }).collect::>(); + + let extrinsics_root = ordered_trie_root::(transactions.iter().map(Encode::encode)).into(); + + println!("root before: {:?}", extrinsics_root); + let mut header = Header { + parent_hash, + number, + state_root, + extrinsics_root, + digest: Digest { logs: vec![], }, + }; + let hash = header.hash(); + let mut overlay = OverlayedChanges::default(); + + execute( + backend, + &mut overlay, + &executor(), + "initialise_block", + &header.encode(), + ExecutionStrategy::NativeWhenPossible, + ).unwrap(); + + for tx in transactions.iter() { + execute( + backend, + &mut overlay, + &executor(), + "apply_extrinsic", + &tx.encode(), + ExecutionStrategy::NativeWhenPossible, + ).unwrap(); + } + + let (ret_data, _) = execute( + backend, + &mut overlay, + &executor(), + "finalise_block", + &[], + ExecutionStrategy::NativeWhenPossible, + ).unwrap(); + header = Header::decode(&mut &ret_data[..]).unwrap(); + println!("root after: {:?}", header.extrinsics_root); + + (vec![].and(&Block { header, extrinsics: transactions }), hash) + } + + fn block1(genesis_hash: Hash, backend: &InMemory) -> (Vec, Hash) { + construct_block( + backend, + 1, + genesis_hash, + hex!("25e5b37074063ab75c889326246640729b40d0c86932edc527bc80db0e04fe5c").into(), + vec![Transfer { + from: Keyring::One.to_raw_public().into(), + to: Keyring::Two.to_raw_public().into(), + amount: 69, + nonce: 0, + }] + ) + } + + #[test] + fn construct_genesis_should_work_with_native() { + let mut storage = GenesisConfig::new_simple( + vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 1000 + ).genesis_map(); + let block = construct_genesis_block::(&storage); + let genesis_hash = block.header.hash(); + storage.extend(additional_storage_with_genesis(&block).into_iter()); + + let backend = InMemory::from(storage); + let (b1data, _b1hash) = block1(genesis_hash, &backend); + + let mut overlay = OverlayedChanges::default(); + let _ = execute( + &backend, + &mut overlay, + &executor(), + "execute_block", + &b1data, + ExecutionStrategy::NativeWhenPossible, + ).unwrap(); + } + + #[test] + fn construct_genesis_should_work_with_wasm() { + let mut storage = GenesisConfig::new_simple( + vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 1000 + ).genesis_map(); + let block = construct_genesis_block::(&storage); + let genesis_hash = block.header.hash(); + storage.extend(additional_storage_with_genesis(&block).into_iter()); + + let backend = InMemory::from(storage); + let (b1data, _b1hash) = block1(genesis_hash, &backend); + + let mut overlay = OverlayedChanges::default(); + let _ = execute( + &backend, + &mut overlay, + &executor(), + "execute_block", + &b1data, + ExecutionStrategy::AlwaysWasm, + ).unwrap(); + } + + #[test] + #[should_panic] + fn construct_genesis_with_bad_transaction_should_panic() { + let mut storage = GenesisConfig::new_simple( + vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 68 + ).genesis_map(); + let block = construct_genesis_block::(&storage); + let genesis_hash = block.header.hash(); + storage.extend(additional_storage_with_genesis(&block).into_iter()); + + let backend = InMemory::from(storage); + let (b1data, _b1hash) = block1(genesis_hash, &backend); + + let mut overlay = OverlayedChanges::default(); + let _ = execute( + &backend, + &mut overlay, + &Executor::new(), + "execute_block", + &b1data, + ExecutionStrategy::NativeWhenPossible, + ).unwrap(); + } +} diff --git a/core/client/src/in_mem.rs b/core/client/src/in_mem.rs new file mode 100644 index 00000000000..74a1fdbabe5 --- /dev/null +++ b/core/client/src/in_mem.rs @@ -0,0 +1,439 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! In memory client backend + +use std::collections::HashMap; +use std::sync::Arc; +use parking_lot::RwLock; +use error; +use backend; +use light; +use primitives::AuthorityId; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, As}; +use runtime_primitives::bft::Justification; +use blockchain::{self, BlockStatus}; +use state_machine::backend::{Backend as StateBackend, InMemory}; +use patricia_trie::NodeCodec; +use hashdb::Hasher; +use heapsize::HeapSizeOf; + +struct PendingBlock { + block: StoredBlock, + is_best: bool, +} + +#[derive(PartialEq, Eq, Clone)] +enum StoredBlock { + Header(B::Header, Option>), + Full(B, Option>), +} + +impl StoredBlock { + fn new(header: B::Header, body: Option>, just: Option>) -> Self { + match body { + Some(body) => StoredBlock::Full(B::new(header, body), just), + None => StoredBlock::Header(header, just), + } + } + + fn header(&self) -> &B::Header { + match *self { + StoredBlock::Header(ref h, _) => h, + StoredBlock::Full(ref b, _) => b.header(), + } + } + + fn justification(&self) -> Option<&Justification> { + match *self { + StoredBlock::Header(_, ref j) | StoredBlock::Full(_, ref j) => j.as_ref() + } + } + + fn extrinsics(&self) -> Option<&[B::Extrinsic]> { + match *self { + StoredBlock::Header(_, _) => None, + StoredBlock::Full(ref b, _) => Some(b.extrinsics()) + } + } + + fn into_inner(self) -> (B::Header, Option>, Option>) { + match self { + StoredBlock::Header(header, just) => (header, None, just), + StoredBlock::Full(block, just) => { + let (header, body) = block.deconstruct(); + (header, Some(body), just) + } + } + } +} + +#[derive(Clone)] +struct BlockchainStorage { + blocks: HashMap>, + hashes: HashMap<<::Header as HeaderT>::Number, Block::Hash>, + best_hash: Block::Hash, + best_number: <::Header as HeaderT>::Number, + genesis_hash: Block::Hash, + cht_roots: HashMap, Block::Hash>, +} + +/// In-memory blockchain. Supports concurrent reads. +pub struct Blockchain { + storage: Arc>>, + cache: Cache, +} + +struct Cache { + storage: Arc>>, + authorities_at: RwLock>>>, +} + +impl Clone for Blockchain { + fn clone(&self) -> Self { + let storage = Arc::new(RwLock::new(self.storage.read().clone())); + Blockchain { + storage: storage.clone(), + cache: Cache { + storage, + authorities_at: RwLock::new(self.cache.authorities_at.read().clone()), + }, + } + } +} + +impl Blockchain { + /// Get header hash of given block. + pub fn id(&self, id: BlockId) -> Option { + match id { + BlockId::Hash(h) => Some(h), + BlockId::Number(n) => self.storage.read().hashes.get(&n).cloned(), + } + } + + /// Create new in-memory blockchain storage. + pub fn new() -> Blockchain { + let storage = Arc::new(RwLock::new( + BlockchainStorage { + blocks: HashMap::new(), + hashes: HashMap::new(), + best_hash: Default::default(), + best_number: Zero::zero(), + genesis_hash: Default::default(), + cht_roots: HashMap::new(), + })); + Blockchain { + storage: storage.clone(), + cache: Cache { + storage: storage, + authorities_at: Default::default(), + }, + } + } + + /// Insert a block header and associated data. + pub fn insert( + &self, + hash: Block::Hash, + header: ::Header, + justification: Option>, + body: Option::Extrinsic>>, + is_new_best: bool + ) { + let number = header.number().clone(); + let mut storage = self.storage.write(); + storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justification)); + storage.hashes.insert(number, hash.clone()); + if is_new_best { + storage.best_hash = hash.clone(); + storage.best_number = number.clone(); + } + if number == Zero::zero() { + storage.genesis_hash = hash; + } + } + + /// Compare this blockchain with another in-mem blockchain + pub fn equals_to(&self, other: &Self) -> bool { + self.canon_equals_to(other) && self.storage.read().blocks == other.storage.read().blocks + } + + /// Compare canonical chain to other canonical chain. + pub fn canon_equals_to(&self, other: &Self) -> bool { + let this = self.storage.read(); + let other = other.storage.read(); + this.hashes == other.hashes + && this.best_hash == other.best_hash + && this.best_number == other.best_number + && this.genesis_hash == other.genesis_hash + } + + /// Insert CHT root. + pub fn insert_cht_root(&self, block: NumberFor, cht_root: Block::Hash) { + self.storage.write().cht_roots.insert(block, cht_root); + } +} + +impl blockchain::HeaderBackend for Blockchain { + fn header(&self, id: BlockId) -> error::Result::Header>> { + Ok(self.id(id).and_then(|hash| { + self.storage.read().blocks.get(&hash).map(|b| b.header().clone()) + })) + } + + fn info(&self) -> error::Result> { + let storage = self.storage.read(); + Ok(blockchain::Info { + best_hash: storage.best_hash, + best_number: storage.best_number, + genesis_hash: storage.genesis_hash, + }) + } + + fn status(&self, id: BlockId) -> error::Result { + match self.id(id).map_or(false, |hash| self.storage.read().blocks.contains_key(&hash)) { + true => Ok(BlockStatus::InChain), + false => Ok(BlockStatus::Unknown), + } + } + + fn number(&self, hash: Block::Hash) -> error::Result>> { + Ok(self.storage.read().blocks.get(&hash).map(|b| *b.header().number())) + } + + fn hash(&self, number: <::Header as HeaderT>::Number) -> error::Result> { + Ok(self.id(BlockId::Number(number))) + } +} + + +impl blockchain::Backend for Blockchain { + fn body(&self, id: BlockId) -> error::Result::Extrinsic>>> { + Ok(self.id(id).and_then(|hash| { + self.storage.read().blocks.get(&hash) + .and_then(|b| b.extrinsics().map(|x| x.to_vec())) + })) + } + + fn justification(&self, id: BlockId) -> error::Result>> { + Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).and_then(|b| + b.justification().map(|x| x.clone())) + )) + } + + fn cache(&self) -> Option<&blockchain::Cache> { + Some(&self.cache) + } +} + +impl light::blockchain::Storage for Blockchain + where + Block::Hash: From<[u8; 32]>, +{ + fn import_header( + &self, + is_new_best: bool, + header: Block::Header, + authorities: Option> + ) -> error::Result<()> { + let hash = header.hash(); + let parent_hash = *header.parent_hash(); + self.insert(hash, header, None, None, is_new_best); + if is_new_best { + self.cache.insert(parent_hash, authorities); + } + Ok(()) + } + + fn cht_root(&self, _cht_size: u64, block: NumberFor) -> error::Result { + self.storage.read().cht_roots.get(&block).cloned() + .ok_or_else(|| error::ErrorKind::Backend(format!("CHT for block {} not exists", block)).into()) + } + + fn cache(&self) -> Option<&blockchain::Cache> { + Some(&self.cache) + } +} + +/// In-memory operation. +pub struct BlockImportOperation> { + pending_block: Option>, + pending_authorities: Option>, + old_state: InMemory, + new_state: Option>, +} + +impl backend::BlockImportOperation for BlockImportOperation +where + Block: BlockT, + H: Hasher, + C: NodeCodec, + H::Out: HeapSizeOf, +{ + type State = InMemory; + + fn state(&self) -> error::Result> { + Ok(Some(&self.old_state)) + } + + fn set_block_data( + &mut self, + header: ::Header, + body: Option::Extrinsic>>, + justification: Option>, + is_new_best: bool + ) -> error::Result<()> { + assert!(self.pending_block.is_none(), "Only one block per operation is allowed"); + self.pending_block = Some(PendingBlock { + block: StoredBlock::new(header, body, justification), + is_best: is_new_best, + }); + Ok(()) + } + + fn update_authorities(&mut self, authorities: Vec) { + self.pending_authorities = Some(authorities); + } + + fn update_storage(&mut self, update: as StateBackend>::Transaction) -> error::Result<()> { + self.new_state = Some(self.old_state.update(update)); + Ok(()) + } + + fn reset_storage, Vec)>>(&mut self, iter: I) -> error::Result<()> { + self.new_state = Some(InMemory::from(iter.collect::>())); + Ok(()) + } +} + +/// In-memory backend. Keeps all states and blocks in memory. Useful for testing. +pub struct Backend +where + Block: BlockT, + H: Hasher, + C: NodeCodec +{ + states: RwLock>>, + blockchain: Blockchain, +} + +impl Backend +where + Block: BlockT, + H: Hasher, + C: NodeCodec +{ + /// Create a new instance of in-mem backend. + pub fn new() -> Backend { + Backend { + states: RwLock::new(HashMap::new()), + blockchain: Blockchain::new(), + } + } +} + +impl backend::Backend for Backend +where + Block: BlockT, + H: Hasher, + H::Out: HeapSizeOf, + C: NodeCodec + Send + Sync, +{ + type BlockImportOperation = BlockImportOperation; + type Blockchain = Blockchain; + type State = InMemory; + + fn begin_operation(&self, block: BlockId) -> error::Result { + let state = match block { + BlockId::Hash(ref h) if h.clone() == Default::default() => Self::State::default(), + _ => self.state_at(block)?, + }; + + Ok(BlockImportOperation { + pending_block: None, + pending_authorities: None, + old_state: state, + new_state: None, + }) + } + + fn commit_operation(&self, operation: Self::BlockImportOperation) -> error::Result<()> { + if let Some(pending_block) = operation.pending_block { + let old_state = &operation.old_state; + let (header, body, justification) = pending_block.block.into_inner(); + let hash = header.hash(); + let parent_hash = *header.parent_hash(); + + self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone())); + self.blockchain.insert(hash, header, justification, body, pending_block.is_best); + // dumb implementation - store value for each block + if pending_block.is_best { + self.blockchain.cache.insert(parent_hash, operation.pending_authorities); + } + } + Ok(()) + } + + fn blockchain(&self) -> &Self::Blockchain { + &self.blockchain + } + + fn state_at(&self, block: BlockId) -> error::Result { + match self.blockchain.id(block).and_then(|id| self.states.read().get(&id).cloned()) { + Some(state) => Ok(state), + None => Err(error::ErrorKind::UnknownBlock(format!("{}", block)).into()), + } + } + + fn revert(&self, _n: NumberFor) -> error::Result> { + Ok(As::sa(0)) + } +} + +impl backend::LocalBackend for Backend +where + Block: BlockT, + H: Hasher, + H::Out: HeapSizeOf, + C: NodeCodec + Send + Sync, +{} + +impl Cache { + fn insert(&self, at: Block::Hash, authorities: Option>) { + self.authorities_at.write().insert(at, authorities); + } +} + +impl blockchain::Cache for Cache { + fn authorities_at(&self, block: BlockId) -> Option> { + let hash = match block { + BlockId::Hash(hash) => hash, + BlockId::Number(number) => self.storage.read().hashes.get(&number).cloned()?, + }; + + self.authorities_at.read().get(&hash).cloned().unwrap_or(None) + } +} + +/// Insert authorities entry into in-memory blockchain cache. Extracted as a separate function to use it in tests. +pub fn cache_authorities_at( + blockchain: &Blockchain, + at: Block::Hash, + authorities: Option> +) { + blockchain.cache.insert(at, authorities); +} diff --git a/core/client/src/lib.rs b/core/client/src/lib.rs new file mode 100644 index 00000000000..fd30eaae72e --- /dev/null +++ b/core/client/src/lib.rs @@ -0,0 +1,69 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate Client and associated logic. + +#![warn(missing_docs)] +#![recursion_limit="128"] + +extern crate substrate_bft as bft; +extern crate substrate_codec as codec; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_support as runtime_support; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_state_machine as state_machine; +#[cfg(test)] extern crate substrate_keyring as keyring; +#[cfg(test)] extern crate substrate_test_client as test_client; +#[macro_use] extern crate substrate_telemetry; +#[macro_use] extern crate slog; // needed until we can reexport `slog_info` from `substrate_telemetry` + +extern crate fnv; +extern crate futures; +extern crate parking_lot; +extern crate triehash; +extern crate patricia_trie; +extern crate hashdb; +extern crate rlp; +extern crate heapsize; + +#[macro_use] extern crate error_chain; +#[macro_use] extern crate log; +#[cfg_attr(test, macro_use)] extern crate substrate_executor as executor; +#[cfg(test)] #[macro_use] extern crate hex_literal; + +pub mod error; +pub mod blockchain; +pub mod backend; +pub mod cht; +pub mod in_mem; +pub mod genesis; +pub mod block_builder; +pub mod light; +mod call_executor; +mod client; +mod notifications; + +pub use blockchain::Info as ChainInfo; +pub use call_executor::{CallResult, CallExecutor, LocalCallExecutor}; +pub use client::{ + new_in_mem, + BlockBody, BlockStatus, BlockOrigin, BlockchainEventStream, BlockchainEvents, + Client, ClientInfo, ChainHead, + ImportResult, JustifiedHeader, +}; +pub use notifications::{StorageEventStream, StorageChangeSet}; +pub use state_machine::ExecutionStrategy; diff --git a/core/client/src/light/backend.rs b/core/client/src/light/backend.rs new file mode 100644 index 00000000000..54291521bb0 --- /dev/null +++ b/core/client/src/light/backend.rs @@ -0,0 +1,229 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Light client backend. Only stores headers and justifications of blocks. +//! Everything else is requested from full nodes on demand. + +use std::sync::{Arc, Weak}; +use futures::{Future, IntoFuture}; +use parking_lot::RwLock; + +use primitives::AuthorityId; +use runtime_primitives::{bft::Justification, generic::BlockId}; +use runtime_primitives::traits::{Block as BlockT, NumberFor}; +use state_machine::{ + Backend as StateBackend, + TrieBackend as StateTrieBackend, + TryIntoTrieBackend as TryIntoStateTrieBackend +}; + +use backend::{Backend as ClientBackend, BlockImportOperation, RemoteBackend}; +use blockchain::HeaderBackend as BlockchainHeaderBackend; +use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult}; +use light::blockchain::{Blockchain, Storage as BlockchainStorage}; +use light::fetcher::{Fetcher, RemoteReadRequest}; +use patricia_trie::NodeCodec; +use hashdb::Hasher; + +/// Light client backend. +pub struct Backend { + blockchain: Arc>, +} + +/// Light block (header and justification) import operation. +pub struct ImportOperation { + is_new_best: bool, + header: Option, + authorities: Option>, + _phantom: ::std::marker::PhantomData<(S, F)>, +} + +/// On-demand state. +pub struct OnDemandState { + fetcher: Weak, + blockchain: Weak>, + block: Block::Hash, + cached_header: RwLock>, +} + +impl Backend { + /// Create new light backend. + pub fn new(blockchain: Arc>) -> Self { + Self { blockchain } + } + + /// Get shared blockchain reference. + pub fn blockchain(&self) -> &Arc> { + &self.blockchain + } +} + +impl ClientBackend for Backend where + Block: BlockT, + S: BlockchainStorage, + F: Fetcher, + H: Hasher, + C: NodeCodec, +{ + type BlockImportOperation = ImportOperation; + type Blockchain = Blockchain; + type State = OnDemandState; + + fn begin_operation(&self, _block: BlockId) -> ClientResult { + Ok(ImportOperation { + is_new_best: false, + header: None, + authorities: None, + _phantom: Default::default(), + }) + } + + fn commit_operation(&self, operation: Self::BlockImportOperation) -> ClientResult<()> { + let header = operation.header.expect("commit is called after set_block_data; set_block_data sets header; qed"); + self.blockchain.storage().import_header(operation.is_new_best, header, operation.authorities) + } + + fn blockchain(&self) -> &Blockchain { + &self.blockchain + } + + fn state_at(&self, block: BlockId) -> ClientResult { + let block_hash = match block { + BlockId::Hash(h) => Some(h), + BlockId::Number(n) => self.blockchain.hash(n).unwrap_or_default(), + }; + + Ok(OnDemandState { + fetcher: self.blockchain.fetcher(), + blockchain: Arc::downgrade(&self.blockchain), + block: block_hash.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", block)))?, + cached_header: RwLock::new(None), + }) + } + + fn revert(&self, _n: NumberFor) -> ClientResult> { + unimplemented!() + } +} + +impl RemoteBackend for Backend +where + Block: BlockT, + S: BlockchainStorage, + F: Fetcher, + H: Hasher, + C: NodeCodec, +{} + +impl BlockImportOperation for ImportOperation +where + Block: BlockT, + F: Fetcher, + S: BlockchainStorage, + H: Hasher, + C: NodeCodec, +{ + type State = OnDemandState; + + fn state(&self) -> ClientResult> { + // None means 'locally-stateless' backend + Ok(None) + } + + fn set_block_data( + &mut self, + header: Block::Header, + _body: Option>, + _justification: Option>, + is_new_best: bool + ) -> ClientResult<()> { + self.is_new_best = is_new_best; + self.header = Some(header); + Ok(()) + } + + fn update_authorities(&mut self, authorities: Vec) { + self.authorities = Some(authorities); + } + + fn update_storage(&mut self, _update: >::Transaction) -> ClientResult<()> { + // we're not storing anything locally => ignore changes + Ok(()) + } + + fn reset_storage, Vec)>>(&mut self, _iter: I) -> ClientResult<()> { + // we're not storing anything locally => ignore changes + Ok(()) + } +} + +impl StateBackend for OnDemandState + where + Block: BlockT, + S: BlockchainStorage, + F: Fetcher, + H: Hasher, + C: NodeCodec, +{ + type Error = ClientError; + type Transaction = (); + + fn storage(&self, key: &[u8]) -> ClientResult>> { + let mut header = self.cached_header.read().clone(); + if header.is_none() { + let cached_header = self.blockchain.upgrade() + .ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", self.block)).into()) + .and_then(|blockchain| blockchain.expect_header(BlockId::Hash(self.block)))?; + header = Some(cached_header.clone()); + *self.cached_header.write() = Some(cached_header); + } + + self.fetcher.upgrade().ok_or(ClientErrorKind::NotAvailableOnLightClient)? + .remote_read(RemoteReadRequest { + block: self.block, + header: header.expect("if block above guarantees that header is_some(); qed"), + key: key.to_vec(), + retry_count: None, + }) + .into_future().wait() + } + + fn for_keys_with_prefix(&self, _prefix: &[u8], _action: A) { + // whole state is not available on light node + } + + fn storage_root(&self, _delta: I) -> (H::Out, Self::Transaction) + where I: IntoIterator, Option>)> { + (H::Out::default(), ()) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + // whole state is not available on light node + Vec::new() + } +} + +impl TryIntoStateTrieBackend for OnDemandState +where + Block: BlockT, + F: Fetcher, + H: Hasher, + C: NodeCodec, +{ + fn try_into_trie_backend(self) -> Option> { + None + } +} diff --git a/core/client/src/light/blockchain.rs b/core/client/src/light/blockchain.rs new file mode 100644 index 00000000000..11f3070aa6d --- /dev/null +++ b/core/client/src/light/blockchain.rs @@ -0,0 +1,142 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Light client blockchin backend. Only stores headers and justifications of recent +//! blocks. CHT roots are stored for headers of ancient blocks. + +use std::sync::Weak; +use futures::{Future, IntoFuture}; +use parking_lot::Mutex; + +use primitives::AuthorityId; +use runtime_primitives::{bft::Justification, generic::BlockId}; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}; + +use blockchain::{Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache, + HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo}; +use cht; +use error::{ErrorKind as ClientErrorKind, Result as ClientResult}; +use light::fetcher::{Fetcher, RemoteHeaderRequest}; + +/// Light client blockchain storage. +pub trait Storage: BlockchainHeaderBackend { + /// Store new header. + fn import_header( + &self, + is_new_best: bool, + header: Block::Header, + authorities: Option> + ) -> ClientResult<()>; + + /// Get CHT root for given block. Fails if the block is not pruned (not a part of any CHT). + fn cht_root(&self, cht_size: u64, block: NumberFor) -> ClientResult; + + /// Get storage cache. + fn cache(&self) -> Option<&BlockchainCache>; +} + +/// Light client blockchain. +pub struct Blockchain { + fetcher: Mutex>, + storage: S, +} + +impl Blockchain { + /// Create new light blockchain backed with given storage. + pub fn new(storage: S) -> Self { + Self { + fetcher: Mutex::new(Default::default()), + storage, + } + } + + /// Sets fetcher reference. + pub fn set_fetcher(&self, fetcher: Weak) { + *self.fetcher.lock() = fetcher; + } + + /// Get fetcher weak reference. + pub fn fetcher(&self) -> Weak { + self.fetcher.lock().clone() + } + + /// Get storage reference. + pub fn storage(&self) -> &S { + &self.storage + } +} + +impl BlockchainHeaderBackend for Blockchain where Block: BlockT, S: Storage, F: Fetcher { + fn header(&self, id: BlockId) -> ClientResult> { + match self.storage.header(id)? { + Some(header) => Ok(Some(header)), + None => { + let number = match id { + BlockId::Hash(hash) => match self.storage.number(hash)? { + Some(number) => number, + None => return Ok(None), + }, + BlockId::Number(number) => number, + }; + + // if the header is from future or genesis (we never prune genesis) => return + if number.is_zero() || self.storage.status(BlockId::Number(number))? != BlockStatus::InChain { + return Ok(None); + } + + self.fetcher().upgrade().ok_or(ClientErrorKind::NotAvailableOnLightClient)? + .remote_header(RemoteHeaderRequest { + cht_root: self.storage.cht_root(cht::SIZE, number)?, + block: number, + retry_count: None, + }) + .into_future().wait() + .map(Some) + } + } + } + + fn info(&self) -> ClientResult> { + self.storage.info() + } + + fn status(&self, id: BlockId) -> ClientResult { + self.storage.status(id) + } + + fn number(&self, hash: Block::Hash) -> ClientResult>> { + self.storage.number(hash) + } + + fn hash(&self, number: <::Header as HeaderT>::Number) -> ClientResult> { + self.storage.hash(number) + } +} + +impl BlockchainBackend for Blockchain where Block: BlockT, S: Storage, F: Fetcher { + fn body(&self, _id: BlockId) -> ClientResult>> { + // TODO [light]: fetch from remote node + Ok(None) + } + + fn justification(&self, _id: BlockId) -> ClientResult>> { + Ok(None) + } + + fn cache(&self) -> Option<&BlockchainCache> { + self.storage.cache() + } +} diff --git a/core/client/src/light/call_executor.rs b/core/client/src/light/call_executor.rs new file mode 100644 index 00000000000..354a2dbe33e --- /dev/null +++ b/core/client/src/light/call_executor.rs @@ -0,0 +1,191 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Light client call exector. Executes methods on remote full nodes, fetching +//! execution proof and checking it locally. + +use std::sync::Arc; +use futures::{IntoFuture, Future}; + +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT}; +use state_machine::{Backend as StateBackend, CodeExecutor, OverlayedChanges, + execution_proof_check, ExecutionManager}; +use primitives::H256; +use patricia_trie::NodeCodec; +use hashdb::Hasher; +use rlp::Encodable; + +use blockchain::Backend as ChainBackend; +use call_executor::{CallExecutor, CallResult}; +use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult}; +use light::fetcher::{Fetcher, RemoteCallRequest}; +use executor::RuntimeVersion; +use codec::Decode; +use heapsize::HeapSizeOf; +use std::marker::PhantomData; + +/// Call executor that executes methods on remote node, querying execution proof +/// and checking proof by re-executing locally. +pub struct RemoteCallExecutor { + blockchain: Arc, + fetcher: Arc, + _hasher: PhantomData, + _codec: PhantomData, +} + +impl Clone for RemoteCallExecutor { + fn clone(&self) -> Self { + RemoteCallExecutor { + blockchain: self.blockchain.clone(), + fetcher: self.fetcher.clone(), + _hasher: Default::default(), + _codec: Default::default(), + } + } +} + +impl RemoteCallExecutor { + /// Creates new instance of remote call executor. + pub fn new(blockchain: Arc, fetcher: Arc) -> Self { + RemoteCallExecutor { blockchain, fetcher, _hasher: PhantomData, _codec: PhantomData } + } +} + +impl CallExecutor for RemoteCallExecutor +where + Block: BlockT, + B: ChainBackend, + F: Fetcher, + H: Hasher, + H::Out: Ord + Encodable, + C: NodeCodec +{ + type Error = ClientError; + + fn call(&self, id: &BlockId, method: &str, call_data: &[u8]) -> ClientResult { + let block_hash = match *id { + BlockId::Hash(hash) => hash, + BlockId::Number(number) => self.blockchain.hash(number)? + .ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", number)))?, + }; + let block_header = self.blockchain.expect_header(id.clone())?; + + self.fetcher.remote_call(RemoteCallRequest { + block: block_hash, + header: block_header, + method: method.into(), + call_data: call_data.to_vec(), + retry_count: None, + }).into_future().wait() + } + + fn runtime_version(&self, id: &BlockId) -> ClientResult { + let call_result = self.call(id, "version", &[])?; + RuntimeVersion::decode(&mut call_result.return_data.as_slice()) + .ok_or_else(|| ClientErrorKind::VersionInvalid.into()) + } + + fn call_at_state< + S: StateBackend, + FF: FnOnce(Result, Self::Error>, Result, Self::Error>) -> Result, Self::Error> + >(&self, + _state: &S, + _changes: &mut OverlayedChanges, + _method: &str, + _call_data: &[u8], + _m: ExecutionManager + ) -> ClientResult<(Vec, S::Transaction)> { + Err(ClientErrorKind::NotAvailableOnLightClient.into()) + } + + fn prove_at_state>( + &self, + _state: S, + _changes: &mut OverlayedChanges, + _method: &str, + _call_data: &[u8] + ) -> ClientResult<(Vec, Vec>)> { + Err(ClientErrorKind::NotAvailableOnLightClient.into()) + } + + fn native_runtime_version(&self) -> Option { + None + } +} + +/// Check remote execution proof using given backend. +pub fn check_execution_proof( + executor: &E, + request: &RemoteCallRequest

, + remote_proof: Vec> +) -> ClientResult + where + Header: HeaderT, + E: CodeExecutor, + H: Hasher, + H::Out: Ord + Encodable + HeapSizeOf + From, + C: NodeCodec, +{ + let local_state_root = request.header.state_root(); + + let mut changes = OverlayedChanges::default(); + let (local_result, _) = execution_proof_check::( + H256::from_slice(local_state_root.as_ref()).into(), + remote_proof, + &mut changes, + executor, + &request.method, + &request.call_data)?; + + Ok(CallResult { return_data: local_result, changes }) +} + +#[cfg(test)] +mod tests { + use test_client; + use executor::NativeExecutionDispatch; + use super::*; + use primitives::RlpCodec; + + #[test] + fn execution_proof_is_generated_and_checked() { + // prepare remote client + let remote_client = test_client::new(); + let remote_block_id = BlockId::Number(0); + let remote_block_storage_root = remote_client.state_at(&remote_block_id) + .unwrap().storage_root(::std::iter::empty()).0; + + // 'fetch' execution proof from remote node + let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap().1; + + // check remote execution proof locally + let local_executor = test_client::LocalExecutor::new(); + check_execution_proof::<_, _, _, RlpCodec>(&local_executor, &RemoteCallRequest { + block: test_client::runtime::Hash::default(), + header: test_client::runtime::Header { + state_root: remote_block_storage_root.into(), + parent_hash: Default::default(), + number: 0, + extrinsics_root: Default::default(), + digest: Default::default(), + }, + method: "authorities".into(), + call_data: vec![], + retry_count: None, + }, remote_execution_proof).unwrap(); + } +} diff --git a/core/client/src/light/fetcher.rs b/core/client/src/light/fetcher.rs new file mode 100644 index 00000000000..02090a4bc60 --- /dev/null +++ b/core/client/src/light/fetcher.rs @@ -0,0 +1,308 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Light client data fetcher. Fetches requested data from remote full nodes. + +use futures::IntoFuture; + +use primitives::H256; +use hashdb::Hasher; +use patricia_trie::NodeCodec; +use rlp::Encodable; +use heapsize::HeapSizeOf; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT}; +use state_machine::{CodeExecutor, read_proof_check}; +use std::marker::PhantomData; + +use call_executor::CallResult; +use cht; +use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult}; +use light::call_executor::check_execution_proof; + +/// Remote call request. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct RemoteCallRequest { + /// Call at state of given block. + pub block: Header::Hash, + /// Head of block at which call is perormed. + pub header: Header, + /// Method to call. + pub method: String, + /// Call data. + pub call_data: Vec, + /// Number of times to retry request. None means that default RETRY_COUNT is used. + pub retry_count: Option, +} + +/// Remote canonical header request. +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +pub struct RemoteHeaderRequest { + /// The root of CHT this block is included in. + pub cht_root: Header::Hash, + /// Number of the header to query. + pub block: Header::Number, + /// Number of times to retry request. None means that default RETRY_COUNT is used. + pub retry_count: Option, +} + +/// Remote storage read request. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct RemoteReadRequest { + /// Read at state of given block. + pub block: Header::Hash, + /// Head of block at which read is perormed. + pub header: Header, + /// Storage key to read. + pub key: Vec, + /// Number of times to retry request. None means that default RETRY_COUNT is used. + pub retry_count: Option, +} + +/// Light client data fetcher. Implementations of this trait must check if remote data +/// is correct (see FetchedDataChecker) and return already checked data. +pub trait Fetcher: Send + Sync { + /// Remote header future. + type RemoteHeaderResult: IntoFuture; + /// Remote storage read future. + type RemoteReadResult: IntoFuture>, Error=ClientError>; + /// Remote call result future. + type RemoteCallResult: IntoFuture; + + /// Fetch remote header. + fn remote_header(&self, request: RemoteHeaderRequest) -> Self::RemoteHeaderResult; + /// Fetch remote storage value. + fn remote_read(&self, request: RemoteReadRequest) -> Self::RemoteReadResult; + /// Fetch remote call result. + fn remote_call(&self, request: RemoteCallRequest) -> Self::RemoteCallResult; +} + +/// Light client remote data checker. +/// +/// Implementations of this trait should not use any blockchain data except that is +/// passed to its methods. +pub trait FetchChecker: Send + Sync { + /// Check remote header proof. + fn check_header_proof( + &self, + request: &RemoteHeaderRequest, + header: Option, + remote_proof: Vec> + ) -> ClientResult; + /// Check remote storage read proof. + fn check_read_proof( + &self, + request: &RemoteReadRequest, + remote_proof: Vec> + ) -> ClientResult>>; + /// Check remote method execution proof. + fn check_execution_proof( + &self, + request: &RemoteCallRequest, + remote_proof: Vec> + ) -> ClientResult; +} + +/// Remote data checker. +pub struct LightDataChecker { + executor: E, + _hasher: PhantomData, + _codec: PhantomData, +} + +impl LightDataChecker { + /// Create new light data checker. + pub fn new(executor: E) -> Self { + Self { + executor, _hasher: PhantomData, _codec: PhantomData + } + } +} + +impl FetchChecker for LightDataChecker + where + Block: BlockT, + Block::Hash: Into + From, + E: CodeExecutor, + H: Hasher, + C: NodeCodec + Sync + Send, + H::Out: Ord + Encodable + HeapSizeOf + From + From, +{ + fn check_header_proof( + &self, + request: &RemoteHeaderRequest, + remote_header: Option, + remote_proof: Vec> + ) -> ClientResult { + let remote_header = remote_header.ok_or_else(|| + ClientError::from(ClientErrorKind::InvalidHeaderProof))?; + let remote_header_hash = remote_header.hash(); + cht::check_proof::( + request.cht_root, + request.block, + remote_header_hash, + remote_proof) + .map(|_| remote_header) + } + + fn check_read_proof( + &self, + request: &RemoteReadRequest, + remote_proof: Vec> + ) -> ClientResult>> { + let local_state_root = request.header.state_root().clone(); + read_proof_check::(local_state_root.into(), remote_proof, &request.key).map_err(Into::into) + } + + fn check_execution_proof( + &self, + request: &RemoteCallRequest, + remote_proof: Vec> + ) -> ClientResult { + check_execution_proof::<_, _, H, C>(&self.executor, request, remote_proof) + } +} + +#[cfg(test)] +pub mod tests { + use futures::future::{ok, err, FutureResult}; + use parking_lot::Mutex; + use call_executor::CallResult; + use executor::{self, NativeExecutionDispatch}; + use error::Error as ClientError; + use test_client::{self, TestClient, runtime::{Hash, Block, Header}}; + use test_client::client::BlockOrigin; + use in_mem::{Blockchain as InMemoryBlockchain}; + use light::fetcher::{Fetcher, FetchChecker, LightDataChecker, + RemoteCallRequest, RemoteHeaderRequest}; + use primitives::{Blake2Hasher, RlpCodec}; + use runtime_primitives::generic::BlockId; + use state_machine::Backend; + use super::*; + + pub type OkCallFetcher = Mutex; + + impl Fetcher for OkCallFetcher { + type RemoteHeaderResult = FutureResult; + type RemoteReadResult = FutureResult>, ClientError>; + type RemoteCallResult = FutureResult; + + fn remote_header(&self, _request: RemoteHeaderRequest
) -> Self::RemoteHeaderResult { + err("Not implemented on test node".into()) + } + + fn remote_read(&self, _request: RemoteReadRequest
) -> Self::RemoteReadResult { + err("Not implemented on test node".into()) + } + + fn remote_call(&self, _request: RemoteCallRequest
) -> Self::RemoteCallResult { + ok((*self.lock()).clone()) + } + } + + fn prepare_for_read_proof_check() -> ( + LightDataChecker, Blake2Hasher, RlpCodec>, + Header, Vec>, usize) + { + // prepare remote client + let remote_client = test_client::new(); + let remote_block_id = BlockId::Number(0); + let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); + let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); + remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap().storage_root(::std::iter::empty()).0.into(); + + // 'fetch' read proof from remote node + let authorities_len = remote_client.authorities_at(&remote_block_id).unwrap().len(); + let remote_read_proof = remote_client.read_proof(&remote_block_id, b":auth:len").unwrap(); + + // check remote read proof locally + let local_storage = InMemoryBlockchain::::new(); + local_storage.insert(remote_block_hash, remote_block_header.clone(), None, None, true); + let local_executor = test_client::LocalExecutor::new(); + let local_checker = LightDataChecker::new(local_executor); + (local_checker, remote_block_header, remote_read_proof, authorities_len) + } + + fn prepare_for_header_proof_check(insert_cht: bool) -> ( + LightDataChecker, Blake2Hasher, RlpCodec>, + Hash, Header, Vec>) + { + // prepare remote client + let remote_client = test_client::new(); + let mut local_headers_hashes = Vec::new(); + for i in 0..4 { + let builder = remote_client.new_block().unwrap(); + remote_client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + local_headers_hashes.push(remote_client.block_hash(i + 1).unwrap()); + } + + // 'fetch' header proof from remote node + let remote_block_id = BlockId::Number(1); + let (remote_block_header, remote_header_proof) = remote_client.header_proof_with_cht_size(&remote_block_id, 4).unwrap(); + + // check remote read proof locally + let local_storage = InMemoryBlockchain::::new(); + let local_cht_root = cht::compute_root::(4, 0, local_headers_hashes.into_iter()).unwrap(); + if insert_cht { + local_storage.insert_cht_root(1, local_cht_root); + } + let local_executor = test_client::LocalExecutor::new(); + let local_checker = LightDataChecker::new(local_executor); + (local_checker, local_cht_root, remote_block_header, remote_header_proof) + } + + #[test] + fn storage_read_proof_is_generated_and_checked() { + let (local_checker, remote_block_header, remote_read_proof, authorities_len) = prepare_for_read_proof_check(); + assert_eq!((&local_checker as &FetchChecker).check_read_proof(&RemoteReadRequest::
{ + block: remote_block_header.hash(), + header: remote_block_header, + key: b":auth:len".to_vec(), + retry_count: None, + }, remote_read_proof).unwrap().unwrap()[0], authorities_len as u8); + } + + #[test] + fn header_proof_is_generated_and_checked() { + let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + assert_eq!((&local_checker as &FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ + cht_root: local_cht_root, + block: 1, + retry_count: None, + }, Some(remote_block_header.clone()), remote_header_proof).unwrap(), remote_block_header); + } + + #[test] + fn check_header_proof_fails_if_cht_root_is_invalid() { + let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + remote_block_header.number = 100; + assert!((&local_checker as &FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ + cht_root: Default::default(), + block: 1, + retry_count: None, + }, Some(remote_block_header.clone()), remote_header_proof).is_err()); + } + + #[test] + fn check_header_proof_fails_if_invalid_header_provided() { + let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + remote_block_header.number = 100; + assert!((&local_checker as &FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ + cht_root: local_cht_root, + block: 1, + retry_count: None, + }, Some(remote_block_header.clone()), remote_header_proof).is_err()); + } +} diff --git a/core/client/src/light/mod.rs b/core/client/src/light/mod.rs new file mode 100644 index 00000000000..d16814ae2fc --- /dev/null +++ b/core/client/src/light/mod.rs @@ -0,0 +1,77 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Light client components. + +pub mod backend; +pub mod blockchain; +pub mod call_executor; +pub mod fetcher; + +use std::sync::Arc; + +use primitives::{Blake2Hasher, RlpCodec}; +use runtime_primitives::BuildStorage; +use runtime_primitives::traits::Block as BlockT; +use state_machine::{CodeExecutor, ExecutionStrategy}; + +use client::Client; +use error::Result as ClientResult; +use light::backend::Backend; +use light::blockchain::{Blockchain, Storage as BlockchainStorage}; +use light::call_executor::RemoteCallExecutor; +use light::fetcher::{Fetcher, LightDataChecker}; +use hashdb::Hasher; +use patricia_trie::NodeCodec; + +/// Create an instance of light client blockchain backend. +pub fn new_light_blockchain, F>(storage: S) -> Arc> { + Arc::new(Blockchain::new(storage)) +} + +/// Create an instance of light client backend. +pub fn new_light_backend, F: Fetcher>(blockchain: Arc>, fetcher: Arc) -> Arc> { + blockchain.set_fetcher(Arc::downgrade(&fetcher)); + Arc::new(Backend::new(blockchain)) +} + +/// Create an instance of light client. +pub fn new_light( + backend: Arc>, + fetcher: Arc, + genesis_storage: GS, +) -> ClientResult, RemoteCallExecutor, F, Blake2Hasher, RlpCodec>, B>> + where + B: BlockT, + S: BlockchainStorage, + F: Fetcher, + GS: BuildStorage, +{ + let executor = RemoteCallExecutor::new(backend.blockchain().clone(), fetcher); + Client::new(backend, executor, genesis_storage, ExecutionStrategy::NativeWhenPossible) +} + +/// Create an instance of fetch data checker. +pub fn new_fetch_checker( + executor: E, +) -> LightDataChecker + where + E: CodeExecutor, + H: Hasher, + C: NodeCodec, +{ + LightDataChecker::new(executor) +} diff --git a/core/client/src/notifications.rs b/core/client/src/notifications.rs new file mode 100644 index 00000000000..390513d37a8 --- /dev/null +++ b/core/client/src/notifications.rs @@ -0,0 +1,289 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Storage notifications + +use std::{ + collections::{HashSet, HashMap}, + sync::Arc, +}; + +use fnv::{FnvHashSet, FnvHashMap}; +use futures::sync::mpsc; +use primitives::storage::{StorageKey, StorageData}; +use runtime_primitives::traits::Block as BlockT; + +/// Storage change set +#[derive(Debug)] +pub struct StorageChangeSet { + changes: Arc)>>, + filter: Option>, +} + +impl StorageChangeSet { + /// Convert the change set into iterator over storage items. + pub fn iter<'a>(&'a self) -> impl Iterator)> + 'a { + self.changes + .iter() + .filter(move |&(key, _)| match self.filter { + Some(ref filter) => filter.contains(key), + None => true, + }) + } +} + +/// Type that implements `futures::Stream` of storage change events. +pub type StorageEventStream = mpsc::UnboundedReceiver<(H, StorageChangeSet)>; + +type SubscriberId = u64; + +/// Manages storage listeners. +#[derive(Debug)] +pub struct StorageNotifications { + next_id: SubscriberId, + wildcard_listeners: FnvHashSet, + listeners: HashMap>, + sinks: FnvHashMap, + Option>, + )>, +} + +impl Default for StorageNotifications { + fn default() -> Self { + StorageNotifications { + next_id: Default::default(), + wildcard_listeners: Default::default(), + listeners: Default::default(), + sinks: Default::default(), + } + } +} + +impl StorageNotifications { + /// Trigger notification to all listeners. + /// + /// Note the changes are going to be filtered by listener's filter key. + /// In fact no event might be sent if clients are not interested in the changes. + pub fn trigger(&mut self, hash: &Block::Hash, changeset: impl Iterator, Option>)>) { + let has_wildcard = !self.wildcard_listeners.is_empty(); + + // early exit if no listeners + if !has_wildcard && self.listeners.is_empty() { + return; + } + + let mut subscribers = self.wildcard_listeners.clone(); + let mut changes = Vec::new(); + + // Collect subscribers and changes + for (k, v) in changeset { + let k = StorageKey(k); + let listeners = self.listeners.get(&k); + + if let Some(ref listeners) = listeners { + subscribers.extend(listeners.iter()); + } + + if has_wildcard || listeners.is_some() { + changes.push((k, v.map(StorageData))); + } + } + + // Don't send empty notifications + if changes.is_empty() { + return; + } + + let changes = Arc::new(changes); + // Trigger the events + for subscriber in subscribers { + let should_remove = { + let &(ref sink, ref filter) = self.sinks.get(&subscriber) + .expect("subscribers returned from self.listeners are always in self.sinks; qed"); + sink.unbounded_send((hash.clone(), StorageChangeSet { + changes: changes.clone(), + filter: filter.clone(), + })).is_err() + }; + + if should_remove { + self.remove_subscriber(subscriber); + } + } + } + + fn remove_subscriber(&mut self, subscriber: SubscriberId) { + if let Some((_, filters)) = self.sinks.remove(&subscriber) { + match filters { + None => { + self.wildcard_listeners.remove(&subscriber); + }, + Some(filters) => { + for key in filters { + let remove_key = match self.listeners.get_mut(&key) { + Some(ref mut set) => { + set.remove(&subscriber); + set.is_empty() + }, + None => false, + }; + + if remove_key { + self.listeners.remove(&key); + } + } + }, + } + } + } + + /// Start listening for particular storage keys. + pub fn listen(&mut self, filter_keys: Option<&[StorageKey]>) -> StorageEventStream { + self.next_id += 1; + + // add subscriber for every key + let keys = match filter_keys { + None => { + self.wildcard_listeners.insert(self.next_id); + None + }, + Some(keys) => Some(keys.iter().map(|key| { + self.listeners + .entry(key.clone()) + .or_insert_with(Default::default) + .insert(self.next_id); + key.clone() + }).collect()) + }; + + // insert sink + let (tx, rx) = mpsc::unbounded(); + self.sinks.insert(self.next_id, (tx, keys)); + rx + } +} + +#[cfg(test)] +mod tests { + use runtime_primitives::testing::{H256 as Hash, Block as RawBlock}; + use super::*; + use futures::Stream; + + + #[cfg(test)] + impl From)>> for StorageChangeSet { + fn from(changes: Vec<(StorageKey, Option)>) -> Self { + StorageChangeSet { + changes: Arc::new(changes), + filter: None, + } + } + } + + #[cfg(test)] + impl PartialEq for StorageChangeSet { + fn eq(&self, other: &Self) -> bool { + self.iter().eq(other.iter()) + } + } + + type Block = RawBlock; + + #[test] + fn triggering_change_should_notify_wildcard_listeners() { + // given + let mut notifications = StorageNotifications::::default(); + let mut recv = notifications.listen(None).wait(); + + // when + let changeset = vec![ + (vec![2], Some(vec![3])), + (vec![3], None), + ]; + notifications.trigger(&1.into(), changeset.into_iter()); + + // then + assert_eq!(recv.next().unwrap(), Ok((1.into(), vec![ + (StorageKey(vec![2]), Some(StorageData(vec![3]))), + (StorageKey(vec![3]), None), + ].into()))); + } + + #[test] + fn should_only_notify_interested_listeners() { + // given + let mut notifications = StorageNotifications::::default(); + let mut recv1 = notifications.listen(Some(&[StorageKey(vec![1])])).wait(); + let mut recv2 = notifications.listen(Some(&[StorageKey(vec![2])])).wait(); + + // when + let changeset = vec![ + (vec![2], Some(vec![3])), + (vec![1], None), + ]; + notifications.trigger(&1.into(), changeset.into_iter()); + + // then + assert_eq!(recv1.next().unwrap(), Ok((1.into(), vec![ + (StorageKey(vec![1]), None), + ].into()))); + assert_eq!(recv2.next().unwrap(), Ok((1.into(), vec![ + (StorageKey(vec![2]), Some(StorageData(vec![3]))), + ].into()))); + } + + #[test] + fn should_cleanup_subscribers_if_dropped() { + // given + let mut notifications = StorageNotifications::::default(); + { + let _recv1 = notifications.listen(Some(&[StorageKey(vec![1])])).wait(); + let _recv2 = notifications.listen(Some(&[StorageKey(vec![2])])).wait(); + let _recv3 = notifications.listen(None).wait(); + assert_eq!(notifications.listeners.len(), 2); + assert_eq!(notifications.wildcard_listeners.len(), 1); + } + + // when + let changeset = vec![ + (vec![2], Some(vec![3])), + (vec![1], None), + ]; + notifications.trigger(&1.into(), changeset.into_iter()); + + // then + assert_eq!(notifications.listeners.len(), 0); + assert_eq!(notifications.wildcard_listeners.len(), 0); + } + + #[test] + fn should_not_send_empty_notifications() { + // given + let mut recv = { + let mut notifications = StorageNotifications::::default(); + let recv = notifications.listen(None).wait(); + + // when + let changeset = vec![]; + notifications.trigger(&1.into(), changeset.into_iter()); + recv + }; + + // then + assert_eq!(recv.next(), None); + } +} diff --git a/core/codec/Cargo.toml b/core/codec/Cargo.toml new file mode 100644 index 00000000000..a228686e90b --- /dev/null +++ b/core/codec/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "substrate-codec" +description = "Serialization and deserialization codec for runtime values" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +arrayvec = { version = "0.4", default_features = false } + +[features] +default = ["std"] +std = [] diff --git a/core/codec/README.adoc b/core/codec/README.adoc new file mode 100644 index 00000000000..12d3953789a --- /dev/null +++ b/core/codec/README.adoc @@ -0,0 +1,13 @@ + += Codec + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/codec/derive/Cargo.toml b/core/codec/derive/Cargo.toml new file mode 100644 index 00000000000..d5ccd9fdd57 --- /dev/null +++ b/core/codec/derive/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "substrate-codec-derive" +description = "Serialization and deserialization derive macro" +version = "0.1.0" +authors = ["Parity Technologies "] + +[lib] +proc-macro = true + +[dependencies] +syn = "0.14" +quote = "0.6" +proc-macro2 = "0.4" + +[dev-dependencies] +substrate-codec = { path = "../" } + +[features] +default = ["std"] +std = [] diff --git a/core/codec/derive/src/decode.rs b/core/codec/derive/src/decode.rs new file mode 100644 index 00000000000..0d33a7c350d --- /dev/null +++ b/core/codec/derive/src/decode.rs @@ -0,0 +1,111 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use proc_macro2::{Span, TokenStream, Ident}; +use syn::{ + Data, Fields, + spanned::Spanned, +}; + +pub fn quote(data: &Data, type_name: &Ident, input: &TokenStream) -> TokenStream { + let call_site = Span::call_site(); + match *data { + Data::Struct(ref data) => match data.fields { + Fields::Named(_) | Fields::Unnamed(_) => create_instance( + call_site, + quote! { #type_name }, + input, + &data.fields, + ), + Fields::Unit => { + quote_spanned! {call_site => + drop(#input); + Some(#type_name) + } + }, + }, + Data::Enum(ref data) => { + assert!(data.variants.len() < 256, "Currently only enums with at most 256 variants are encodable."); + + let recurse = data.variants.iter().enumerate().map(|(i, v)| { + let name = &v.ident; + let index = super::index(v, i); + + let create = create_instance( + call_site, + quote! { #type_name :: #name }, + input, + &v.fields, + ); + + quote_spanned! { v.span() => + x if x == #index as u8 => { + #create + }, + } + }); + + quote! { + match #input.read_byte()? { + #( #recurse )* + _ => None, + } + + } + + }, + Data::Union(_) => panic!("Union types are not supported."), + } +} + +fn create_instance(call_site: Span, name: TokenStream, input: &TokenStream, fields: &Fields) -> TokenStream { + match *fields { + Fields::Named(ref fields) => { + let recurse = fields.named.iter().map(|f| { + let name = &f.ident; + let field = quote_spanned!(call_site => #name); + + quote_spanned! { f.span() => + #field: ::codec::Decode::decode(#input)? + } + }); + + quote_spanned! {call_site => + Some(#name { + #( #recurse, )* + }) + } + }, + Fields::Unnamed(ref fields) => { + let recurse = fields.unnamed.iter().map(|f| { + quote_spanned! { f.span() => + ::codec::Decode::decode(#input)? + } + }); + + quote_spanned! {call_site => + Some(#name ( + #( #recurse, )* + )) + } + }, + Fields::Unit => { + quote_spanned! {call_site => + Some(#name) + } + }, + } +} diff --git a/core/codec/derive/src/encode.rs b/core/codec/derive/src/encode.rs new file mode 100644 index 00000000000..c378c9af510 --- /dev/null +++ b/core/codec/derive/src/encode.rs @@ -0,0 +1,153 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[cfg(not(feature = "std"))] +use core::str::from_utf8; +#[cfg(feature = "std")] +use std::str::from_utf8; + +use proc_macro2::{Span, TokenStream}; +use syn::{ + Data, Field, Fields, Ident, Index, + punctuated::Punctuated, + spanned::Spanned, + token::Comma, +}; + +type FieldsList = Punctuated; + +fn encode_fields( + dest: &TokenStream, + fields: &FieldsList, + field_name: F, +) -> TokenStream where + F: Fn(usize, &Option) -> TokenStream, +{ + let recurse = fields.iter().enumerate().map(|(i, f)| { + let field = field_name(i, &f.ident); + + quote_spanned! { f.span() => + #dest.push(#field); + } + }); + + quote! { + #( #recurse )* + } +} + +pub fn quote(data: &Data, type_name: &Ident, self_: &TokenStream, dest: &TokenStream) -> TokenStream { + let call_site = Span::call_site(); + match *data { + Data::Struct(ref data) => match data.fields { + Fields::Named(ref fields) => encode_fields( + dest, + &fields.named, + |_, name| quote_spanned!(call_site => &#self_.#name), + ), + Fields::Unnamed(ref fields) => encode_fields( + dest, + &fields.unnamed, + |i, _| { + let index = Index { index: i as u32, span: call_site }; + quote_spanned!(call_site => &#self_.#index) + }, + ), + Fields::Unit => quote_spanned! { call_site => + drop(#dest); + }, + }, + Data::Enum(ref data) => { + assert!(data.variants.len() < 256, "Currently only enums with at most 256 variants are encodable."); + + let recurse = data.variants.iter().enumerate().map(|(i, f)| { + let name = &f.ident; + let index = super::index(f, i); + + match f.fields { + Fields::Named(ref fields) => { + let field_name = |_, ident: &Option| quote_spanned!(call_site => #ident); + let names = fields.named + .iter() + .enumerate() + .map(|(i, f)| field_name(i, &f.ident)); + + let encode_fields = encode_fields( + dest, + &fields.named, + |a, b| field_name(a, b), + ); + + quote_spanned! { f.span() => + #type_name :: #name { #( ref #names, )* } => { + #dest.push_byte(#index as u8); + #encode_fields + } + } + }, + Fields::Unnamed(ref fields) => { + let field_name = |i, _: &Option| { + let data = stringify(i as u8); + let ident = from_utf8(&data).expect("We never go beyond ASCII"); + let ident = Ident::new(ident, call_site); + quote_spanned!(call_site => #ident) + }; + let names = fields.unnamed + .iter() + .enumerate() + .map(|(i, f)| field_name(i, &f.ident)); + + let encode_fields = encode_fields( + dest, + &fields.unnamed, + |a, b| field_name(a, b), + ); + + quote_spanned! { f.span() => + #type_name :: #name ( #( ref #names, )* ) => { + #dest.push_byte(#index as u8); + #encode_fields + } + } + }, + Fields::Unit => { + quote_spanned! { f.span() => + #type_name :: #name => { + #dest.push_byte(#index as u8); + } + } + }, + } + }); + + quote! { + match *#self_ { + #( #recurse )*, + } + } + }, + Data::Union(_) => panic!("Union types are not supported."), + } +} +pub fn stringify(id: u8) -> [u8; 2] { + const CHARS: &[u8] = b"abcdefghijklmnopqrstuvwxyz"; + let len = CHARS.len() as u8; + let symbol = |id: u8| CHARS[(id % len) as usize]; + let a = symbol(id); + let b = symbol(id / len); + + [a, b] +} diff --git a/core/codec/derive/src/lib.rs b/core/codec/derive/src/lib.rs new file mode 100644 index 00000000000..1bf3aebe752 --- /dev/null +++ b/core/codec/derive/src/lib.rs @@ -0,0 +1,126 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Derives serialization and deserialization codec for complex structs for simple marshalling. + +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate proc_macro; +extern crate proc_macro2; + +#[macro_use] +extern crate syn; + +#[macro_use] +extern crate quote; + +use proc_macro::TokenStream; +use syn::{DeriveInput, Generics, GenericParam, Ident}; + +mod decode; +mod encode; + +const ENCODE_ERR: &str = "derive(Encode) failed"; + +#[proc_macro_derive(Encode, attributes(codec))] +pub fn encode_derive(input: TokenStream) -> TokenStream { + let input: DeriveInput = syn::parse(input).expect(ENCODE_ERR); + let name = &input.ident; + + let generics = add_trait_bounds(input.generics, parse_quote!(::codec::Encode)); + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let self_ = quote!(self); + let dest_ = quote!(dest); + let encoding = encode::quote(&input.data, name, &self_, &dest_); + + let expanded = quote! { + impl #impl_generics ::codec::Encode for #name #ty_generics #where_clause { + fn encode_to(&#self_, #dest_: &mut EncOut) { + #encoding + } + } + }; + + expanded.into() +} + +#[proc_macro_derive(Decode, attributes(codec))] +pub fn decode_derive(input: TokenStream) -> TokenStream { + let input: DeriveInput = syn::parse(input).expect(ENCODE_ERR); + let name = &input.ident; + + let generics = add_trait_bounds(input.generics, parse_quote!(::codec::Decode)); + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let input_ = quote!(input); + let decoding = decode::quote(&input.data, name, &input_); + + let expanded = quote! { + impl #impl_generics ::codec::Decode for #name #ty_generics #where_clause { + fn decode(#input_: &mut DecIn) -> Option { + #decoding + } + } + }; + + expanded.into() +} + +fn add_trait_bounds(mut generics: Generics, bounds: syn::TypeParamBound) -> Generics { + for param in &mut generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param.bounds.push(bounds.clone()); + } + } + generics +} + +fn index(v: &syn::Variant, i: usize) -> proc_macro2::TokenStream { + // look for an index in attributes + let index = v.attrs.iter().filter_map(|attr| { + let pair = attr.path.segments.first()?; + let seg = pair.value(); + + if seg.ident == Ident::new("codec", seg.ident.span()) { + assert_eq!(attr.path.segments.len(), 1); + + let meta = attr.interpret_meta(); + if let Some(syn::Meta::List(ref l)) = meta { + if let syn::NestedMeta::Meta(syn::Meta::NameValue(ref nv)) = l.nested.last().unwrap().value() { + assert_eq!(nv.ident, Ident::new("index", nv.ident.span())); + if let syn::Lit::Str(ref s) = nv.lit { + let byte: u8 = s.value().parse().expect("Numeric index expected."); + return Some(byte) + } + panic!("Invalid syntax for `codec` attribute: Expected string literal.") + } + } + panic!("Invalid syntax for `codec` attribute: Expected `name = value` pair.") + } else { + None + } + }).next(); + + // then fallback to discriminant or just index + index.map(|i| quote! { #i }) + .unwrap_or_else(|| v.discriminant + .as_ref() + .map(|&(_, ref expr)| quote! { #expr }) + .unwrap_or_else(|| quote! { #i }) + ) +} + diff --git a/core/codec/derive/tests/mod.rs b/core/codec/derive/tests/mod.rs new file mode 100644 index 00000000000..6b3260f8f64 --- /dev/null +++ b/core/codec/derive/tests/mod.rs @@ -0,0 +1,151 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +extern crate substrate_codec as codec; + +#[macro_use] +extern crate substrate_codec_derive; + +use codec::{Encode, Decode}; + +#[derive(Debug, PartialEq, Encode, Decode)] +struct Unit; + +#[derive(Debug, PartialEq, Encode, Decode)] +struct Indexed(u32, u64); + +#[derive(Debug, PartialEq, Encode, Decode)] +struct Struct { + pub a: A, + pub b: B, + pub c: C, +} + +type TestType = Struct>; + +impl Struct { + fn new(a: A, b: B, c: C) -> Self { + Self { a, b, c } + } +} + +#[derive(Debug, PartialEq, Encode, Decode)] +enum EnumType { + #[codec(index = "15")] + A, + B(u32, u64), + C { + a: u32, + b: u64, + }, +} + +#[derive(Debug, PartialEq, Encode, Decode)] +enum EnumWithDiscriminant { + A = 1, + B = 15, + C = 255, +} + +#[test] +fn should_work_for_simple_enum() { + let a = EnumType::A; + let b = EnumType::B(1, 2); + let c = EnumType::C { a: 1, b: 2 }; + + a.using_encoded(|ref slice| { + assert_eq!(slice, &b"\x0f"); + }); + b.using_encoded(|ref slice| { + assert_eq!(slice, &b"\x01\x01\0\0\0\x02\0\0\0\0\0\0\0"); + }); + c.using_encoded(|ref slice| { + assert_eq!(slice, &b"\x02\x01\0\0\0\x02\0\0\0\0\0\0\0"); + }); + + let mut da: &[u8] = b"\x0f"; + assert_eq!(EnumType::decode(&mut da), Some(a)); + let mut db: &[u8] = b"\x01\x01\0\0\0\x02\0\0\0\0\0\0\0"; + assert_eq!(EnumType::decode(&mut db), Some(b)); + let mut dc: &[u8] = b"\x02\x01\0\0\0\x02\0\0\0\0\0\0\0"; + assert_eq!(EnumType::decode(&mut dc), Some(c)); + let mut dz: &[u8] = &[0]; + assert_eq!(EnumType::decode(&mut dz), None); +} + +#[test] +fn should_work_for_enum_with_discriminant() { + EnumWithDiscriminant::A.using_encoded(|ref slice| { + assert_eq!(slice, &[1]); + }); + EnumWithDiscriminant::B.using_encoded(|ref slice| { + assert_eq!(slice, &[15]); + }); + EnumWithDiscriminant::C.using_encoded(|ref slice| { + assert_eq!(slice, &[255]); + }); + + let mut da: &[u8] = &[1]; + assert_eq!(EnumWithDiscriminant::decode(&mut da), Some(EnumWithDiscriminant::A)); + let mut db: &[u8] = &[15]; + assert_eq!(EnumWithDiscriminant::decode(&mut db), Some(EnumWithDiscriminant::B)); + let mut dc: &[u8] = &[255]; + assert_eq!(EnumWithDiscriminant::decode(&mut dc), Some(EnumWithDiscriminant::C)); + let mut dz: &[u8] = &[2]; + assert_eq!(EnumWithDiscriminant::decode(&mut dz), None); +} + +#[test] +fn should_derive_encode() { + let v = TestType::new(15, 9, b"Hello world".to_vec()); + + v.using_encoded(|ref slice| { + assert_eq!(slice, &b"\x0f\0\0\0\x09\0\0\0\0\0\0\0\x0b\0\0\0Hello world") + }); +} + +#[test] +fn should_derive_decode() { + let slice = b"\x0f\0\0\0\x09\0\0\0\0\0\0\0\x0b\0\0\0Hello world".to_vec(); + + let v = TestType::decode(&mut &*slice); + + assert_eq!(v, Some(TestType::new(15, 9, b"Hello world".to_vec()))); +} + +#[test] +fn should_work_for_unit() { + let v = Unit; + + v.using_encoded(|ref slice| { + assert_eq!(slice, &[]); + }); + + let mut a: &[u8] = &[]; + assert_eq!(Unit::decode(&mut a), Some(Unit)); +} + +#[test] +fn should_work_for_indexed() { + let v = Indexed(1, 2); + + v.using_encoded(|ref slice| { + assert_eq!(slice, &b"\x01\0\0\0\x02\0\0\0\0\0\0\0") + }); + + let mut v: &[u8] = b"\x01\0\0\0\x02\0\0\0\0\0\0\0"; + assert_eq!(Indexed::decode(&mut v), Some(Indexed(1, 2))); +} diff --git a/core/codec/src/codec.rs b/core/codec/src/codec.rs new file mode 100644 index 00000000000..c92790f097a --- /dev/null +++ b/core/codec/src/codec.rs @@ -0,0 +1,540 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Serialisation. + +use alloc::vec::Vec; +use alloc::boxed::Box; +use core::{mem, slice}; +use arrayvec::ArrayVec; + +/// Trait that allows reading of data into a slice. +pub trait Input { + /// Read into the provided input slice. Returns the number of bytes read. + fn read(&mut self, into: &mut [u8]) -> usize; + + /// Read a single byte from the input. + fn read_byte(&mut self) -> Option { + let mut buf = [0u8]; + match self.read(&mut buf[..]) { + 0 => None, + 1 => Some(buf[0]), + _ => unreachable!(), + } + } +} + +#[cfg(not(feature = "std"))] +impl<'a> Input for &'a [u8] { + fn read(&mut self, into: &mut [u8]) -> usize { + let len = ::core::cmp::min(into.len(), self.len()); + into[..len].copy_from_slice(&self[..len]); + *self = &self[len..]; + len + } +} + +#[cfg(feature = "std")] +impl Input for R { + fn read(&mut self, into: &mut [u8]) -> usize { + match (self as &mut ::std::io::Read).read_exact(into) { + Ok(()) => into.len(), + Err(_) => 0, + } + } +} + +/// Trait that allows writing of data. +pub trait Output: Sized { + /// Write to the output. + fn write(&mut self, bytes: &[u8]); + + fn push_byte(&mut self, byte: u8) { + self.write(&[byte]); + } + + fn push(&mut self, value: &V) { + value.encode_to(self); + } +} + +#[cfg(not(feature = "std"))] +impl Output for Vec { + fn write(&mut self, bytes: &[u8]) { + self.extend(bytes); + } +} + +#[cfg(feature = "std")] +impl Output for W { + fn write(&mut self, bytes: &[u8]) { + (self as &mut ::std::io::Write).write_all(bytes).expect("Codec outputs are infallible"); + } +} + +/// Trait that allows zero-copy write of value-references to slices in LE format. +/// Implementations should override `using_encoded` for value types and `encode_to` for allocating types. +pub trait Encode { + /// Convert self to a slice and append it to the destination. + fn encode_to(&self, dest: &mut T) { + self.using_encoded(|buf| dest.write(buf)); + } + + /// Convert self to an owned vector. + fn encode(&self) -> Vec { + let mut r = Vec::new(); + self.encode_to(&mut r); + r + } + + /// Convert self to a slice and then invoke the given closure with it. + fn using_encoded R>(&self, f: F) -> R { + f(&self.encode()) + } +} + +/// Trait that allows zero-copy read of value-references from slices in LE format. +pub trait Decode: Sized { + /// Attempt to deserialise the value from input. + fn decode(value: &mut I) -> Option; +} + +/// Trait that allows zero-copy read/write of value-references to/from slices in LE format. +pub trait Codec: Decode + Encode {} + +impl Codec for S {} + +impl Encode for Result { + fn encode_to(&self, dest: &mut W) { + match *self { + Ok(ref t) => { + dest.push_byte(0); + t.encode_to(dest); + } + Err(ref e) => { + dest.push_byte(1); + e.encode_to(dest); + } + } + } +} + +impl Decode for Result { + fn decode(input: &mut I) -> Option { + match input.read_byte()? { + 0 => Some(Ok(T::decode(input)?)), + 1 => Some(Err(E::decode(input)?)), + _ => None, + } + } +} + +/// Shim type because we can't do a specialised implementation for `Option` directly. +pub struct OptionBool(pub Option); + +impl Encode for OptionBool { + fn using_encoded R>(&self, f: F) -> R { + f(&[match *self { + OptionBool(None) => 0u8, + OptionBool(Some(true)) => 1u8, + OptionBool(Some(false)) => 2u8, + }]) + } +} + +impl Decode for OptionBool { + fn decode(input: &mut I) -> Option { + match input.read_byte()? { + 0 => Some(OptionBool(None)), + 1 => Some(OptionBool(Some(true))), + 2 => Some(OptionBool(Some(false))), + _ => None, + } + } +} + +impl Encode for Option { + fn encode_to(&self, dest: &mut W) { + match *self { + Some(ref t) => { + dest.push_byte(1); + t.encode_to(dest); + } + None => dest.push_byte(0), + } + } +} + +impl Decode for Option { + fn decode(input: &mut I) -> Option { + match input.read_byte()? { + 0 => Some(None), + 1 => Some(Some(T::decode(input)?)), + _ => None, + } + } +} + +macro_rules! impl_array { + ( $( $n:expr )* ) => { $( + impl Encode for [T; $n] { + fn encode_to(&self, dest: &mut W) { + for item in self.iter() { + item.encode_to(dest); + } + } + } + + impl Decode for [T; $n] { + fn decode(input: &mut I) -> Option { + let mut r = ArrayVec::new(); + for _ in 0..$n { + r.push(T::decode(input)?); + } + r.into_inner().ok() + } + } + )* } +} + +impl_array!(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 + 40 48 56 64 72 96 128 160 192 224 256); + +impl Encode for Box { + fn encode_to(&self, dest: &mut W) { + self.as_ref().encode_to(dest) + } +} + +impl Decode for Box { + fn decode(input: &mut I) -> Option { + Some(Box::new(T::decode(input)?)) + } +} + +impl Encode for [u8] { + fn encode_to(&self, dest: &mut W) { + let len = self.len(); + assert!(len <= u32::max_value() as usize, "Attempted to serialize a collection with too many elements."); + (len as u32).encode_to(dest); + dest.write(self) + } +} + +impl Encode for Vec { + fn encode_to(&self, dest: &mut W) { + self.as_slice().encode_to(dest) + } +} + +impl Decode for Vec { + fn decode(input: &mut I) -> Option { + u32::decode(input).and_then(move |len| { + let len = len as usize; + let mut vec = vec![0; len]; + if input.read(&mut vec[..len]) != len { + None + } else { + Some(vec) + } + }) + } +} + +impl<'a> Encode for &'a str { + fn encode_to(&self, dest: &mut W) { + self.as_bytes().encode_to(dest) + } +} + +#[cfg(feature = "std")] +impl<'a > Encode for ::std::borrow::Cow<'a, str> { + fn encode_to(&self, dest: &mut W) { + self.as_bytes().encode_to(dest) + } +} + +#[cfg(feature = "std")] +impl<'a> Decode for ::std::borrow::Cow<'a, str> { + fn decode(input: &mut I) -> Option { + Some(::std::borrow::Cow::Owned(String::from_utf8_lossy(&Vec::decode(input)?).into())) + } +} + +#[cfg(feature = "std")] +impl Encode for String { + fn encode_to(&self, dest: &mut W) { + self.as_bytes().encode_to(dest) + } +} + +#[cfg(feature = "std")] +impl Decode for String { + fn decode(input: &mut I) -> Option { + Some(Self::from_utf8_lossy(&Vec::decode(input)?).into()) + } +} + +impl Encode for [T] { + fn encode_to(&self, dest: &mut W) { + let len = self.len(); + assert!(len <= u32::max_value() as usize, "Attempted to serialize a collection with too many elements."); + (len as u32).encode_to(dest); + for item in self { + item.encode_to(dest); + } + } +} + +impl Encode for Vec { + fn encode_to(&self, dest: &mut W) { + self.as_slice().encode_to(dest) + } +} + +impl Decode for Vec { + fn decode(input: &mut I) -> Option { + u32::decode(input).and_then(move |len| { + let mut r = Vec::with_capacity(len as usize); + for _ in 0..len { + r.push(T::decode(input)?); + } + Some(r) + }) + } +} + +impl Encode for () { + fn encode_to(&self, _dest: &mut T) { + } + + fn using_encoded R>(&self, f: F) -> R { + f(&[]) + } + + fn encode(&self) -> Vec { + Vec::new() + } +} + +impl<'a, T: 'a + Encode + ?Sized> Encode for &'a T { + fn encode_to(&self, dest: &mut D) { + (&**self).encode_to(dest) + } + + fn using_encoded R>(&self, f: F) -> R { + (&**self).using_encoded(f) + } + + fn encode(&self) -> Vec { + (&**self).encode() + } +} + +impl Decode for () { + fn decode(_: &mut I) -> Option<()> { + Some(()) + } +} + +macro_rules! tuple_impl { + ($one:ident,) => { + impl<$one: Encode> Encode for ($one,) { + fn encode_to(&self, dest: &mut T) { + self.0.encode_to(dest); + } + } + + impl<$one: Decode> Decode for ($one,) { + fn decode(input: &mut I) -> Option { + match $one::decode(input) { + None => None, + Some($one) => Some(($one,)), + } + } + } + }; + ($first:ident, $($rest:ident,)+) => { + impl<$first: Encode, $($rest: Encode),+> + Encode for + ($first, $($rest),+) { + fn encode_to(&self, dest: &mut T) { + let ( + ref $first, + $(ref $rest),+ + ) = *self; + + $first.encode_to(dest); + $($rest.encode_to(dest);)+ + } + } + + impl<$first: Decode, $($rest: Decode),+> + Decode for + ($first, $($rest),+) { + fn decode(input: &mut INPUT) -> Option { + Some(( + match $first::decode(input) { + Some(x) => x, + None => return None, + }, + $(match $rest::decode(input) { + Some(x) => x, + None => return None, + },)+ + )) + } + } + + tuple_impl!($($rest,)+); + } +} + +#[allow(non_snake_case)] +mod inner_tuple_impl { + use super::{Input, Output, Decode, Encode}; + tuple_impl!(A, B, C, D, E, F, G, H, I, J, K,); +} + +/// Trait to allow conversion to a know endian representation when sensitive. +/// Types implementing this trait must have a size > 0. +// note: the copy bound and static lifetimes are necessary for safety of `Codec` blanket +// implementation. +trait EndianSensitive: Copy + 'static { + fn to_le(self) -> Self { self } + fn to_be(self) -> Self { self } + fn from_le(self) -> Self { self } + fn from_be(self) -> Self { self } + fn as_be_then T>(&self, f: F) -> T { f(&self) } + fn as_le_then T>(&self, f: F) -> T { f(&self) } +} + +macro_rules! impl_endians { + ( $( $t:ty ),* ) => { $( + impl EndianSensitive for $t { + fn to_le(self) -> Self { <$t>::to_le(self) } + fn to_be(self) -> Self { <$t>::to_be(self) } + fn from_le(self) -> Self { <$t>::from_le(self) } + fn from_be(self) -> Self { <$t>::from_be(self) } + fn as_be_then T>(&self, f: F) -> T { let d = self.to_be(); f(&d) } + fn as_le_then T>(&self, f: F) -> T { let d = self.to_le(); f(&d) } + } + + impl Encode for $t { + fn using_encoded R>(&self, f: F) -> R { + self.as_le_then(|le| { + let size = mem::size_of::<$t>(); + let value_slice = unsafe { + let ptr = le as *const _ as *const u8; + if size != 0 { + slice::from_raw_parts(ptr, size) + } else { + &[] + } + }; + + f(value_slice) + }) + } + } + + impl Decode for $t { + fn decode(input: &mut I) -> Option { + let size = mem::size_of::<$t>(); + assert!(size > 0, "EndianSensitive can never be implemented for a zero-sized type."); + let mut val: $t = unsafe { mem::zeroed() }; + + unsafe { + let raw: &mut [u8] = slice::from_raw_parts_mut( + &mut val as *mut $t as *mut u8, + size + ); + if input.read(raw) != size { return None } + } + Some(val.from_le()) + } + } + )* } +} +macro_rules! impl_non_endians { + ( $( $t:ty ),* ) => { $( + impl EndianSensitive for $t {} + + impl Encode for $t { + fn using_encoded R>(&self, f: F) -> R { + self.as_le_then(|le| { + let size = mem::size_of::<$t>(); + let value_slice = unsafe { + let ptr = le as *const _ as *const u8; + if size != 0 { + slice::from_raw_parts(ptr, size) + } else { + &[] + } + }; + + f(value_slice) + }) + } + } + + impl Decode for $t { + fn decode(input: &mut I) -> Option { + let size = mem::size_of::<$t>(); + assert!(size > 0, "EndianSensitive can never be implemented for a zero-sized type."); + let mut val: $t = unsafe { mem::zeroed() }; + + unsafe { + let raw: &mut [u8] = slice::from_raw_parts_mut( + &mut val as *mut $t as *mut u8, + size + ); + if input.read(raw) != size { return None } + } + Some(val.from_le()) + } + } + )* } +} + +impl_endians!(u16, u32, u64, u128, usize, i16, i32, i64, i128, isize); +impl_non_endians!(i8, [u8; 1], [u8; 2], [u8; 3], [u8; 4], [u8; 5], [u8; 6], [u8; 7], [u8; 8], + [u8; 10], [u8; 12], [u8; 14], [u8; 16], [u8; 20], [u8; 24], [u8; 28], [u8; 32], [u8; 40], + [u8; 48], [u8; 56], [u8; 64], [u8; 80], [u8; 96], [u8; 112], [u8; 128], bool); + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vec_is_slicable() { + let v = b"Hello world".to_vec(); + v.using_encoded(|ref slice| + assert_eq!(slice, &b"\x0b\0\0\0Hello world") + ); + } + + #[test] + fn encode_borrowed_tuple() { + let x = vec![1u8, 2, 3, 4]; + let y = 128i64; + + let encoded = (&x, &y).encode(); + + assert_eq!((x, y), Decode::decode(&mut &encoded[..]).unwrap()); + } +} diff --git a/core/codec/src/joiner.rs b/core/codec/src/joiner.rs new file mode 100644 index 00000000000..81105b060ce --- /dev/null +++ b/core/codec/src/joiner.rs @@ -0,0 +1,33 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Trait + +use core::iter::Extend; +use super::Codec; + +/// Trait to allow itself to be serialised into a value which can be extended +/// by bytes. +pub trait Joiner { + fn and(self, value: &V) -> Self; +} + +impl Joiner for T where T: for<'a> Extend<&'a u8> { + fn and(mut self, value: &V) -> Self { + value.using_encoded(|s| self.extend(s)); + self + } +} diff --git a/core/codec/src/keyedvec.rs b/core/codec/src/keyedvec.rs new file mode 100644 index 00000000000..37298674a21 --- /dev/null +++ b/core/codec/src/keyedvec.rs @@ -0,0 +1,36 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Serialiser and prepender. + +use Codec; +use core::iter::Extend; +use alloc::vec::Vec; + +/// Trait to allow itself to be serialised and prepended by a given slice. +pub trait KeyedVec { + fn to_keyed_vec(&self, prepend_key: &[u8]) -> Vec; +} + +impl KeyedVec for T { + fn to_keyed_vec(&self, prepend_key: &[u8]) -> Vec { + self.using_encoded(|slice| { + let mut r = prepend_key.to_vec(); + r.extend(slice); + r + }) + } +} diff --git a/core/codec/src/lib.rs b/core/codec/src/lib.rs new file mode 100644 index 00000000000..c1c29ccaa0a --- /dev/null +++ b/core/codec/src/lib.rs @@ -0,0 +1,45 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Implements a serialization and deserialization codec for simple marshalling. +// end::description[] + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +#[cfg(not(feature = "std"))] +#[macro_use] +extern crate alloc; + +#[cfg(feature = "std")] +extern crate core; + +extern crate arrayvec; + +#[cfg(feature = "std")] +pub mod alloc { + pub use std::boxed; + pub use std::vec; +} + +mod codec; +mod joiner; +mod keyedvec; + +pub use self::codec::{Input, Output, Encode, Decode, Codec}; +pub use self::joiner::Joiner; +pub use self::keyedvec::KeyedVec; diff --git a/core/environmental/Cargo.toml b/core/environmental/Cargo.toml new file mode 100644 index 00000000000..971dd38b676 --- /dev/null +++ b/core/environmental/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "environmental" +version = "0.1.0" +authors = ["Parity Technologies "] + +[features] +default = ["std"] +std = [] diff --git a/core/environmental/README.adoc b/core/environmental/README.adoc new file mode 100644 index 00000000000..b12a0f9f5fa --- /dev/null +++ b/core/environmental/README.adoc @@ -0,0 +1,13 @@ + += Environmental + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/environmental/src/lib.rs b/core/environmental/src/lib.rs new file mode 100644 index 00000000000..ea7bbfa4f0a --- /dev/null +++ b/core/environmental/src/lib.rs @@ -0,0 +1,383 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Safe global references to stack variables. +//! +//! Set up a global reference with environmental! macro giving it a name and type. +//! Use the `using` function scoped under its name to name a reference and call a function that +//! takes no parameters yet can access said reference through the similarly placed `with` function. +//! +//! # Examples +//! +//! ``` +//! #[macro_use] extern crate environmental; +//! // create a place for the global reference to exist. +//! environmental!(counter: u32); +//! fn stuff() { +//! // do some stuff, accessing the named reference as desired. +//! counter::with(|i| *i += 1); +//! } +//! fn main() { +//! // declare a stack variable of the same type as our global declaration. +//! let mut counter_value = 41u32; +//! // call stuff, setting up our `counter` environment as a reference to our counter_value var. +//! counter::using(&mut counter_value, stuff); +//! println!("The answer is {:?}", counter_value); // will print 42! +//! stuff(); // safe! doesn't do anything. +//! } +//! ``` +// end::description[] + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(const_fn))] + +#[cfg(feature = "std")] +include!("../with_std.rs"); + +#[cfg(not(feature = "std"))] +include!("../without_std.rs"); + +#[doc(hidden)] +pub fn using R>( + global: &'static imp::LocalKey>>, + protected: &mut T, + f: F +) -> R { + // store the `protected` reference as a pointer so we can provide it to logic running within + // `f`. + // while we record this pointer (while it's non-zero) we guarantee: + // - it will only be used once at any time (no reentrancy); + // - that no other thread will use it; and + // - that we do not use the original mutating reference while the pointer. + // exists. + global.with(|r| { + let original = { + let mut global = r.borrow_mut(); + imp::replace(&mut *global, Some(protected as _)) + }; + + // even if `f` panics the original will be replaced. + struct ReplaceOriginal<'a, T: 'a + ?Sized> { + original: Option<*mut T>, + global: &'a imp::RefCell>, + } + + impl<'a, T: 'a + ?Sized> Drop for ReplaceOriginal<'a, T> { + fn drop(&mut self) { + *self.global.borrow_mut() = self.original.take(); + } + } + + let _guard = ReplaceOriginal { + original, + global: r, + }; + + f() + }) +} + +#[doc(hidden)] +pub fn with R>( + global: &'static imp::LocalKey>>, + mutator: F, +) -> Option { + global.with(|r| unsafe { + let ptr = r.borrow_mut(); + match *ptr { + Some(ptr) => { + // safe because it's only non-zero when it's being called from using, which + // is holding on to the underlying reference (and not using it itself) safely. + Some(mutator(&mut *ptr)) + } + None => None, + } + }) +} + +/// Declare a new global reference module whose underlying value does not contain references. +/// +/// Will create a module of a given name that contains two functions: +/// +/// * `pub fn using R>(protected: &mut $t, f: F) -> R` +/// This executes `f`, returning its value. During the call, the module's reference is set to +/// be equal to `protected`. +/// * `pub fn with R>(f: F) -> Option` +/// This executes `f`, returning `Some` of its value if called from code that is being executed +/// as part of a `using` call. If not, it returns `None`. `f` is provided with one argument: the +/// same reference as provided to the most recent `using` call. +/// +/// # Examples +/// +/// Initializing the global context with a given value. +/// +/// ```rust +/// #[macro_use] extern crate environmental; +/// environmental!(counter: u32); +/// fn main() { +/// let mut counter_value = 41u32; +/// counter::using(&mut counter_value, || { +/// let odd = counter::with(|value| +/// if *value % 2 == 1 { +/// *value += 1; true +/// } else { +/// *value -= 3; false +/// }).unwrap(); // safe because we're inside a counter::using +/// println!("counter was {}", match odd { true => "odd", _ => "even" }); +/// }); +/// +/// println!("The answer is {:?}", counter_value); // 42 +/// } +/// ``` +/// +/// Roughly the same, but with a trait object: +/// +/// ```rust +/// #[macro_use] extern crate environmental; +/// +/// trait Increment { fn increment(&mut self); } +/// +/// impl Increment for i32 { +/// fn increment(&mut self) { *self += 1 } +/// } +/// +/// environmental!(val: Increment + 'static); +/// +/// fn main() { +/// let mut local = 0i32; +/// val::using(&mut local, || { +/// val::with(|v| for _ in 0..5 { v.increment() }); +/// }); +/// +/// assert_eq!(local, 5); +/// } +/// ``` +#[macro_export] +macro_rules! environmental { + ($name:ident : $t:ty) => { + #[allow(non_camel_case_types)] + struct $name { __private_field: () } + + thread_local_impl!(static GLOBAL: ::std::cell::RefCell> + = ::std::cell::RefCell::new(None)); + + impl $name { + #[allow(unused_imports)] + + pub fn using R>( + protected: &mut $t, + f: F + ) -> R { + $crate::using(&GLOBAL, protected, f) + } + + pub fn with R>( + f: F + ) -> Option { + $crate::with(&GLOBAL, |x| f(x)) + } + } + }; + ($name:ident : trait @$t:ident [$($args:ty,)*]) => { + #[allow(non_camel_case_types, dead_code)] + struct $name { __private_field: () } + + thread_local_impl!(static GLOBAL: $crate::imp::RefCell + 'static)>> + = $crate::imp::RefCell::new(None)); + + impl $name { + #[allow(unused_imports)] + + pub fn using R>( + protected: &mut $t<$($args),*>, + f: F + ) -> R { + let lifetime_extended = unsafe { + $crate::imp::transmute::<&mut $t<$($args),*>, &mut ($t<$($args),*> + 'static)>(protected) + }; + $crate::using(&GLOBAL, lifetime_extended, f) + } + + pub fn with FnOnce(&'a mut ($t<$($args),*> + 'a)) -> R>( + f: F + ) -> Option { + $crate::with(&GLOBAL, |x| f(x)) + } + } + }; + ($name:ident<$traittype:ident> : trait $t:ident <$concretetype:ty>) => { + #[allow(non_camel_case_types, dead_code)] + struct $name { _private_field: $crate::imp::PhantomData } + + thread_local_impl!(static GLOBAL: $crate::imp::RefCell + 'static)>> + = $crate::imp::RefCell::new(None)); + + impl $name { + #[allow(unused_imports)] + pub fn using R>( + protected: &mut $t, + f: F + ) -> R { + let lifetime_extended = unsafe { + $crate::imp::transmute::<&mut $t, &mut ($t<$concretetype> + 'static)>(protected) + }; + $crate::using(&GLOBAL, lifetime_extended, f) + } + + pub fn with FnOnce(&'a mut ($t<$concretetype> + 'a)) -> R>( + f: F + ) -> Option { + $crate::with(&GLOBAL, |x| f(x)) + } + } + }; + ($name:ident : trait $t:ident <>) => { environmental! { $name : trait @$t [] } }; + ($name:ident : trait $t:ident < $($args:ty),* $(,)* >) => { environmental! { $name : trait @$t [$($args,)*] } }; + ($name:ident : trait $t:ident) => { environmental! { $name : trait @$t [] } }; +} + +#[cfg(test)] +mod tests { + + #[test] + fn simple_works() { + environmental!(counter: u32); + + fn stuff() { counter::with(|value| *value += 1); }; + + // declare a stack variable of the same type as our global declaration. + let mut local = 41u32; + + // call stuff, setting up our `counter` environment as a reference to our local counter var. + counter::using(&mut local, stuff); + assert_eq!(local, 42); + stuff(); // safe! doesn't do anything. + assert_eq!(local, 42); + } + + #[test] + fn overwrite_with_lesser_lifetime() { + environmental!(items: Vec); + + let mut local_items = vec![1, 2, 3]; + items::using(&mut local_items, || { + let dies_at_end = vec![4, 5, 6]; + items::with(|items| *items = dies_at_end); + }); + + assert_eq!(local_items, vec![4, 5, 6]); + } + + #[test] + fn declare_with_trait_object() { + trait Foo { + fn get(&self) -> i32; + fn set(&mut self, x: i32); + } + + impl Foo for i32 { + fn get(&self) -> i32 { *self } + fn set(&mut self, x: i32) { *self = x } + } + + environmental!(foo: Foo + 'static); + + fn stuff() { + foo::with(|value| { + let new_val = value.get() + 1; + value.set(new_val); + }); + } + + let mut local = 41i32; + foo::using(&mut local, stuff); + + assert_eq!(local, 42); + + stuff(); // doesn't do anything. + + assert_eq!(local, 42); + } + + #[test] + fn unwind_recursive() { + use std::panic; + + environmental!(items: Vec); + + let panicked = panic::catch_unwind(|| { + let mut local_outer = vec![1, 2, 3]; + + items::using(&mut local_outer, || { + let mut local_inner = vec![4, 5, 6]; + items::using(&mut local_inner, || { + panic!("are you unsafe?"); + }) + }); + }).is_err(); + + assert!(panicked); + + let mut was_cleared = true; + items::with(|_items| was_cleared = false); + + assert!(was_cleared); + } + + #[test] + fn use_non_static_trait() { + trait Sum { fn sum(&self) -> usize; } + impl<'a> Sum for &'a [usize] { + fn sum(&self) -> usize { + self.iter().fold(0, |a, c| a + c) + } + } + + environmental!(sum: trait Sum); + let numbers = vec![1, 2, 3, 4, 5]; + let mut numbers = &numbers[..]; + let got_sum = sum::using(&mut numbers, || { + sum::with(|x| x.sum()) + }).unwrap(); + + assert_eq!(got_sum, 15); + } + + #[test] + fn use_generic_trait() { + trait Plus { fn plus42() -> usize; } + struct ConcretePlus; + impl Plus for ConcretePlus { + fn plus42() -> usize { 42 } + } + trait Multiplier { fn mul_and_add(&self) -> usize; } + impl<'a, P: Plus> Multiplier

for &'a [usize] { + fn mul_and_add(&self) -> usize { + self.iter().fold(1, |a, c| a * c) + P::plus42() + } + } + + let numbers = vec![1, 2, 3]; + let mut numbers = &numbers[..]; + let out = foo::::using(&mut numbers, || { + foo::::with(|x| x.mul_and_add() ) + }).unwrap(); + + assert_eq!(out, 6 + 42); + environmental!(foo: trait Multiplier); + } +} diff --git a/core/environmental/with_std.rs b/core/environmental/with_std.rs new file mode 100644 index 00000000000..63ec5a71af3 --- /dev/null +++ b/core/environmental/with_std.rs @@ -0,0 +1,32 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[doc(hidden)] +pub mod imp { + pub use std::cell::RefCell; + pub use std::thread::LocalKey; + pub use std::mem::transmute; + pub use std::mem::replace; + pub use std::marker::PhantomData; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! thread_local_impl { + ($(#[$attr:meta])* static $name:ident: $t:ty = $init:expr) => ( + thread_local!($(#[$attr])* static $name: $t = $init); + ); +} diff --git a/core/environmental/without_std.rs b/core/environmental/without_std.rs new file mode 100644 index 00000000000..0576419d1d3 --- /dev/null +++ b/core/environmental/without_std.rs @@ -0,0 +1,70 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[doc(hidden)] +pub mod imp { + pub use core::cell::RefCell; + pub use core::mem::transmute; + pub use core::mem::replace; + pub use core::marker::PhantomData; + + // This code is a simplified version of [`LocalKey`] and it's wasm32 specialization: [`statik::Key`]. + // [`LocalKey`]: https://github.com/alexcrichton/rust/blob/98931165a23a1c2860d99759385f45d6807c8982/src/libstd/thread/local.rs#L89 + // [`statik::Key`]: https://github.com/alexcrichton/rust/blob/98931165a23a1c2860d99759385f45d6807c8982/src/libstd/thread/local.rs#L310-L312 + + pub struct LocalKey { + pub init: fn() -> T, + pub inner: RefCell>, + } + + // This is safe as long there is no threads in wasm32. + unsafe impl ::core::marker::Sync for LocalKey { } + + impl LocalKey { + pub const fn new(init: fn() -> T) -> LocalKey { + LocalKey { + init, + inner: RefCell::new(None), + } + } + + pub fn with(&'static self, f: F) -> R + where F: FnOnce(&T) -> R + { + if self.inner.borrow().is_none() { + let v = (self.init)(); + *self.inner.borrow_mut() = Some(v); + } + // This code can't panic because: + // 1. `inner` can be borrowed mutably only once at the initialization time. + // 2. After the initialization `inner` is always `Some`. + f(&*self.inner.borrow().as_ref().unwrap()) + } + } +} + +#[doc(hidden)] +#[macro_export] +macro_rules! thread_local_impl { + ($(#[$attr:meta])* static $name:ident: $t:ty = $init:expr) => ( + $(#[$attr])* + static $name: $crate::imp::LocalKey<$t> = { + fn __init() -> $t { $init } + + $crate::imp::LocalKey::new(__init) + }; + ); +} diff --git a/core/executor/Cargo.toml b/core/executor/Cargo.toml new file mode 100644 index 00000000000..4c30827dc1b --- /dev/null +++ b/core/executor/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "substrate-executor" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +error-chain = "0.12" +substrate-codec = { path = "../codec" } +substrate-runtime-io = { path = "../runtime-io" } +substrate-primitives = { path = "../primitives" } +substrate-serializer = { path = "../serializer" } +substrate-state-machine = { path = "../state-machine" } +substrate-runtime-version = { path = "../../runtime/version" } + +serde = "1.0" +serde_derive = "1.0" +wasmi = "0.4" +byteorder = "1.1" +triehash = "0.2" +twox-hash = "1.1.0" +lazy_static = "1.0" +parking_lot = "*" +log = "0.3" +hashdb = "0.2.1" + +[dev-dependencies] +assert_matches = "1.1" +wabt = "0.4" +hex-literal = "0.1.0" + +[features] +default = [] +wasm-extern-trace = [] diff --git a/core/executor/README.adoc b/core/executor/README.adoc new file mode 100644 index 00000000000..6a0ee23565a --- /dev/null +++ b/core/executor/README.adoc @@ -0,0 +1,13 @@ + += Executor + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/executor/src/error.rs b/core/executor/src/error.rs new file mode 100644 index 00000000000..1729ed2bafa --- /dev/null +++ b/core/executor/src/error.rs @@ -0,0 +1,81 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Rust executor possible errors. + +use state_machine; +use serializer; +use wasmi; + +error_chain! { + foreign_links { + InvalidData(serializer::Error) #[doc = "Unserializable Data"]; + Trap(wasmi::Trap) #[doc = "Trap occured during execution"]; + Wasmi(wasmi::Error) #[doc = "Wasmi loading/instantiating error"]; + } + + errors { + /// Method is not found + MethodNotFound(t: String) { + description("method not found"), + display("Method not found: '{}'", t), + } + + /// Code is invalid (expected single byte) + InvalidCode(c: Vec) { + description("invalid code"), + display("Invalid Code: {:?}", c), + } + + /// Could not get runtime version. + VersionInvalid { + description("Runtime version error"), + display("On-chain runtime does not specify version"), + } + + /// Externalities have failed. + Externalities { + description("externalities failure"), + display("Externalities error"), + } + + /// Invalid index. + InvalidIndex { + description("index given was not in range"), + display("Invalid index provided"), + } + + /// Invalid return type. + InvalidReturn { + description("u64 was not returned"), + display("Invalid type returned (should be u64)"), + } + + /// Runtime failed. + Runtime { + description("runtime failure"), + display("Runtime error"), + } + + /// Runtime failed. + InvalidMemoryReference { + description("invalid memory reference"), + display("Invalid memory reference"), + } + } +} + +impl state_machine::Error for Error {} diff --git a/core/executor/src/lib.rs b/core/executor/src/lib.rs new file mode 100644 index 00000000000..7b9779a9b64 --- /dev/null +++ b/core/executor/src/lib.rs @@ -0,0 +1,93 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Temporary crate for contracts implementations. +//! +//! This will be replaced with WASM contracts stored on-chain. +//! ** NOTE *** +//! This is entirely deprecated with the idea of a single-module Wasm module for state transition. +//! The dispatch table should be replaced with the specific functions needed: +//! - execute_block(bytes) +//! - init_block(PrevBlock?) -> InProgressBlock +//! - add_transaction(InProgressBlock) -> InProgressBlock +//! I leave it as is for now as it might be removed before this is ever done. +// end::description[] + +#![warn(missing_docs)] +#![recursion_limit="128"] + +extern crate substrate_codec as codec; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_primitives as primitives; +extern crate substrate_serializer as serializer; +extern crate substrate_state_machine as state_machine; +extern crate substrate_runtime_version as runtime_version; + +extern crate serde; +extern crate wasmi; +extern crate byteorder; +extern crate triehash; +extern crate parking_lot; +extern crate twox_hash; +extern crate hashdb; + +#[macro_use] extern crate log; + +#[macro_use] +extern crate lazy_static; + +#[macro_use] +extern crate error_chain; + +#[cfg(test)] +extern crate assert_matches; + +#[cfg(test)] +extern crate wabt; + +#[cfg(test)] +#[macro_use] +extern crate hex_literal; + +#[macro_use] +mod wasm_utils; +mod wasm_executor; +#[macro_use] +mod native_executor; +mod sandbox; + +pub mod error; +pub use wasm_executor::WasmExecutor; +pub use native_executor::{with_native_environment, NativeExecutor, NativeExecutionDispatch}; +pub use state_machine::Externalities; +pub use runtime_version::RuntimeVersion; +pub use codec::Codec; +use primitives::Blake2Hasher; + +/// Provides runtime information. +pub trait RuntimeInfo { + /// Native runtime information if any. + const NATIVE_VERSION: Option; + + /// Extract RuntimeVersion of given :code block + fn runtime_version> ( + &self, + ext: &mut E, + heap_pages: usize, + code: &[u8] + ) -> Option; +} diff --git a/core/executor/src/native_executor.rs b/core/executor/src/native_executor.rs new file mode 100644 index 00000000000..bc928364a60 --- /dev/null +++ b/core/executor/src/native_executor.rs @@ -0,0 +1,224 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use error::{Error, ErrorKind, Result}; +use state_machine::{CodeExecutor, Externalities}; +use wasm_executor::WasmExecutor; +use wasmi::Module as WasmModule; +use runtime_version::RuntimeVersion; +use std::collections::HashMap; +use codec::Decode; +use primitives::hashing::blake2_256; +use parking_lot::{Mutex, MutexGuard}; +use RuntimeInfo; +use primitives::Blake2Hasher; + +// For the internal Runtime Cache: +// Is it compatible enough to run this natively or do we need to fall back on the WasmModule + +enum RuntimePreproc { + InvalidCode, + ValidCode(WasmModule, Option), +} + +type CacheType = HashMap<[u8; 32], RuntimePreproc>; + +lazy_static! { + static ref RUNTIMES_CACHE: Mutex = Mutex::new(HashMap::new()); +} + +// helper function to generate low-over-head caching_keys +// it is asserted that part of the audit process that any potential on-chain code change +// will have done is to ensure that the two-x hash is different to that of any other +// :code value from the same chain +fn gen_cache_key(code: &[u8]) -> [u8; 32] { + blake2_256(code) +} + +/// fetch a runtime version from the cache or if there is no cached version yet, create +/// the runtime version entry for `code`, determines whether `Compatibility::IsCompatible` +/// can be used by comparing returned RuntimeVersion to `ref_version` +fn fetch_cached_runtime_version<'a, E: Externalities>( + wasm_executor: &WasmExecutor, + cache: &'a mut MutexGuard, + ext: &mut E, + heap_pages: usize, + code: &[u8] +) -> Result<(&'a WasmModule, &'a Option)> { + let maybe_runtime_preproc = cache.entry(gen_cache_key(code)) + .or_insert_with(|| match WasmModule::from_buffer(code) { + Ok(module) => { + let version = wasm_executor.call_in_wasm_module(ext, heap_pages, &module, "version", &[]) + .ok() + .and_then(|v| RuntimeVersion::decode(&mut v.as_slice())); + RuntimePreproc::ValidCode(module, version) + } + Err(e) => { + trace!(target: "executor", "Invalid code presented to executor ({:?})", e); + RuntimePreproc::InvalidCode + } + }); + match maybe_runtime_preproc { + RuntimePreproc::InvalidCode => Err(ErrorKind::InvalidCode(code.into()).into()), + RuntimePreproc::ValidCode(m, v) => Ok((m, v)), + } +} + +fn safe_call(f: F) -> Result + where F: ::std::panic::UnwindSafe + FnOnce() -> U +{ + // Substrate uses custom panic hook that terminates process on panic. Disable it for the native call. + let hook = ::std::panic::take_hook(); + let result = ::std::panic::catch_unwind(f).map_err(|_| ErrorKind::Runtime.into()); + ::std::panic::set_hook(hook); + result +} + +/// Set up the externalities and safe calling environment to execute calls to a native runtime. +/// +/// If the inner closure panics, it will be caught and return an error. +pub fn with_native_environment(ext: &mut Externalities, f: F) -> Result +where F: ::std::panic::UnwindSafe + FnOnce() -> U +{ + ::runtime_io::with_externalities(ext, move || safe_call(f)) +} + +/// Delegate for dispatching a CodeExecutor call to native code. +pub trait NativeExecutionDispatch: Send + Sync { + /// Get the wasm code that the native dispatch will be equivalent to. + fn native_equivalent() -> &'static [u8]; + + /// Dispatch a method and input data to be executed natively. Returns `Some` result or `None` + /// if the `method` is unknown. Panics if there's an unrecoverable error. + // fn dispatch(ext: &mut Externalities, method: &str, data: &[u8]) -> Result>; + fn dispatch(ext: &mut Externalities, method: &str, data: &[u8]) -> Result>; + + /// Get native runtime version. + const VERSION: RuntimeVersion; + + /// Construct corresponding `NativeExecutor` + fn new() -> NativeExecutor where Self: Sized { + NativeExecutor::new() + } +} + +/// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence +/// and dispatch to native code when possible, falling back on `WasmExecutor` when not. +#[derive(Debug)] +pub struct NativeExecutor { + /// Dummy field to avoid the compiler complaining about us not using `D`. + _dummy: ::std::marker::PhantomData, + /// The fallback executor in case native isn't available. + fallback: WasmExecutor, +} + +impl NativeExecutor { + /// Create new instance. + pub fn new() -> Self { + NativeExecutor { + _dummy: Default::default(), + fallback: WasmExecutor::new(), + } + } +} + +impl Clone for NativeExecutor { + fn clone(&self) -> Self { + NativeExecutor { + _dummy: Default::default(), + fallback: self.fallback.clone(), + } + } +} + +impl RuntimeInfo for NativeExecutor { + const NATIVE_VERSION: Option = Some(D::VERSION); + + fn runtime_version>( + &self, + ext: &mut E, + heap_pages: usize, + code: &[u8], + ) -> Option { + fetch_cached_runtime_version(&self.fallback, &mut RUNTIMES_CACHE.lock(), ext, heap_pages, code).ok()?.1.clone() + } +} + +impl CodeExecutor for NativeExecutor { + type Error = Error; + + fn call>( + &self, + ext: &mut E, + heap_pages: usize, + code: &[u8], + method: &str, + data: &[u8], + use_native: bool, + ) -> (Result>, bool) { + let mut c = RUNTIMES_CACHE.lock(); + let (module, onchain_version) = match fetch_cached_runtime_version(&self.fallback, &mut c, ext, heap_pages, code) { + Ok((module, onchain_version)) => (module, onchain_version), + Err(_) => return (Err(ErrorKind::InvalidCode(code.into()).into()), false), + }; + match (use_native, onchain_version.as_ref().map_or(false, |v| v.can_call_with(&D::VERSION))) { + (_, false) => { + trace!(target: "executor", "Request for native execution failed (native: {}, chain: {})", D::VERSION, onchain_version.as_ref().map_or_else(||"".into(), |v| format!("{}", v))); + (self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false) + } + (false, _) => { + (self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false) + } + _ => { + trace!(target: "executor", "Request for native execution succeeded (native: {}, chain: {})", D::VERSION, onchain_version.as_ref().map_or_else(||"".into(), |v| format!("{}", v))); + (D::dispatch(ext, method, data), true) + } + } + } +} + +#[macro_export] +macro_rules! native_executor_instance { + (pub $name:ident, $dispatcher:path, $version:path, $code:expr) => { + pub struct $name; + native_executor_instance!(IMPL $name, $dispatcher, $version, $code); + }; + ($name:ident, $dispatcher:path, $version:path, $code:expr) => { + /// A unit struct which implements `NativeExecutionDispatch` feeding in the hard-coded runtime. + struct $name; + native_executor_instance!(IMPL $name, $dispatcher, $version, $code); + }; + (IMPL $name:ident, $dispatcher:path, $version:path, $code:expr) => { + // TODO: this is not so great – I think I should go back to have dispatch take a type param and modify this macro to accept a type param and then pass it in from the test-client instead + use primitives::Blake2Hasher as _Blake2Hasher; + impl $crate::NativeExecutionDispatch for $name { + const VERSION: $crate::RuntimeVersion = $version; + fn native_equivalent() -> &'static [u8] { + // WARNING!!! This assumes that the runtime was built *before* the main project. Until we + // get a proper build script, this must be strictly adhered to or things will go wrong. + $code + } + fn dispatch(ext: &mut $crate::Externalities<_Blake2Hasher>, method: &str, data: &[u8]) -> $crate::error::Result> { + $crate::with_native_environment(ext, move || $dispatcher(method, data))? + .ok_or_else(|| $crate::error::ErrorKind::MethodNotFound(method.to_owned()).into()) + } + + fn new() -> $crate::NativeExecutor<$name> { + $crate::NativeExecutor::new() + } + } + } +} diff --git a/core/executor/src/sandbox.rs b/core/executor/src/sandbox.rs new file mode 100644 index 00000000000..7bedde70dc3 --- /dev/null +++ b/core/executor/src/sandbox.rs @@ -0,0 +1,676 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![warn(missing_docs)] + +//! This module implements sandboxing support in the runtime. + +use std::collections::HashMap; +use std::rc::Rc; +use codec::{Decode, Encode}; +use primitives::sandbox as sandbox_primitives; +use wasm_utils::UserError; +use wasmi; +use wasmi::memory_units::Pages; +use wasmi::{ + Externals, FuncRef, ImportResolver, MemoryInstance, MemoryRef, Module, ModuleInstance, + ModuleRef, RuntimeArgs, RuntimeValue, Trap, TrapKind +}; + +/// Index of a function inside the supervisor. +/// +/// This is a typically an index in the default table of the supervisor, however +/// the exact meaning of this index is depends on the implementation of dispatch function. +#[derive(Copy, Clone, Debug, PartialEq)] +struct SupervisorFuncIndex(usize); + +/// Index of a function within guest index space. +/// +/// This index is supposed to be used with as index for `Externals`. +#[derive(Copy, Clone, Debug, PartialEq)] +struct GuestFuncIndex(usize); + +/// This struct holds a mapping from guest index space to supervisor. +struct GuestToSupervisorFunctionMapping { + funcs: Vec, +} + +impl GuestToSupervisorFunctionMapping { + fn new() -> GuestToSupervisorFunctionMapping { + GuestToSupervisorFunctionMapping { funcs: Vec::new() } + } + + fn define(&mut self, supervisor_func: SupervisorFuncIndex) -> GuestFuncIndex { + let idx = self.funcs.len(); + self.funcs.push(supervisor_func); + GuestFuncIndex(idx) + } + + fn func_by_guest_index(&self, guest_func_idx: GuestFuncIndex) -> Option { + self.funcs.get(guest_func_idx.0).cloned() + } +} + +struct Imports { + func_map: HashMap<(Vec, Vec), GuestFuncIndex>, + memories_map: HashMap<(Vec, Vec), MemoryRef>, +} + +impl ImportResolver for Imports { + fn resolve_func( + &self, + module_name: &str, + field_name: &str, + signature: &::wasmi::Signature, + ) -> Result { + let key = ( + module_name.as_bytes().to_owned(), + field_name.as_bytes().to_owned(), + ); + let idx = *self.func_map.get(&key).ok_or_else(|| { + ::wasmi::Error::Instantiation(format!( + "Export {}:{} not found", + module_name, field_name + )) + })?; + Ok(::wasmi::FuncInstance::alloc_host(signature.clone(), idx.0)) + } + + fn resolve_memory( + &self, + module_name: &str, + field_name: &str, + _memory_type: &::wasmi::MemoryDescriptor, + ) -> Result { + let key = ( + module_name.as_bytes().to_vec(), + field_name.as_bytes().to_vec(), + ); + let mem = self.memories_map + .get(&key) + .ok_or_else(|| { + ::wasmi::Error::Instantiation(format!( + "Export {}:{} not found", + module_name, field_name + )) + })? + .clone(); + Ok(mem) + } + + fn resolve_global( + &self, + module_name: &str, + field_name: &str, + _global_type: &::wasmi::GlobalDescriptor, + ) -> Result<::wasmi::GlobalRef, ::wasmi::Error> { + Err(::wasmi::Error::Instantiation(format!( + "Export {}:{} not found", + module_name, field_name + ))) + } + + fn resolve_table( + &self, + module_name: &str, + field_name: &str, + _table_type: &::wasmi::TableDescriptor, + ) -> Result<::wasmi::TableRef, ::wasmi::Error> { + Err(::wasmi::Error::Instantiation(format!( + "Export {}:{} not found", + module_name, field_name + ))) + } +} + +/// This trait encapsulates sandboxing capabilities. +/// +/// Note that this functions are only called in the `supervisor` context. +pub trait SandboxCapabilities { + /// Returns a reference to an associated sandbox `Store`. + fn store(&self) -> &Store; + + /// Returns a mutable reference to an associated sandbox `Store`. + fn store_mut(&mut self) -> &mut Store; + + /// Allocate space of the specified length in the supervisor memory. + /// + /// Returns pointer to the allocated block. + fn allocate(&mut self, len: u32) -> u32; + + /// Deallocate space specified by the pointer that was previously returned by [`allocate`]. + /// + /// [`allocate`]: #tymethod.allocate + fn deallocate(&mut self, ptr: u32); + + /// Write `data` into the supervisor memory at offset specified by `ptr`. + /// + /// # Errors + /// + /// Returns `Err` if `ptr + data.len()` is out of bounds. + fn write_memory(&mut self, ptr: u32, data: &[u8]) -> Result<(), UserError>; + + /// Read `len` bytes from the supervisor memory. + /// + /// # Errors + /// + /// Returns `Err` if `ptr + len` is out of bounds. + fn read_memory(&self, ptr: u32, len: u32) -> Result, UserError>; +} + +/// Implementation of [`Externals`] that allows execution of guest module with +/// [externals][`Externals`] that might refer functions defined by supervisor. +/// +/// [`Externals`]: ../../wasmi/trait.Externals.html +pub struct GuestExternals<'a, FE: SandboxCapabilities + Externals + 'a> { + supervisor_externals: &'a mut FE, + sandbox_instance: &'a SandboxInstance, + state: u32, +} + +fn trap() -> Trap { + TrapKind::Host(Box::new(UserError("Sandbox error"))).into() +} + +fn deserialize_result(serialized_result: &[u8]) -> Result, Trap> { + use self::sandbox_primitives::{HostError, ReturnValue}; + let result_val = Result::::decode(&mut &serialized_result[..]) + .ok_or_else(|| trap())?; + + match result_val { + Ok(return_value) => Ok(match return_value { + ReturnValue::Unit => None, + ReturnValue::Value(typed_value) => Some(RuntimeValue::from(typed_value)), + }), + Err(HostError) => Err(trap()), + } +} + +impl<'a, FE: SandboxCapabilities + Externals + 'a> Externals for GuestExternals<'a, FE> { + fn invoke_index( + &mut self, + index: usize, + args: RuntimeArgs, + ) -> Result, Trap> { + // Make `index` typesafe again. + let index = GuestFuncIndex(index); + + let dispatch_thunk = self.sandbox_instance.dispatch_thunk.clone(); + let func_idx = self.sandbox_instance + .guest_to_supervisor_mapping + .func_by_guest_index(index) + .expect( + "`invoke_index` is called with indexes registered via `FuncInstance::alloc_host`; + `FuncInstance::alloc_host` is called with indexes that was obtained from `guest_to_supervisor_mapping`; + `func_by_guest_index` called with `index` can't return `None`; + qed" + ); + + // Serialize arguments into a byte vector. + let invoke_args_data: Vec = args.as_ref() + .iter() + .cloned() + .map(sandbox_primitives::TypedValue::from) + .collect::>() + .encode(); + + let state = self.state; + + // Move serialized arguments inside the memory and invoke dispatch thunk and + // then free allocated memory. + let invoke_args_ptr = self.supervisor_externals + .allocate(invoke_args_data.len() as u32); + self.supervisor_externals + .write_memory(invoke_args_ptr, &invoke_args_data)?; + let result = ::wasmi::FuncInstance::invoke( + &dispatch_thunk, + &[ + RuntimeValue::I32(invoke_args_ptr as i32), + RuntimeValue::I32(invoke_args_data.len() as i32), + RuntimeValue::I32(state as i32), + RuntimeValue::I32(func_idx.0 as i32), + ], + self.supervisor_externals, + ); + self.supervisor_externals.deallocate(invoke_args_ptr); + + // dispatch_thunk returns pointer to serialized arguments. + let (serialized_result_val_ptr, serialized_result_val_len) = match result { + // Unpack pointer and len of the serialized result data. + Ok(Some(RuntimeValue::I64(v))) => { + // Cast to u64 to use zero-extension. + let v = v as u64; + let ptr = (v as u64 >> 32) as u32; + let len = (v & 0xFFFFFFFF) as u32; + (ptr, len) + } + _ => return Err(trap()), + }; + + let serialized_result_val = self.supervisor_externals + .read_memory(serialized_result_val_ptr, serialized_result_val_len)?; + self.supervisor_externals + .deallocate(serialized_result_val_ptr); + + // We do not have to check the signature here, because it's automatically + // checked by wasmi. + + deserialize_result(&serialized_result_val) + } +} + +fn with_guest_externals( + supervisor_externals: &mut FE, + sandbox_instance: &SandboxInstance, + state: u32, + f: F, +) -> R +where + FE: SandboxCapabilities + Externals, + F: FnOnce(&mut GuestExternals) -> R, +{ + let mut guest_externals = GuestExternals { + supervisor_externals, + sandbox_instance, + state, + }; + f(&mut guest_externals) +} + +/// Sandboxed instance of a wasm module. +/// +/// It's primary purpose is to [`invoke`] exported functions on it. +/// +/// All imports of this instance are specified at the creation time and +/// imports are implemented by the supervisor. +/// +/// Hence, in order to invoke an exported function on a sandboxed module instance, +/// it's required to provide supervisor externals: it will be used to execute +/// code in the supervisor context. +/// +/// [`invoke`]: #method.invoke +pub struct SandboxInstance { + instance: ModuleRef, + dispatch_thunk: FuncRef, + guest_to_supervisor_mapping: GuestToSupervisorFunctionMapping, +} + +impl SandboxInstance { + /// Invoke an exported function by a name. + /// + /// `supervisor_externals` is required to execute the implementations + /// of the syscalls that published to a sandboxed module instance. + /// + /// The `state` parameter can be used to provide custom data for + /// these syscall implementations. + pub fn invoke( + &self, + export_name: &str, + args: &[RuntimeValue], + supervisor_externals: &mut FE, + state: u32, + ) -> Result, wasmi::Error> { + with_guest_externals( + supervisor_externals, + self, + state, + |guest_externals| { + self.instance + .invoke_export(export_name, args, guest_externals) + }, + ) + } +} + +fn decode_environment_definition( + raw_env_def: &[u8], + memories: &[Option], +) -> Result<(Imports, GuestToSupervisorFunctionMapping), UserError> { + let env_def = sandbox_primitives::EnvironmentDefinition::decode(&mut &raw_env_def[..]).ok_or_else(|| UserError("Sandbox error"))?; + + let mut func_map = HashMap::new(); + let mut memories_map = HashMap::new(); + let mut guest_to_supervisor_mapping = GuestToSupervisorFunctionMapping::new(); + + for entry in &env_def.entries { + let module = entry.module_name.clone(); + let field = entry.field_name.clone(); + + match entry.entity { + sandbox_primitives::ExternEntity::Function(func_idx) => { + let externals_idx = + guest_to_supervisor_mapping.define(SupervisorFuncIndex(func_idx as usize)); + func_map.insert((module, field), externals_idx); + } + sandbox_primitives::ExternEntity::Memory(memory_idx) => { + let memory_ref = memories + .get(memory_idx as usize) + .cloned() + .ok_or_else(|| UserError("Sandbox error"))? + .ok_or_else(|| UserError("Sandbox error"))?; + memories_map.insert((module, field), memory_ref); + } + } + } + + Ok(( + Imports { + func_map, + memories_map, + }, + guest_to_supervisor_mapping, + )) +} + +/// Instantiate a guest module and return it's index in the store. +/// +/// The guest module's code is specified in `wasm`. Environment that will be available to +/// guest module is specified in `raw_env_def` (serialized version of [`EnvironmentDefinition`]). +/// `dispatch_thunk` is used as function that handle calls from guests. +/// +/// # Errors +/// +/// Returns `Err` if any of the following conditions happens: +/// +/// - `raw_env_def` can't be deserialized as a [`EnvironmentDefinition`]. +/// - Module in `wasm` is invalid or couldn't be instantiated. +/// +/// [`EnvironmentDefinition`]: ../../sandbox/struct.EnvironmentDefinition.html +pub fn instantiate( + supervisor_externals: &mut FE, + dispatch_thunk: FuncRef, + wasm: &[u8], + raw_env_def: &[u8], + state: u32, +) -> Result { + let (imports, guest_to_supervisor_mapping) = + decode_environment_definition(raw_env_def, &supervisor_externals.store().memories)?; + + let module = Module::from_buffer(wasm).map_err(|_| UserError("Sandbox error"))?; + let instance = ModuleInstance::new(&module, &imports).map_err(|_| UserError("Sandbox error"))?; + + let sandbox_instance = Rc::new(SandboxInstance { + // In general, it's not a very good idea to use `.not_started_instance()` for anything + // but for extracting memory and tables. But in this particular case, we are extracting + // for the purpose of running `start` function which should be ok. + instance: instance.not_started_instance().clone(), + dispatch_thunk, + guest_to_supervisor_mapping, + }); + + with_guest_externals( + supervisor_externals, + &sandbox_instance, + state, + |guest_externals| { + instance + .run_start(guest_externals) + .map_err(|_| UserError("Sandbox error")) + }, + )?; + + let instance_idx = supervisor_externals + .store_mut() + .register_sandbox_instance(sandbox_instance); + + Ok(instance_idx) +} + +/// This struct keeps track of all sandboxed components. +pub struct Store { + // Memories and instances are `Some` untill torndown. + instances: Vec>>, + memories: Vec>, +} + +impl Store { + /// Create a new empty sandbox store. + pub fn new() -> Store { + Store { + instances: Vec::new(), + memories: Vec::new(), + } + } + + /// Create a new memory instance and return it's index. + /// + /// # Errors + /// + /// Returns `Err` if the memory couldn't be created. + /// Typically happens if `initial` is more than `maximum`. + pub fn new_memory(&mut self, initial: u32, maximum: u32) -> Result { + let maximum = match maximum { + sandbox_primitives::MEM_UNLIMITED => None, + specified_limit => Some(Pages(specified_limit as usize)), + }; + + let mem = + MemoryInstance::alloc(Pages(initial as usize), maximum).map_err(|_| UserError("Sandbox error"))?; + let mem_idx = self.memories.len(); + self.memories.push(Some(mem)); + Ok(mem_idx as u32) + } + + /// Returns `SandboxInstance` by `instance_idx`. + /// + /// # Errors + /// + /// Returns `Err` If `instance_idx` isn't a valid index of an instance or + /// instance is already torndown. + pub fn instance(&self, instance_idx: u32) -> Result, UserError> { + self.instances + .get(instance_idx as usize) + .cloned() + .ok_or_else(|| UserError("Sandbox error"))? + .ok_or_else(|| UserError("Sandbox error")) + } + + /// Returns reference to a memory instance by `memory_idx`. + /// + /// # Errors + /// + /// Returns `Err` If `memory_idx` isn't a valid index of an memory or + /// memory is already torndown. + pub fn memory(&self, memory_idx: u32) -> Result { + self.memories + .get(memory_idx as usize) + .cloned() + .ok_or_else(|| UserError("Sandbox error"))? + .ok_or_else(|| UserError("Sandbox error")) + } + + /// Teardown the memory at the specified index. + /// + /// # Errors + /// + /// Returns `Err` if `memory_idx` isn't a valid index of an memory. + pub fn memory_teardown(&mut self, memory_idx: u32) -> Result<(), UserError> { + if memory_idx as usize >= self.memories.len() { + return Err(UserError("Sandbox error")); + } + self.memories[memory_idx as usize] = None; + Ok(()) + } + + /// Teardown the instance at the specified index. + pub fn instance_teardown(&mut self, instance_idx: u32) -> Result<(), UserError> { + if instance_idx as usize >= self.instances.len() { + return Err(UserError("Sandbox error")); + } + self.instances[instance_idx as usize] = None; + Ok(()) + } + + fn register_sandbox_instance(&mut self, sandbox_instance: Rc) -> u32 { + let instance_idx = self.instances.len(); + self.instances.push(Some(sandbox_instance)); + instance_idx as u32 + } +} + +#[cfg(test)] +mod tests { + use wasm_executor::WasmExecutor; + use state_machine::TestExternalities; + use wabt; + + #[test] + fn sandbox_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let code = wabt::wat2wasm(r#" + (module + (import "env" "assert" (func $assert (param i32))) + (import "env" "inc_counter" (func $inc_counter (param i32) (result i32))) + (func (export "call") + (drop + (call $inc_counter (i32.const 5)) + ) + + (call $inc_counter (i32.const 3)) + ;; current counter value is on the stack + + ;; check whether current == 8 + i32.const 8 + i32.eq + + call $assert + ) + ) + "#).unwrap(); + + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), + vec![1], + ); + } + + #[test] + fn sandbox_trap() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let code = wabt::wat2wasm(r#" + (module + (import "env" "assert" (func $assert (param i32))) + (func (export "call") + i32.const 0 + call $assert + ) + ) + "#).unwrap(); + + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), + vec![0], + ); + } + + #[test] + fn start_called() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let code = wabt::wat2wasm(r#" + (module + (import "env" "assert" (func $assert (param i32))) + (import "env" "inc_counter" (func $inc_counter (param i32) (result i32))) + + ;; Start function + (start $start) + (func $start + ;; Increment counter by 1 + (drop + (call $inc_counter (i32.const 1)) + ) + ) + + (func (export "call") + ;; Increment counter by 1. The current value is placed on the stack. + (call $inc_counter (i32.const 1)) + + ;; Counter is incremented twice by 1, once there and once in `start` func. + ;; So check the returned value is equal to 2. + i32.const 2 + i32.eq + call $assert + ) + ) + "#).unwrap(); + + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), + vec![1], + ); + } + + #[test] + fn invoke_args() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let code = wabt::wat2wasm(r#" + (module + (import "env" "assert" (func $assert (param i32))) + + (func (export "call") (param $x i32) (param $y i64) + ;; assert that $x = 0x12345678 + (call $assert + (i32.eq + (get_local $x) + (i32.const 0x12345678) + ) + ) + + (call $assert + (i64.eq + (get_local $y) + (i64.const 0x1234567887654321) + ) + ) + ) + ) + "#).unwrap(); + + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_args", &code).unwrap(), + vec![1], + ); + } + + #[test] + fn return_val() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let code = wabt::wat2wasm(r#" + (module + (func (export "call") (param $x i32) (result i32) + (i32.add + (get_local $x) + (i32.const 1) + ) + ) + ) + "#).unwrap(); + + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_return_val", &code).unwrap(), + vec![1], + ); + } +} diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs new file mode 100644 index 00000000000..18280c797df --- /dev/null +++ b/core/executor/src/wasm_executor.rs @@ -0,0 +1,718 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Rust implementation of Substrate contracts. + +use std::cmp::Ordering; +use std::collections::HashMap; + +use wasmi::{ + Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder +}; +use wasmi::RuntimeValue::{I32, I64}; +use wasmi::memory_units::{Pages, Bytes}; +use state_machine::Externalities; +use error::{Error, ErrorKind, Result}; +use wasm_utils::UserError; +use primitives::{blake2_256, twox_128, twox_256, ed25519, Blake2Hasher, hexdisplay::HexDisplay}; +use primitives::sandbox as sandbox_primitives; +use triehash::ordered_trie_root; +use sandbox; + + +struct Heap { + end: u32, +} + +impl Heap { + /// Construct new `Heap` struct with a given number of pages. + /// + /// Returns `Err` if the heap couldn't allocate required + /// number of pages. + /// + /// This could mean that wasm binary specifies memory + /// limit and we are trying to allocate beyond that limit. + fn new(memory: &MemoryRef, pages: usize) -> Result { + let prev_page_count = memory.initial(); + memory.grow(Pages(pages)).map_err(|_| Error::from(ErrorKind::Runtime))?; + Ok(Heap { + end: Bytes::from(prev_page_count).0 as u32, + }) + } + + fn allocate(&mut self, size: u32) -> u32 { + let r = self.end; + self.end += size; + r + } + + fn deallocate(&mut self, _offset: u32) { + } +} + +#[cfg(feature="wasm-extern-trace")] +macro_rules! debug_trace { + ( $( $x:tt )* ) => ( trace!( $( $x )* ) ) +} +#[cfg(not(feature="wasm-extern-trace"))] +macro_rules! debug_trace { + ( $( $x:tt )* ) => () +} + +struct FunctionExecutor<'e, E: Externalities + 'e> { + sandbox_store: sandbox::Store, + heap: Heap, + memory: MemoryRef, + table: Option, + ext: &'e mut E, + hash_lookup: HashMap, Vec>, +} + +impl<'e, E: Externalities> FunctionExecutor<'e, E> { + fn new(m: MemoryRef, heap_pages: usize, t: Option, e: &'e mut E) -> Result { + Ok(FunctionExecutor { + sandbox_store: sandbox::Store::new(), + heap: Heap::new(&m, heap_pages)?, + memory: m, + table: t, + ext: e, + hash_lookup: HashMap::new(), + }) + } +} + +impl<'e, E: Externalities> sandbox::SandboxCapabilities for FunctionExecutor<'e, E> { + fn store(&self) -> &sandbox::Store { + &self.sandbox_store + } + fn store_mut(&mut self) -> &mut sandbox::Store { + &mut self.sandbox_store + } + fn allocate(&mut self, len: u32) -> u32 { + self.heap.allocate(len) + } + fn deallocate(&mut self, ptr: u32) { + self.heap.deallocate(ptr) + } + fn write_memory(&mut self, ptr: u32, data: &[u8]) -> ::std::result::Result<(), UserError> { + self.memory.set(ptr, data).map_err(|_| UserError("Invalid attempt to write_memory")) + } + fn read_memory(&self, ptr: u32, len: u32) -> ::std::result::Result, UserError> { + self.memory.get(ptr, len as usize).map_err(|_| UserError("Invalid attempt to write_memory")) + } +} + +trait WritePrimitive { + fn write_primitive(&self, offset: u32, t: T) -> ::std::result::Result<(), UserError>; +} + +impl WritePrimitive for MemoryInstance { + fn write_primitive(&self, offset: u32, t: u32) -> ::std::result::Result<(), UserError> { + use byteorder::{LittleEndian, ByteOrder}; + let mut r = [0u8; 4]; + LittleEndian::write_u32(&mut r, t); + self.set(offset, &r).map_err(|_| UserError("Invalid attempt to write_primitive")) + } +} + +trait ReadPrimitive { + fn read_primitive(&self, offset: u32) -> ::std::result::Result; +} + +impl ReadPrimitive for MemoryInstance { + fn read_primitive(&self, offset: u32) -> ::std::result::Result { + use byteorder::{LittleEndian, ByteOrder}; + Ok(LittleEndian::read_u32(&self.get(offset, 4).map_err(|_| UserError("Invalid attempt to read_primitive"))?)) + } +} + +// TODO: this macro does not support `where` clauses and that seems somewhat tricky to add +impl_function_executor!(this: FunctionExecutor<'e, E>, + ext_print_utf8(utf8_data: *const u8, utf8_len: u32) => { + if let Ok(utf8) = this.memory.get(utf8_data, utf8_len as usize) { + if let Ok(message) = String::from_utf8(utf8) { + println!("{}", message); + } + } + Ok(()) + }, + ext_print_hex(data: *const u8, len: u32) => { + if let Ok(hex) = this.memory.get(data, len as usize) { + println!("{}", HexDisplay::from(&hex)); + } + Ok(()) + }, + ext_print_num(number: u64) => { + println!("{}", number); + Ok(()) + }, + ext_memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 => { + let sl1 = this.memory.get(s1, n as usize).map_err(|_| UserError("Invalid attempt to read from memory in first arg of ext_memcmp"))?; + let sl2 = this.memory.get(s2, n as usize).map_err(|_| UserError("Invalid attempt to read from memory in second arg of ext_memcmp"))?; + Ok(match sl1.cmp(&sl2) { + Ordering::Greater => 1, + Ordering::Less => -1, + Ordering::Equal => 0, + }) + }, + ext_memcpy(dest: *mut u8, src: *const u8, count: usize) -> *mut u8 => { + this.memory.copy_nonoverlapping(src as usize, dest as usize, count as usize) + .map_err(|_| UserError("Invalid attempt to copy_nonoverlapping in ext_memcpy"))?; + debug_trace!(target: "runtime-io", "memcpy {} from {}, {} bytes", dest, src, count); + Ok(dest) + }, + ext_memmove(dest: *mut u8, src: *const u8, count: usize) -> *mut u8 => { + this.memory.copy(src as usize, dest as usize, count as usize) + .map_err(|_| UserError("Invalid attempt to copy in ext_memmove"))?; + debug_trace!(target: "runtime-io", "memmove {} from {}, {} bytes", dest, src, count); + Ok(dest) + }, + ext_memset(dest: *mut u8, val: u32, count: usize) -> *mut u8 => { + debug_trace!(target: "runtime-io", "memset {} with {}, {} bytes", dest, val, count); + this.memory.clear(dest as usize, val as u8, count as usize) + .map_err(|_| UserError("Invalid attempt to clear in ext_memset"))?; + Ok(dest) + }, + ext_malloc(size: usize) -> *mut u8 => { + let r = this.heap.allocate(size); + debug_trace!(target: "runtime-io", "malloc {} bytes at {}", size, r); + Ok(r) + }, + ext_free(addr: *mut u8) => { + this.heap.deallocate(addr); + debug_trace!(target: "runtime-io", "free {}", addr); + Ok(()) + }, + ext_set_storage(key_data: *const u8, key_len: u32, value_data: *const u8, value_len: u32) => { + let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to determine key in ext_set_storage"))?; + let value = this.memory.get(value_data, value_len as usize).map_err(|_| UserError("Invalid attempt to determine value in ext_set_storage"))?; + if let Some(_preimage) = this.hash_lookup.get(&key) { + debug_trace!(target: "wasm-trace", "*** Setting storage: %{} -> {} [k={}]", ::primitives::hexdisplay::ascii_format(&_preimage), HexDisplay::from(&value), HexDisplay::from(&key)); + } else { + debug_trace!(target: "wasm-trace", "*** Setting storage: {} -> {} [k={}]", ::primitives::hexdisplay::ascii_format(&key), HexDisplay::from(&value), HexDisplay::from(&key)); + } + this.ext.set_storage(key, value); + Ok(()) + }, + ext_clear_storage(key_data: *const u8, key_len: u32) => { + let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to determine key in ext_clear_storage"))?; + debug_trace!(target: "wasm-trace", "*** Clearing storage: {} [k={}]", + if let Some(_preimage) = this.hash_lookup.get(&key) { + format!("%{}", ::primitives::hexdisplay::ascii_format(&_preimage)) + } else { + format!(" {}", ::primitives::hexdisplay::ascii_format(&key)) + }, HexDisplay::from(&key)); + this.ext.clear_storage(&key); + Ok(()) + }, + ext_exists_storage(key_data: *const u8, key_len: u32) -> u32 => { + let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to determine key in ext_exists_storage"))?; + Ok(if this.ext.exists_storage(&key) { 1 } else { 0 }) + }, + ext_clear_prefix(prefix_data: *const u8, prefix_len: u32) => { + let prefix = this.memory.get(prefix_data, prefix_len as usize).map_err(|_| UserError("Invalid attempt to determine prefix in ext_clear_prefix"))?; + this.ext.clear_prefix(&prefix); + Ok(()) + }, + // return 0 and place u32::max_value() into written_out if no value exists for the key. + ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8 => { + let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to determine key in ext_get_allocated_storage"))?; + let maybe_value = this.ext.storage(&key); + + debug_trace!(target: "wasm-trace", "*** Getting storage: {} == {} [k={}]", + if let Some(_preimage) = this.hash_lookup.get(&key) { + format!("%{}", ::primitives::hexdisplay::ascii_format(&_preimage)) + } else { + format!(" {}", ::primitives::hexdisplay::ascii_format(&key)) + }, + if let Some(ref b) = maybe_value { + format!("{}", HexDisplay::from(b)) + } else { + "".to_owned() + }, + HexDisplay::from(&key) + ); + + if let Some(value) = maybe_value { + let offset = this.heap.allocate(value.len() as u32) as u32; + this.memory.set(offset, &value).map_err(|_| UserError("Invalid attempt to set memory in ext_get_allocated_storage"))?; + this.memory.write_primitive(written_out, value.len() as u32) + .map_err(|_| UserError("Invalid attempt to write written_out in ext_get_allocated_storage"))?; + Ok(offset) + } else { + this.memory.write_primitive(written_out, u32::max_value()) + .map_err(|_| UserError("Invalid attempt to write failed written_out in ext_get_allocated_storage"))?; + Ok(0) + } + }, + // return u32::max_value() if no value exists for the key. + ext_get_storage_into(key_data: *const u8, key_len: u32, value_data: *mut u8, value_len: u32, value_offset: u32) -> u32 => { + let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to get key in ext_get_storage_into"))?; + let maybe_value = this.ext.storage(&key); + debug_trace!(target: "wasm-trace", "*** Getting storage: {} == {} [k={}]", + if let Some(_preimage) = this.hash_lookup.get(&key) { + format!("%{}", ::primitives::hexdisplay::ascii_format(&_preimage)) + } else { + format!(" {}", ::primitives::hexdisplay::ascii_format(&key)) + }, + if let Some(ref b) = maybe_value { + format!("{}", HexDisplay::from(b)) + } else { + "".to_owned() + }, + HexDisplay::from(&key) + ); + + if let Some(value) = maybe_value { + let value = &value[value_offset as usize..]; + let written = ::std::cmp::min(value_len as usize, value.len()); + this.memory.set(value_data, &value[..written]).map_err(|_| UserError("Invalid attempt to set value in ext_get_storage_into"))?; + Ok(written as u32) + } else { + Ok(u32::max_value()) + } + }, + ext_storage_root(result: *mut u8) => { + let r = this.ext.storage_root(); + this.memory.set(result, r.as_ref()).map_err(|_| UserError("Invalid attempt to set memory in ext_storage_root"))?; + Ok(()) + }, + ext_blake2_256_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8) => { + let values = (0..lens_len) + .map(|i| this.memory.read_primitive(lens_data + i * 4)) + .collect::<::std::result::Result, UserError>>()? + .into_iter() + .scan(0u32, |acc, v| { let o = *acc; *acc += v; Some((o, v)) }) + .map(|(offset, len)| + this.memory.get(values_data + offset, len as usize) + .map_err(|_| UserError("Invalid attempt to get memory in ext_blake2_256_enumerated_trie_root")) + ) + .collect::<::std::result::Result, UserError>>()?; + let r = ordered_trie_root::(values.into_iter()); + this.memory.set(result, &r[..]).map_err(|_| UserError("Invalid attempt to set memory in ext_blake2_256_enumerated_trie_root"))?; + Ok(()) + }, + ext_chain_id() -> u64 => { + Ok(this.ext.chain_id()) + }, + ext_twox_128(data: *const u8, len: u32, out: *mut u8) => { + let result = if len == 0 { + let hashed = twox_128(&[0u8; 0]); + debug_trace!(target: "xxhash", "XXhash: '' -> {}", HexDisplay::from(&hashed)); + this.hash_lookup.insert(hashed.to_vec(), vec![]); + hashed + } else { + let key = this.memory.get(data, len as usize).map_err(|_| UserError("Invalid attempt to get key in ext_twox_128"))?; + let hashed_key = twox_128(&key); + debug_trace!(target: "xxhash", "XXhash: {} -> {}", + if let Ok(_skey) = ::std::str::from_utf8(&key) { + _skey.to_owned() + } else { + format!("{}", HexDisplay::from(&key)) + }, + HexDisplay::from(&hashed_key) + ); + this.hash_lookup.insert(hashed_key.to_vec(), key); + hashed_key + }; + + this.memory.set(out, &result).map_err(|_| UserError("Invalid attempt to set result in ext_twox_128"))?; + Ok(()) + }, + ext_twox_256(data: *const u8, len: u32, out: *mut u8) => { + let result = if len == 0 { + twox_256(&[0u8; 0]) + } else { + twox_256(&this.memory.get(data, len as usize).map_err(|_| UserError("Invalid attempt to get data in ext_twox_256"))?) + }; + this.memory.set(out, &result).map_err(|_| UserError("Invalid attempt to set result in ext_twox_256"))?; + Ok(()) + }, + ext_blake2_256(data: *const u8, len: u32, out: *mut u8) => { + let result = if len == 0 { + blake2_256(&[0u8; 0]) + } else { + blake2_256(&this.memory.get(data, len as usize).map_err(|_| UserError("Invalid attempt to get data in ext_blake2_256"))?) + }; + this.memory.set(out, &result).map_err(|_| UserError("Invalid attempt to set result in ext_blake2_256"))?; + Ok(()) + }, + ext_ed25519_verify(msg_data: *const u8, msg_len: u32, sig_data: *const u8, pubkey_data: *const u8) -> u32 => { + let mut sig = [0u8; 64]; + this.memory.get_into(sig_data, &mut sig[..]).map_err(|_| UserError("Invalid attempt to get signature in ext_ed25519_verify"))?; + let mut pubkey = [0u8; 32]; + this.memory.get_into(pubkey_data, &mut pubkey[..]).map_err(|_| UserError("Invalid attempt to get pubkey in ext_ed25519_verify"))?; + let msg = this.memory.get(msg_data, msg_len as usize).map_err(|_| UserError("Invalid attempt to get message in ext_ed25519_verify"))?; + + Ok(if ed25519::verify(&sig, &msg, &pubkey) { + 0 + } else { + 5 + }) + }, + ext_sandbox_instantiate(dispatch_thunk_idx: usize, wasm_ptr: *const u8, wasm_len: usize, imports_ptr: *const u8, imports_len: usize, state: usize) -> u32 => { + let wasm = this.memory.get(wasm_ptr, wasm_len as usize).map_err(|_| UserError("Sandbox error"))?; + let raw_env_def = this.memory.get(imports_ptr, imports_len as usize).map_err(|_| UserError("Sandbox error"))?; + + // Extract a dispatch thunk from instance's table by the specified index. + let dispatch_thunk = { + let table = this.table.as_ref().ok_or_else(|| UserError("Sandbox error"))?; + table.get(dispatch_thunk_idx) + .map_err(|_| UserError("Sandbox error"))? + .ok_or_else(|| UserError("Sandbox error"))? + .clone() + }; + + let instance_idx = sandbox::instantiate(this, dispatch_thunk, &wasm, &raw_env_def, state)?; + + Ok(instance_idx as u32) + }, + ext_sandbox_instance_teardown(instance_idx: u32) => { + this.sandbox_store.instance_teardown(instance_idx)?; + Ok(()) + }, + ext_sandbox_invoke(instance_idx: u32, export_ptr: *const u8, export_len: usize, args_ptr: *const u8, args_len: usize, return_val_ptr: *const u8, return_val_len: usize, state: usize) -> u32 => { + use codec::{Decode, Encode}; + + trace!(target: "runtime-sandbox", "invoke, instance_idx={}", instance_idx); + let export = this.memory.get(export_ptr, export_len as usize) + .map_err(|_| UserError("Sandbox error")) + .and_then(|b| + String::from_utf8(b) + .map_err(|_| UserError("Sandbox error")) + )?; + + // Deserialize arguments and convert them into wasmi types. + let serialized_args = this.memory.get(args_ptr, args_len as usize) + .map_err(|_| UserError("Sandbox error"))?; + let args = Vec::::decode(&mut &serialized_args[..]) + .ok_or_else(|| UserError("Sandbox error"))? + .into_iter() + .map(Into::into) + .collect::>(); + + let instance = this.sandbox_store.instance(instance_idx)?; + let result = instance.invoke(&export, &args, this, state); + + match result { + Ok(None) => Ok(sandbox_primitives::ERR_OK), + Ok(Some(val)) => { + // Serialize return value and write it back into the memory. + sandbox_primitives::ReturnValue::Value(val.into()).using_encoded(|val| { + if val.len() > return_val_len as usize { + Err(UserError("Sandbox error"))?; + } + this.memory + .set(return_val_ptr, val) + .map_err(|_| UserError("Sandbox error"))?; + Ok(sandbox_primitives::ERR_OK) + }) + } + Err(_) => Ok(sandbox_primitives::ERR_EXECUTION), + } + }, + ext_sandbox_memory_new(initial: u32, maximum: u32) -> u32 => { + let mem_idx = this.sandbox_store.new_memory(initial, maximum)?; + Ok(mem_idx) + }, + ext_sandbox_memory_get(memory_idx: u32, offset: u32, buf_ptr: *mut u8, buf_len: usize) -> u32 => { + let dst_memory = this.sandbox_store.memory(memory_idx)?; + + let data: Vec = match dst_memory.get(offset, buf_len as usize) { + Ok(data) => data, + Err(_) => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + match this.memory.set(buf_ptr, &data) { + Err(_) => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + _ => {}, + } + + Ok(sandbox_primitives::ERR_OK) + }, + ext_sandbox_memory_set(memory_idx: u32, offset: u32, val_ptr: *const u8, val_len: usize) -> u32 => { + let dst_memory = this.sandbox_store.memory(memory_idx)?; + + let data = match this.memory.get(offset, val_len as usize) { + Ok(data) => data, + Err(_) => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + match dst_memory.set(val_ptr, &data) { + Err(_) => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + _ => {}, + } + + Ok(sandbox_primitives::ERR_OK) + }, + ext_sandbox_memory_teardown(memory_idx: u32) => { + this.sandbox_store.memory_teardown(memory_idx)?; + Ok(()) + }, + => <'e, E: Externalities + 'e> +); + +/// Wasm rust executor for contracts. +/// +/// Executes the provided code in a sandboxed wasm runtime. +#[derive(Debug, Clone)] +pub struct WasmExecutor { +} + +impl WasmExecutor { + + /// Create a new instance. + pub fn new() -> Self { + WasmExecutor{} + } + + /// Call a given method in the given code. + /// This should be used for tests only. + pub fn call>( + &self, + ext: &mut E, + heap_pages: usize, + code: &[u8], + method: &str, + data: &[u8], + ) -> Result> { + let module = ::wasmi::Module::from_buffer(code).expect("all modules compiled with rustc are valid wasm code; qed"); + self.call_in_wasm_module(ext, heap_pages, &module, method, data) + } + + /// Call a given method in the given wasm-module runtime. + pub fn call_in_wasm_module>( + &self, + ext: &mut E, + heap_pages: usize, + module: &Module, + method: &str, + data: &[u8], + ) -> Result> { + // start module instantiation. Don't run 'start' function yet. + let intermediate_instance = ModuleInstance::new( + module, + &ImportsBuilder::new() + .with_resolver("env", FunctionExecutor::::resolver()) + )?; + + // extract a reference to a linear memory, optional reference to a table + // and then initialize FunctionExecutor. + let memory = intermediate_instance + .not_started_instance() + .export_by_name("memory") + // TODO: with code coming from the blockchain it isn't strictly been compiled with rustc anymore. + // these assumptions are probably not true anymore + .expect("all modules compiled with rustc should have an export named 'memory'; qed") + .as_memory() + .expect("in module generated by rustc export named 'memory' should be a memory; qed") + .clone(); + let table: Option = intermediate_instance + .not_started_instance() + .export_by_name("__indirect_function_table") + .and_then(|e| e.as_table().cloned()); + + let mut fec = FunctionExecutor::new(memory.clone(), heap_pages, table, ext)?; + + // finish instantiation by running 'start' function (if any). + let instance = intermediate_instance.run_start(&mut fec)?; + + let size = data.len() as u32; + let offset = fec.heap.allocate(size); + memory.set(offset, &data)?; + + let result = instance.invoke_export( + method, + &[ + I32(offset as i32), + I32(size as i32) + ], + &mut fec + ); + + let returned = match result { + Ok(x) => x, + Err(e) => { + trace!(target: "wasm-executor", "Failed to execute code with {} pages", heap_pages); + return Err(e.into()) + }, + }; + + if let Some(I64(r)) = returned { + let offset = r as u32; + let length = (r >> 32) as u32 as usize; + memory.get(offset, length) + .map_err(|_| ErrorKind::Runtime.into()) + } else { + Err(ErrorKind::InvalidReturn.into()) + } + } +} + + +#[cfg(test)] +mod tests { + use super::*; + use codec::Encode; + use state_machine::TestExternalities; + + // TODO: move into own crate. + macro_rules! map { + ($( $name:expr => $value:expr ),*) => ( + vec![ $( ( $name, $value ) ),* ].into_iter().collect() + ) + } + + #[test] + fn returning_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_empty_return", &[]).unwrap(); + assert_eq!(output, vec![0u8; 0]); + } + + #[test] + fn panicking_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_panic", &[]); + assert!(output.is_err()); + + let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_conditional_panic", &[2]); + assert!(output.is_err()); + } + + #[test] + fn storage_should_work() { + let mut ext = TestExternalities::default(); + ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_data_in", b"Hello world").unwrap(); + + assert_eq!(output, b"all ok!".to_vec()); + + let expected : TestExternalities<_> = map![ + b"input".to_vec() => b"Hello world".to_vec(), + b"foo".to_vec() => b"bar".to_vec(), + b"baz".to_vec() => b"bar".to_vec() + ]; + assert_eq!(expected, ext); + } + + #[test] + fn clear_prefix_should_work() { + let mut ext = TestExternalities::default(); + ext.set_storage(b"aaa".to_vec(), b"1".to_vec()); + ext.set_storage(b"aab".to_vec(), b"2".to_vec()); + ext.set_storage(b"aba".to_vec(), b"3".to_vec()); + ext.set_storage(b"abb".to_vec(), b"4".to_vec()); + ext.set_storage(b"bbb".to_vec(), b"5".to_vec()); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + + // This will clear all entries which prefix is "ab". + let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_clear_prefix", b"ab").unwrap(); + + assert_eq!(output, b"all ok!".to_vec()); + + let expected: TestExternalities<_> = map![ + b"aaa".to_vec() => b"1".to_vec(), + b"aab".to_vec() => b"2".to_vec(), + b"bbb".to_vec() => b"5".to_vec() + ]; + assert_eq!(expected, ext); + } + + #[test] + fn blake2_256_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_256", &[]).unwrap(), + blake2_256(&b""[..]).encode() + ); + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_256", b"Hello world!").unwrap(), + blake2_256(&b"Hello world!"[..]).encode() + ); + } + + #[test] + fn twox_256_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_256", &[]).unwrap(), + hex!("99e9d85137db46ef4bbea33613baafd56f963c64b1f3685a4eb4abd67ff6203a") + ); + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_256", b"Hello world!").unwrap(), + hex!("b27dfd7f223f177f2a13647b533599af0c07f68bda23d96d059da2b451a35a74") + ); + } + + #[test] + fn twox_128_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_128", &[]).unwrap(), + hex!("99e9d85137db46ef4bbea33613baafd5") + ); + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_128", b"Hello world!").unwrap(), + hex!("b27dfd7f223f177f2a13647b533599af") + ); + } + + #[test] + fn ed25519_verify_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + let key = ed25519::Pair::from_seed(&blake2_256(b"test")); + let sig = key.sign(b"all ok!"); + let mut calldata = vec![]; + calldata.extend_from_slice(key.public().as_ref()); + calldata.extend_from_slice(sig.as_ref()); + + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_ed25519_verify", &calldata).unwrap(), + vec![1] + ); + + let other_sig = key.sign(b"all is not ok!"); + let mut calldata = vec![]; + calldata.extend_from_slice(key.public().as_ref()); + calldata.extend_from_slice(other_sig.as_ref()); + + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_ed25519_verify", &calldata).unwrap(), + vec![0] + ); + } + + #[test] + fn enumerated_trie_root_should_work() { + let mut ext = TestExternalities::default(); + let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm"); + assert_eq!( + WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_enumerated_trie_root", &[]).unwrap(), + ordered_trie_root::(vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()]).0.encode() + ); + } + + +} diff --git a/core/executor/src/wasm_utils.rs b/core/executor/src/wasm_utils.rs new file mode 100644 index 00000000000..5459266ba46 --- /dev/null +++ b/core/executor/src/wasm_utils.rs @@ -0,0 +1,199 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Rust implementation of Substrate contracts. + +use wasmi::{ValueType, RuntimeValue, HostError}; +use wasmi::nan_preserving_float::{F32, F64}; +use std::fmt; + +#[derive(Debug)] +pub struct UserError(pub &'static str); +impl fmt::Display for UserError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "UserError: {}", self.0) + } +} +impl HostError for UserError { +} + +pub trait ConvertibleToWasm { const VALUE_TYPE: ValueType; type NativeType; fn to_runtime_value(self) -> RuntimeValue; } +impl ConvertibleToWasm for i32 { type NativeType = i32; const VALUE_TYPE: ValueType = ValueType::I32; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I32(self) } } +impl ConvertibleToWasm for u32 { type NativeType = u32; const VALUE_TYPE: ValueType = ValueType::I32; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I32(self as i32) } } +impl ConvertibleToWasm for i64 { type NativeType = i64; const VALUE_TYPE: ValueType = ValueType::I64; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I64(self) } } +impl ConvertibleToWasm for u64 { type NativeType = u64; const VALUE_TYPE: ValueType = ValueType::I64; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I64(self as i64) } } +impl ConvertibleToWasm for F32 { type NativeType = F32; const VALUE_TYPE: ValueType = ValueType::F32; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::F32(self) } } +impl ConvertibleToWasm for F64 { type NativeType = F64; const VALUE_TYPE: ValueType = ValueType::F64; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::F64(self) } } +impl ConvertibleToWasm for isize { type NativeType = i32; const VALUE_TYPE: ValueType = ValueType::I32; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I32(self as i32) } } +impl ConvertibleToWasm for usize { type NativeType = u32; const VALUE_TYPE: ValueType = ValueType::I32; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I32(self as u32 as i32) } } +impl ConvertibleToWasm for *const T { type NativeType = u32; const VALUE_TYPE: ValueType = ValueType::I32; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I32(self as isize as i32) } } +impl ConvertibleToWasm for *mut T { type NativeType = u32; const VALUE_TYPE: ValueType = ValueType::I32; fn to_runtime_value(self) -> RuntimeValue { RuntimeValue::I32(self as isize as i32) } } + +#[macro_export] +macro_rules! convert_args { + () => ([]); + ( $( $t:ty ),* ) => ( [ $( { use $crate::wasm_utils::ConvertibleToWasm; <$t>::VALUE_TYPE }, )* ] ); +} + +#[macro_export] +macro_rules! gen_signature { + ( ( $( $params: ty ),* ) ) => ( + { + $crate::wasmi::Signature::new(&convert_args!($($params),*)[..], None) + } + ); + + ( ( $( $params: ty ),* ) -> $returns: ty ) => ( + { + $crate::wasmi::Signature::new(&convert_args!($($params),*)[..], Some({ + use $crate::wasm_utils::ConvertibleToWasm; <$returns>::VALUE_TYPE + })) + } + ); +} + +macro_rules! resolve_fn { + (@iter $index:expr, $sig_var:ident, $name_var:ident) => (); + (@iter $index:expr, $sig_var:ident, $name_var:ident $name:ident ( $( $params:ty ),* ) $( -> $returns:ty )* => $($tail:tt)* ) => ( + if $name_var == stringify!($name) { + let signature = gen_signature!( ( $( $params ),* ) $( -> $returns )* ); + if $sig_var != &signature { + return Err($crate::wasmi::Error::Instantiation( + format!("Export {} has different signature {:?}", $name_var, $sig_var), + )); + } + return Ok($crate::wasmi::FuncInstance::alloc_host(signature, $index)); + } + resolve_fn!(@iter $index + 1, $sig_var, $name_var $($tail)*) + ); + + ($sig_var:ident, $name_var:ident, $($tail:tt)* ) => ( + resolve_fn!(@iter 0, $sig_var, $name_var $($tail)*); + ); +} + +#[macro_export] +macro_rules! unmarshall_args { + ( $body:tt, $objectname:ident, $args_iter:ident, $( $names:ident : $params:ty ),*) => ({ + $( + let $names : <$params as $crate::wasm_utils::ConvertibleToWasm>::NativeType = + $args_iter.next() + .and_then(|rt_val| rt_val.try_into()) + .expect( + "`$args_iter` comes from an argument of Externals::invoke_index; + args to an external call always matches the signature of the external; + external signatures are built with count and types and in order defined by `$params`; + here, we iterating on `$params`; + qed; + " + ); + )* + $body + }) +} + +/// Since we can't specify the type of closure directly at binding site: +/// +/// ```rust,ignore +/// let f: FnOnce() -> Result<::NativeType, _> = || { /* ... */ }; +/// ``` +/// +/// we use this function to constrain the type of the closure. +#[inline(always)] +pub fn constrain_closure(f: F) -> F +where + F: FnOnce() -> Result +{ + f +} + +#[macro_export] +macro_rules! marshall { + ( $args_iter:ident, $objectname:ident, ( $( $names:ident : $params:ty ),* ) -> $returns:ty => $body:tt ) => ({ + let body = $crate::wasm_utils::constrain_closure::< + <$returns as $crate::wasm_utils::ConvertibleToWasm>::NativeType, _ + >(|| { + unmarshall_args!($body, $objectname, $args_iter, $( $names : $params ),*) + }); + let r = body()?; + return Ok(Some({ use $crate::wasm_utils::ConvertibleToWasm; r.to_runtime_value() })) + }); + ( $args_iter:ident, $objectname:ident, ( $( $names:ident : $params:ty ),* ) => $body:tt ) => ({ + let body = $crate::wasm_utils::constrain_closure::<(), _>(|| { + unmarshall_args!($body, $objectname, $args_iter, $( $names : $params ),*) + }); + body()?; + return Ok(None) + }) +} + +macro_rules! dispatch_fn { + ( @iter $index:expr, $index_ident:ident, $objectname:ident, $args_iter:ident) => { + // `$index` comes from an argument of Externals::invoke_index; + // externals are always invoked with index given by resolve_fn! at resolve time; + // For each next function resolve_fn! gives new index, starting from 0; + // Both dispatch_fn! and resolve_fn! are called with the same list of functions; + // qed; + panic!("fn with index {} is undefined", $index); + }; + + ( @iter $index:expr, $index_ident:ident, $objectname:ident, $args_iter:ident, $name:ident ( $( $names:ident : $params:ty ),* ) $( -> $returns:ty )* => $body:tt $($tail:tt)*) => ( + if $index_ident == $index { + { marshall!($args_iter, $objectname, ( $( $names : $params ),* ) $( -> $returns )* => $body) } + } + dispatch_fn!( @iter $index + 1, $index_ident, $objectname, $args_iter $($tail)*) + ); + + ( $index_ident:ident, $objectname:ident, $args_iter:ident, $($tail:tt)* ) => ( + dispatch_fn!( @iter 0, $index_ident, $objectname, $args_iter, $($tail)*); + ); +} + +#[macro_export] +macro_rules! impl_function_executor { + ( $objectname:ident : $structname:ty, + $( $name:ident ( $( $names:ident : $params:ty ),* ) $( -> $returns:ty )* => $body:tt , )* + => $($pre:tt)+ ) => ( + impl $( $pre ) + $structname { + #[allow(unused)] + fn resolver() -> &'static $crate::wasmi::ModuleImportResolver { + struct Resolver; + impl $crate::wasmi::ModuleImportResolver for Resolver { + fn resolve_func(&self, name: &str, signature: &$crate::wasmi::Signature) -> ::std::result::Result<$crate::wasmi::FuncRef, $crate::wasmi::Error> { + resolve_fn!(signature, name, $( $name( $( $params ),* ) $( -> $returns )* => )*); + + Err($crate::wasmi::Error::Instantiation( + format!("Export {} not found", name), + )) + } + } + &Resolver + } + } + + impl $( $pre ) + $crate::wasmi::Externals for $structname { + fn invoke_index( + &mut self, + index: usize, + args: $crate::wasmi::RuntimeArgs, + ) -> ::std::result::Result, $crate::wasmi::Trap> { + let $objectname = self; + let mut args = args.as_ref().iter(); + dispatch_fn!(index, $objectname, args, $( $name( $( $names : $params ),* ) $( -> $returns )* => $body ),*); + } + } + ); +} diff --git a/core/executor/wasm/Cargo.lock b/core/executor/wasm/Cargo.lock new file mode 100644 index 00000000000..ed26df817ed --- /dev/null +++ b/core/executor/wasm/Cargo.lock @@ -0,0 +1,216 @@ +[[package]] +name = "arrayvec" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byteorder" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "crunchy" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fixed-hash" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hashdb" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nodrop" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "plain_hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pwasm-alloc" +version = "0.1.0" +dependencies = [ + "pwasm-libc 0.1.0", + "rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pwasm-libc" +version = "0.1.0" + +[[package]] +name = "quote" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "runtime-test" +version = "0.1.0" +dependencies = [ + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-sandbox 0.1.0", +] + +[[package]] +name = "rustc-hex" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "substrate-codec" +version = "0.1.0" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-codec-derive" +version = "0.1.0" +dependencies = [ + "proc-macro2 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.14.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-primitives" +version = "0.1.0" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-runtime-std 0.1.0", + "uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-runtime-io" +version = "0.1.0" +dependencies = [ + "hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "substrate-runtime-sandbox" +version = "0.1.0" +dependencies = [ + "rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "substrate-runtime-std" +version = "0.1.0" +dependencies = [ + "pwasm-alloc 0.1.0", + "pwasm-libc 0.1.0", + "rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "uint" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23" +"checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" +"checksum fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0d5ec8112f00ea8a483e04748a85522184418fd1cf02890b626d8fc28683f7de" +"checksum hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f1c71fc577cde89b3345d5f2880fecaf462a32e96c619f431279bdaf1ba5ddb1" +"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" +"checksum plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95fa6386b1d34aaf0adb9b7dd2885dbe7c34190e6263785e5a7ec2b19044a90f" +"checksum proc-macro2 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "cccdc7557a98fe98453030f077df7f3a042052fae465bb61d2c2c41435cfd9b6" +"checksum quote 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3372dc35766b36a99ce2352bd1b6ea0137c38d215cc0c8780bf6de6df7842ba9" +"checksum rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b03280c2813907a030785570c577fb27d3deec8da4c18566751ade94de0ace" +"checksum rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9743a7670d88d5d52950408ecdb7c71d8986251ab604d4689dd2ca25c9bca69" +"checksum semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a3186ec9e65071a2095434b1f5bb24838d4e8e130f584c790f6033c79943537" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "db99f3919e20faa51bb2996057f5031d8685019b5a06139b1ce761da671b8526" +"checksum syn 0.14.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e13df71f29f9440b50261a5882c86eac334f1badb3134ec26f0de2f1418e44" +"checksum uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "754ba11732b9161b94c41798e5197e5e75388d012f760c42adb5000353e98646" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" diff --git a/core/executor/wasm/Cargo.toml b/core/executor/wasm/Cargo.toml new file mode 100644 index 00000000000..44e0dafe5d9 --- /dev/null +++ b/core/executor/wasm/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "runtime-test" +version = "0.1.0" +authors = ["Parity Technologies "] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +substrate-runtime-io = { path = "../../runtime-io", version = "0.1", default_features = false } +substrate-runtime-sandbox = { path = "../../runtime-sandbox", version = "0.1", default_features = false } +substrate-primitives = { path = "../../primitives", default_features = false } + +[profile.release] +panic = "abort" +lto = true + +[workspace] +members = [] diff --git a/core/executor/wasm/build.sh b/core/executor/wasm/build.sh new file mode 100755 index 00000000000..ab718644817 --- /dev/null +++ b/core/executor/wasm/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e + +cargo +nightly build --target=wasm32-unknown-unknown --release +for i in test +do + wasm-gc target/wasm32-unknown-unknown/release/runtime_$i.wasm target/wasm32-unknown-unknown/release/runtime_$i.compact.wasm +done diff --git a/core/executor/wasm/src/lib.rs b/core/executor/wasm/src/lib.rs new file mode 100644 index 00000000000..3046fd1f166 --- /dev/null +++ b/core/executor/wasm/src/lib.rs @@ -0,0 +1,122 @@ +#![no_std] +#![feature(panic_handler)] +#![cfg_attr(feature = "strict", deny(warnings))] + +#![feature(alloc)] +extern crate alloc; +use alloc::vec::Vec; + +#[macro_use] +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_sandbox as sandbox; +extern crate substrate_primitives; + +use runtime_io::{ + set_storage, storage, clear_prefix, print, blake2_256, + twox_128, twox_256, ed25519_verify, enumerated_trie_root +}; + +impl_stubs!( + test_data_in NO_DECODE => |input| { + print("set_storage"); + set_storage(b"input", input); + + print("storage"); + let foo = storage(b"foo").unwrap(); + + print("set_storage"); + set_storage(b"baz", &foo); + + print("finished!"); + b"all ok!".to_vec() + }, + test_clear_prefix NO_DECODE => |input| { + clear_prefix(input); + b"all ok!".to_vec() + }, + test_empty_return NO_DECODE => |_| Vec::new(), + test_panic NO_DECODE => |_| panic!("test panic"), + test_conditional_panic NO_DECODE => |input: &[u8]| { + if input.len() > 0 { + panic!("test panic") + } + input.to_vec() + }, + test_blake2_256 NO_DECODE => |input| blake2_256(input).to_vec(), + test_twox_256 NO_DECODE => |input| twox_256(input).to_vec(), + test_twox_128 NO_DECODE => |input| twox_128(input).to_vec(), + test_ed25519_verify NO_DECODE => |input: &[u8]| { + let mut pubkey = [0; 32]; + let mut sig = [0; 64]; + + pubkey.copy_from_slice(&input[0..32]); + sig.copy_from_slice(&input[32..96]); + + let msg = b"all ok!"; + [ed25519_verify(&sig, &msg[..], &pubkey) as u8].to_vec() + }, + test_enumerated_trie_root NO_DECODE => |_| { + enumerated_trie_root::(&[&b"zero"[..], &b"one"[..], &b"two"[..]]).to_vec() + }, + test_sandbox NO_DECODE => |code: &[u8]| { + let ok = execute_sandboxed(code, &[]).is_ok(); + [ok as u8].to_vec() + }, + test_sandbox_args NO_DECODE => |code: &[u8]| { + let ok = execute_sandboxed( + code, + &[ + sandbox::TypedValue::I32(0x12345678), + sandbox::TypedValue::I64(0x1234567887654321), + ] + ).is_ok(); + [ok as u8].to_vec() + }, + test_sandbox_return_val NO_DECODE => |code: &[u8]| { + let result = execute_sandboxed( + code, + &[ + sandbox::TypedValue::I32(0x1336), + ] + ); + let ok = if let Ok(sandbox::ReturnValue::Value(sandbox::TypedValue::I32(0x1337))) = result { true } else { false }; + [ok as u8].to_vec() + } +); + +fn execute_sandboxed(code: &[u8], args: &[sandbox::TypedValue]) -> Result { + struct State { + counter: u32, + } + + fn env_assert(_e: &mut State, args: &[sandbox::TypedValue]) -> Result { + if args.len() != 1 { + return Err(sandbox::HostError); + } + let condition = args[0].as_i32().ok_or_else(|| sandbox::HostError)?; + if condition != 0 { + Ok(sandbox::ReturnValue::Unit) + } else { + Err(sandbox::HostError) + } + } + fn env_inc_counter(e: &mut State, args: &[sandbox::TypedValue]) -> Result { + if args.len() != 1 { + return Err(sandbox::HostError); + } + let inc_by = args[0].as_i32().ok_or_else(|| sandbox::HostError)?; + e.counter += inc_by as u32; + Ok(sandbox::ReturnValue::Value(sandbox::TypedValue::I32(e.counter as i32))) + } + + let mut state = State { counter: 0 }; + + let mut env_builder = sandbox::EnvironmentDefinitionBuilder::new(); + env_builder.add_host_func("env", "assert", env_assert); + env_builder.add_host_func("env", "inc_counter", env_inc_counter); + + let mut instance = sandbox::Instance::new(code, &env_builder, &mut state)?; + let result = instance.invoke(b"call", args, &mut state); + + result.map_err(|_| sandbox::HostError) +} diff --git a/core/extrinsic-pool/Cargo.toml b/core/extrinsic-pool/Cargo.toml new file mode 100644 index 00000000000..8fc4f491865 --- /dev/null +++ b/core/extrinsic-pool/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "substrate-extrinsic-pool" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +serde = "1.0" +serde_derive = "1.0" +error-chain = "0.12" +futures = "0.1" +log = "0.3" +parking_lot = "0.4" +transaction-pool = "1.13.2" +substrate-runtime-primitives = { path = "../../runtime/primitives" } + +[dev-dependencies] +substrate-test-client = { path = "../test-client" } +substrate-keyring = { path = "../keyring" } +substrate-codec = { path = "../codec" } diff --git a/core/extrinsic-pool/README.adoc b/core/extrinsic-pool/README.adoc new file mode 100644 index 00000000000..e128d64c9bf --- /dev/null +++ b/core/extrinsic-pool/README.adoc @@ -0,0 +1,13 @@ + += extrinsic-pool + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/extrinsic-pool/src/error.rs b/core/extrinsic-pool/src/error.rs new file mode 100644 index 00000000000..047041d18d5 --- /dev/null +++ b/core/extrinsic-pool/src/error.rs @@ -0,0 +1,33 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! External Error trait for extrinsic pool. + +use txpool; + +/// Extrinsic pool error. +pub trait IntoPoolError: ::std::error::Error + Send + Sized { + /// Try to extract original `txpool::Error` + /// + /// This implementation is optional and used only to + /// provide more descriptive error messages for end users + /// of RPC API. + fn into_pool_error(self) -> Result { Err(self) } +} + +impl IntoPoolError for txpool::Error { + fn into_pool_error(self) -> Result { Ok(self) } +} diff --git a/core/extrinsic-pool/src/lib.rs b/core/extrinsic-pool/src/lib.rs new file mode 100644 index 00000000000..bcd2b03c80e --- /dev/null +++ b/core/extrinsic-pool/src/lib.rs @@ -0,0 +1,49 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Generic extrinsic pool. +// end::description[] + +#![warn(missing_docs)] +#![warn(unused_extern_crates)] + +extern crate futures; +extern crate parking_lot; +extern crate substrate_runtime_primitives as runtime_primitives; + +#[macro_use] +extern crate log; +extern crate serde; +#[macro_use] +extern crate serde_derive; +extern crate transaction_pool as txpool; +#[cfg(test)] extern crate substrate_test_client as test_client; +#[cfg(test)] extern crate substrate_keyring as keyring; +#[cfg(test)] extern crate substrate_codec as codec; + +pub mod watcher; +mod error; +mod listener; +mod pool; +mod rotator; + +pub use listener::Listener; +pub use pool::{Pool, ChainApi, EventStream, Verified, VerifiedFor, ExtrinsicFor, ExHash, AllExtrinsics}; +pub use txpool::scoring; +pub use txpool::{Error, ErrorKind}; +pub use error::IntoPoolError; +pub use txpool::{Options, Status, LightStatus, VerifiedTransaction, Readiness, Transaction}; diff --git a/core/extrinsic-pool/src/listener.rs b/core/extrinsic-pool/src/listener.rs new file mode 100644 index 00000000000..8badb331a5b --- /dev/null +++ b/core/extrinsic-pool/src/listener.rs @@ -0,0 +1,95 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{ + sync::Arc, + fmt, + collections::HashMap, +}; +use txpool; + +use watcher; + +/// Extrinsic pool default listener. +#[derive(Default)] +pub struct Listener { + watchers: HashMap> +} + +impl Listener { + /// Creates a new watcher for given verified extrinsic. + /// + /// The watcher can be used to subscribe to lifecycle events of that extrinsic. + pub fn create_watcher>(&mut self, xt: Arc) -> watcher::Watcher { + let sender = self.watchers.entry(*xt.hash()).or_insert_with(watcher::Sender::default); + sender.new_watcher() + } + + /// Notify the listeners about extrinsic broadcast. + pub fn broadcasted(&mut self, hash: &H, peers: Vec) { + self.fire(hash, |watcher| watcher.broadcast(peers)); + } + + fn fire(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender) { + let clean = if let Some(h) = self.watchers.get_mut(hash) { + fun(h); + h.is_done() + } else { + false + }; + + if clean { + self.watchers.remove(hash); + } + } +} + +impl txpool::Listener for Listener where + H: ::std::hash::Hash + Eq + Copy + fmt::Debug + fmt::LowerHex + Default, + T: txpool::VerifiedTransaction, +{ + fn added(&mut self, tx: &Arc, old: Option<&Arc>) { + if let Some(old) = old { + let hash = tx.hash(); + self.fire(old.hash(), |watcher| watcher.usurped(*hash)); + } + } + + fn dropped(&mut self, tx: &Arc, by: Option<&T>) { + self.fire(tx.hash(), |watcher| match by { + Some(t) => watcher.usurped(*t.hash()), + None => watcher.dropped(), + }) + } + + fn rejected(&mut self, tx: &Arc, reason: &txpool::ErrorKind) { + warn!(target: "extrinsic-pool", "Extrinsic rejected ({}): {:?}", reason, tx); + } + + fn invalid(&mut self, tx: &Arc) { + warn!(target: "extrinsic-pool", "Extrinsic invalid: {:?}", tx); + } + + fn canceled(&mut self, tx: &Arc) { + debug!(target: "extrinsic-pool", "Extrinsic canceled: {:?}", tx); + } + + fn culled(&mut self, tx: &Arc) { + // TODO [ToDr] latest block number? + let header_hash = Default::default(); + self.fire(tx.hash(), |watcher| watcher.finalised(header_hash)) + } +} diff --git a/core/extrinsic-pool/src/pool.rs b/core/extrinsic-pool/src/pool.rs new file mode 100644 index 00000000000..642171b3760 --- /dev/null +++ b/core/extrinsic-pool/src/pool.rs @@ -0,0 +1,606 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{ + collections::{BTreeMap, HashMap}, + fmt, + sync::Arc, + time, +}; +use futures::sync::mpsc; +use parking_lot::{Mutex, RwLock}; +use serde::{Serialize, de::DeserializeOwned}; +use txpool::{self, Scoring, Readiness}; + +use error::IntoPoolError; +use listener::Listener; +use rotator::PoolRotator; +use watcher::Watcher; + +use runtime_primitives::{generic::BlockId, traits::Block as BlockT}; + +/// Modification notification event stream type; +pub type EventStream = mpsc::UnboundedReceiver<()>; + +/// Extrinsic hash type for a pool. +pub type ExHash = ::Hash; +/// Extrinsic type for a pool. +pub type ExtrinsicFor = <::Block as BlockT>::Extrinsic; +/// Verified extrinsic data for `ChainApi`. +pub type VerifiedFor = Verified, ::VEx>; +/// A collection of all extrinsics. +pub type AllExtrinsics = BTreeMap<<::VEx as txpool::VerifiedTransaction>::Sender, Vec>>; + +/// Verified extrinsic struct. Wraps original extrinsic and verification info. +#[derive(Debug)] +pub struct Verified { + /// Original extrinsic. + pub original: Ex, + /// Verification data. + pub verified: VEx, + /// Pool deadline, after it's reached we remove the extrinsic from the pool. + pub valid_till: time::Instant, +} + +impl txpool::VerifiedTransaction for Verified +where + Ex: fmt::Debug, + VEx: txpool::VerifiedTransaction, +{ + type Hash = ::Hash; + type Sender = ::Sender; + + fn hash(&self) -> &Self::Hash { + self.verified.hash() + } + + fn sender(&self) -> &Self::Sender { + self.verified.sender() + } + + fn mem_usage(&self) -> usize { + // TODO: add `original` mem usage. + self.verified.mem_usage() + } +} + +/// Concrete extrinsic validation and query logic. +pub trait ChainApi: Send + Sync { + /// Block type. + type Block: BlockT; + /// Extrinsic hash type. + type Hash: ::std::hash::Hash + Eq + Copy + fmt::Debug + fmt::LowerHex + Serialize + DeserializeOwned + ::std::str::FromStr + Send + Sync + Default + 'static; + /// Extrinsic sender type. + type Sender: ::std::hash::Hash + fmt::Debug + Serialize + DeserializeOwned + Eq + Clone + Send + Sync + Ord + Default; + /// Unchecked extrinsic type. + /// Verified extrinsic type. + type VEx: txpool::VerifiedTransaction + Send + Sync + Clone; + /// Readiness evaluator + type Ready; + /// Error type. + type Error: From + IntoPoolError; + /// Score type. + type Score: ::std::cmp::Ord + Clone + Default + fmt::Debug + Send + Send + Sync + fmt::LowerHex; + /// Custom scoring update event type. + type Event: ::std::fmt::Debug; + /// Verify extrinsic at given block. + fn verify_transaction(&self, at: &BlockId, uxt: &ExtrinsicFor) -> Result; + + /// Create new readiness evaluator. + fn ready(&self) -> Self::Ready; + + /// Check readiness for verified extrinsic at given block. + fn is_ready(&self, at: &BlockId, context: &mut Self::Ready, xt: &VerifiedFor) -> Readiness; + + /// Decides on ordering of `T`s from a particular sender. + fn compare(old: &VerifiedFor, other: &VerifiedFor) -> ::std::cmp::Ordering; + + /// Decides how to deal with two transactions from a sender that seem to occupy the same slot in the queue. + fn choose(old: &VerifiedFor, new: &VerifiedFor) -> txpool::scoring::Choice; + + /// Updates the transaction scores given a list of transactions and a change to previous scoring. + /// NOTE: you can safely assume that both slices have the same length. + /// (i.e. score at index `i` represents transaction at the same index) + fn update_scores(xts: &[txpool::Transaction>], scores: &mut [Self::Score], change: txpool::scoring::Change); + + /// Decides if `new` should push out `old` transaction from the pool. + /// + /// NOTE returning `InsertNew` here can lead to some transactions being accepted above pool limits. + fn should_replace(old: &VerifiedFor, new: &VerifiedFor) -> txpool::scoring::Choice; +} + +pub struct Ready<'a, 'b, B: 'a + ChainApi> { + api: &'a B, + at: &'b BlockId, + context: B::Ready, + rotator: &'a PoolRotator, + now: time::Instant, +} + +impl<'a, 'b, B: ChainApi> txpool::Ready> for Ready<'a, 'b, B> { + fn is_ready(&mut self, xt: &VerifiedFor) -> Readiness { + if self.rotator.ban_if_stale(&self.now, xt) { + debug!(target: "extrinsic-pool", "[{:?}] Banning as stale.", txpool::VerifiedTransaction::hash(xt)); + return Readiness::Stale; + } + + self.api.is_ready(self.at, &mut self.context, xt) + } +} + +pub struct ScoringAdapter(::std::marker::PhantomData); + +impl ::std::fmt::Debug for ScoringAdapter { + fn fmt(&self, _f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + Ok(()) + } +} + +impl Scoring> for ScoringAdapter { + type Score = ::Score; + type Event = ::Event; + + fn compare(&self, old: &VerifiedFor, other: &VerifiedFor) -> ::std::cmp::Ordering { + T::compare(old, other) + } + + fn choose(&self, old: &VerifiedFor, new: &VerifiedFor) -> txpool::scoring::Choice { + T::choose(old, new) + } + + fn update_scores(&self, xts: &[txpool::Transaction>], scores: &mut [Self::Score], change: txpool::scoring::Change) { + T::update_scores(xts, scores, change) + } + + fn should_replace(&self, old: &VerifiedFor, new: &VerifiedFor) -> txpool::scoring::Choice { + T::should_replace(old, new) + } +} + +/// Maximum time the transaction will be kept in the pool. +/// +/// Transactions that don't get included within the limit are removed from the pool. +const POOL_TIME: time::Duration = time::Duration::from_secs(60 * 5); + +/// Extrinsics pool. +pub struct Pool { + api: B, + pool: RwLock, + ScoringAdapter, + Listener, + >>, + import_notification_sinks: Mutex>>, + rotator: PoolRotator, +} + +impl Pool { + /// Create a new transaction pool. + pub fn new(options: txpool::Options, api: B) -> Self { + Pool { + pool: RwLock::new(txpool::Pool::new(Listener::default(), ScoringAdapter::(Default::default()), options)), + import_notification_sinks: Default::default(), + api, + rotator: Default::default(), + } + } + + /// Imports a pre-verified extrinsic to the pool. + pub fn import(&self, xt: VerifiedFor) -> Result>, B::Error> { + let result = self.pool.write().import(xt)?; + + self.import_notification_sinks.lock() + .retain(|sink| sink.unbounded_send(()).is_ok()); + + Ok(result) + } + + /// Return an event stream of transactions imported to the pool. + pub fn import_notification_stream(&self) -> EventStream { + let (sink, stream) = mpsc::unbounded(); + self.import_notification_sinks.lock().push(sink); + stream + } + + /// Invoked when extrinsics are broadcasted. + pub fn on_broadcasted(&self, propagated: HashMap>) { + for (hash, peers) in propagated.into_iter() { + self.pool.write().listener_mut().broadcasted(&hash, peers); + } + } + + /// Imports a bunch of unverified extrinsics to the pool + pub fn submit_at(&self, at: &BlockId, xts: T) -> Result>>, B::Error> where + T: IntoIterator> + { + xts + .into_iter() + .map(|xt| { + match self.api.verify_transaction(at, &xt) { + Ok(ref verified) if self.rotator.is_banned(txpool::VerifiedTransaction::hash(verified)) => { + return (Err(txpool::Error::from("Temporarily Banned".to_owned()).into()), xt) + }, + result => (result, xt), + } + }) + .map(|(v, xt)| { + let xt = Verified { + original: xt, + verified: v?, + valid_till: time::Instant::now() + POOL_TIME, + }; + Ok(self.pool.write().import(xt)?) + }) + .collect() + } + + /// Imports one unverified extrinsic to the pool + pub fn submit_one(&self, at: &BlockId, xt: ExtrinsicFor) -> Result>, B::Error> { + Ok(self.submit_at(at, ::std::iter::once(xt))?.pop().expect("One extrinsic passed; one result returned; qed")) + } + + /// Import a single extrinsic and starts to watch their progress in the pool. + pub fn submit_and_watch(&self, at: &BlockId, xt: ExtrinsicFor) -> Result, B::Error> { + let xt = self.submit_at(at, Some(xt))?.pop().expect("One extrinsic passed; one result returned; qed"); + Ok(self.pool.write().listener_mut().create_watcher(xt)) + } + + /// Remove from the pool. + pub fn remove(&self, hashes: &[B::Hash], is_valid: bool) -> Vec>>> { + let mut pool = self.pool.write(); + let mut results = Vec::with_capacity(hashes.len()); + + // temporarily ban invalid transactions + if !is_valid { + debug!(target: "transaction-pool", "Banning invalid transactions: {:?}", hashes); + self.rotator.ban(&time::Instant::now(), hashes); + } + + for hash in hashes { + results.push(pool.remove(hash, is_valid)); + } + + results + } + + /// Cull transactions from the queue. + pub fn cull_from( + &self, + at: &BlockId, + senders: Option<&[::Sender]>, + ) -> usize + { + self.rotator.clear_timeouts(&time::Instant::now()); + let ready = self.ready(at); + self.pool.write().cull(senders, ready) + } + + /// Cull old transactions from the queue. + pub fn cull(&self, at: &BlockId) -> Result { + Ok(self.cull_from(at, None)) + } + + /// Cull transactions from the queue and then compute the pending set. + pub fn cull_and_get_pending(&self, at: &BlockId, f: F) -> Result where + F: FnOnce(txpool::PendingIterator, Ready, ScoringAdapter, Listener>) -> T, + { + self.cull_from(at, None); + Ok(self.pending(at, f)) + } + + /// Get the full status of the queue (including readiness) + pub fn status>>(&self, ready: R) -> txpool::Status { + self.pool.read().status(ready) + } + + /// Returns light status of the pool. + pub fn light_status(&self) -> txpool::LightStatus { + self.pool.read().light_status() + } + + /// Removes all transactions from given sender + pub fn remove_sender(&self, sender: ::Sender) -> Vec>> { + let mut pool = self.pool.write(); + let pending = pool.pending_from_sender(|_: &VerifiedFor| txpool::Readiness::Ready, &sender).collect(); + // remove all transactions from this sender + pool.cull(Some(&[sender]), |_: &VerifiedFor| txpool::Readiness::Stale); + pending + } + + /// Retrieve the pending set. Be careful to not leak the pool `ReadGuard` to prevent deadlocks. + pub fn pending(&self, at: &BlockId, f: F) -> T where + F: FnOnce(txpool::PendingIterator, Ready, ScoringAdapter, Listener>) -> T, + { + let ready = self.ready(at); + f(self.pool.read().pending(ready)) + } + + /// Retry to import all verified transactions from given sender. + pub fn retry_verification(&self, at: &BlockId, sender: ::Sender) -> Result<(), B::Error> { + let to_reverify = self.remove_sender(sender); + self.submit_at(at, to_reverify.into_iter().map(|ex| Arc::try_unwrap(ex).expect("Removed items have no references").original))?; + Ok(()) + } + + /// Reverify transaction that has been reported incorrect. + /// + /// Returns `Ok(None)` in case the hash is missing, `Err(e)` in case of verification error and new transaction + /// reference otherwise. + /// + /// TODO [ToDr] That method is currently unused, should be used together with BlockBuilder + /// when we detect that particular transaction has failed. + /// In such case we will attempt to remove or re-verify it. + pub fn reverify_transaction(&self, at: &BlockId, hash: B::Hash) -> Result>>, B::Error> { + let result = self.remove(&[hash], false).pop().expect("One hash passed; one result received; qed"); + if let Some(ex) = result { + self.submit_one(at, Arc::try_unwrap(ex).expect("Removed items have no references").original).map(Some) + } else { + Ok(None) + } + } + + /// Retrieve all transactions in the pool grouped by sender. + pub fn all(&self) -> AllExtrinsics { + use txpool::VerifiedTransaction; + let pool = self.pool.read(); + let all = pool.unordered_pending(AlwaysReady); + all.fold(Default::default(), |mut map: AllExtrinsics, tx| { + // Map with `null` key is not serializable, so we fallback to default accountId. + map.entry(tx.verified.sender().clone()) + .or_insert_with(Vec::new) + // use bytes type to make it serialize nicer. + .push(tx.original.clone()); + map + }) + } + + fn ready<'a, 'b>(&'a self, at: &'b BlockId) -> Ready<'a, 'b, B> { + Ready { + api: &self.api, + rotator: &self.rotator, + context: self.api.ready(), + at, + now: time::Instant::now(), + } + } +} + + /// A Readiness implementation that returns `Ready` for all transactions. +pub struct AlwaysReady; +impl txpool::Ready for AlwaysReady { + fn is_ready(&mut self, _tx: &VEx) -> txpool::Readiness { + txpool::Readiness::Ready + } +} + +#[cfg(test)] +pub mod tests { + use txpool; + use super::{VerifiedFor, ExtrinsicFor}; + use std::collections::HashMap; + use std::cmp::Ordering; + use {Pool, ChainApi, scoring, Readiness}; + use keyring::Keyring::{self, *}; + use codec::Encode; + use test_client::runtime::{AccountId, Block, Hash, Index, Extrinsic, Transfer}; + use runtime_primitives::{generic, traits::{Hash as HashT, BlindCheckable, BlakeTwo256}}; + use VerifiedTransaction as VerifiedExtrinsic; + + type BlockId = generic::BlockId; + + #[derive(Clone, Debug)] + pub struct VerifiedTransaction { + pub hash: Hash, + pub sender: AccountId, + pub nonce: u64, + } + + impl txpool::VerifiedTransaction for VerifiedTransaction { + type Hash = Hash; + type Sender = AccountId; + + fn hash(&self) -> &Self::Hash { + &self.hash + } + + fn sender(&self) -> &Self::Sender { + &self.sender + } + + fn mem_usage(&self) -> usize { + 256 + } + } + + struct TestApi; + + impl TestApi { + fn default() -> Self { + TestApi + } + } + + impl ChainApi for TestApi { + type Block = Block; + type Hash = Hash; + type Sender = AccountId; + type Error = txpool::Error; + type VEx = VerifiedTransaction; + type Ready = HashMap; + type Score = u64; + type Event = (); + + fn verify_transaction(&self, _at: &BlockId, uxt: &ExtrinsicFor) -> Result { + let hash = BlakeTwo256::hash(&uxt.encode()); + let xt = uxt.clone().check()?; + Ok(VerifiedTransaction { + hash, + sender: xt.transfer.from, + nonce: xt.transfer.nonce, + }) + } + + fn is_ready(&self, at: &BlockId, nonce_cache: &mut Self::Ready, xt: &VerifiedFor) -> Readiness { + let sender = xt.verified.sender; + let next_index = nonce_cache.entry(sender) + .or_insert_with(|| index(at, sender)); + + let result = match xt.original.transfer.nonce.cmp(&next_index) { + Ordering::Greater => Readiness::Future, + Ordering::Equal => Readiness::Ready, + Ordering::Less => Readiness::Stale, + }; + + // remember to increment `next_index` + *next_index = next_index.saturating_add(1); + + result + } + + fn ready(&self) -> Self::Ready { + HashMap::default() + } + + fn compare(old: &VerifiedFor, other: &VerifiedFor) -> Ordering { + old.original.transfer.nonce.cmp(&other.original.transfer.nonce) + } + + fn choose(old: &VerifiedFor, new: &VerifiedFor) -> scoring::Choice { + assert!(new.verified.sender == old.verified.sender, "Scoring::choose called with transactions from different senders"); + if old.original.transfer.nonce == new.original.transfer.nonce { + return scoring::Choice::RejectNew; + } + scoring::Choice::InsertNew + } + + fn update_scores( + xts: &[txpool::Transaction>], + scores: &mut [Self::Score], + _change: scoring::Change<()> + ) { + for i in 0..xts.len() { + scores[i] = xts[i].original.transfer.amount; + } + } + + fn should_replace(_old: &VerifiedFor, _new: &VerifiedFor) -> scoring::Choice { + scoring::Choice::InsertNew + } + } + + fn index(at: &BlockId, _account: AccountId) -> u64 { + (_account[0] as u64) + number_of(at) + } + + fn number_of(at: &BlockId) -> u64 { + match at { + generic::BlockId::Number(n) => *n as u64, + _ => 0, + } + } + + fn uxt(who: Keyring, nonce: Index) -> Extrinsic { + let transfer = Transfer { + from: who.to_raw_public().into(), + to: AccountId::default(), + nonce, + amount: 1, + }; + let signature = transfer.using_encoded(|e| who.sign(e)); + Extrinsic { + transfer, + signature: signature.into(), + } + } + + fn pool() -> Pool { + Pool::new(Default::default(), TestApi::default()) + } + + #[test] + fn submission_should_work() { + let pool = pool(); + assert_eq!(209, index(&BlockId::number(0), Alice.to_raw_public().into())); + pool.submit_one(&BlockId::number(0), uxt(Alice, 209)).unwrap(); + + let pending: Vec<_> = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| (*a.sender(), a.original.transfer.nonce)).collect()).unwrap(); + assert_eq!(pending, vec![(Alice.to_raw_public().into(), 209)]); + } + + #[test] + fn multiple_submission_should_work() { + let pool = pool(); + pool.submit_one(&BlockId::number(0), uxt(Alice, 209)).unwrap(); + pool.submit_one(&BlockId::number(0), uxt(Alice, 210)).unwrap(); + + let pending: Vec<_> = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| (*a.sender(), a.original.transfer.nonce)).collect()).unwrap(); + assert_eq!(pending, vec![(Alice.to_raw_public().into(), 209), (Alice.to_raw_public().into(), 210)]); + } + + #[test] + fn early_nonce_should_be_culled() { + let pool = pool(); + pool.submit_one(&BlockId::number(0), uxt(Alice, 208)).unwrap(); + + let pending: Vec<_> = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| (*a.sender(), a.original.transfer.nonce)).collect()).unwrap(); + assert_eq!(pending, vec![]); + } + + #[test] + fn late_nonce_should_be_queued() { + let pool = pool(); + + pool.submit_one(&BlockId::number(0), uxt(Alice, 210)).unwrap(); + let pending: Vec<_> = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| (*a.sender(), a.original.transfer.nonce)).collect()).unwrap(); + assert_eq!(pending, vec![]); + + pool.submit_one(&BlockId::number(0), uxt(Alice, 209)).unwrap(); + let pending: Vec<_> = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| (*a.sender(), a.original.transfer.nonce)).collect()).unwrap(); + assert_eq!(pending, vec![(Alice.to_raw_public().into(), 209), (Alice.to_raw_public().into(), 210)]); + } + + #[test] + fn retrying_verification_might_not_change_anything() { + let pool = pool(); + pool.submit_one(&BlockId::number(0), uxt(Alice, 209)).unwrap(); + pool.submit_one(&BlockId::number(0), uxt(Alice, 210)).unwrap(); + + let pending: Vec<_> = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| (*a.sender(), a.original.transfer.nonce)).collect()).unwrap(); + assert_eq!(pending, vec![(Alice.to_raw_public().into(), 209), (Alice.to_raw_public().into(), 210)]); + + pool.retry_verification(&BlockId::number(1), Alice.to_raw_public().into()).unwrap(); + + let pending: Vec<_> = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| (*a.sender(), a.original.transfer.nonce)).collect()).unwrap(); + assert_eq!(pending, vec![(Alice.to_raw_public().into(), 209), (Alice.to_raw_public().into(), 210)]); + } + + #[test] + fn should_ban_invalid_transactions() { + let pool = pool(); + let uxt = uxt(Alice, 209); + let hash = *pool.submit_one(&BlockId::number(0), uxt.clone()).unwrap().hash(); + pool.remove(&[hash], true); + pool.submit_one(&BlockId::number(0), uxt.clone()).unwrap(); + + // when + pool.remove(&[hash], false); + let pending: Vec = pool.cull_and_get_pending(&BlockId::number(0), |p| p.map(|a| *a.sender()).collect()).unwrap(); + assert_eq!(pending, vec![]); + + // then + pool.submit_one(&BlockId::number(0), uxt.clone()).unwrap_err(); + } +} diff --git a/core/extrinsic-pool/src/rotator.rs b/core/extrinsic-pool/src/rotator.rs new file mode 100644 index 00000000000..93acce9e913 --- /dev/null +++ b/core/extrinsic-pool/src/rotator.rs @@ -0,0 +1,209 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Rotate extrinsic inside the pool. +//! +//! Keeps only recent extrinsic and discard the ones kept for a significant amount of time. +//! Discarded extrinsics are banned so that they don't get re-imported again. + +use std::{ + collections::HashMap, + fmt, + hash, + time::{Duration, Instant}, +}; +use parking_lot::RwLock; +use txpool::VerifiedTransaction; +use Verified; + +/// Expected size of the banned extrinsics cache. +const EXPECTED_SIZE: usize = 2048; + +/// Pool rotator is responsible to only keep fresh extrinsics in the pool. +/// +/// Extrinsics that occupy the pool for too long are culled and temporarily banned from entering +/// the pool again. +pub struct PoolRotator { + /// How long the extrinsic is banned for. + ban_time: Duration, + /// Currently banned extrinsics. + banned_until: RwLock>, +} + +impl Default for PoolRotator { + fn default() -> Self { + PoolRotator { + ban_time: Duration::from_secs(60 * 30), + banned_until: Default::default(), + } + } +} + +impl PoolRotator { + /// Returns `true` if extrinsic hash is currently banned. + pub fn is_banned(&self, hash: &Hash) -> bool { + self.banned_until.read().contains_key(hash) + } + + /// Bans given set of hashes. + pub fn ban(&self, now: &Instant, hashes: &[Hash]) { + let mut banned = self.banned_until.write(); + + for hash in hashes { + banned.insert(hash.clone(), *now + self.ban_time); + } + + if banned.len() > 2 * EXPECTED_SIZE { + while banned.len() > EXPECTED_SIZE { + if let Some(key) = banned.keys().next().cloned() { + banned.remove(&key); + } + } + } + } + + /// Bans extrinsic if it's stale. + /// + /// Returns `true` if extrinsic is stale and got banned. + pub fn ban_if_stale(&self, now: &Instant, xt: &Verified) -> bool where + VEx: VerifiedTransaction, + Hash: fmt::Debug + fmt::LowerHex, + { + if &xt.valid_till > now { + return false; + } + + self.ban(now, &[xt.verified.hash().clone()]); + true + } + + /// Removes timed bans. + pub fn clear_timeouts(&self, now: &Instant) { + let mut banned = self.banned_until.write(); + + banned.retain(|_, &mut v| v >= *now); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pool::tests::VerifiedTransaction; + use test_client::runtime::Hash; + + fn rotator() -> PoolRotator { + PoolRotator { + ban_time: Duration::from_millis(10), + ..Default::default() + } + } + + fn tx() -> (Hash, Verified) { + let hash = 5.into(); + let tx = Verified { + original: 5, + verified: VerifiedTransaction { + hash, + sender: Default::default(), + nonce: Default::default(), + }, + valid_till: Instant::now(), + }; + + (hash, tx) + } + + #[test] + fn should_not_ban_if_not_stale() { + // given + let (hash, tx) = tx(); + let rotator = rotator(); + assert!(!rotator.is_banned(&hash)); + let past = Instant::now() - Duration::from_millis(1000); + + // when + assert!(!rotator.ban_if_stale(&past, &tx)); + + // then + assert!(!rotator.is_banned(&hash)); + } + + #[test] + fn should_ban_stale_extrinsic() { + // given + let (hash, tx) = tx(); + let rotator = rotator(); + assert!(!rotator.is_banned(&hash)); + + // when + assert!(rotator.ban_if_stale(&Instant::now(), &tx)); + + // then + assert!(rotator.is_banned(&hash)); + } + + + #[test] + fn should_clear_banned() { + // given + let (hash, tx) = tx(); + let rotator = rotator(); + assert!(rotator.ban_if_stale(&Instant::now(), &tx)); + assert!(rotator.is_banned(&hash)); + + // when + let future = Instant::now() + rotator.ban_time + rotator.ban_time; + rotator.clear_timeouts(&future); + + // then + assert!(!rotator.is_banned(&hash)); + } + + #[test] + fn should_garbage_collect() { + // given + fn tx_with(i: u64, time: Instant) -> Verified { + let hash = i.into(); + Verified { + original: i, + verified: VerifiedTransaction { + hash, + sender: Default::default(), + nonce: Default::default(), + }, + valid_till: time, + } + } + + let rotator = rotator(); + + let now = Instant::now(); + let past = now - Duration::from_secs(1); + + // when + for i in 0..2*EXPECTED_SIZE { + let tx = tx_with(i as u64, past); + assert!(rotator.ban_if_stale(&now, &tx)); + } + assert_eq!(rotator.banned_until.read().len(), 2*EXPECTED_SIZE); + + // then + let tx = tx_with(2*EXPECTED_SIZE as u64, past); + // trigger a garbage collection + assert!(rotator.ban_if_stale(&now, &tx)); + assert_eq!(rotator.banned_until.read().len(), EXPECTED_SIZE); + } +} diff --git a/core/extrinsic-pool/src/watcher.rs b/core/extrinsic-pool/src/watcher.rs new file mode 100644 index 00000000000..78b27326f86 --- /dev/null +++ b/core/extrinsic-pool/src/watcher.rs @@ -0,0 +1,103 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Extrinsics status updates. + +use futures::{ + Stream, + sync::mpsc, +}; + +/// Possible extrinsic status events +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Status { + /// Extrinsic has been finalised in block with given hash. + Finalised(H), + /// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid. + Usurped(H), + /// The extrinsic has been broadcast to the given peers. + Broadcast(Vec), + /// Extrinsic has been dropped from the pool because of the limit. + Dropped, +} + +/// Extrinsic watcher. +/// +/// Represents a stream of status updates for particular extrinsic. +#[derive(Debug)] +pub struct Watcher { + receiver: mpsc::UnboundedReceiver>, +} + +impl Watcher { + /// Pipe the notifications to given sink. + /// + /// Make sure to drive the future to completion. + pub fn into_stream(self) -> impl Stream, Error=()> { + // we can safely ignore the error here, `UnboundedReceiver` never fails. + self.receiver.map_err(|_| ()) + } +} + +/// Sender part of the watcher. Exposed only for testing purposes. +#[derive(Debug, Default)] +pub struct Sender { + receivers: Vec>>, + finalised: bool, +} + +impl Sender { + /// Add a new watcher to this sender object. + pub fn new_watcher(&mut self) -> Watcher { + let (tx, receiver) = mpsc::unbounded(); + self.receivers.push(tx); + Watcher { + receiver, + } + } + + /// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid. + pub fn usurped(&mut self, hash: H) { + self.send(Status::Usurped(hash)) + } + + /// Extrinsic has been finalised in block with given hash. + pub fn finalised(&mut self, hash: H) { + self.send(Status::Finalised(hash)); + self.finalised = true; + } + + /// Transaction has been dropped from the pool because of the limit. + pub fn dropped(&mut self) { + self.send(Status::Dropped); + } + + /// The extrinsic has been broadcast to the given peers. + pub fn broadcast(&mut self, peers: Vec) { + self.send(Status::Broadcast(peers)) + } + + + /// Returns true if the are no more listeners for this extrinsic or it was finalised. + pub fn is_done(&self) -> bool { + self.finalised || self.receivers.is_empty() + } + + fn send(&mut self, status: Status) { + self.receivers.retain(|sender| sender.unbounded_send(status.clone()).is_ok()) + } +} diff --git a/core/keyring/Cargo.toml b/core/keyring/Cargo.toml new file mode 100644 index 00000000000..04d6c2bd134 --- /dev/null +++ b/core/keyring/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "substrate-keyring" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +substrate-primitives = { path = "../primitives" } +hex-literal = { version = "0.1.0" } +lazy_static = { version = "1.0" } diff --git a/core/keyring/README.adoc b/core/keyring/README.adoc new file mode 100644 index 00000000000..0118fe883d1 --- /dev/null +++ b/core/keyring/README.adoc @@ -0,0 +1,13 @@ + += Keyring + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/keyring/src/lib.rs b/core/keyring/src/lib.rs new file mode 100644 index 00000000000..321ea6c04cf --- /dev/null +++ b/core/keyring/src/lib.rs @@ -0,0 +1,176 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Support code for the runtime. +// end::description[] + +#[macro_use] extern crate hex_literal; +#[macro_use] extern crate lazy_static; +pub extern crate substrate_primitives; + +use std::collections::HashMap; +use std::ops::Deref; +use substrate_primitives::ed25519::{Pair, Public, Signature}; + +/// Set of test accounts. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub enum Keyring { + Alice, + Bob, + Charlie, + Dave, + Eve, + Ferdie, + One, + Two, +} + +impl Keyring { + pub fn from_public(who: Public) -> Option { + [ + Keyring::Alice, + Keyring::Bob, + Keyring::Charlie, + Keyring::Dave, + Keyring::Eve, + Keyring::Ferdie, + Keyring::One, + Keyring::Two, + ].iter() + .map(|i| *i) + .find(|&k| Public::from(k) == who) + } + + pub fn from_raw_public(who: [u8; 32]) -> Option { + Self::from_public(Public::from_raw(who)) + } + + pub fn to_raw_public(self) -> [u8; 32] { + *Public::from(self).as_array_ref() + } + + pub fn to_raw_public_vec(self) -> Vec { + Public::from(self).to_raw_vec() + } + + pub fn sign(self, msg: &[u8]) -> Signature { + Pair::from(self).sign(msg) + } + + pub fn pair(self) -> Pair { + match self { + Keyring::Alice => Pair::from_seed(b"Alice "), + Keyring::Bob => Pair::from_seed(b"Bob "), + Keyring::Charlie => Pair::from_seed(b"Charlie "), + Keyring::Dave => Pair::from_seed(b"Dave "), + Keyring::Eve => Pair::from_seed(b"Eve "), + Keyring::Ferdie => Pair::from_seed(b"Ferdie "), + Keyring::One => Pair::from_seed(b"12345678901234567890123456789012"), + Keyring::Two => Pair::from_seed(&hex!("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60")), + } + } +} + +impl From for &'static str { + fn from(k: Keyring) -> Self { + match k { + Keyring::Alice => "Alice", + Keyring::Bob => "Bob", + Keyring::Charlie => "Charlie", + Keyring::Dave => "Dave", + Keyring::Eve => "Eve", + Keyring::Ferdie => "Ferdie", + Keyring::One => "one", + Keyring::Two => "two", + } + } +} + +lazy_static! { + static ref PRIVATE_KEYS: HashMap = { + [ + Keyring::Alice, + Keyring::Bob, + Keyring::Charlie, + Keyring::Dave, + Keyring::Eve, + Keyring::Ferdie, + Keyring::One, + Keyring::Two, + ].iter().map(|&i| (i, i.pair())).collect() + }; + + static ref PUBLIC_KEYS: HashMap = { + PRIVATE_KEYS.iter().map(|(&name, pair)| (name, pair.public())).collect() + }; +} + +impl From for Public { + fn from(k: Keyring) -> Self { + (*PUBLIC_KEYS).get(&k).unwrap().clone() + } +} + +impl From for Pair { + fn from(k: Keyring) -> Self { + k.pair() + } +} + +impl From for [u8; 32] { + fn from(k: Keyring) -> Self { + *(*PUBLIC_KEYS).get(&k).unwrap().as_array_ref() + } +} + +impl From for &'static [u8; 32] { + fn from(k: Keyring) -> Self { + (*PUBLIC_KEYS).get(&k).unwrap().as_array_ref() + } +} + +impl AsRef<[u8; 32]> for Keyring { + fn as_ref(&self) -> &[u8; 32] { + (*PUBLIC_KEYS).get(self).unwrap().as_array_ref() + } +} + +impl AsRef for Keyring { + fn as_ref(&self) -> &Public { + (*PUBLIC_KEYS).get(self).unwrap() + } +} + +impl Deref for Keyring { + type Target = [u8; 32]; + fn deref(&self) -> &[u8; 32] { + (*PUBLIC_KEYS).get(self).unwrap().as_array_ref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use substrate_primitives::ed25519::Verifiable; + + #[test] + fn should_work() { + assert!(Keyring::Alice.sign(b"I am Alice!").verify(b"I am Alice!", Keyring::Alice)); + assert!(!Keyring::Alice.sign(b"I am Alice!").verify(b"I am Bob!", Keyring::Alice)); + assert!(!Keyring::Alice.sign(b"I am Alice!").verify(b"I am Alice!", Keyring::Bob)); + } +} diff --git a/core/keystore/Cargo.toml b/core/keystore/Cargo.toml new file mode 100644 index 00000000000..a464f7005d6 --- /dev/null +++ b/core/keystore/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "substrate-keystore" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +substrate-primitives = { path = "../primitives" } +parity-crypto = { version = "0.1", default_features = false } + +error-chain = "0.12" +hex = "0.3" +rand = "0.4" +serde_json = "1.0" +serde = "1.0" +serde_derive = "1.0" +subtle = "0.5" + +[dev-dependencies] +tempdir = "0.3" diff --git a/core/keystore/README.adoc b/core/keystore/README.adoc new file mode 100644 index 00000000000..5a66a882ff0 --- /dev/null +++ b/core/keystore/README.adoc @@ -0,0 +1,13 @@ + += Keystore + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/keystore/src/lib.rs b/core/keystore/src/lib.rs new file mode 100644 index 00000000000..f3eeb4be3f8 --- /dev/null +++ b/core/keystore/src/lib.rs @@ -0,0 +1,293 @@ +// Copyright 2017-2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Keystore (and session key management) for ed25519 based chains like Polkadot. +// end::description[] + +extern crate substrate_primitives; +extern crate parity_crypto as crypto; +extern crate subtle; +extern crate rand; +extern crate serde_json; +extern crate serde; +extern crate hex; + +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate error_chain; + +#[cfg(test)] +extern crate tempdir; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::fs::{self, File}; +use std::io::{self, Write}; + +use substrate_primitives::{hashing::blake2_256, ed25519::{Pair, Public, PKCS_LEN}}; + +pub use crypto::KEY_ITERATIONS; + +error_chain! { + foreign_links { + Io(io::Error); + Json(serde_json::Error); + } + + errors { + InvalidPassword { + description("Invalid password"), + display("Invalid password"), + } + InvalidPKCS8 { + description("Invalid PKCS#8 data"), + display("Invalid PKCS#8 data"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InvalidPassword; + +#[derive(Serialize, Deserialize)] +struct EncryptedKey { + mac: [u8; 32], + salt: [u8; 32], + ciphertext: Vec, // TODO: switch to fixed-size when serde supports + iv: [u8; 16], + iterations: u32, +} + +impl EncryptedKey { + fn encrypt(plain: &[u8; PKCS_LEN], password: &str, iterations: u32) -> Self { + use rand::{Rng, OsRng}; + + let mut rng = OsRng::new().expect("OS Randomness available on all supported platforms; qed"); + + let salt: [u8; 32] = rng.gen(); + let iv: [u8; 16] = rng.gen(); + + // two parts of derived key + // DK = [ DK[0..15] DK[16..31] ] = [derived_left_bits, derived_right_bits] + let (derived_left_bits, derived_right_bits) = crypto::derive_key_iterations(password.as_bytes(), &salt, iterations); + + // preallocated (on-stack in case of `Secret`) buffer to hold cipher + // length = length(plain) as we are using CTR-approach + let mut ciphertext = vec![0; PKCS_LEN]; + + // aes-128-ctr with initial vector of iv + crypto::aes::encrypt_128_ctr(&derived_left_bits, &iv, plain, &mut *ciphertext) + .expect("input lengths of key and iv are both 16; qed"); + + // Blake2_256(DK[16..31] ++ ), where DK[16..31] - derived_right_bits + let mac = blake2_256(&crypto::derive_mac(&derived_right_bits, &*ciphertext)); + + EncryptedKey { + salt, + iv, + mac, + iterations, + ciphertext, + } + } + + fn decrypt(&self, password: &str) -> Result<[u8; PKCS_LEN]> { + let (derived_left_bits, derived_right_bits) = + crypto::derive_key_iterations(password.as_bytes(), &self.salt, self.iterations); + + let mac = blake2_256(&crypto::derive_mac(&derived_right_bits, &self.ciphertext)); + + if subtle::slices_equal(&mac[..], &self.mac[..]) != 1 { + return Err(ErrorKind::InvalidPassword.into()); + } + + let mut plain = [0; PKCS_LEN]; + crypto::aes::decrypt_128_ctr(&derived_left_bits, &self.iv, &self.ciphertext, &mut plain[..]) + .expect("input lengths of key and iv are both 16; qed"); + Ok(plain) + } +} + +type Seed = [u8; 32]; + +/// Key store. +pub struct Store { + path: PathBuf, + additional: HashMap, +} + +impl Store { + /// Create a new store at the given path. + pub fn open(path: PathBuf) -> Result { + fs::create_dir_all(&path)?; + Ok(Store { path, additional: HashMap::new() }) + } + + /// Generate a new key, placing it into the store. + pub fn generate(&self, password: &str) -> Result { + let (pair, pkcs_bytes) = Pair::generate_with_pkcs8(); + let key_file = EncryptedKey::encrypt(&pkcs_bytes, password, KEY_ITERATIONS as u32); + + let mut file = File::create(self.key_file_path(&pair.public()))?; + ::serde_json::to_writer(&file, &key_file)?; + + file.flush()?; + + Ok(pair) + } + + /// Create a new key from seed. Do not place it into the store. + /// Only the first 32 bytes of the sead are used. This is meant to be used for testing only. + // TODO: Remove this + pub fn generate_from_seed(&mut self, seed: &str) -> Result { + let mut s: [u8; 32] = [' ' as u8; 32]; + + let was_hex = if seed.len() == 66 && &seed[0..2] == "0x" { + if let Ok(d) = hex::decode(&seed[2..]) { + s.copy_from_slice(&d); + true + } else { false } + } else { false }; + + if !was_hex { + let len = ::std::cmp::min(32, seed.len()); + &mut s[..len].copy_from_slice(&seed.as_bytes()[..len]); + } + + let pair = Pair::from_seed(&s); + self.additional.insert(pair.public(), s); + Ok(pair) + } + + /// Load a key file with given public key. + pub fn load(&self, public: &Public, password: &str) -> Result { + if let Some(ref seed) = self.additional.get(public) { + let pair = Pair::from_seed(seed); + return Ok(pair); + } + let path = self.key_file_path(public); + let file = File::open(path)?; + + let encrypted_key: EncryptedKey = ::serde_json::from_reader(&file)?; + let pkcs_bytes = encrypted_key.decrypt(password)?; + + Pair::from_pkcs8(&pkcs_bytes[..]).map_err(|_| ErrorKind::InvalidPKCS8.into()) + } + + /// Get public keys of all stored keys. + pub fn contents(&self) -> Result> { + let mut public_keys: Vec = self.additional.keys().cloned().collect(); + for entry in fs::read_dir(&self.path)? { + let entry = entry?; + let path = entry.path(); + + // skip directories and non-unicode file names (hex is unicode) + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.len() != 64 { continue } + + match hex::decode(name) { + Ok(ref hex) if hex.len() == 32 => { + let mut buf = [0; 32]; + buf.copy_from_slice(&hex[..]); + + public_keys.push(Public(buf)); + } + _ => continue, + } + } + } + + Ok(public_keys) + } + + fn key_file_path(&self, public: &Public) -> PathBuf { + let mut buf = self.path.clone(); + buf.push(hex::encode(public.as_slice())); + buf + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempdir::TempDir; + + #[test] + fn encrypt_and_decrypt() { + let plain = [1; PKCS_LEN]; + let encrypted_key = EncryptedKey::encrypt(&plain, "thepassword", KEY_ITERATIONS as u32); + + let decrypted_key = encrypted_key.decrypt("thepassword").unwrap(); + + assert_eq!(&plain[..], &decrypted_key[..]); + } + + #[test] + fn decrypt_wrong_password_fails() { + let plain = [1; PKCS_LEN]; + let encrypted_key = EncryptedKey::encrypt(&plain, "thepassword", KEY_ITERATIONS as u32); + + assert!(encrypted_key.decrypt("thepassword2").is_err()); + } + + #[test] + fn decrypt_wrong_iterations_fails() { + let plain = [1; PKCS_LEN]; + let mut encrypted_key = EncryptedKey::encrypt(&plain, "thepassword", KEY_ITERATIONS as u32); + + encrypted_key.iterations -= 64; + + assert!(encrypted_key.decrypt("thepassword").is_err()); + } + + #[test] + fn basic_store() { + let temp_dir = TempDir::new("keystore").unwrap(); + let store = Store::open(temp_dir.path().to_owned()).unwrap(); + + assert!(store.contents().unwrap().is_empty()); + + let key = store.generate("thepassword").unwrap(); + let key2 = store.load(&key.public(), "thepassword").unwrap(); + + assert!(store.load(&key.public(), "notthepassword").is_err()); + + assert_eq!(key.public(), key2.public()); + + assert_eq!(store.contents().unwrap()[0], key.public()); + } + + #[test] + fn test_generate_from_seed() { + let temp_dir = TempDir::new("keystore").unwrap(); + let mut store = Store::open(temp_dir.path().to_owned()).unwrap(); + + let pair = store.generate_from_seed("0x1").unwrap(); + assert_eq!("5GqhgbUd2S9uc5Tm7hWhw29Tw2jBnuHshmTV1fDF4V1w3G2z", pair.public().to_ss58check()); + + let pair = store.generate_from_seed("0x3d97c819d68f9bafa7d6e79cb991eebcd77d966c5334c0b94d9e1fa7ad0869dc").unwrap(); + assert_eq!("5DKUrgFqCPV8iAXx9sjy1nyBygQCeiUYRFWurZGhnrn3HBL8", pair.public().to_ss58check()); + + let pair = store.generate_from_seed("12345678901234567890123456789022").unwrap(); + assert_eq!("5DscZvfjnM5im7oKRXXP9xtCG1SEwfMb8J5eGLmw5EHhoHR3", pair.public().to_ss58check()); + + let pair = store.generate_from_seed("1").unwrap(); + assert_eq!("5DYnksEZFc7kgtfyNM1xK2eBtW142gZ3Ho3NQubrF2S6B2fq", pair.public().to_ss58check()); + } +} diff --git a/core/misbehavior-check/Cargo.toml b/core/misbehavior-check/Cargo.toml new file mode 100644 index 00000000000..d455a4fc045 --- /dev/null +++ b/core/misbehavior-check/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "substrate-misbehavior-check" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +substrate-codec = { path = "../codec", default-features = false } +substrate-primitives = { path = "../primitives", default-features = false } +substrate-runtime-primitives = { path = "../../runtime/primitives", default-features = false } +substrate-runtime-io = { path = "../runtime-io", default-features = false } + +[dev-dependencies] +substrate-bft = { path = "../bft" } +rhododendron = "0.3" +substrate-keyring = { path = "../keyring" } + +[features] +default = ["std"] +std = ["substrate-codec/std", "substrate-primitives/std", "substrate-runtime-primitives/std", "substrate-runtime-io/std"] diff --git a/core/misbehavior-check/README.adoc b/core/misbehavior-check/README.adoc new file mode 100644 index 00000000000..e5b52b954b6 --- /dev/null +++ b/core/misbehavior-check/README.adoc @@ -0,0 +1,13 @@ + += Misbehavior-check + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/misbehavior-check/src/lib.rs b/core/misbehavior-check/src/lib.rs new file mode 100644 index 00000000000..eff87ab1e5e --- /dev/null +++ b/core/misbehavior-check/src/lib.rs @@ -0,0 +1,206 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Utility for substrate-based runtimes that want to check misbehavior reports. +// end::description[] + +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate substrate_codec as codec; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_primitives as runtime_primitives; + +#[cfg(test)] +extern crate substrate_bft; +#[cfg(test)] +extern crate substrate_keyring as keyring; +#[cfg(test)] +extern crate rhododendron; + +use codec::{Codec, Encode}; +use primitives::{AuthorityId, Signature}; + +use runtime_primitives::bft::{Action, Message, MisbehaviorKind}; + +// check a message signature. returns true if signed by that authority. +fn check_message_sig( + message: Message, + signature: &Signature, + from: &AuthorityId +) -> bool { + let msg: Vec = message.encode(); + runtime_io::ed25519_verify(&signature.0, &msg, from) +} + +fn prepare(parent: H, round_number: u32, hash: H) -> Message { + Message { + parent, + action: Action::Prepare(round_number, hash), + } +} + +fn commit(parent: H, round_number: u32, hash: H) -> Message { + Message { + parent, + action: Action::Commit(round_number, hash), + } +} + +/// Evaluate misbehavior. +/// +/// Doesn't check that the header hash in question is +/// valid or whether the misbehaving authority was part of +/// the set at that block. +pub fn evaluate_misbehavior( + misbehaved: &AuthorityId, + parent_hash: H, + kind: &MisbehaviorKind, +) -> bool { + match *kind { + MisbehaviorKind::BftDoublePrepare(round, (h_1, ref s_1), (h_2, ref s_2)) => { + s_1 != s_2 && + check_message_sig::(prepare::(parent_hash, round, h_1), s_1, misbehaved) && + check_message_sig::(prepare::(parent_hash, round, h_2), s_2, misbehaved) + } + MisbehaviorKind::BftDoubleCommit(round, (h_1, ref s_1), (h_2, ref s_2)) => { + s_1 != s_2 && + check_message_sig::(commit::(parent_hash, round, h_1), s_1, misbehaved) && + check_message_sig::(commit::(parent_hash, round, h_2), s_2, misbehaved) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use keyring::ed25519; + use keyring::Keyring; + + use runtime_primitives::testing::{H256, Block as RawBlock}; + + type Block = RawBlock; + + fn sign_prepare(key: &ed25519::Pair, round: u32, hash: H256, parent_hash: H256) -> (H256, Signature) { + let msg = substrate_bft::sign_message::( + rhododendron::Message::Vote(rhododendron::Vote::Prepare(round as _, hash)), + key, + parent_hash + ); + + match msg { + rhododendron::LocalizedMessage::Vote(vote) => (hash, vote.signature.signature), + _ => panic!("signing vote leads to signed vote"), + } + } + + fn sign_commit(key: &ed25519::Pair, round: u32, hash: H256, parent_hash: H256) -> (H256, Signature) { + let msg = substrate_bft::sign_message::( + rhododendron::Message::Vote(rhododendron::Vote::Commit(round as _, hash)), + key, + parent_hash + ); + + match msg { + rhododendron::LocalizedMessage::Vote(vote) => (hash, vote.signature.signature), + _ => panic!("signing vote leads to signed vote"), + } + } + + #[test] + fn evaluates_double_prepare() { + let key: ed25519::Pair = Keyring::One.into(); + let parent_hash = [0xff; 32].into(); + let hash_1 = [0; 32].into(); + let hash_2 = [1; 32].into(); + + assert!(evaluate_misbehavior::( + &key.public().into(), + parent_hash, + &MisbehaviorKind::BftDoublePrepare( + 1, + sign_prepare(&key, 1, hash_1, parent_hash), + sign_prepare(&key, 1, hash_2, parent_hash) + ) + )); + + // same signature twice is not misbehavior. + let signed = sign_prepare(&key, 1, hash_1, parent_hash); + assert!(evaluate_misbehavior::( + &key.public().into(), + parent_hash, + &MisbehaviorKind::BftDoublePrepare( + 1, + signed, + signed, + ) + ) == false); + + // misbehavior has wrong target. + assert!(evaluate_misbehavior::( + &Keyring::Two.to_raw_public().into(), + parent_hash, + &MisbehaviorKind::BftDoublePrepare( + 1, + sign_prepare(&key, 1, hash_1, parent_hash), + sign_prepare(&key, 1, hash_2, parent_hash), + ) + ) == false); + } + + #[test] + fn evaluates_double_commit() { + let key: ed25519::Pair = Keyring::One.into(); + let parent_hash = [0xff; 32].into(); + let hash_1 = [0; 32].into(); + let hash_2 = [1; 32].into(); + + assert!(evaluate_misbehavior::( + &key.public().into(), + parent_hash, + &MisbehaviorKind::BftDoubleCommit( + 1, + sign_commit(&key, 1, hash_1, parent_hash), + sign_commit(&key, 1, hash_2, parent_hash) + ) + )); + + // same signature twice is not misbehavior. + let signed = sign_commit(&key, 1, hash_1, parent_hash); + assert!(evaluate_misbehavior::( + &key.public().into(), + parent_hash, + &MisbehaviorKind::BftDoubleCommit( + 1, + signed, + signed, + ) + ) == false); + + // misbehavior has wrong target. + assert!(evaluate_misbehavior::( + &Keyring::Two.to_raw_public().into(), + parent_hash, + &MisbehaviorKind::BftDoubleCommit( + 1, + sign_commit(&key, 1, hash_1, parent_hash), + sign_commit(&key, 1, hash_2, parent_hash), + ) + ) == false); + } +} diff --git a/core/network-libp2p/Cargo.toml b/core/network-libp2p/Cargo.toml new file mode 100644 index 00000000000..b002fa82e96 --- /dev/null +++ b/core/network-libp2p/Cargo.toml @@ -0,0 +1,34 @@ +[package] +description = "libp2p implementation of the ethcore network library" +homepage = "http://parity.io" +license = "GPL-3.0" +name = "substrate-network-libp2p" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +bytes = "0.4" +error-chain = { version = "0.12", default-features = false } +fnv = "1.0" +futures = "0.1" +libp2p = { git = "https://github.com/libp2p/rust-libp2p", rev = "304e9c72c88bc97824f2734dc19d1b5f4556d346", default-features = false, features = ["libp2p-secio", "libp2p-secio-secp256k1"] } +ethcore-io = { git = "https://github.com/paritytech/parity.git" } +ethkey = { git = "https://github.com/paritytech/parity.git" } +ethereum-types = "0.3" +parking_lot = "0.5" +libc = "0.2" +log = "0.3" +rand = "0.5.0" +serde = "1.0.70" +serde_derive = "1.0.70" +serde_json = "1.0.24" +tokio = "0.1" +tokio-io = "0.1" +tokio-timer = "0.2" +unsigned-varint = { version = "0.2.1", features = ["codec"] } + +[dev-dependencies] +assert_matches = "1.2" +parity-bytes = "0.1" +ethcore-io = { git = "https://github.com/paritytech/parity.git" } +ethcore-logger = { git = "https://github.com/paritytech/parity.git" } diff --git a/core/network-libp2p/README.adoc b/core/network-libp2p/README.adoc new file mode 100644 index 00000000000..2f340aa397e --- /dev/null +++ b/core/network-libp2p/README.adoc @@ -0,0 +1,13 @@ + += Network libp2p + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/network-libp2p/src/connection_filter.rs b/core/network-libp2p/src/connection_filter.rs new file mode 100644 index 00000000000..46d9d86b58d --- /dev/null +++ b/core/network-libp2p/src/connection_filter.rs @@ -0,0 +1,31 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Connection filter trait. + +use super::NodeId; + +/// Filtered connection direction. +pub enum ConnectionDirection { + Inbound, + Outbound, +} + +/// Connection filter. Each connection is checked against `connection_allowed`. +pub trait ConnectionFilter : Send + Sync { + /// Filter a connection. Returns `true` if connection should be allowed. `false` if rejected. + fn connection_allowed(&self, own_id: &NodeId, connecting_id: &NodeId, direction: ConnectionDirection) -> bool; +} diff --git a/core/network-libp2p/src/custom_proto.rs b/core/network-libp2p/src/custom_proto.rs new file mode 100644 index 00000000000..72807f21e8f --- /dev/null +++ b/core/network-libp2p/src/custom_proto.rs @@ -0,0 +1,290 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use bytes::{Bytes, BytesMut}; +use ProtocolId; +use libp2p::core::{Multiaddr, ConnectionUpgrade, Endpoint}; +use PacketId; +use std::io::Error as IoError; +use std::vec::IntoIter as VecIntoIter; +use futures::{future, Future, stream, Stream, Sink}; +use futures::sync::mpsc; +use tokio_io::{AsyncRead, AsyncWrite}; +use unsigned_varint::codec::UviBytes; + +/// Connection upgrade for a single protocol. +/// +/// Note that "a single protocol" here refers to `par` for example. However +/// each protocol can have multiple different versions for networking purposes. +#[derive(Clone)] +pub struct RegisteredProtocol { + /// Id of the protocol for API purposes. + id: ProtocolId, + /// Base name of the protocol as advertised on the network. + /// Ends with `/` so that we can append a version number behind. + base_name: Bytes, + /// List of protocol versions that we support, plus their packet count. + /// Ordered in descending order so that the best comes first. + /// The packet count is used to filter out invalid messages. + supported_versions: Vec<(u8, u8)>, + /// Custom data. + custom_data: T, +} + +/// Output of a `RegisteredProtocol` upgrade. +pub struct RegisteredProtocolOutput { + /// Data passed to `RegisteredProtocol::new`. + pub custom_data: T, + + /// Id of the protocol. + pub protocol_id: ProtocolId, + + /// Endpoint of the connection. + pub endpoint: Endpoint, + + /// Version of the protocol that was negotiated. + pub protocol_version: u8, + + /// Channel to sender outgoing messages to. + // TODO: consider assembling packet_id here + pub outgoing: mpsc::UnboundedSender, + + /// Stream where incoming messages are received. The stream ends whenever + /// either side is closed. + pub incoming: Box + Send>, +} + +impl RegisteredProtocol { + /// Creates a new `RegisteredProtocol`. The `custom_data` parameter will be + /// passed inside the `RegisteredProtocolOutput`. + pub fn new(custom_data: T, protocol: ProtocolId, versions: &[(u8, u8)]) + -> Self { + let mut proto_name = Bytes::from_static(b"/substrate/"); + proto_name.extend_from_slice(&protocol); + proto_name.extend_from_slice(b"/"); + + RegisteredProtocol { + base_name: proto_name, + id: protocol, + supported_versions: { + let mut tmp: Vec<_> = versions.iter().rev().cloned().collect(); + tmp.sort_unstable_by(|a, b| b.1.cmp(&a.1)); + tmp + }, + custom_data: custom_data, + } + } + + /// Returns the ID of the protocol. + pub fn id(&self) -> ProtocolId { + self.id + } + + /// Returns the custom data that was passed to `new`. + pub fn custom_data(&self) -> &T { + &self.custom_data + } +} + +// `Maf` is short for `MultiaddressFuture` +impl ConnectionUpgrade for RegisteredProtocol +where C: AsyncRead + AsyncWrite + Send + 'static, // TODO: 'static :-/ + Maf: Future + Send + 'static, // TODO: 'static :( +{ + type NamesIter = VecIntoIter<(Bytes, Self::UpgradeIdentifier)>; + type UpgradeIdentifier = u8; // Protocol version + + #[inline] + fn protocol_names(&self) -> Self::NamesIter { + // Report each version as an individual protocol. + self.supported_versions.iter().map(|&(ver, _)| { + let num = ver.to_string(); + let mut name = self.base_name.clone(); + name.extend_from_slice(num.as_bytes()); + (name, ver) + }).collect::>().into_iter() + } + + type Output = RegisteredProtocolOutput; + type MultiaddrFuture = Maf; + type Future = future::FutureResult<(Self::Output, Self::MultiaddrFuture), IoError>; + + #[allow(deprecated)] + fn upgrade( + self, + socket: C, + protocol_version: Self::UpgradeIdentifier, + endpoint: Endpoint, + remote_addr: Maf + ) -> Self::Future { + let packet_count = self.supported_versions + .iter() + .find(|&(v, _)| *v == protocol_version) + .expect("negotiated protocol version that wasn't advertised ; \ + programmer error") + .1; + + // This function is called whenever we successfully negotiated a + // protocol with a remote (both if initiated by us or by the remote) + + // This channel is used to send outgoing packets to the custom_data + // for this open substream. + let (msg_tx, msg_rx) = mpsc::unbounded(); + + // Build the sink for outgoing network bytes, and the stream for + // incoming instructions. `stream` implements `Stream`. + enum Message { + /// Received data from the network. + RecvSocket(BytesMut), + /// Data to send to the network. + /// The packet_id must already be inside the `Bytes`. + SendReq(Bytes), + /// The socket has been closed. + Finished, + } + + let (sink, stream) = { + let framed = AsyncRead::framed(socket, UviBytes::default()); + let msg_rx = msg_rx.map(Message::SendReq) + .map_err(|()| unreachable!("mpsc::UnboundedReceiver never errors")); + let (sink, stream) = framed.split(); + let stream = stream.map(Message::RecvSocket) + .chain(stream::once(Ok(Message::Finished))); + (sink, msg_rx.select(stream)) + }; + + let incoming = stream::unfold((sink, stream, false), move |(sink, stream, finished)| { + if finished { + return None + } + + Some(stream + .into_future() + .map_err(|(err, _)| err) + .and_then(move |(message, stream)| + match message { + Some(Message::RecvSocket(mut data)) => { + // The `data` should be prefixed by the packet ID, + // therefore an empty packet is invalid. + if data.is_empty() { + debug!(target: "sub-libp2p", "ignoring incoming \ + packet because it was empty"); + let f = future::ok((None, (sink, stream, false))); + return future::Either::A(f) + } + + let packet_id = data[0]; + let data = data.split_off(1); + + if packet_id >= packet_count { + debug!(target: "sub-libp2p", "ignoring incoming packet \ + because packet_id {} is too large", packet_id); + let f = future::ok((None, (sink, stream, false))); + future::Either::A(f) + } else { + let out = Some((packet_id, data.freeze())); + let f = future::ok((out, (sink, stream, false))); + future::Either::A(f) + } + }, + + Some(Message::SendReq(data)) => { + let fut = sink.send(data) + .map(move |sink| (None, (sink, stream, false))); + future::Either::B(fut) + }, + + Some(Message::Finished) | None => { + let f = future::ok((None, (sink, stream, true))); + future::Either::A(f) + }, + } + )) + }).filter_map(|v| v); + + let out = RegisteredProtocolOutput { + custom_data: self.custom_data, + protocol_id: self.id, + endpoint, + protocol_version: protocol_version, + outgoing: msg_tx, + incoming: Box::new(incoming), + }; + + future::ok((out, remote_addr)) + } +} + +// Connection upgrade for all the protocols contained in it. +#[derive(Clone)] +pub struct RegisteredProtocols(pub Vec>); + +impl RegisteredProtocols { + /// Finds a protocol in the list by its id. + pub fn find_protocol(&self, protocol: ProtocolId) + -> Option<&RegisteredProtocol> { + self.0.iter().find(|p| p.id == protocol) + } + + /// Returns true if the given protocol is in the list. + pub fn has_protocol(&self, protocol: ProtocolId) -> bool { + self.0.iter().any(|p| p.id == protocol) + } +} + +impl Default for RegisteredProtocols { + fn default() -> Self { + RegisteredProtocols(Vec::new()) + } +} + +impl ConnectionUpgrade for RegisteredProtocols +where C: AsyncRead + AsyncWrite + Send + 'static, // TODO: 'static :-/ + Maf: Future + Send + 'static, // TODO: 'static :( +{ + type NamesIter = VecIntoIter<(Bytes, Self::UpgradeIdentifier)>; + type UpgradeIdentifier = (usize, + as ConnectionUpgrade>::UpgradeIdentifier); + + fn protocol_names(&self) -> Self::NamesIter { + // We concat the lists of `RegisteredProtocol::protocol_names` for + // each protocol. + self.0.iter().enumerate().flat_map(|(n, proto)| + ConnectionUpgrade::::protocol_names(proto) + .map(move |(name, id)| (name, (n, id))) + ).collect::>().into_iter() + } + + type Output = as ConnectionUpgrade>::Output; + type MultiaddrFuture = as + ConnectionUpgrade>::MultiaddrFuture; + type Future = as ConnectionUpgrade>::Future; + + #[inline] + fn upgrade( + self, + socket: C, + upgrade_identifier: Self::UpgradeIdentifier, + endpoint: Endpoint, + remote_addr: Maf + ) -> Self::Future { + let (protocol_index, inner_proto_id) = upgrade_identifier; + self.0.into_iter() + .nth(protocol_index) + .expect("invalid protocol index ; programmer logic error") + .upgrade(socket, inner_proto_id, endpoint, remote_addr) + } +} diff --git a/core/network-libp2p/src/error.rs b/core/network-libp2p/src/error.rs new file mode 100644 index 00000000000..d095858b122 --- /dev/null +++ b/core/network-libp2p/src/error.rs @@ -0,0 +1,221 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{io, net, fmt}; +use libc::{ENFILE, EMFILE}; +use io::IoError; +use ethkey; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum DisconnectReason +{ + DisconnectRequested, + TCPError, + BadProtocol, + UselessPeer, + TooManyPeers, + DuplicatePeer, + IncompatibleProtocol, + NullIdentity, + ClientQuit, + UnexpectedIdentity, + LocalIdentity, + PingTimeout, + Unknown, +} + +impl DisconnectReason { + pub fn from_u8(n: u8) -> DisconnectReason { + match n { + 0 => DisconnectReason::DisconnectRequested, + 1 => DisconnectReason::TCPError, + 2 => DisconnectReason::BadProtocol, + 3 => DisconnectReason::UselessPeer, + 4 => DisconnectReason::TooManyPeers, + 5 => DisconnectReason::DuplicatePeer, + 6 => DisconnectReason::IncompatibleProtocol, + 7 => DisconnectReason::NullIdentity, + 8 => DisconnectReason::ClientQuit, + 9 => DisconnectReason::UnexpectedIdentity, + 10 => DisconnectReason::LocalIdentity, + 11 => DisconnectReason::PingTimeout, + _ => DisconnectReason::Unknown, + } + } +} + +impl fmt::Display for DisconnectReason { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use self::DisconnectReason::*; + + let msg = match *self { + DisconnectRequested => "disconnect requested", + TCPError => "TCP error", + BadProtocol => "bad protocol", + UselessPeer => "useless peer", + TooManyPeers => "too many peers", + DuplicatePeer => "duplicate peer", + IncompatibleProtocol => "incompatible protocol", + NullIdentity => "null identity", + ClientQuit => "client quit", + UnexpectedIdentity => "unexpected identity", + LocalIdentity => "local identity", + PingTimeout => "ping timeout", + Unknown => "unknown", + }; + + f.write_str(msg) + } +} + +error_chain! { + foreign_links { + SocketIo(IoError) #[doc = "Socket IO error."]; + } + + errors { + #[doc = "Error concerning the network address parsing subsystem."] + AddressParse { + description("Failed to parse network address"), + display("Failed to parse network address"), + } + + #[doc = "Error concerning the network address resolution subsystem."] + AddressResolve(err: Option) { + description("Failed to resolve network address"), + display("Failed to resolve network address {}", err.as_ref().map_or("".to_string(), |e| e.to_string())), + } + + #[doc = "Authentication failure"] + Auth { + description("Authentication failure"), + display("Authentication failure"), + } + + #[doc = "Unrecognised protocol"] + BadProtocol { + description("Bad protocol"), + display("Bad protocol"), + } + + #[doc = "Expired message"] + Expired { + description("Expired message"), + display("Expired message"), + } + + #[doc = "Peer not found"] + PeerNotFound { + description("Peer not found"), + display("Peer not found"), + } + + #[doc = "Peer is disconnected"] + Disconnect(reason: DisconnectReason) { + description("Peer disconnected"), + display("Peer disconnected: {}", reason), + } + + #[doc = "Invalid node id"] + InvalidNodeId { + description("Invalid node id"), + display("Invalid node id"), + } + + #[doc = "Packet size is over the protocol limit"] + OversizedPacket { + description("Packet is too large"), + display("Packet is too large"), + } + + #[doc = "Reached system resource limits for this process"] + ProcessTooManyFiles { + description("Too many open files in process."), + display("Too many open files in this process. Check your resource limits and restart parity"), + } + + #[doc = "Reached system wide resource limits"] + SystemTooManyFiles { + description("Too many open files on system."), + display("Too many open files on system. Consider closing some processes/release some file handlers or increas the system-wide resource limits and restart parity."), + } + + #[doc = "An unknown IO error occurred."] + Io(err: io::Error) { + description("IO Error"), + display("Unexpected IO error: {}", err), + } + } +} + +impl From for Error { + fn from(err: io::Error) -> Self { + match err.raw_os_error() { + Some(ENFILE) => ErrorKind::ProcessTooManyFiles.into(), + Some(EMFILE) => ErrorKind::SystemTooManyFiles.into(), + _ => Error::from_kind(ErrorKind::Io(err)) + } + } +} + +impl From for Error { + fn from(_err: ethkey::Error) -> Self { + ErrorKind::Auth.into() + } +} + +impl From for Error { + fn from(_err: ethkey::crypto::Error) -> Self { + ErrorKind::Auth.into() + } +} + +impl From for Error { + fn from(_err: net::AddrParseError) -> Self { ErrorKind::AddressParse.into() } +} + +#[test] +fn test_errors() { + assert_eq!(DisconnectReason::ClientQuit, DisconnectReason::from_u8(8)); + let mut r = DisconnectReason::DisconnectRequested; + for i in 0 .. 20 { + r = DisconnectReason::from_u8(i); + } + assert_eq!(DisconnectReason::Unknown, r); +} + +#[test] +fn test_io_errors() { + use libc::{EMFILE, ENFILE}; + + assert_matches!( + >::from( + io::Error::from_raw_os_error(ENFILE) + ).kind(), + ErrorKind::ProcessTooManyFiles); + + assert_matches!( + >::from( + io::Error::from_raw_os_error(EMFILE) + ).kind(), + ErrorKind::SystemTooManyFiles); + + assert_matches!( + >::from( + io::Error::from_raw_os_error(0) + ).kind(), + ErrorKind::Io(_)); +} diff --git a/core/network-libp2p/src/lib.rs b/core/network-libp2p/src/lib.rs new file mode 100644 index 00000000000..f06c8fefd86 --- /dev/null +++ b/core/network-libp2p/src/lib.rs @@ -0,0 +1,75 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! TODO: Missing doc +// end::description[] + +#![recursion_limit="128"] +#![type_length_limit = "268435456"] + +extern crate parking_lot; +extern crate fnv; +extern crate futures; +extern crate tokio; +extern crate tokio_io; +extern crate tokio_timer; +extern crate ethkey; +extern crate libc; +extern crate libp2p; +extern crate rand; +extern crate serde; +#[macro_use] +extern crate serde_derive; +extern crate serde_json; +extern crate bytes; +extern crate unsigned_varint; + +extern crate ethcore_io as io; +extern crate ethereum_types; + +#[macro_use] +extern crate error_chain; +#[macro_use] +extern crate log; +#[cfg(test)] #[macro_use] +extern crate assert_matches; + +pub use connection_filter::{ConnectionFilter, ConnectionDirection}; +pub use io::TimerToken; +pub use error::{Error, ErrorKind, DisconnectReason}; +pub use libp2p::{Multiaddr, multiaddr::AddrComponent}; +pub use traits::*; + +mod connection_filter; +mod custom_proto; +mod error; +mod network_state; +mod service; +mod timeouts; +mod topology; +mod traits; +mod transport; + +pub use service::NetworkService; + +/// Check if node url is valid +pub fn validate_node_url(url: &str) -> Result<(), Error> { + match url.parse::() { + Ok(_) => Ok(()), + Err(_) => Err(ErrorKind::InvalidNodeId.into()), + } +} diff --git a/core/network-libp2p/src/network_state.rs b/core/network-libp2p/src/network_state.rs new file mode 100644 index 00000000000..e06735f647f --- /dev/null +++ b/core/network-libp2p/src/network_state.rs @@ -0,0 +1,953 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use bytes::Bytes; +use fnv::{FnvHashMap, FnvHashSet}; +use futures::sync::mpsc; +use libp2p::core::{multiaddr::ToMultiaddr, Multiaddr, AddrComponent, Endpoint, UniqueConnec}; +use libp2p::core::{UniqueConnecState, PeerId, PublicKey}; +use libp2p::kad::KadConnecController; +use libp2p::ping::Pinger; +use libp2p::secio; +use {Error, ErrorKind, NetworkConfiguration, NonReservedPeerMode}; +use {NodeIndex, ProtocolId, SessionInfo}; +use parking_lot::{Mutex, RwLock}; +use rand::{self, Rng}; +use topology::{DisconnectReason, NetTopology}; +use std::fs; +use std::io::{Error as IoError, ErrorKind as IoErrorKind, Read, Write}; +use std::path::Path; +use std::sync::atomic; +use std::time::{Duration, Instant}; + +// File where the peers are stored. +const NODES_FILE: &str = "nodes.json"; +// File where the private key is stored. +const SECRET_FILE: &str = "secret"; +// Duration during which a peer is disabled. +const PEER_DISABLE_DURATION: Duration = Duration::from_secs(5 * 60); + +// Common struct shared throughout all the components of the service. +pub struct NetworkState { + /// Contains the information about the network. + topology: RwLock, + + /// Active connections. + connections: RwLock, + + /// Maximum incoming peers. + max_incoming_peers: u32, + /// Maximum outgoing peers. + max_outgoing_peers: u32, + + /// If true, only reserved peers can connect. + reserved_only: atomic::AtomicBool, + /// List of the IDs of the reserved peers. + reserved_peers: RwLock>, + + /// Each node we discover gets assigned a new unique ID. This ID increases linearly. + next_node_index: atomic::AtomicUsize, + + /// List of the IDs of the disabled peers. These peers will see their + /// connections refused. Includes the time when the disabling expires. + disabled_nodes: Mutex>, + + /// Local private key. + local_private_key: secio::SecioKeyPair, + /// Local public key. + local_public_key: PublicKey, +} + +struct Connections { + /// For each libp2p peer ID, the ID of the peer in the API we expose. + /// Also corresponds to the index in `info_by_peer`. + peer_by_nodeid: FnvHashMap, + + /// For each peer ID, information about our connection to this peer. + info_by_peer: FnvHashMap, +} + +struct PeerConnectionInfo { + /// A list of protocols, and the potential corresponding connection. + /// The `UniqueConnec` contains a sender and the protocol version. + /// The sender can be used to transmit data for the remote. Note that the + /// packet_id has to be inside the `Bytes`. + protocols: Vec<(ProtocolId, UniqueConnec<(mpsc::UnboundedSender, u8)>)>, + + /// The Kademlia connection to this node. + kad_connec: UniqueConnec, + + /// The ping connection to this node. + ping_connec: UniqueConnec, + + /// Id of the peer. + id: PeerId, + + /// True if this connection was initiated by us. `None` if we're not connected. + /// Note that it is theoretically possible that we dial the remote at the + /// same time they dial us, in which case the protocols may be dispatched + /// between both connections, and in which case the value here will be racy. + originated: Option, + + /// Latest known ping duration. + ping: Option, + + /// The client version of the remote, or `None` if not known. + client_version: Option, + + /// The multiaddresses of the remote, or `None` if not known. + remote_addresses: Vec, + + /// The local multiaddress used to communicate with the remote, or `None` + /// if not known. + // TODO: never filled ; also shouldn't be an `Option` + local_address: Option, +} + +/// Simplified, POD version of PeerConnectionInfo. +#[derive(Debug, Clone)] +pub struct PeerInfo { + /// Id of the peer. + pub id: PeerId, + + /// True if this connection was initiated by us. + /// Note that it is theoretically possible that we dial the remote at the + /// same time they dial us, in which case the protocols may be dispatched + /// between both connections, and in which case the value here will be racy. + pub originated: bool, + + /// Latest known ping duration. + pub ping: Option, + + /// The client version of the remote, or `None` if not known. + pub client_version: Option, + + /// The multiaddress of the remote. + pub remote_address: Option, + + /// The local multiaddress used to communicate with the remote, or `None` + /// if not known. + pub local_address: Option, +} + +impl<'a> From<&'a PeerConnectionInfo> for PeerInfo { + fn from(i: &'a PeerConnectionInfo) -> PeerInfo { + PeerInfo { + id: i.id.clone(), + originated: i.originated.unwrap_or(true), + ping: i.ping, + client_version: i.client_version.clone(), + remote_address: i.remote_addresses.get(0).map(|a| a.clone()), + local_address: i.local_address.clone(), + } + } +} + +impl NetworkState { + pub fn new(config: &NetworkConfiguration) -> Result { + // Private and public keys configuration. + let local_private_key = obtain_private_key(&config)?; + let local_public_key = local_private_key.to_public_key(); + + // Build the storage for peers, including the bootstrap nodes. + let mut topology = if let Some(ref path) = config.net_config_path { + let path = Path::new(path).join(NODES_FILE); + debug!(target: "sub-libp2p", "Initializing peer store for JSON file {:?}", path); + NetTopology::from_file(path) + } else { + debug!(target: "sub-libp2p", "No peers file configured ; peers won't be saved"); + NetTopology::memory() + }; + + let reserved_peers = { + let mut reserved_peers = FnvHashSet::with_capacity_and_hasher( + config.reserved_nodes.len(), + Default::default() + ); + for peer in config.reserved_nodes.iter() { + let (id, _) = parse_and_add_to_topology(peer, &mut topology)?; + reserved_peers.insert(id); + } + RwLock::new(reserved_peers) + }; + + let expected_max_peers = config.max_peers as usize + config.reserved_nodes.len(); + + Ok(NetworkState { + topology: RwLock::new(topology), + max_outgoing_peers: config.min_peers, + max_incoming_peers: config.max_peers.saturating_sub(config.min_peers), + connections: RwLock::new(Connections { + peer_by_nodeid: FnvHashMap::with_capacity_and_hasher(expected_max_peers, Default::default()), + info_by_peer: FnvHashMap::with_capacity_and_hasher(expected_max_peers, Default::default()), + }), + reserved_only: atomic::AtomicBool::new(config.non_reserved_mode == NonReservedPeerMode::Deny), + reserved_peers, + next_node_index: atomic::AtomicUsize::new(0), + disabled_nodes: Mutex::new(Default::default()), + local_private_key, + local_public_key, + }) + } + + /// Returns the private key of the local node. + pub fn local_private_key(&self) -> &secio::SecioKeyPair { + &self.local_private_key + } + + /// Returns the public key of the local node. + pub fn local_public_key(&self) -> &PublicKey { + &self.local_public_key + } + + /// Returns a list of peers and addresses which we should try connect to. + /// + /// Because of expiration and back-off mechanisms, this list can change + /// by itself over time. The `Instant` that is returned corresponds to + /// the earlier known time when a new entry will be added automatically to + /// the list. + pub fn outgoing_connections_to_attempt(&self) -> (Vec<(PeerId, Multiaddr)>, Instant) { + // TODO: handle better + let connections = self.connections.read(); + + let num_to_attempt = if self.reserved_only.load(atomic::Ordering::Relaxed) { + 0 + } else { + let num_open_custom_connections = num_open_custom_connections(&connections, &self.reserved_peers.read()); + self.max_outgoing_peers.saturating_sub(num_open_custom_connections.unreserved_outgoing) + }; + + let topology = self.topology.read(); + let (list, change) = topology.addrs_to_attempt(); + let list = list + .filter(|&(peer, _)| { + // Filter out peers which we are already connected to. + let cur = match connections.peer_by_nodeid.get(peer) { + Some(e) => e, + None => return true + }; + + let infos = match connections.info_by_peer.get(&cur) { + Some(i) => i, + None => return true + }; + + !infos.protocols.iter().any(|(_, conn)| conn.is_alive()) + }) + .take(num_to_attempt as usize) + .map(|(addr, peer)| (addr.clone(), peer.clone())) + .collect(); + (list, change) + } + + /// Returns true if we are connected to any peer at all. + pub fn has_connected_peer(&self) -> bool { + !self.connections.read().peer_by_nodeid.is_empty() + } + + /// Get a list of all connected peers by id. + pub fn connected_peers(&self) -> Vec { + self.connections.read().peer_by_nodeid.values().cloned().collect() + } + + /// Returns true if the given `NodeIndex` is valid. + /// + /// `NodeIndex`s are never reused, so once this function returns `false` it + /// will never return `true` again for the same `NodeIndex`. + pub fn is_peer_connected(&self, peer: NodeIndex) -> bool { + self.connections.read().info_by_peer.contains_key(&peer) + } + + /// Reports the ping of the peer. Returned later by `session_info()`. + /// No-op if the `who` is not valid/expired. + pub fn report_ping_duration(&self, who: NodeIndex, ping: Duration) { + let mut connections = self.connections.write(); + let info = match connections.info_by_peer.get_mut(&who) { + Some(info) => info, + None => return, + }; + info.ping = Some(ping); + } + + /// If we're connected to a peer with the given protocol, returns + /// information about the connection. Otherwise, returns `None`. + pub fn session_info(&self, peer: NodeIndex, protocol: ProtocolId) -> Option { + let connections = self.connections.read(); + let info = match connections.info_by_peer.get(&peer) { + Some(info) => info, + None => return None, + }; + + let protocol_version = match info.protocols.iter().find(|&(ref p, _)| p == &protocol) { + Some(&(_, ref unique_connec)) => + if let Some(val) = unique_connec.poll() { + val.1 as u32 + } else { + return None + } + None => return None, + }; + + Some(SessionInfo { + id: None, // TODO: ???? what to do??? wrong format! + client_version: info.client_version.clone().take().unwrap_or(String::new()), + protocol_version, + capabilities: Vec::new(), // TODO: list of supported protocols ; hard + peer_capabilities: Vec::new(), // TODO: difference with `peer_capabilities`? + ping: info.ping, + originated: info.originated.unwrap_or(true), + remote_address: info.remote_addresses.get(0).map(|a| a.to_string()).unwrap_or_default(), + local_address: info.local_address.as_ref().map(|a| a.to_string()) + .unwrap_or(String::new()), + }) + } + + /// If we're connected to a peer with the given protocol, returns the + /// protocol version. Otherwise, returns `None`. + pub fn protocol_version(&self, peer: NodeIndex, protocol: ProtocolId) -> Option { + let connections = self.connections.read(); + let peer = match connections.info_by_peer.get(&peer) { + Some(peer) => peer, + None => return None, + }; + + peer.protocols.iter() + .find(|p| p.0 == protocol) + .and_then(|p| p.1.poll()) + .map(|(_, version)| version) + } + + /// Equivalent to `session_info(peer).map(|info| info.client_version)`. + pub fn peer_client_version(&self, peer: NodeIndex, protocol: ProtocolId) -> Option { + // TODO: implement more directly, without going through `session_info` + self.session_info(peer, protocol) + .map(|info| info.client_version) + } + + /// Adds an address discovered by Kademlia. + /// Note that we don't have to be connected to a peer to add an address. + /// If `connectable` is `true`, that means we have a hint from a remote that this node can be + /// connected to. + pub fn add_kad_discovered_addr(&self, node_id: &PeerId, addr: Multiaddr, connectable: bool) { + self.topology.write().add_kademlia_discovered_addr(node_id, addr, connectable) + } + + /// Returns the known multiaddresses of a peer. + /// + /// The boolean associated to each address indicates whether we're connected to it. + pub fn addrs_of_peer(&self, node_id: &PeerId) -> Vec<(Multiaddr, bool)> { + let topology = self.topology.read(); + // Note: I have no idea why, but fusing the two lines below fails the + // borrow check + let out: Vec<_> = topology + .addrs_of_peer(node_id).map(|(a, c)| (a.clone(), c)).collect(); + out + } + + /// Sets information about a peer. + /// + /// No-op if the node index is invalid. + pub fn set_node_info( + &self, + node_index: NodeIndex, + client_version: String + ) { + let mut connections = self.connections.write(); + let infos = match connections.info_by_peer.get_mut(&node_index) { + Some(i) => i, + None => return + }; + + infos.client_version = Some(client_version); + } + + /// Adds a peer to the internal peer store. + /// Returns an error if the peer address is invalid. + pub fn add_bootstrap_peer(&self, peer: &str) -> Result<(PeerId, Multiaddr), Error> { + parse_and_add_to_topology(peer, &mut self.topology.write()) + } + + /// Adds a reserved peer to the list of reserved peers. + /// Returns an error if the peer address is invalid. + pub fn add_reserved_peer(&self, peer: &str) -> Result<(), Error> { + let (id, _) = parse_and_add_to_topology(peer, &mut self.topology.write())?; + self.reserved_peers.write().insert(id); + Ok(()) + } + + /// Removes the peer from the list of reserved peers. If we're in reserved mode, drops any + /// active connection to this peer. + /// Returns an error if the peer address is invalid. + pub fn remove_reserved_peer(&self, peer: &str) -> Result<(), Error> { + let (id, _) = parse_and_add_to_topology(peer, &mut self.topology.write())?; + self.reserved_peers.write().remove(&id); + + // Dropping the peer if we're in reserved mode. + if self.reserved_only.load(atomic::Ordering::SeqCst) { + let mut connections = self.connections.write(); + if let Some(who) = connections.peer_by_nodeid.remove(&id) { + connections.info_by_peer.remove(&who); + // TODO: use drop_peer instead + } + } + + Ok(()) + } + + /// Set the non-reserved peer mode. + pub fn set_non_reserved_mode(&self, mode: NonReservedPeerMode) { + match mode { + NonReservedPeerMode::Accept => + self.reserved_only.store(false, atomic::Ordering::SeqCst), + NonReservedPeerMode::Deny => + // TODO: drop existing peers? + self.reserved_only.store(true, atomic::Ordering::SeqCst), + } + } + + /// Reports that we tried to connect to the given address but failed. + /// + /// This decreases the chance this address will be tried again in the future. + #[inline] + pub fn report_failed_to_connect(&self, addr: &Multiaddr) { + trace!(target: "sub-libp2p", "Failed to connect to {:?}", addr); + self.topology.write().report_failed_to_connect(addr); + } + + /// Returns the `NodeIndex` corresponding to a node id, or assigns a `NodeIndex` if none + /// exists. + /// + /// Returns an error if this node is on the list of disabled/banned nodes.. + pub fn assign_node_index( + &self, + node_id: &PeerId + ) -> Result { + // Check whether node is disabled. + // TODO: figure out the locking strategy here to avoid possible deadlocks + // TODO: put disabled_nodes in connections? + let mut disabled_nodes = self.disabled_nodes.lock(); + if let Some(timeout) = disabled_nodes.get(node_id).cloned() { + if timeout > Instant::now() { + debug!(target: "sub-libp2p", "Refusing peer {:?} because it is disabled", node_id); + return Err(IoError::new(IoErrorKind::ConnectionRefused, "peer is disabled")); + } else { + disabled_nodes.remove(node_id); + } + } + drop(disabled_nodes); + + let mut connections = self.connections.write(); + let connections = &mut *connections; + let peer_by_nodeid = &mut connections.peer_by_nodeid; + let info_by_peer = &mut connections.info_by_peer; + + let who = *peer_by_nodeid.entry(node_id.clone()).or_insert_with(|| { + let new_id = self.next_node_index.fetch_add(1, atomic::Ordering::Relaxed); + trace!(target: "sub-libp2p", "Creating new peer #{:?} for {:?}", new_id, node_id); + + info_by_peer.insert(new_id, PeerConnectionInfo { + protocols: Vec::new(), // TODO: Vec::with_capacity(num_registered_protocols), + kad_connec: UniqueConnec::empty(), + ping_connec: UniqueConnec::empty(), + id: node_id.clone(), + originated: None, + ping: None, + client_version: None, + local_address: None, + remote_addresses: Vec::with_capacity(1), + }); + + new_id + }); + + Ok(who) + } + + /// Notifies that we're connected to a node through an address. + /// + /// Returns an error if we refuse the connection. + /// + /// Note that is it legal to connection multiple times to the same node id through different + /// addresses and endpoints. + pub fn report_connected( + &self, + node_index: NodeIndex, + addr: &Multiaddr, + endpoint: Endpoint + ) -> Result<(), IoError> { + let mut connections = self.connections.write(); + + // TODO: double locking in this function ; although this has been reviewed to not deadlock + // as of the writing of this code, it is possible that a later change that isn't carefully + // reviewed triggers one + + if endpoint == Endpoint::Listener { + let stats = num_open_custom_connections(&connections, &self.reserved_peers.read()); + if stats.unreserved_incoming >= self.max_incoming_peers { + debug!(target: "sub-libp2p", "Refusing incoming connection from {} because we \ + reached max incoming peers", addr); + return Err(IoError::new(IoErrorKind::ConnectionRefused, + "maximum incoming peers reached")); + } + } + + let infos = match connections.info_by_peer.get_mut(&node_index) { + Some(i) => i, + None => return Ok(()) + }; + + if !infos.remote_addresses.iter().any(|a| a == addr) { + infos.remote_addresses.push(addr.clone()); + } + + if infos.originated.is_none() { + infos.originated = Some(endpoint == Endpoint::Dialer); + } + + self.topology.write().report_connected(addr, &infos.id); + + Ok(()) + } + + /// Returns the node id from a node index. + /// + /// Returns `None` if the node index is invalid. + pub fn node_id_from_index( + &self, + node_index: NodeIndex + ) -> Option { + let mut connections = self.connections.write(); + let infos = match connections.info_by_peer.get_mut(&node_index) { + Some(i) => i, + None => return None + }; + Some(infos.id.clone()) + } + + /// Obtains the `UniqueConnec` corresponding to the Kademlia connection to a peer. + /// + /// Returns `None` if the node index is invalid. + pub fn kad_connection( + &self, + node_index: NodeIndex + ) -> Option> { + let mut connections = self.connections.write(); + let infos = match connections.info_by_peer.get_mut(&node_index) { + Some(i) => i, + None => return None + }; + Some(infos.kad_connec.clone()) + } + + /// Obtains the `UniqueConnec` corresponding to the Ping connection to a peer. + /// + /// Returns `None` if the node index is invalid. + pub fn ping_connection( + &self, + node_index: NodeIndex + ) -> Option> { + let mut connections = self.connections.write(); + let infos = match connections.info_by_peer.get_mut(&node_index) { + Some(i) => i, + None => return None + }; + Some(infos.ping_connec.clone()) + } + + /// Cleans up inactive connections and returns a list of + /// connections to ping and identify. + pub fn cleanup_and_prepare_updates( + &self + ) -> Vec { + self.topology.write().cleanup(); + + let mut connections = self.connections.write(); + let connections = &mut *connections; + let peer_by_nodeid = &mut connections.peer_by_nodeid; + let info_by_peer = &mut connections.info_by_peer; + + let mut ret = Vec::with_capacity(info_by_peer.len()); + info_by_peer.retain(|&who, infos| { + // Remove the peer if neither Kad nor any protocol is alive. + if !infos.kad_connec.is_alive() && + !infos.protocols.iter().any(|(_, conn)| conn.is_alive()) + { + peer_by_nodeid.remove(&infos.id); + trace!(target: "sub-libp2p", "Cleaning up expired peer \ + #{:?} ({:?})", who, infos.id); + return false; + } + + if let Some(addr) = infos.remote_addresses.get(0) { + ret.push(PeriodicUpdate { + node_index: who, + peer_id: infos.id.clone(), + address: addr.clone(), + pinger: infos.ping_connec.clone(), + identify: infos.client_version.is_none(), + }); + } + true + }); + ret + } + + /// Obtains the `UniqueConnec` corresponding to a custom protocol connection to a peer. + /// + /// Returns `None` if the node index is invalid. + pub fn custom_proto( + &self, + node_index: NodeIndex, + protocol_id: ProtocolId, + ) -> Option, u8)>> { + let mut connections = self.connections.write(); + let infos = match connections.info_by_peer.get_mut(&node_index) { + Some(i) => i, + None => return None + }; + + if let Some((_, ref uconn)) = infos.protocols.iter().find(|&(prot, _)| prot == &protocol_id) { + return Some(uconn.clone()) + } + + let unique_connec = UniqueConnec::empty(); + infos.protocols.push((protocol_id.clone(), unique_connec.clone())); + Some(unique_connec) + } + + /// Sends some data to the given peer, using the sender that was passed + /// to the `UniqueConnec` of `custom_proto`. + pub fn send(&self, who: NodeIndex, protocol: ProtocolId, message: Bytes) -> Result<(), Error> { + if let Some(peer) = self.connections.read().info_by_peer.get(&who) { + let sender = peer.protocols.iter().find(|elem| elem.0 == protocol) + .and_then(|e| e.1.poll()) + .map(|e| e.0); + if let Some(sender) = sender { + sender.unbounded_send(message) + .map_err(|err| ErrorKind::Io(IoError::new(IoErrorKind::Other, err)))?; + Ok(()) + } else { + // We are connected to this peer, but not with the current + // protocol. + debug!(target: "sub-libp2p", + "Tried to send message to peer {} for which we aren't connected with the requested protocol", + who + ); + return Err(ErrorKind::PeerNotFound.into()) + } + } else { + debug!(target: "sub-libp2p", "Tried to send message to invalid peer ID {}", who); + return Err(ErrorKind::PeerNotFound.into()) + } + } + + /// Get the info on a peer, if there's an active connection. + pub fn peer_info(&self, who: NodeIndex) -> Option { + self.connections.read().info_by_peer.get(&who).map(Into::into) + } + + /// Reports that an attempt to make a low-level ping of the peer failed. + pub fn report_ping_failed(&self, who: NodeIndex) { + self.drop_peer(who); + } + + /// Disconnects a peer, if a connection exists (ie. drops the Kademlia + /// controller, and the senders that were stored in the `UniqueConnec` of + /// `custom_proto`). + pub fn drop_peer(&self, who: NodeIndex) { + let mut connections = self.connections.write(); + if let Some(peer_info) = connections.info_by_peer.remove(&who) { + trace!(target: "sub-libp2p", "Destroying peer #{} {:?} ; kademlia = {:?} ; num_protos = {:?}", + who, + peer_info.id, + peer_info.kad_connec.is_alive(), + peer_info.protocols.iter().filter(|c| c.1.is_alive()).count()); + let old = connections.peer_by_nodeid.remove(&peer_info.id); + debug_assert_eq!(old, Some(who)); + for addr in &peer_info.remote_addresses { + self.topology.write().report_disconnected(addr, + DisconnectReason::ClosedGracefully); // TODO: wrong reason + } + } + } + + /// Disconnects all the peers. + /// This destroys all the Kademlia controllers and the senders that were + /// stored in the `UniqueConnec` of `custom_proto`. + pub fn disconnect_all(&self) { + let mut connec = self.connections.write(); + *connec = Connections { + info_by_peer: FnvHashMap::with_capacity_and_hasher( + connec.peer_by_nodeid.capacity(), Default::default()), + peer_by_nodeid: FnvHashMap::with_capacity_and_hasher( + connec.peer_by_nodeid.capacity(), Default::default()), + }; + } + + /// Disables a peer for `PEER_DISABLE_DURATION`. This adds the peer to the + /// list of disabled peers, and drops any existing connections if + /// necessary (ie. drops the sender that was stored in the `UniqueConnec` + /// of `custom_proto`). + pub fn ban_peer(&self, who: NodeIndex, reason: &str) { + // TODO: what do we do if the peer is reserved? + // TODO: same logging as in drop_peer + let mut connections = self.connections.write(); + let peer_info = if let Some(peer_info) = connections.info_by_peer.remove(&who) { + if let &Some(ref client_version) = &peer_info.client_version { + info!(target: "network", "Peer {} (version: {}, addresses: {:?}) disabled. {}", who, client_version, peer_info.remote_addresses, reason); + } else { + info!(target: "network", "Peer {} (addresses: {:?}) disabled. {}", who, peer_info.remote_addresses, reason); + } + let old = connections.peer_by_nodeid.remove(&peer_info.id); + debug_assert_eq!(old, Some(who)); + peer_info + } else { + return + }; + + drop(connections); + let timeout = Instant::now() + PEER_DISABLE_DURATION; + self.disabled_nodes.lock().insert(peer_info.id.clone(), timeout); + } + + /// Flushes the caches to the disk. + /// + /// This is done in an atomical way, so that an error doesn't corrupt + /// anything. + pub fn flush_caches_to_disk(&self) -> Result<(), IoError> { + match self.topology.read().flush_to_disk() { + Ok(()) => { + debug!(target: "sub-libp2p", "Flushed JSON peer store to disk"); + Ok(()) + } + Err(err) => { + warn!(target: "sub-libp2p", "Failed to flush changes to JSON peer store: {}", err); + Err(err) + } + } + } +} + +impl Drop for NetworkState { + fn drop(&mut self) { + let _ = self.flush_caches_to_disk(); + } +} + +/// Periodic update that should be performed by the user of the network state. +pub struct PeriodicUpdate { + /// Index of the node in the network state. + pub node_index: NodeIndex, + /// Id of the peer. + pub peer_id: PeerId, + /// Address of the node to ping. + pub address: Multiaddr, + /// Object that allows pinging the node. + pub pinger: UniqueConnec, + /// The node should be identified as well. + pub identify: bool, +} + +struct OpenCustomConnectionsNumbers { + /// Total number of open and pending connections. + pub total: u32, + /// Unreserved incoming number of open and pending connections. + pub unreserved_incoming: u32, + /// Unreserved outgoing number of open and pending connections. + pub unreserved_outgoing: u32, +} + +/// Returns the number of open and pending connections with +/// custom protocols. +fn num_open_custom_connections(connections: &Connections, reserved_peers: &FnvHashSet) -> OpenCustomConnectionsNumbers { + let filtered = connections + .info_by_peer + .values() + .filter(|info| + info.protocols.iter().any(|&(_, ref connec)| + match connec.state() { + UniqueConnecState::Pending | UniqueConnecState::Full => true, + _ => false + } + ) + ); + + let mut total: u32 = 0; + let mut unreserved_incoming: u32 = 0; + let mut unreserved_outgoing: u32 = 0; + + for info in filtered { + total += 1; + let node_is_reserved = reserved_peers.contains(&info.id); + if !node_is_reserved { + if !info.originated.unwrap_or(true) { + unreserved_incoming += 1; + } else { + unreserved_outgoing += 1; + } + } + } + + OpenCustomConnectionsNumbers { + total, + unreserved_incoming, + unreserved_outgoing, + } +} + +/// Parses an address of the form `/ip4/x.x.x.x/tcp/x/p2p/xxxxxx`, and adds it +/// to the given topology. Returns the corresponding peer ID and multiaddr. +fn parse_and_add_to_topology( + addr_str: &str, + topology: &mut NetTopology +) -> Result<(PeerId, Multiaddr), Error> { + + let mut addr = addr_str.to_multiaddr().map_err(|_| ErrorKind::AddressParse)?; + let who = match addr.pop() { + Some(AddrComponent::P2P(key)) => + PeerId::from_multihash(key).map_err(|_| ErrorKind::AddressParse)?, + _ => return Err(ErrorKind::AddressParse.into()), + }; + + topology.add_bootstrap_addr(&who, addr.clone()); + Ok((who, addr)) +} + +/// Obtains or generates the local private key using the configuration. +fn obtain_private_key(config: &NetworkConfiguration) + -> Result { + if let Some(ref secret) = config.use_secret { + // Key was specified in the configuration. + secio::SecioKeyPair::secp256k1_raw_key(&secret[..]) + .map_err(|err| IoError::new(IoErrorKind::InvalidData, err)) + + } else { + if let Some(ref path) = config.net_config_path { + fs::create_dir_all(Path::new(path))?; + + // Try fetch the key from a the file containing th esecret. + let secret_path = Path::new(path).join(SECRET_FILE); + match load_private_key_from_file(&secret_path) { + Ok(s) => Ok(s), + Err(err) => { + // Failed to fetch existing file ; generate a new key + trace!(target: "sub-libp2p", + "Failed to load existing secret key file {:?}, generating new key ; err = {:?}", + secret_path, + err + ); + Ok(gen_key_and_try_write_to_file(&secret_path)) + } + } + + } else { + // No path in the configuration, nothing we can do except generate + // a new key. + let mut key: [u8; 32] = [0; 32]; + rand::rngs::EntropyRng::new().fill(&mut key); + Ok(secio::SecioKeyPair::secp256k1_raw_key(&key) + .expect("randomly-generated key with correct len should always be valid")) + } + } +} + +/// Tries to load a private key from a file located at the given path. +fn load_private_key_from_file

(path: P) + -> Result + where P: AsRef + { + fs::File::open(path) + .and_then(|mut file| { + // We are in 2018 and there is still no method on `std::io::Read` + // that directly returns a `Vec`. + let mut buf = Vec::new(); + file.read_to_end(&mut buf).map(|_| buf) + }) + .and_then(|content| + secio::SecioKeyPair::secp256k1_raw_key(&content) + .map_err(|err| IoError::new(IoErrorKind::InvalidData, err)) + ) +} + +/// Generates a new secret key and tries to write it to the given file. +/// Doesn't error if we couldn't open or write to the file. +fn gen_key_and_try_write_to_file

(path: P) -> secio::SecioKeyPair + where P: AsRef { + let raw_key: [u8; 32] = rand::rngs::EntropyRng::new().gen(); + let secio_key = secio::SecioKeyPair::secp256k1_raw_key(&raw_key) + .expect("randomly-generated key with correct len should always be valid"); + + // And store the newly-generated key in the file if possible. + // Errors that happen while doing so are ignored. + match open_priv_key_file(&path) { + Ok(mut file) => + match file.write_all(&raw_key) { + Ok(()) => (), + Err(err) => warn!(target: "sub-libp2p", + "Failed to write secret key in file {:?} ; err = {:?}", + path.as_ref(), + err + ), + }, + Err(err) => warn!(target: "sub-libp2p", + "Failed to store secret key in file {:?} ; err = {:?}", + path.as_ref(), + err + ), + } + + secio_key +} + +/// Opens a file containing a private key in write mode. +#[cfg(unix)] +fn open_priv_key_file

(path: P) -> Result + where P: AsRef +{ + use std::os::unix::fs::OpenOptionsExt; + fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(256 | 128) // 0o600 in decimal + .open(path) +} +/// Opens a file containing a private key in write mode. +#[cfg(not(unix))] +fn open_priv_key_file

(path: P) -> Result + where P: AsRef +{ + fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) +} + +#[cfg(test)] +mod tests { + use libp2p::core::PublicKey; + use network_state::NetworkState; + + #[test] + fn refuse_disabled_peer() { + let state = NetworkState::new(&Default::default()).unwrap(); + let example_peer = PublicKey::Rsa(vec![1, 2, 3, 4]).into_peer_id(); + + let who = state.assign_node_index(&example_peer).unwrap(); + state.ban_peer(who, "Just a test"); + + assert!(state.assign_node_index(&example_peer).is_err()); + } +} diff --git a/core/network-libp2p/src/service.rs b/core/network-libp2p/src/service.rs new file mode 100644 index 00000000000..4e829738932 --- /dev/null +++ b/core/network-libp2p/src/service.rs @@ -0,0 +1,1437 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use bytes::Bytes; +use {Error, ErrorKind, NetworkConfiguration, NetworkProtocolHandler}; +use {NonReservedPeerMode, NetworkContext, Severity, NodeIndex, ProtocolId}; +use parking_lot::RwLock; +use libp2p; +use libp2p::multiaddr::{AddrComponent, Multiaddr}; +use libp2p::kad::{KadSystem, KadConnecConfig, KadSystemConfig}; +use libp2p::kad::{KadIncomingRequest, KadConnecController, KadPeer}; +use libp2p::kad::{KadConnectionType, KadQueryEvent}; +use libp2p::identify::{IdentifyInfo, IdentifyOutput, IdentifySender}; +use libp2p::identify::{IdentifyProtocolConfig}; +use libp2p::core::{upgrade, Transport, MuxedTransport, ConnectionUpgrade}; +use libp2p::core::{Endpoint, PeerId as PeerstorePeerId, PublicKey}; +use libp2p::core::{SwarmController, UniqueConnecState}; +use libp2p::ping; +use libp2p::transport_timeout::TransportTimeout; +use {PacketId, SessionInfo, TimerToken}; +use rand; +use std::io::{Error as IoError, ErrorKind as IoErrorKind}; +use std::iter; +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::mpsc as sync_mpsc; +use std::thread; +use std::time::{Duration, Instant}; +use futures::{future, Future, stream, Stream, select_all}; +use futures::sync::{mpsc, oneshot}; +use tokio::runtime::current_thread; +use tokio_io::{AsyncRead, AsyncWrite}; +use tokio_timer::{Interval, Timeout}; + +use custom_proto::{RegisteredProtocol, RegisteredProtocols}; +use custom_proto::RegisteredProtocolOutput; +use network_state::{NetworkState, PeriodicUpdate}; +use timeouts; +use transport; + +/// IO Service with networking. +pub struct NetworkService { + shared: Arc, + + /// Holds the networking-running background thread alive. The `Option` is + /// only set to `None` in the destructor. + /// Sending a message on the channel will trigger the end of the + /// background thread. We can then wait on the join handle. + bg_thread: Option<(oneshot::Sender<()>, thread::JoinHandle<()>)>, +} + +/// Common struct shared throughout all the components of the service. +struct Shared { + /// Original configuration of the service. + config: NetworkConfiguration, + + /// Contains the state of the network. + network_state: NetworkState, + + /// Kademlia system. Contains the DHT. + kad_system: KadSystem, + + /// Configuration for the Kademlia upgrade. + kad_upgrade: KadConnecConfig, + + /// List of protocols available on the network. It is a logic error to + /// remove protocols from this list, and the code may assume that protocols + /// stay at the same index forever. + protocols: RegisteredProtocols>, + + /// Use this channel to send a timeout request to the background thread's + /// events loop. After the timeout, elapsed, it will call `timeout` on the + /// `NetworkProtocolHandler`. This can be closed if the background thread + /// is not running. The sender will be overwritten every time we start + /// the service. + timeouts_register_tx: mpsc::UnboundedSender<(Duration, (Arc, ProtocolId, TimerToken))>, + + /// Original address from the configuration, after being adjusted by the `Transport`. + // TODO: because we create the `Shared` before starting to listen, this + // has to be set later ; sort this out + original_listened_addr: RwLock>, + + /// Contains the addresses we known about ourselves. + listened_addrs: RwLock>, +} + +impl NetworkService { + /// Starts the networking service. + /// + /// Note that we could use an iterator for `protocols`, but having a + /// generic here is too much and crashes the Rust compiler. + pub fn new( + config: NetworkConfiguration, + protocols: Vec<(Arc, ProtocolId, &[(u8, u8)])> + ) -> Result { + let network_state = NetworkState::new(&config)?; + + let local_peer_id = network_state.local_public_key().clone() + .into_peer_id(); + for mut addr in config.listen_addresses.iter().cloned() { + addr.append(AddrComponent::P2P(local_peer_id.clone().into())); + info!(target: "sub-libp2p", "Local node address is: {}", addr); + } + + let kad_system = KadSystem::without_init(KadSystemConfig { + parallelism: 3, + local_peer_id: local_peer_id.clone(), + kbuckets_timeout: Duration::from_secs(600), + request_timeout: Duration::from_secs(10), + known_initial_peers: iter::empty(), + }); + + // Channel we use to signal success or failure of the bg thread + // initialization process. + let (init_tx, init_rx) = sync_mpsc::channel(); + // Channel the main thread uses to signal the bg thread that it + // should stop + let (close_tx, close_rx) = oneshot::channel(); + let (timeouts_register_tx, timeouts_register_rx) = mpsc::unbounded(); + + let listened_addrs = config.public_addresses.clone(); + + let shared = Arc::new(Shared { + network_state, + protocols: RegisteredProtocols(protocols.into_iter() + .map(|(handler, protocol, versions)| + RegisteredProtocol::new(handler.clone(), protocol, versions)) + .collect() + ), + kad_system, + kad_upgrade: KadConnecConfig::new(), + config, + timeouts_register_tx, + original_listened_addr: RwLock::new(Vec::new()), + listened_addrs: RwLock::new(listened_addrs), + }); + + // Initialize all the protocols now. + // TODO: what about failure to initialize? we can't uninitialize a protocol + // TODO: remove this `initialize` method eventually, as it's only used for timers + for protocol in shared.protocols.0.iter() { + protocol.custom_data().initialize(&NetworkContextImpl { + inner: shared.clone(), + protocol: protocol.id().clone(), + current_peer: None, + }); + } + + let shared_clone = shared.clone(); + let join_handle = thread::spawn(move || { + // Tokio runtime that is going to run everything in this thread. + let mut runtime = match current_thread::Runtime::new() { + Ok(c) => c, + Err(err) => { + let _ = init_tx.send(Err(err.into())); + return + } + }; + + let fut = match init_thread(shared_clone, timeouts_register_rx, close_rx) { + Ok(future) => { + debug!(target: "sub-libp2p", "Successfully started networking service"); + let _ = init_tx.send(Ok(())); + future + }, + Err(err) => { + let _ = init_tx.send(Err(err)); + return + } + }; + + match runtime.block_on(fut) { + Ok(()) => debug!(target: "sub-libp2p", "libp2p future finished"), + Err(err) => error!(target: "sub-libp2p", "error while running libp2p: {:?}", err), + } + }); + + init_rx.recv().expect("libp2p background thread panicked")?; + + Ok(NetworkService { + shared, + bg_thread: Some((close_tx, join_handle)), + }) + } + + /// Returns network configuration. + pub fn config(&self) -> &NetworkConfiguration { + &self.shared.config + } + + pub fn external_url(&self) -> Option { + // TODO: in the context of libp2p, it is hard to define what an external + // URL is, as different nodes can have multiple different ways to + // reach us + self.shared.original_listened_addr.read().get(0) + .map(|addr| + format!("{}/p2p/{}", addr, self.shared.kad_system.local_peer_id().to_base58()) + ) + } + + /// Get a list of all connected peers by id. + pub fn connected_peers(&self) -> Vec { + self.shared.network_state.connected_peers() + } + + /// Try to add a reserved peer. + pub fn add_reserved_peer(&self, peer: &str) -> Result<(), Error> { + // TODO: try to dial the peer? + self.shared.network_state.add_reserved_peer(peer) + } + + /// Try to remove a reserved peer. + pub fn remove_reserved_peer(&self, peer: &str) -> Result<(), Error> { + self.shared.network_state.remove_reserved_peer(peer) + } + + /// Set the non-reserved peer mode. + pub fn set_non_reserved_mode(&self, mode: NonReservedPeerMode) { + self.shared.network_state.set_non_reserved_mode(mode) + } + + /// Executes action in the network context + pub fn with_context(&self, protocol: ProtocolId, action: F) + where F: FnOnce(&NetworkContext) { + self.with_context_eval(protocol, action); + } + + /// Evaluates function in the network context + pub fn with_context_eval(&self, protocol: ProtocolId, action: F) + -> Option + where F: FnOnce(&NetworkContext) -> T { + if !self.shared.protocols.has_protocol(protocol) { + return None + } + + Some(action(&NetworkContextImpl { + inner: self.shared.clone(), + protocol: protocol.clone(), + current_peer: None, + })) + } +} + +impl Drop for NetworkService { + fn drop(&mut self) { + if let Some((close_tx, join)) = self.bg_thread.take() { + let _ = close_tx.send(()); + if let Err(e) = join.join() { + warn!(target: "sub-libp2p", "error while waiting on libp2p background thread: {:?}", e); + } + } + + debug_assert!(!self.shared.network_state.has_connected_peer()); + } +} + +#[derive(Clone)] +struct NetworkContextImpl { + inner: Arc, + protocol: ProtocolId, + current_peer: Option, +} + +impl NetworkContext for NetworkContextImpl { + fn send(&self, peer: NodeIndex, packet_id: PacketId, data: Vec) { + self.send_protocol(self.protocol, peer, packet_id, data) + } + + fn send_protocol( + &self, + protocol: ProtocolId, + peer: NodeIndex, + packet_id: PacketId, + data: Vec + ) { + debug_assert!(self.inner.protocols.has_protocol(protocol), + "invalid protocol id requested in the API of the libp2p networking"); + // TODO: could be "optimized" by building `message` only after checking the validity of + // the peer, but that's probably not worth the effort + let mut message = Bytes::with_capacity(1 + data.len()); + message.extend_from_slice(&[packet_id]); + message.extend_from_slice(&data); + if self.inner.network_state.send(peer, protocol, message).is_err() { + debug!(target: "sub-libp2p", "Sending to peer {} failed. Dropping.", peer); + self.inner.network_state.drop_peer(peer); + } + } + + fn respond(&self, packet_id: PacketId, data: Vec) { + if let Some(peer) = self.current_peer { + self.send_protocol(self.protocol, peer, packet_id, data) + } else { + panic!("respond() called outside of a received message"); + } + } + + fn report_peer(&self, peer: NodeIndex, reason: Severity) { + if let Some(info) = self.inner.network_state.peer_info(peer) { + if let Some(client_version) = info.client_version { + info!(target: "sub-libp2p", + "Peer {} ({:?} {}) reported by client: {}", + peer, + info.remote_address, + client_version, + reason + ); + } else { + info!(target: "sub-libp2p", "Peer {} reported by client: {}", peer, reason); + } + } + match reason { + Severity::Bad(reason) => self.inner.network_state.ban_peer(peer, reason), + Severity::Useless(_) => self.inner.network_state.drop_peer(peer), + Severity::Timeout => self.inner.network_state.drop_peer(peer), + } + } + + fn is_expired(&self) -> bool { + if let Some(current_peer) = self.current_peer { + !self.inner.network_state.is_peer_connected(current_peer) + } else { + // TODO: is this correct? + true + } + } + + fn register_timer(&self, token: usize, duration: Duration) + -> Result<(), Error> { + let handler = self.inner.protocols + .find_protocol(self.protocol) + .ok_or(ErrorKind::BadProtocol)? + .custom_data() + .clone(); + self.inner.timeouts_register_tx + .unbounded_send((duration, (handler, self.protocol, token))) + .map_err(|err| ErrorKind::Io(IoError::new(IoErrorKind::Other, err)))?; + Ok(()) + } + + fn peer_client_version(&self, peer: NodeIndex) -> String { + // Devp2p returns "unknown" on unknown peer ID, so we do the same. + self.inner.network_state.peer_client_version(peer, self.protocol) + .unwrap_or_else(|| "unknown".to_string()) + } + + fn session_info(&self, peer: NodeIndex) -> Option { + self.inner.network_state.session_info(peer, self.protocol) + } + + fn protocol_version(&self, protocol: ProtocolId, peer: NodeIndex) -> Option { + self.inner.network_state.protocol_version(peer, protocol) + } + + fn subprotocol_name(&self) -> ProtocolId { + self.protocol.clone() + } +} + +/// Builds the main `Future` for the network service. +/// +/// - `timeouts_register_rx` should receive newly-registered timeouts. +/// - `close_rx` should be triggered when we want to close the network. +fn init_thread( + shared: Arc, + timeouts_register_rx: mpsc::UnboundedReceiver< + (Duration, (Arc, ProtocolId, TimerToken)) + >, + close_rx: oneshot::Receiver<()> +) -> Result, Error> { + // Build the transport layer. + let transport = { + let base = transport::build_transport( + shared.network_state.local_private_key().clone() + ); + + let base = base.map_err_dial({ + let shared = shared.clone(); + move |err, addr| { + trace!(target: "sub-libp2p", "Failed to dial {}: {:?}", addr, err); + shared.network_state.report_failed_to_connect(&addr); + err + } + }); + + let shared = shared.clone(); + Transport::and_then(base, move |(peer_id, stream), endpoint, remote_addr| { + remote_addr.and_then(move |remote_addr| { + if &peer_id == shared.kad_system.local_peer_id() { + // TODO: this happens very frequently for now and floods the logs + //warn!(target: "sub-libp2p", "Refusing connection from our local peer id"); + return Err(IoErrorKind::ConnectionRefused.into()) + } + + // TODO: there's a possible race condition here if `cleanup_and_prepare_updates` is + // called between `assign_node_index` and one of `kad_connec`, `unique_connec`, + // etc. ; in practice though, it is very unlikely to happen + let node_index = shared.network_state.assign_node_index(&peer_id)?; + shared.network_state.report_connected(node_index, &remote_addr, endpoint)?; + let out = TransportOutput { + socket: stream, + node_index, + original_addr: remote_addr.clone(), + }; + Ok((out, future::ok(remote_addr))) + }) + }) + }; + + // Build the swarm. The swarm is the single entry point where successfully + // negotiated protocols arrive. + let (swarm_controller, swarm_events) = { + let upgraded_transport = transport.clone() + .and_then({ + let shared = shared.clone(); + move |out, endpoint, client_addr| { + let node_index = out.node_index; + let original_addr = out.original_addr; + let listener_upgrade = upgrade::or(upgrade::or(upgrade::or( + upgrade::map(shared.kad_upgrade.clone(), move |(c, f)| FinalUpgrade::Kad(node_index, c, f)), + upgrade::map(IdentifyProtocolConfig, move |id| FinalUpgrade::from((node_index, id, original_addr)))), + upgrade::map(ping::Ping, move |out| FinalUpgrade::from((node_index, out)))), + upgrade::map(DelayedProtosList(shared), move |c| FinalUpgrade::Custom(node_index, c))); + upgrade::apply(out.socket, listener_upgrade, endpoint, client_addr) + } + }); + let shared = shared.clone(); + + libp2p::core::swarm( + upgraded_transport, + move |upgrade, _client_addr| + listener_handle(shared.clone(), upgrade) + ) + }; + + // Listen on multiaddresses. + for addr in &shared.config.listen_addresses { + match swarm_controller.listen_on(addr.clone()) { + Ok(new_addr) => { + debug!(target: "sub-libp2p", "Libp2p listening on {}", new_addr); + shared.original_listened_addr.write().push(new_addr.clone()); + }, + Err(_) => { + warn!(target: "sub-libp2p", "Can't listen on {}, protocol not supported", addr); + return Err(ErrorKind::BadProtocol.into()) + }, + } + } + + // Explicitely connect to _all_ the boostrap nodes as a temporary measure. + for bootnode in shared.config.boot_nodes.iter() { + match shared.network_state.add_bootstrap_peer(bootnode) { + Ok((who, addr)) => { + trace!(target: "sub-libp2p", "Dialing bootnode {:?} through {}", who, addr); + for proto in shared.protocols.0.clone().into_iter() { + open_peer_custom_proto( + shared.clone(), + transport.clone(), + addr.clone(), + Some(who.clone()), + proto, + &swarm_controller + ) + } + }, + Err(Error(ErrorKind::AddressParse, _)) => { + // fallback: trying with IP:Port + let multi = match bootnode.parse::() { + Ok(SocketAddr::V4(socket)) => + format!("/ip4/{}/tcp/{}", socket.ip(), socket.port()).parse::(), + Ok(SocketAddr::V6(socket)) => + format!("/ip6/{}/tcp/{}", socket.ip(), socket.port()).parse::(), + _ => { + warn!(target: "sub-libp2p", "Not a valid Bootnode Address {:}", bootnode); + continue; + } + }; + + if let Ok(addr) = multi { + trace!(target: "sub-libp2p", "Missing NodeIndex for Bootnode {:}. Querying", bootnode); + for proto in shared.protocols.0.clone().into_iter() { + open_peer_custom_proto( + shared.clone(), + transport.clone(), + addr.clone(), + None, + proto, + &swarm_controller + ) + } + } else { + warn!(target: "sub-libp2p", "Not a valid Bootnode Address {:}", bootnode); + continue; + } + }, + Err(err) => warn!(target:"sub-libp2p", "Couldn't parse Bootnode Address: {}", err), + } + } + + let outgoing_connections = Interval::new(Instant::now(), Duration::from_secs(5)) + .map_err(|err| IoError::new(IoErrorKind::Other, err)) + .for_each({ + let shared = shared.clone(); + let transport = transport.clone(); + let swarm_controller = swarm_controller.clone(); + move |_| { + connect_to_nodes(shared.clone(), transport.clone(), &swarm_controller); + Ok(()) + } + }); + + // Build the timeouts system for the `register_timeout` function. + // (note: this has nothing to do with socket timeouts) + let timeouts = timeouts::build_timeouts_stream(timeouts_register_rx) + .for_each({ + let shared = shared.clone(); + move |(handler, protocol_id, timer_token)| { + handler.timeout(&NetworkContextImpl { + inner: shared.clone(), + protocol: protocol_id, + current_peer: None, + }, timer_token); + Ok(()) + } + }) + .then(|val| { + warn!(target: "sub-libp2p", "Timeouts stream closed unexpectedly: {:?}", val); + val + }); + + // Start the process of periodically discovering nodes to connect to. + let discovery = start_kademlia_discovery(shared.clone(), + transport.clone(), swarm_controller.clone()); + + // Start the process of pinging the active nodes on the network. + let periodic = start_periodic_updates(shared.clone(), transport, swarm_controller); + + let futures: Vec>> = vec![ + Box::new(swarm_events.for_each(|_| Ok(()))), + Box::new(discovery), + Box::new(periodic), + Box::new(outgoing_connections), + Box::new(timeouts), + Box::new(close_rx.map_err(|err| IoError::new(IoErrorKind::Other, err))), + ]; + + Ok( + select_all(futures) + .and_then(move |_| { + debug!(target: "sub-libp2p", "Networking ended ; disconnecting all peers"); + shared.network_state.disconnect_all(); + Ok(()) + }) + .map_err(|(r, _, _)| r) + ) +} + +/// Output of the common transport layer. +struct TransportOutput { + socket: S, + node_index: NodeIndex, + original_addr: Multiaddr, +} + +/// Enum of all the possible protocols our service handles. +enum FinalUpgrade { + Kad(NodeIndex, KadConnecController, Box + Send>), + /// The remote identification system, and the multiaddress we see the remote as. + IdentifyListener(NodeIndex, IdentifySender, Multiaddr), + /// The remote information about the address they see us as. + IdentifyDialer(NodeIndex, IdentifyInfo, Multiaddr), + PingDialer(NodeIndex, ping::Pinger, Box + Send>), + PingListener(NodeIndex, Box + Send>), + /// `Custom` means anything not in the core libp2p and is handled + /// by `CustomProtoConnectionUpgrade`. + Custom(NodeIndex, RegisteredProtocolOutput>), +} + +impl From<(NodeIndex, ping::PingOutput)> for FinalUpgrade { + fn from((node_index, out): (NodeIndex, ping::PingOutput)) -> FinalUpgrade { + match out { + ping::PingOutput::Ponger(processing) => + FinalUpgrade::PingListener(node_index, processing), + ping::PingOutput::Pinger { pinger, processing } => + FinalUpgrade::PingDialer(node_index, pinger, processing), + } + } +} + +impl From<(NodeIndex, IdentifyOutput, Multiaddr)> for FinalUpgrade { + fn from((node_index, out, addr): (NodeIndex, IdentifyOutput, Multiaddr)) -> FinalUpgrade { + match out { + IdentifyOutput::RemoteInfo { info, observed_addr } => + FinalUpgrade::IdentifyDialer(node_index, info, observed_addr), + IdentifyOutput::Sender { sender } => + FinalUpgrade::IdentifyListener(node_index, sender, addr), + } + } +} + +/// Called whenever we successfully open a multistream with a remote. +fn listener_handle<'a, C>( + shared: Arc, + upgrade: FinalUpgrade, +) -> Box + Send + 'a> + where C: AsyncRead + AsyncWrite + Send + 'a { + match upgrade { + FinalUpgrade::Kad(node_index, controller, kademlia_stream) => { + trace!(target: "sub-libp2p", "Opened kademlia substream with #{:?}", node_index); + match handle_kademlia_connection(shared, node_index, controller, kademlia_stream) { + Ok(fut) => Box::new(fut) as Box<_>, + Err(err) => Box::new(future::err(err)) as Box<_>, + } + }, + + FinalUpgrade::IdentifyListener(node_index, sender, original_addr) => { + trace!(target: "sub-libp2p", "Sending back identification info to #{}", node_index); + sender.send( + IdentifyInfo { + public_key: shared.network_state.local_public_key().clone(), + protocol_version: concat!("substrate/", + env!("CARGO_PKG_VERSION")).to_owned(), // TODO: ? + agent_version: concat!("substrate/", + env!("CARGO_PKG_VERSION")).to_owned(), + listen_addrs: shared.listened_addrs.read().clone(), + protocols: Vec::new(), // TODO: protocols_to_report, + }, + &original_addr + ) + }, + + FinalUpgrade::IdentifyDialer(node_index, info, observed_addr) => { + process_identify_info(&shared, node_index, &info, &observed_addr); + Box::new(future::ok(())) + }, + + FinalUpgrade::PingListener(node_index, future) => { + trace!(target: "sub-libp2p", "Received ping substream from #{}", node_index); + future + }, + + FinalUpgrade::PingDialer(node_index, pinger, future) => { + let ping_connec = match shared.network_state.ping_connection(node_index) { + Some(p) => p, + None => return Box::new(future::ok(())) as Box<_> + }; + trace!(target: "sub-libp2p", "Successfully opened ping substream with #{}", node_index); + let fut = ping_connec.tie_or_passthrough(pinger, future); + Box::new(fut) as Box<_> + }, + + FinalUpgrade::Custom(node_index, custom_proto_out) => { + // A "custom" protocol is one that is part of substrate and not part of libp2p. + let shared = shared.clone(); + let fut = handle_custom_connection(shared, node_index, custom_proto_out); + Box::new(fut) as Box<_> + }, + } +} + +/// Handles a newly-opened Kademlia connection. +fn handle_kademlia_connection( + shared: Arc, + node_index: NodeIndex, + controller: KadConnecController, + kademlia_stream: Box + Send> +) -> Result, IoError> { + let kad_connec = match shared.network_state.kad_connection(node_index) { + Some(kad) => kad, + None => return Err(IoError::new(IoErrorKind::Other, "node no longer exists")), + }; + + let node_id = match shared.network_state.node_id_from_index(node_index) { + Some(id) => id, + None => return Err(IoError::new(IoErrorKind::Other, "node no longer exists")), + }; + + let node_id2 = node_id.clone(); + let future = future::loop_fn(kademlia_stream, move |kademlia_stream| { + let shared = shared.clone(); + let node_id = node_id.clone(); + + let next = kademlia_stream + .into_future() + .map_err(|(err, _)| err); + + Timeout::new(next, Duration::from_secs(20)) + .map_err(|err| + // TODO: improve the error reporting here, but tokio-timer's API is bad + IoError::new(IoErrorKind::Other, err) + ) + .and_then(move |(req, rest)| { + shared.kad_system.update_kbuckets(node_id); + match req { + Some(KadIncomingRequest::FindNode { searched, responder }) => { + let resp = build_kademlia_response(&shared, &searched); + trace!(target: "sub-libp2p", "Responding to Kad {:?} with {:?}", searched, resp); + responder.respond(resp) + }, + Some(KadIncomingRequest::PingPong) => (), + None => return Ok(future::Loop::Break(())) + } + Ok(future::Loop::Continue(rest)) + }) + }).then(move |val| { + trace!(target: "sub-libp2p", "Closed Kademlia connection with #{} {:?} => {:?}", node_index, node_id2, val); + val + }); + + Ok(kad_connec.tie_or_passthrough(controller, future)) +} + +/// When a remote performs a `FIND_NODE` Kademlia request for `searched`, +/// this function builds the response to send back. +fn build_kademlia_response( + shared: &Arc, + searched: &PeerstorePeerId +) -> Vec { + shared.kad_system + .known_closest_peers(searched) + .map(move |who| { + if who == *shared.kad_system.local_peer_id() { + KadPeer { + node_id: who.clone(), + multiaddrs: shared.listened_addrs.read().clone(), + connection_ty: KadConnectionType::Connected, + } + } else { + let mut addrs = shared.network_state.addrs_of_peer(&who); + let connected = addrs.iter().any(|&(_, conn)| conn); + // The Kademlia protocol of libp2p doesn't allow specifying which address is valid + // and which is outdated, therefore in order to stay honest towards the network + // we only report the addresses we're connected to if we're connected to any. + if connected { + addrs = addrs.into_iter() + .filter_map(|(a, c)| if c { Some((a, c)) } else { None }) + .collect(); + } + + KadPeer { + node_id: who.clone(), + multiaddrs: addrs.into_iter().map(|(a, _)| a).collect(), + connection_ty: if connected { + KadConnectionType::Connected + } else { + KadConnectionType::NotConnected + }, + } + } + }) + // TODO: we really want to remove nodes with no multiaddress from + // the results, but a flaw in the Kad protocol of libp2p makes it + // impossible to return empty results ; therefore we must at least + // return ourselves + .filter(|p| p.node_id == *shared.kad_system.local_peer_id() || + !p.multiaddrs.is_empty()) + .take(20) + .collect::>() +} + +/// Handles a newly-opened connection to a remote with a custom protocol +/// (eg. `/substrate/dot/0`). +/// Returns a future that corresponds to when the handling is finished. +fn handle_custom_connection( + shared: Arc, + node_index: NodeIndex, + custom_proto_out: RegisteredProtocolOutput> +) -> Box + Send> { + let handler = custom_proto_out.custom_data; + let protocol_id = custom_proto_out.protocol_id; + + // Determine the ID of this peer, or drop the connection if the peer is disabled, + // if we reached `max_peers`, or a similar reason. + // TODO: is there a better way to refuse connections than to drop the + // newly-opened substream? should we refuse the connection + // beforehand? + let unique_connec = match shared.network_state.custom_proto( + node_index, + protocol_id, + ) { + Some(c) => c, + None => return Box::new(future::err(IoErrorKind::Other.into())) as Box<_>, + }; + + if let UniqueConnecState::Full = unique_connec.state() { + debug!(target: "sub-libp2p", + "Interrupting connection attempt to #{} with {:?} because we're already connected", + node_index, + custom_proto_out.protocol_id + ); + return Box::new(future::ok(())) as Box<_> + } + + struct ProtoDisconnectGuard { + inner: Arc, + who: NodeIndex, + handler: Arc, + protocol: ProtocolId, + print_log_message: bool, + } + + impl Drop for ProtoDisconnectGuard { + fn drop(&mut self) { + if self.print_log_message { + info!(target: "sub-libp2p", + "Node {:?} with peer ID {} through protocol {:?} disconnected", + self.inner.network_state.node_id_from_index(self.who), + self.who, + self.protocol + ); + } + self.handler.disconnected(&NetworkContextImpl { + inner: self.inner.clone(), + protocol: self.protocol, + current_peer: Some(self.who), + }, &self.who); + + // When any custom protocol drops, we drop the peer entirely. + // TODO: is this correct? + self.inner.network_state.drop_peer(self.who); + } + } + + let mut dc_guard = ProtoDisconnectGuard { + inner: shared.clone(), + who: node_index, + handler: handler.clone(), + protocol: protocol_id, + print_log_message: true, + }; + + let fut = custom_proto_out.incoming + .for_each({ + let shared = shared.clone(); + let handler = handler.clone(); + move |(packet_id, data)| { + if let Some(id) = shared.network_state.node_id_from_index(node_index) { + shared.kad_system.update_kbuckets(id); + } + handler.read(&NetworkContextImpl { + inner: shared.clone(), + protocol: protocol_id, + current_peer: Some(node_index.clone()), + }, &node_index, packet_id, &data); + Ok(()) + } + }); + + let val = (custom_proto_out.outgoing, custom_proto_out.protocol_version); + let final_fut = unique_connec.tie_or_stop(val, fut) + .then(move |val| { + info!(target: "sub-libp2p", "Finishing future for proto {:?} with {:?} => {:?}", + protocol_id, node_index, val); + // Makes sure that `dc_guard` is kept alive until here. + dc_guard.print_log_message = false; + drop(dc_guard); + val + }); + + debug!(target: "sub-libp2p", + "Successfully connected to {:?} (peer id #{}) with protocol {:?} version {}", + shared.network_state.node_id_from_index(node_index), + node_index, + protocol_id, + custom_proto_out.protocol_version + ); + + handler.connected(&NetworkContextImpl { + inner: shared.clone(), + protocol: protocol_id, + current_peer: Some(node_index), + }, &node_index); + + Box::new(final_fut) as Box<_> +} + +/// Randomly discovers peers to connect to. +/// This works by running a round at a regular interval, and skipping if we +/// reached `min_peers`. When we are over `min_peers`, we stop trying to dial +/// nodes and only accept incoming connections. +fn start_kademlia_discovery( + shared: Arc, + transport: T, + swarm_controller: SwarmController + Send>> +) -> Box + Send> + where T: MuxedTransport> + Clone + Send + 'static, + T::Dial: Send, + T::MultiaddrFuture: Send + 'static, + T::Listener: Send, + T::ListenerUpgrade: Send, + T::Incoming: Send, + T::IncomingUpgrade: Send, + To: AsyncRead + AsyncWrite + Send + 'static, + St: MuxedTransport> + Clone + Send + 'static, + St::Dial: Send, + St::MultiaddrFuture: Send, + St::Listener: Send, + St::ListenerUpgrade: Send, + St::Incoming: Send, + St::IncomingUpgrade: Send, + C: Send + 'static { + let kad_init = shared.kad_system.perform_initialization({ + let shared = shared.clone(); + let transport = transport.clone(); + let swarm_controller = swarm_controller.clone(); + move |who| + obtain_kad_connection( + shared.clone(), + who.clone(), + transport.clone(), + swarm_controller.clone() + ) + }); + + // We perform a random Kademlia query at a regular interval. + let discovery = Interval::new(Instant::now(), Duration::from_secs(32)) + // TODO: add a timeout to the lookups? + .map_err(|err| IoError::new(IoErrorKind::Other, err)) + .for_each({ + let shared = shared.clone(); + let transport = transport.clone(); + let swarm_controller = swarm_controller.clone(); + move |_| { + let _ = shared.network_state.flush_caches_to_disk(); + perform_kademlia_query(shared.clone(), transport.clone(), swarm_controller.clone()) + } + }); + + let final_future = kad_init + .select(discovery) + .map_err(|(err, _)| err) + .and_then(|(_, rest)| rest); + + // Note that we use a Box in order to speed compilation time. + Box::new(final_future) as Box + Send> +} + +/// Performs a kademlia request to a random node. +/// Note that we don't actually care about the results, so the future +/// produces `()`. +fn perform_kademlia_query( + shared: Arc, + transport: T, + swarm_controller: SwarmController + Send>> +) -> Box + Send> + where T: MuxedTransport> + Clone + Send + 'static, + T::MultiaddrFuture: Send + 'static, + T::Dial: Send, + T::Listener: Send, + T::ListenerUpgrade: Send, + T::Incoming: Send, + T::IncomingUpgrade: Send, + To: AsyncRead + AsyncWrite + Send + 'static, + St: MuxedTransport> + Send + Clone + 'static, + St::Dial: Send, + St::MultiaddrFuture: Send, + St::Listener: Send, + St::ListenerUpgrade: Send, + St::Incoming: Send, + St::IncomingUpgrade: Send, + C: Send + 'static { + // Query the node IDs that are closest to a random ID. + // Note that the randomness doesn't have to be secure, as this only + // influences which nodes we end up being connected to. + let random_key = PublicKey::Ed25519((0 .. 32) + .map(|_| -> u8 { rand::random() }).collect()); + let random_peer_id = random_key.into_peer_id(); + trace!(target: "sub-libp2p", "Start kademlia discovery for {:?}", random_peer_id); + + let future = shared.clone() + .kad_system + .find_node(random_peer_id, { + let shared = shared.clone(); + let transport = transport.clone(); + let swarm_controller = swarm_controller.clone(); + move |who| obtain_kad_connection(shared.clone(), who.clone(), + transport.clone(), swarm_controller.clone()) + }) + .filter_map(move |event| + match event { + KadQueryEvent::PeersReported(peers) => { + for peer in peers { + let connected = match peer.connection_ty { + KadConnectionType::NotConnected => false, + KadConnectionType::Connected => true, + KadConnectionType::CanConnect => true, + KadConnectionType::CannotConnect => continue, + }; + + for addr in peer.multiaddrs { + shared.network_state.add_kad_discovered_addr( + &peer.node_id, + addr, + connected + ); + } + } + None + }, + KadQueryEvent::Finished(_) => Some(()), + } + ) + .into_future() + .map_err(|(err, _)| err) + .map(|_| ()); + + // Note that we use a `Box` in order to speed up compilation. + Box::new(future) as Box + Send> +} + +/// Connects to additional nodes, if necessary. +fn connect_to_nodes( + shared: Arc, + base_transport: T, + swarm_controller: &SwarmController + Send>> +) + where T: MuxedTransport> + Clone + Send + 'static, + T::MultiaddrFuture: Send + 'static, + T::Dial: Send, + T::Listener: Send, + T::ListenerUpgrade: Send, + T::Incoming: Send, + T::IncomingUpgrade: Send, + To: AsyncRead + AsyncWrite + Send + 'static, + St: MuxedTransport> + Clone + Send + 'static, + St::Dial: Send, + St::MultiaddrFuture: Send, + St::Listener: Send, + St::ListenerUpgrade: Send, + St::Incoming: Send, + St::IncomingUpgrade: Send, + C: Send + 'static { + let (addrs, _will_change) = shared.network_state.outgoing_connections_to_attempt(); + + for (peer, addr) in addrs.into_iter() { + // Try to dial that node for each registered protocol. Since dialing + // upgrades the connection to use multiplexing, dialing multiple times + // should automatically open multiple substreams. + for proto in shared.protocols.0.clone().into_iter() { + open_peer_custom_proto( + shared.clone(), + base_transport.clone(), + addr.clone(), + Some(peer.clone()), + proto, + swarm_controller + ) + } + } +} + +/// Dials the given address for the given protocol and using the given `swarm_controller`. +/// +/// This function *always* performs a dial, and doesn't check whether we already have an existing +/// connection to the remote. This is expected to be checked by the caller. +/// +/// The dialing will fail if the obtained peer ID doesn't match the expected ID. This is an +/// opinionated decision, as we could just let the new connection through. But we decide not to. +/// If `None` is passed for the expected peer ID, we always accept the connection. +fn open_peer_custom_proto( + shared: Arc, + base_transport: T, + addr: Multiaddr, + expected_peer_id: Option, + proto: RegisteredProtocol>, + swarm_controller: &SwarmController + Send>> +) + where T: MuxedTransport> + Clone + Send + 'static, + T::MultiaddrFuture: Send + 'static, + T::Dial: Send, + T::Listener: Send, + T::ListenerUpgrade: Send, + T::Incoming: Send, + T::IncomingUpgrade: Send, + To: AsyncRead + AsyncWrite + Send + 'static, + St: MuxedTransport> + Clone + Send + 'static, + St::Dial: Send, + St::MultiaddrFuture: Send, + St::Listener: Send, + St::ListenerUpgrade: Send, + St::Incoming: Send, + St::IncomingUpgrade: Send, + C: Send + 'static, +{ + let proto_id = proto.id(); + + let with_proto = base_transport + .and_then(move |out, endpoint, client_addr| { + let node_index = out.node_index; + upgrade::apply(out.socket, proto, endpoint, client_addr) + .map(move |(custom, client_addr)| + ((node_index, FinalUpgrade::Custom(node_index, custom)), client_addr)) + }); + + let with_timeout = TransportTimeout::new(with_proto, Duration::from_secs(20)); + + if let Some(expected_peer_id) = expected_peer_id { + let expected_node_index = match shared.network_state.assign_node_index(&expected_peer_id) { + Ok(i) => i, + Err(_) => return, + }; + + let unique_connec = match shared.network_state.custom_proto(expected_node_index, proto_id) { + Some(uc) => uc, + None => return, + }; + + let with_peer_check = with_timeout + .and_then(move |(node_index, custom), _, client_addr| { + if node_index == expected_node_index { + future::ok((custom, client_addr)) + } else { + future::err(IoError::new(IoErrorKind::ConnectionRefused, "Peer id mismatch")) + } + }); + + trace!(target: "sub-libp2p", + "Opening connection to {:?} through {} with proto {:?}", + expected_peer_id, + addr, + proto_id + ); + + let _ = unique_connec.dial(swarm_controller, &addr, with_peer_check); + + } else { + let trans = with_timeout.map(|(_, out), _| out); + if let Err(addr) = swarm_controller.dial(addr, trans) { + debug!(target: "sub-libp2p", "Failed to dial {:?}", addr); + } + } +} + +/// Obtain a Kademlia connection to the given peer. +fn obtain_kad_connection( + shared: Arc, + who: PeerstorePeerId, + transport: T, + swarm_controller: SwarmController + Send>> +) -> Box + Send> + where T: MuxedTransport> + Clone + Send + 'static, + T::MultiaddrFuture: Send + 'static, + T::Dial: Send, + T::Listener: Send, + T::ListenerUpgrade: Send, + T::Incoming: Send, + T::IncomingUpgrade: Send, + To: AsyncRead + AsyncWrite + Send + 'static, + St: MuxedTransport> + Clone + Send + 'static, + St::Dial: Send, + St::MultiaddrFuture: Send, + St::Listener: Send, + St::ListenerUpgrade: Send, + St::Incoming: Send, + St::IncomingUpgrade: Send, + C: Send + 'static { + let kad_upgrade = shared.kad_upgrade.clone(); + let transport = transport + .and_then(move |out, endpoint, client_addr| { + let node_index = out.node_index; + upgrade::apply(out.socket, kad_upgrade.clone(), endpoint, client_addr) + .map(move |((ctrl, fut), addr)| (FinalUpgrade::Kad(node_index, ctrl, fut), addr)) + }); + + // This function consists in trying all the addresses we know one by one until we find + // one that works. + // + // This `future` returns a Kad controller, or an error if all dialing attempts failed. + let future = stream::iter_ok(shared.network_state.addrs_of_peer(&who)) + .and_then(move |addr| { + let node_index = shared.network_state.assign_node_index(&who)?; + let kad = match shared.network_state.kad_connection(node_index) { + Some(kad) => kad, + None => return Err(IoError::new(IoErrorKind::Other, "node no longer exists")), + }; + Ok((kad, addr)) + }) + .and_then(move |(unique_connec, addr)| { + unique_connec.dial(&swarm_controller, &addr.0, transport.clone()) + }) + .then(|result| -> Result<_, ()> { Ok(result.ok()) }) + .filter_map(|result| result) + .into_future() + .map_err(|_| -> IoError { unreachable!("all items always succeed") }) + .and_then(|(kad, _)| kad.ok_or_else(|| IoErrorKind::ConnectionRefused.into())); + + // Note that we use a Box in order to speed up compilation. + Box::new(future) as Box + Send> +} + +/// Processes the identification information that we received about a node. +fn process_identify_info( + shared: &Shared, + node_index: NodeIndex, + info: &IdentifyInfo, + observed_addr: &Multiaddr, +) { + trace!(target: "sub-libp2p", "Received identification info from #{}", node_index); + + shared.network_state.set_node_info(node_index, info.agent_version.clone()); + + for original_listened_addr in &*shared.original_listened_addr.read() { + // TODO: we're using a hack here ; ideally we would call `nat_traversal` on our + // `Transport` ; but that's complicated to pass around ; we could put it in a `Box` in + // `Shared`, but since our transport doesn't implement `Send` (libp2p doesn't implement + // `Send` on modifiers), we can't. Instead let's just recreate a transport locally every + // time. + let transport = libp2p::tcp::TcpConfig::new(); + if let Some(mut ext_addr) = transport.nat_traversal(original_listened_addr, &observed_addr) { + let mut listened_addrs = shared.listened_addrs.write(); + if !listened_addrs.iter().any(|a| a == &ext_addr) { + trace!(target: "sub-libp2p", + "NAT traversal: remote observes us as {}; registering {} as one of our own addresses", + observed_addr, + ext_addr + ); + listened_addrs.push(ext_addr.clone()); + ext_addr.append(AddrComponent::P2P(shared.kad_system + .local_peer_id().clone().into())); + info!(target: "sub-libp2p", "New external node address: {}", ext_addr); + } + } + } + + for addr in info.listen_addrs.iter() { + if let Some(node_id) = shared.network_state.node_id_from_index(node_index) { + shared.network_state.add_kad_discovered_addr(&node_id, addr.clone(), true); + } + } +} + +/// Returns a future that regularly pings every peer we're connected to. +/// If a peer doesn't respond after a while, we disconnect it. +fn start_periodic_updates( + shared: Arc, + transport: T, + swarm_controller: SwarmController + Send>> +) -> Box + Send> + where T: MuxedTransport> + Clone + Send + 'static, + T::MultiaddrFuture: Send + 'static, + T::Dial: Send, + T::Listener: Send, + T::ListenerUpgrade: Send, + T::Incoming: Send, + T::IncomingUpgrade: Send, + To: AsyncRead + AsyncWrite + Send + 'static, + St: MuxedTransport> + Clone + Send + 'static, + St::Dial: Send, + St::MultiaddrFuture: Send, + St::Listener: Send, + St::ListenerUpgrade: Send, + St::Incoming: Send, + St::IncomingUpgrade: Send, + C: Send + 'static { + let ping_transport = transport.clone() + .and_then(move |out, endpoint, client_addr| { + let node_index = out.node_index; + upgrade::apply(out.socket, ping::Ping, endpoint, client_addr) + .map(move |(stream, addr)| (FinalUpgrade::from((node_index, stream)), addr)) + }); + + let identify_transport = transport + .and_then(move |out, endpoint, client_addr| { + let node_index = out.node_index; + upgrade::apply(out.socket, IdentifyProtocolConfig, endpoint, client_addr) + .map(move |(id, addr)| { + let fin = match id { + IdentifyOutput::RemoteInfo { info, observed_addr } => + FinalUpgrade::IdentifyDialer(node_index, info, observed_addr), + IdentifyOutput::Sender { .. } => unreachable!("can't reach that on the dialer side"), + }; + (fin, addr) + }) + }); + + let fut = Interval::new(Instant::now() + Duration::from_secs(5), Duration::from_secs(30)) + .map_err(|err| IoError::new(IoErrorKind::Other, err)) + .for_each(move |_| periodic_updates( + shared.clone(), + ping_transport.clone(), + identify_transport.clone(), + &swarm_controller + )) + .then(|val| { + warn!(target: "sub-libp2p", "Periodic updates stream has stopped: {:?}", val); + val + }); + + // Note that we use a Box in order to speed compilation time. + Box::new(fut) as Box + Send> +} + +/// Pings all the nodes we're connected to and disconnects any node that +/// doesn't respond. Identifies nodes that need to be identified. Returns +/// a `Future` when all the pings have either suceeded or timed out. +fn periodic_updates( + shared: Arc, + ping_transport: Tp, + identify_transport: Tid, + swarm_controller: &SwarmController + Send>> +) -> Box + Send> + where Tp: MuxedTransport> + Clone + Send + 'static, + Tp::MultiaddrFuture: Send + 'static, + Tp::Dial: Send, + Tp::MultiaddrFuture: Send, + Tp::Listener: Send, + Tp::ListenerUpgrade: Send, + Tp::Incoming: Send, + Tp::IncomingUpgrade: Send, + Tid: MuxedTransport> + Clone + Send + 'static, + Tid::MultiaddrFuture: Send + 'static, + Tid::Dial: Send, + Tid::MultiaddrFuture: Send, + Tid::Listener: Send, + Tid::ListenerUpgrade: Send, + Tid::Incoming: Send, + Tid::IncomingUpgrade: Send, + St: MuxedTransport> + Clone + Send + 'static, + St::Dial: Send, + St::MultiaddrFuture: Send, + St::Listener: Send, + St::ListenerUpgrade: Send, + St::Incoming: Send, + St::IncomingUpgrade: Send, + C: Send + 'static { + trace!(target: "sub-libp2p", "Periodic update cycle"); + + let mut ping_futures = Vec::new(); + + for PeriodicUpdate { node_index, peer_id, address, pinger, identify } in + shared.network_state.cleanup_and_prepare_updates() { + let shared = shared.clone(); + + let fut = pinger + .dial(&swarm_controller, &address, ping_transport.clone()) + .and_then(move |mut p| { + trace!(target: "sub-libp2p", "Pinging peer #{} aka. {:?}", node_index, peer_id); + p.ping() + .map(move |()| peer_id) + .map_err(|err| IoError::new(IoErrorKind::Other, err)) + }); + let ping_start_time = Instant::now(); + let fut = Timeout::new_at(fut, ping_start_time + Duration::from_secs(30)) + .then(move |val| + match val { + Err(err) => { + trace!(target: "sub-libp2p", "Error while pinging #{:?} => {:?}", node_index, err); + shared.network_state.report_ping_failed(node_index); + // Return Ok, otherwise we would close the ping service + Ok(()) + }, + Ok(who) => { + let elapsed = ping_start_time.elapsed(); + trace!(target: "sub-libp2p", "Pong from #{:?} in {:?}", who, elapsed); + shared.network_state.report_ping_duration(node_index, elapsed); + shared.kad_system.update_kbuckets(who); + Ok(()) + } + } + ); + ping_futures.push(fut); + + if identify { + // Ignore dialing errors, as identifying is only about diagnostics. + trace!(target: "sub-libp2p", "Attempting to identify #{}", node_index); + let _ = swarm_controller.dial(address, identify_transport.clone()); + } + } + + let future = future::loop_fn(ping_futures, |ping_futures| { + if ping_futures.is_empty() { + let fut = future::ok(future::Loop::Break(())); + return future::Either::A(fut) + } + + let fut = future::select_all(ping_futures) + .map(|((), _, rest)| future::Loop::Continue(rest)) + .map_err(|(err, _, _)| err); + future::Either::B(fut) + }); + + // Note that we use a Box in order to speed up compilation. + Box::new(future) as Box + Send> +} + +/// Since new protocols are added after the networking starts, we have to load the protocols list +/// in a lazy way. This is what this wrapper does. +#[derive(Clone)] +struct DelayedProtosList(Arc); +// `Maf` is short for `MultiaddressFuture` +impl ConnectionUpgrade for DelayedProtosList +where C: AsyncRead + AsyncWrite + Send + 'static, // TODO: 'static :-/ + Maf: Future + Send + 'static, // TODO: 'static :( +{ + type NamesIter = > as ConnectionUpgrade>::NamesIter; + type UpgradeIdentifier = > as ConnectionUpgrade>::UpgradeIdentifier; + + fn protocol_names(&self) -> Self::NamesIter { + ConnectionUpgrade::::protocol_names(&self.0.protocols) + } + + type Output = > as ConnectionUpgrade>::Output; + type MultiaddrFuture = > as ConnectionUpgrade>::MultiaddrFuture; + type Future = > as ConnectionUpgrade>::Future; + + #[inline] + fn upgrade(self, socket: C, id: Self::UpgradeIdentifier, endpoint: Endpoint, + remote_addr: Maf) -> Self::Future + { + self.0.protocols + .clone() + .upgrade(socket, id, endpoint, remote_addr) + } +} + +#[cfg(test)] +mod tests { + use super::NetworkService; + + #[test] + fn builds_and_finishes_in_finite_time() { + // Checks that merely starting the network doesn't end up in an infinite loop. + let _service = NetworkService::new(Default::default(), vec![]).unwrap(); + } +} diff --git a/core/network-libp2p/src/timeouts.rs b/core/network-libp2p/src/timeouts.rs new file mode 100644 index 00000000000..9b5615b0d54 --- /dev/null +++ b/core/network-libp2p/src/timeouts.rs @@ -0,0 +1,115 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use futures::{Async, future, Future, Poll, stream, Stream, sync::mpsc}; +use std::io::{Error as IoError, ErrorKind as IoErrorKind}; +use std::marker::PhantomData; +use std::time::{Duration, Instant}; +use tokio_timer::{self, Delay}; + +/// Builds the timeouts system. +/// +/// The `timeouts_rx` should be a stream receiving newly-created timeout +/// requests. Returns a stream that produces items as their timeout elapses. +/// `T` can be anything you want, as it is transparently passed from the input +/// to the output. Timeouts continue to fire forever, as there is no way to +/// unregister them. +pub fn build_timeouts_stream<'a, T>( + timeouts_rx: mpsc::UnboundedReceiver<(Duration, T)> +) -> Box + 'a> + where T: Clone + 'a { + let next_timeout = next_in_timeouts_stream(timeouts_rx); + + // The `unfold` function is essentially a loop turned into a stream. The + // first parameter is the initial state, and the closure returns the new + // state and an item. + let stream = stream::unfold(vec![future::Either::A(next_timeout)], move |timeouts| { + // `timeouts` is a `Vec` of futures that produce an `Out`. + + // `select_ok` panics if `timeouts` is empty anyway. + if timeouts.is_empty() { + return None + } + + Some(future::select_ok(timeouts.into_iter()) + .and_then(move |(item, mut timeouts)| + match item { + Out::NewTimeout((Some((duration, item)), next_timeouts)) => { + // Received a new timeout request on the channel. + let next_timeout = next_in_timeouts_stream(next_timeouts); + let timeout = Delay::new(Instant::now() + duration); + let timeout = TimeoutWrapper(timeout, duration, Some(item), PhantomData); + timeouts.push(future::Either::B(timeout)); + timeouts.push(future::Either::A(next_timeout)); + Ok((None, timeouts)) + }, + Out::NewTimeout((None, _)) => + // The channel has been closed. + Ok((None, timeouts)), + Out::Timeout(duration, item) => { + // A timeout has happened. + let returned = item.clone(); + let timeout = Delay::new(Instant::now() + duration); + let timeout = TimeoutWrapper(timeout, duration, Some(item), PhantomData); + timeouts.push(future::Either::B(timeout)); + Ok((Some(returned), timeouts)) + }, + } + ) + ) + }).filter_map(|item| item); + + // Note that we use a `Box` in order to speed up compilation time. + Box::new(stream) as Box> +} + +/// Local enum representing the output of the selection. +enum Out { + NewTimeout(A), + Timeout(Duration, B), +} + +/// Convenience function that calls `.into_future()` on the timeouts stream, +/// and applies some modifiers. +/// This function is necessary. Otherwise if we copy-paste its content we run +/// into errors because the type of the copy-pasted closures differs. +fn next_in_timeouts_stream( + stream: mpsc::UnboundedReceiver +) -> impl Future, mpsc::UnboundedReceiver), B>, Error = IoError> { + stream + .into_future() + .map(Out::NewTimeout) + .map_err(|_| unreachable!("an UnboundedReceiver can never error")) +} + +/// Does the equivalent to `future.map(move |()| (duration, item)).map_err(|err| to_io_err(err))`. +struct TimeoutWrapper(F, Duration, Option, PhantomData); +impl Future for TimeoutWrapper + where F: Future { + type Item = Out; + type Error = IoError; + + fn poll(&mut self) -> Poll { + match self.0.poll() { + Ok(Async::Ready(())) => (), + Ok(Async::NotReady) => return Ok(Async::NotReady), + Err(err) => return Err(IoError::new(IoErrorKind::Other, err.to_string())), + } + + let out = Out::Timeout(self.1, self.2.take().expect("poll() called again after success")); + Ok(Async::Ready(out)) + } +} diff --git a/core/network-libp2p/src/topology.rs b/core/network-libp2p/src/topology.rs new file mode 100644 index 00000000000..d36eb07624b --- /dev/null +++ b/core/network-libp2p/src/topology.rs @@ -0,0 +1,655 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see .? + +use fnv::FnvHashMap; +use parking_lot::Mutex; +use libp2p::{Multiaddr, PeerId}; +use serde_json; +use std::{cmp, fs}; +use std::io::{Read, Cursor, Error as IoError, ErrorKind as IoErrorKind, Write, BufReader, BufWriter}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +/// For each address we're connected to, a period of this duration increases the score by 1. +const CONNEC_DURATION_PER_SCORE: Duration = Duration::from_secs(10); +/// Maximum value for the score. +const MAX_SCORE: u32 = 100; +/// When we successfully connect to a node, raises its score to the given minimum value. +const CONNECTED_MINIMUM_SCORE: u32 = 20; +/// Initial score that a node discovered through Kademlia receives, where we have a hint that the +/// node is reachable. +const KADEMLIA_DISCOVERY_INITIAL_SCORE_CONNECTABLE: u32 = 15; +/// Initial score that a node discovered through Kademlia receives, without any hint. +const KADEMLIA_DISCOVERY_INITIAL_SCORE: u32 = 10; +/// Score adjustement when we fail to connect to an address. +const SCORE_DIFF_ON_FAILED_TO_CONNECT: i32 = -1; +/// Default time-to-live for addresses discovered through Kademlia. +/// After this time has elapsed and no connection has succeeded, the address will be removed. +const KADEMLIA_DISCOVERY_EXPIRATION: Duration = Duration::from_secs(2 * 3600); +/// After a successful connection, the TTL is set to a minimum at this amount. +const EXPIRATION_PUSH_BACK_CONNEC: Duration = Duration::from_secs(2 * 3600); +/// Initial score that a bootstrap node receives when registered. +const BOOTSTRAP_NODE_SCORE: u32 = 100; +/// Time to live of a boostrap node. This only applies if you start the node later *without* +/// that bootstrap node configured anymore. +const BOOTSTRAP_NODE_EXPIRATION: Duration = Duration::from_secs(24 * 3600); +/// The first time we fail to connect to an address, wait this duration before trying again. +const FIRST_CONNECT_FAIL_BACKOFF: Duration = Duration::from_secs(2); +/// Every time we fail to connect to an address, multiply the backoff by this constant. +const FAIL_BACKOFF_MULTIPLIER: u32 = 2; +/// We need a maximum value for the backoff, overwise we risk an overflow. +const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60); + +// TODO: should be merged with the Kademlia k-buckets + +/// Stores information about the topology of the network. +#[derive(Debug)] +pub struct NetTopology { + store: FnvHashMap, + cache_path: Option, +} + +impl Default for NetTopology { + #[inline] + fn default() -> NetTopology { + NetTopology::memory() + } +} + +impl NetTopology { + /// Initializes a new `NetTopology` that isn't tied to any file. + /// + /// `flush_to_disk()` will be a no-op. + #[inline] + pub fn memory() -> NetTopology { + NetTopology { + store: Default::default(), + cache_path: None, + } + } + + /// Builds a `NetTopology` that will use `path` as a cache. + /// + /// This function tries to load a known topology from the file. If the file doesn't exist + /// or contains garbage data, the execution still continues. + /// + /// Calling `flush_to_disk()` in the future writes to the given path. + pub fn from_file>(path: P) -> NetTopology { + let path = path.as_ref(); + debug!(target: "sub-libp2p", "Initializing peer store for JSON file {:?}", path); + NetTopology { + store: try_load(path), + cache_path: Some(path.to_owned()), + } + } + + /// Writes the topology into the path passed to `from_file`. + /// + /// No-op if the object was created with `memory()`. + pub fn flush_to_disk(&self) -> Result<(), IoError> { + let path = match self.cache_path { + Some(ref p) => p, + None => return Ok(()) + }; + + let file = fs::File::create(path)?; + // TODO: the capacity of the BufWriter is kind of arbitrary ; decide better + serialize(BufWriter::with_capacity(1024 * 1024, file), &self.store) + } + + /// Perform a cleanup pass, removing all obsolete addresses and peers. + /// + /// This should be done from time to time. + pub fn cleanup(&mut self) { + let now_systime = SystemTime::now(); + self.store.retain(|_, peer| { + peer.addrs.retain(|a| { + a.expires > now_systime || a.is_connected() + }); + !peer.addrs.is_empty() + }); + } + + /// Returns the known potential addresses of a peer, ordered by score. + /// + /// The boolean associated to each address indicates whether we're connected to it. + // TODO: filter out backed off ones? + pub fn addrs_of_peer(&self, peer: &PeerId) -> impl Iterator { + let peer = if let Some(peer) = self.store.get(peer) { + peer + } else { + // TODO: use an EitherIterator or something + return Vec::new().into_iter(); + }; + + let now = SystemTime::now(); + let mut list = peer.addrs.iter().filter_map(move |addr| { + let (score, connected) = addr.score_and_is_connected(); + if (addr.expires >= now && score > 0) || connected { + Some((score, connected, &addr.addr)) + } else { + None + } + }).collect::>(); + list.sort_by(|a, b| a.0.cmp(&b.0)); + // TODO: meh, optimize + let l = list.into_iter().map(|(_, connec, addr)| (addr, connec)).collect::>(); + l.into_iter() + } + + /// Returns a list of all the known addresses of peers, ordered by the + /// order in which we should attempt to connect to them. + /// + /// Because of expiration and back-off mechanisms, this list can grow + /// by itself over time. The `Instant` that is returned corresponds to + /// the earlier known time when a new entry will be added automatically to + /// the list. + pub fn addrs_to_attempt(&self) -> (impl Iterator, Instant) { + // TODO: optimize + let now = Instant::now(); + let now_systime = SystemTime::now(); + let mut instant = now + Duration::from_secs(3600); + let mut addrs_out = Vec::new(); + + for (peer, info) in &self.store { + for addr in &info.addrs { + let (score, is_connected) = addr.score_and_is_connected(); + if score == 0 || addr.expires < now_systime { + continue; + } + if !is_connected && addr.back_off_until > now { + instant = cmp::min(instant, addr.back_off_until); + continue; + } + + addrs_out.push(((peer, &addr.addr), score)); + } + } + + addrs_out.sort_by(|a, b| b.1.cmp(&a.1)); + (addrs_out.into_iter().map(|a| a.0), instant) + } + + /// Adds an address corresponding to a boostrap node. + /// + /// We assume that the address is valid, so its score starts very high. + pub fn add_bootstrap_addr(&mut self, peer: &PeerId, addr: Multiaddr) { + let now_systime = SystemTime::now(); + let now = Instant::now(); + + let peer = peer_access(&mut self.store, peer); + + let mut found = false; + peer.addrs.retain(|a| { + if a.expires < now_systime && !a.is_connected() { + return false; + } + if a.addr == addr { + found = true; + } + true + }); + + if !found { + peer.addrs.push(Addr { + addr, + expires: now_systime + BOOTSTRAP_NODE_EXPIRATION, + back_off_until: now, + next_back_off: FIRST_CONNECT_FAIL_BACKOFF, + score: Mutex::new(AddrScore { + connected_since: None, + score: BOOTSTRAP_NODE_SCORE, + latest_score_update: now, + }), + }); + } + } + + /// Adds an address discovered through the Kademlia DHT. + /// + /// This address is not necessarily valid and should expire after a TTL. + /// + /// If `connectable` is true, that means we have some sort of hint that this node can + /// be reached. + pub fn add_kademlia_discovered_addr( + &mut self, + peer_id: &PeerId, + addr: Multiaddr, + connectable: bool + ) { + let now_systime = SystemTime::now(); + let now = Instant::now(); + + let peer = peer_access(&mut self.store, peer_id); + + let mut found = false; + peer.addrs.retain(|a| { + if a.expires < now_systime && !a.is_connected() { + return false; + } + if a.addr == addr { + found = true; + } + true + }); + + if !found { + trace!( + target: "sub-libp2p", + "Peer store: adding address {} for {:?} (connectable hint: {:?})", + addr, + peer_id, + connectable + ); + + let initial_score = if connectable { + KADEMLIA_DISCOVERY_INITIAL_SCORE_CONNECTABLE + } else { + KADEMLIA_DISCOVERY_INITIAL_SCORE + }; + + peer.addrs.push(Addr { + addr, + expires: now_systime + KADEMLIA_DISCOVERY_EXPIRATION, + back_off_until: now, + next_back_off: FIRST_CONNECT_FAIL_BACKOFF, + score: Mutex::new(AddrScore { + connected_since: None, + score: initial_score, + latest_score_update: now, + }), + }); + } + } + + /// Indicates the peer store that we're connected to this given address. + /// + /// This increases the score of the address that we connected to. Since we assume that only + /// one peer can be reached with any specific address, we also remove all addresses from other + /// peers that match the one we connected to. + pub fn report_connected(&mut self, addr: &Multiaddr, peer: &PeerId) { + let now = Instant::now(); + + // Just making sure that we have an entry for this peer in `store`, but don't use it. + let _ = peer_access(&mut self.store, peer); + + for (peer_in_store, info_in_store) in self.store.iter_mut() { + if peer == peer_in_store { + if let Some(addr) = info_in_store.addrs.iter_mut().find(|a| &a.addr == addr) { + addr.connected_now(CONNECTED_MINIMUM_SCORE); + addr.back_off_until = now; + addr.next_back_off = FIRST_CONNECT_FAIL_BACKOFF; + continue; + } + + // TODO: a else block would be better, but we get borrowck errors + info_in_store.addrs.push(Addr { + addr: addr.clone(), + expires: SystemTime::now() + EXPIRATION_PUSH_BACK_CONNEC, + back_off_until: now, + next_back_off: FIRST_CONNECT_FAIL_BACKOFF, + score: Mutex::new(AddrScore { + connected_since: Some(now), + latest_score_update: now, + score: CONNECTED_MINIMUM_SCORE, + }), + }); + + } else { + // Set the score to 0 for any address that matches the one we connected to. + for addr_in_store in &mut info_in_store.addrs { + if &addr_in_store.addr == addr { + addr_in_store.adjust_score(-(MAX_SCORE as i32)); + } + } + } + } + } + + /// Indicates the peer store that we're disconnected from an address. + /// + /// There's no need to indicate a peer ID, as each address can only have one peer ID. + /// If we were indeed connected to this addr, then we can find out which peer ID it is. + pub fn report_disconnected(&mut self, addr: &Multiaddr, reason: DisconnectReason) { + let score_diff = match reason { + DisconnectReason::ClosedGracefully => -1, + }; + + for info in self.store.values_mut() { + for a in info.addrs.iter_mut() { + if &a.addr == addr { + a.disconnected_now(score_diff); + a.back_off_until = Instant::now() + a.next_back_off; + a.next_back_off = cmp::min(a.next_back_off * FAIL_BACKOFF_MULTIPLIER, MAX_BACKOFF); + let expires_push_back = SystemTime::now() + EXPIRATION_PUSH_BACK_CONNEC; + if a.expires < expires_push_back { + a.expires = expires_push_back; + } + return; + } + } + } + } + + /// Indicates the peer store that we failed to connect to an address. + /// + /// We don't care about which peer is supposed to be behind that address. If we failed to dial + /// it for a specific peer, we would also fail to dial it for all peers that have this + /// address. + pub fn report_failed_to_connect(&mut self, addr: &Multiaddr) { + for info in self.store.values_mut() { + for a in info.addrs.iter_mut() { + if &a.addr == addr { + a.adjust_score(SCORE_DIFF_ON_FAILED_TO_CONNECT); + a.back_off_until = Instant::now() + a.next_back_off; + a.next_back_off = cmp::min(a.next_back_off * FAIL_BACKOFF_MULTIPLIER, MAX_BACKOFF); + } + } + } + } +} + +/// Reason why we disconnected from a peer. +pub enum DisconnectReason { + /// The disconnection was graceful. + ClosedGracefully, +} + +fn peer_access<'a>(store: &'a mut FnvHashMap, peer: &PeerId) -> &'a mut PeerInfo { + // TODO: should be optimizable if HashMap gets a better API + store.entry(peer.clone()).or_insert_with(Default::default) +} + +#[derive(Debug, Clone, Default)] +struct PeerInfo { + /// Addresses of that peer. + addrs: Vec, +} + +#[derive(Debug)] +struct Addr { + /// The multiaddress. + addr: Multiaddr, + /// When the address expires. + expires: SystemTime, + next_back_off: Duration, + /// Don't try to connect to this node until `Instant`. + back_off_until: Instant, + score: Mutex, +} + +impl Clone for Addr { + fn clone(&self) -> Addr { + Addr { + addr: self.addr.clone(), + expires: self.expires.clone(), + next_back_off: self.next_back_off.clone(), + back_off_until: self.back_off_until.clone(), + score: Mutex::new(self.score.lock().clone()), + } + } +} + +#[derive(Debug, Clone)] +struct AddrScore { + /// If connected, contains the moment when we connected. `None` if we're not connected. + connected_since: Option, + /// Score of this address. Potentially needs to be updated based on `latest_score_update`. + score: u32, + /// When we last updated the score. + latest_score_update: Instant, +} + +impl Addr { + /// Sets the addr to connected. If the score is lower than the given value, raises it to this + /// value. + fn connected_now(&self, raise_to_min: u32) { + let mut score = self.score.lock(); + let now = Instant::now(); + Addr::flush(&mut score, now); + score.connected_since = Some(now); + if score.score < raise_to_min { + score.score = raise_to_min; + } + } + + /// Applies a modification to the score. + fn adjust_score(&self, score_diff: i32) { + let mut score = self.score.lock(); + Addr::flush(&mut score, Instant::now()); + if score_diff >= 0 { + score.score = cmp::min(MAX_SCORE, score.score + score_diff as u32); + } else { + score.score = score.score.saturating_sub(-score_diff as u32); + } + } + + /// Sets the addr to disconnected and applies a modification to the score. + fn disconnected_now(&self, score_diff: i32) { + let mut score = self.score.lock(); + Addr::flush(&mut score, Instant::now()); + score.connected_since = None; + if score_diff >= 0 { + score.score = cmp::min(MAX_SCORE, score.score + score_diff as u32); + } else { + score.score = score.score.saturating_sub(-score_diff as u32); + } + } + + /// Returns true if we are connected to this addr. + fn is_connected(&self) -> bool { + let score = self.score.lock(); + score.connected_since.is_some() + } + + /// Returns the score, and true if we are connected to this addr. + fn score_and_is_connected(&self) -> (u32, bool) { + let mut score = self.score.lock(); + Addr::flush(&mut score, Instant::now()); + let is_connected = score.connected_since.is_some(); + (score.score, is_connected) + } + + /// Updates `score` and `latest_score_update`, and returns the score. + fn score(&self) -> u32 { + let mut score = self.score.lock(); + Addr::flush(&mut score, Instant::now()); + score.score + } + + fn flush(score: &mut AddrScore, now: Instant) { + if let Some(connected_since) = score.connected_since { + let potential_score: u32 = div_dur_with_dur(now - connected_since, CONNEC_DURATION_PER_SCORE); + // We flush when we connect to an address. + debug_assert!(score.latest_score_update >= connected_since); + let effective_score: u32 = + div_dur_with_dur(score.latest_score_update - connected_since, CONNEC_DURATION_PER_SCORE); + let to_add = potential_score.saturating_sub(effective_score); + score.score = cmp::min(MAX_SCORE, score.score + to_add); + } + + score.latest_score_update = now; + } +} + +/// Divides a `Duration` with a `Duration`. This exists in the stdlib but isn't stable yet. +// TODO: remove this function once stable +fn div_dur_with_dur(a: Duration, b: Duration) -> u32 { + let a_ms = a.as_secs() * 1_000_000 + (a.subsec_nanos() / 1_000) as u64; + let b_ms = b.as_secs() * 1_000_000 + (b.subsec_nanos() / 1_000) as u64; + (a_ms / b_ms) as u32 +} + +/// Serialized version of a `PeerInfo`. Suitable for storage in the cache file. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SerializedPeerInfo { + addrs: Vec, +} + +/// Serialized version of an `Addr`. Suitable for storage in the cache file. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SerializedAddr { + addr: String, + expires: SystemTime, + score: u32, +} + +impl<'a> From<&'a Addr> for SerializedAddr { + fn from(addr: &'a Addr) -> SerializedAddr { + SerializedAddr { + addr: addr.addr.to_string(), + expires: addr.expires, + score: addr.score(), + } + } +} + +/// Attempts to load storage from a file. +/// Deletes the file and returns an empty map if the file doesn't exist, cannot be opened +/// or is corrupted. +fn try_load(path: impl AsRef) -> FnvHashMap { + let path = path.as_ref(); + if !path.exists() { + debug!(target: "sub-libp2p", "Peer storage file {:?} doesn't exist", path); + return Default::default() + } + + let mut file = match fs::File::open(path) { + // TODO: the capacity of the BufReader is kind of arbitrary ; decide better + Ok(f) => BufReader::with_capacity(1024 * 1024, f), + Err(err) => { + warn!(target: "sub-libp2p", "Failed to open peer storage file: {:?}", err); + info!(target: "sub-libp2p", "Deleting peer storage file {:?}", path); + let _ = fs::remove_file(path); + return Default::default() + } + }; + + // We want to support empty files (and treat them as an empty recordset). Unfortunately + // `serde_json` will always produce an error if we do this ("unexpected EOF at line 0 + // column 0"). Therefore we start by reading one byte from the file in order to check + // for EOF. + + let mut first_byte = [0]; + let num_read = match file.read(&mut first_byte) { + Ok(f) => f, + Err(err) => { + // TODO: DRY + warn!(target: "sub-libp2p", "Failed to read peer storage file: {:?}", err); + info!(target: "sub-libp2p", "Deleting peer storage file {:?}", path); + let _ = fs::remove_file(path); + return Default::default() + } + }; + + if num_read == 0 { + // File is empty. + debug!(target: "sub-libp2p", "Peer storage file {:?} is empty", path); + Default::default() + + } else { + let data = Cursor::new(first_byte).chain(file); + match serde_json::from_reader::<_, serde_json::Value>(data) { + Ok(serde_json::Value::Null) => Default::default(), + Ok(serde_json::Value::Object(map)) => deserialize_tolerant(map.into_iter()), + Ok(_) | Err(_) => { + // The `Ok(_)` case means that the file doesn't contain a map. + let _ = fs::remove_file(path); + Default::default() + }, + } + } +} + +/// Attempts to turn a deserialized version of the storage into the final version. +/// +/// Skips entries that are invalid. +fn deserialize_tolerant( + iter: impl Iterator +) -> FnvHashMap { + let now = Instant::now(); + let now_systime = SystemTime::now(); + + let mut out = FnvHashMap::default(); + for (peer, info) in iter { + let peer: PeerId = match peer.parse() { + Ok(p) => p, + Err(_) => continue, + }; + + let info: SerializedPeerInfo = match serde_json::from_value(info) { + Ok(i) => i, + Err(_) => continue, + }; + + let mut addrs = Vec::with_capacity(info.addrs.len()); + for addr in info.addrs { + let multiaddr = match addr.addr.parse() { + Ok(a) => a, + Err(_) => continue, + }; + + if addr.expires < now_systime { + continue + } + + addrs.push(Addr { + addr: multiaddr, + expires: addr.expires, + next_back_off: FIRST_CONNECT_FAIL_BACKOFF, + back_off_until: now, + score: Mutex::new(AddrScore { + connected_since: None, + score: addr.score, + latest_score_update: now, + }), + }); + } + + if addrs.is_empty() { + continue; + } + + out.insert(peer, PeerInfo { addrs }); + } + + out +} + +/// Attempts to turn a deserialized version of the storage into the final version. +/// +/// Skips entries that are invalid or expired. +fn serialize(out: W, map: &FnvHashMap) -> Result<(), IoError> { + let now = SystemTime::now(); + let array: FnvHashMap<_, _> = map.iter().filter_map(|(peer, info)| { + if info.addrs.is_empty() { + return None + } + + let peer = peer.to_base58(); + let info = SerializedPeerInfo { + addrs: info.addrs.iter() + .filter(|a| a.expires > now || a.is_connected()) + .map(Into::into) + .collect(), + }; + + Some((peer, info)) + }).collect(); + + serde_json::to_writer_pretty(out, &array) + .map_err(|err| IoError::new(IoErrorKind::Other, err)) +} diff --git a/core/network-libp2p/src/traits.rs b/core/network-libp2p/src/traits.rs new file mode 100644 index 00000000000..de18f083c55 --- /dev/null +++ b/core/network-libp2p/src/traits.rs @@ -0,0 +1,299 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt; +use std::cmp::Ordering; +use std::iter; +use std::net::Ipv4Addr; +use std::str; +use std::time::Duration; +use io::TimerToken; +use libp2p::{multiaddr::AddrComponent, Multiaddr}; +use error::Error; +use ethkey::Secret; +use ethereum_types::H512; + +/// Protocol handler level packet id +pub type PacketId = u8; +/// Protocol / handler id +pub type ProtocolId = [u8; 3]; + +/// Node public key +pub type NodeId = H512; + +/// Local (temporary) peer session ID. +pub type NodeIndex = usize; + +/// Shared session information +#[derive(Debug, Clone)] +pub struct SessionInfo { + /// Peer public key + pub id: Option, + /// Peer client ID + pub client_version: String, + /// Peer RLPx protocol version + pub protocol_version: u32, + /// Session protocol capabilities + pub capabilities: Vec, + /// Peer protocol capabilities + pub peer_capabilities: Vec, + /// Peer ping delay + pub ping: Option, + /// True if this session was originated by us. + pub originated: bool, + /// Remote endpoint address of the session + pub remote_address: String, + /// Local endpoint address of the session + pub local_address: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PeerCapabilityInfo { + pub protocol: ProtocolId, + pub version: u8, +} + +impl ToString for PeerCapabilityInfo { + fn to_string(&self) -> String { + format!("{}/{}", str::from_utf8(&self.protocol[..]).unwrap_or("???"), self.version) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SessionCapabilityInfo { + pub protocol: [u8; 3], + pub version: u8, + pub packet_count: u8, + pub id_offset: u8, +} + +impl PartialOrd for SessionCapabilityInfo { + fn partial_cmp(&self, other: &SessionCapabilityInfo) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SessionCapabilityInfo { + fn cmp(&self, b: &SessionCapabilityInfo) -> Ordering { + // By protocol id first + if self.protocol != b.protocol { + return self.protocol.cmp(&b.protocol); + } + // By version + self.version.cmp(&b.version) + } +} + +/// Network service configuration +#[derive(Debug, PartialEq, Clone)] +pub struct NetworkConfiguration { + /// Directory path to store general network configuration. None means nothing will be saved + pub config_path: Option, + /// Directory path to store network-specific configuration. None means nothing will be saved + pub net_config_path: Option, + /// Multiaddresses to listen for incoming connections. + pub listen_addresses: Vec, + /// Multiaddresses to advertise. Detected automatically if empty. + pub public_addresses: Vec, + /// List of initial node addresses + pub boot_nodes: Vec, + /// Use provided node key instead of default + pub use_secret: Option, + /// Minimum number of connected peers to maintain + pub min_peers: u32, + /// Maximum allowed number of peers + pub max_peers: u32, + /// List of reserved node addresses. + pub reserved_nodes: Vec, + /// The non-reserved peer mode. + pub non_reserved_mode: NonReservedPeerMode, + /// Client identifier + pub client_version: String, +} + +impl Default for NetworkConfiguration { + fn default() -> Self { + NetworkConfiguration::new() + } +} + +impl NetworkConfiguration { + /// Create a new instance of default settings. + pub fn new() -> Self { + NetworkConfiguration { + config_path: None, + net_config_path: None, + listen_addresses: vec![ + iter::once(AddrComponent::IP4(Ipv4Addr::new(0, 0, 0, 0))) + .chain(iter::once(AddrComponent::TCP(30333))) + .collect() + ], + public_addresses: Vec::new(), + boot_nodes: Vec::new(), + use_secret: None, + min_peers: 25, + max_peers: 50, + reserved_nodes: Vec::new(), + non_reserved_mode: NonReservedPeerMode::Accept, + client_version: "Parity-network".into(), + } + } + + /// Create new default configuration for localhost-only connection with random port (useful for testing) + pub fn new_local() -> NetworkConfiguration { + let mut config = NetworkConfiguration::new(); + config.listen_addresses = vec![ + iter::once(AddrComponent::IP4(Ipv4Addr::new(127, 0, 0, 1))) + .chain(iter::once(AddrComponent::TCP(0))) + .collect() + ]; + config + } +} + +/// The severity of misbehaviour of a peer that is reported. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum Severity<'a> { + /// Peer is timing out. Could be bad connectivity of overload of work on either of our sides. + Timeout, + /// Peer has been notably useless. E.g. unable to answer a request that we might reasonably consider + /// it could answer. + Useless(&'a str), + /// Peer has behaved in an invalid manner. This doesn't necessarily need to be Byzantine, but peer + /// must have taken concrete action in order to behave in such a way which is wantanly invalid. + Bad(&'a str), +} + +impl<'a> fmt::Display for Severity<'a> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + match *self { + Severity::Timeout => write!(fmt, "Timeout"), + Severity::Useless(r) => write!(fmt, "Useless ({})", r), + Severity::Bad(r) => write!(fmt, "Bad ({})", r), + } + } +} + +/// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. +pub trait NetworkContext { + /// Send a packet over the network to another peer. + fn send(&self, peer: NodeIndex, packet_id: PacketId, data: Vec); + + /// Send a packet over the network to another peer using specified protocol. + fn send_protocol(&self, protocol: ProtocolId, peer: NodeIndex, packet_id: PacketId, data: Vec); + + /// Respond to a current network message. Panics if no there is no packet in the context. If the session is expired returns nothing. + fn respond(&self, packet_id: PacketId, data: Vec); + + /// Report peer. Depending on the report, peer may be disconnected and possibly banned. + fn report_peer(&self, peer: NodeIndex, reason: Severity); + + /// Check if the session is still active. + fn is_expired(&self) -> bool; + + /// Register a new IO timer. 'IoHandler::timeout' will be called with the token. + fn register_timer(&self, token: TimerToken, delay: Duration) -> Result<(), Error>; + + /// Returns peer identification string + fn peer_client_version(&self, peer: NodeIndex) -> String; + + /// Returns information on p2p session + fn session_info(&self, peer: NodeIndex) -> Option; + + /// Returns max version for a given protocol. + fn protocol_version(&self, protocol: ProtocolId, peer: NodeIndex) -> Option; + + /// Returns this object's subprotocol name. + fn subprotocol_name(&self) -> ProtocolId; +} + +impl<'a, T> NetworkContext for &'a T where T: ?Sized + NetworkContext { + fn send(&self, peer: NodeIndex, packet_id: PacketId, data: Vec) { + (**self).send(peer, packet_id, data) + } + + fn send_protocol(&self, protocol: ProtocolId, peer: NodeIndex, packet_id: PacketId, data: Vec) { + (**self).send_protocol(protocol, peer, packet_id, data) + } + + fn respond(&self, packet_id: PacketId, data: Vec) { + (**self).respond(packet_id, data) + } + + fn report_peer(&self, peer: NodeIndex, reason: Severity) { + (**self).report_peer(peer, reason) + } + + fn is_expired(&self) -> bool { + (**self).is_expired() + } + + fn register_timer(&self, token: TimerToken, delay: Duration) -> Result<(), Error> { + (**self).register_timer(token, delay) + } + + fn peer_client_version(&self, peer: NodeIndex) -> String { + (**self).peer_client_version(peer) + } + + fn session_info(&self, peer: NodeIndex) -> Option { + (**self).session_info(peer) + } + + fn protocol_version(&self, protocol: ProtocolId, peer: NodeIndex) -> Option { + (**self).protocol_version(protocol, peer) + } + + fn subprotocol_name(&self) -> ProtocolId { + (**self).subprotocol_name() + } +} + +/// Network IO protocol handler. This needs to be implemented for each new subprotocol. +/// All the handler function are called from within IO event loop. +/// `Message` is the type for message data. +pub trait NetworkProtocolHandler: Sync + Send { + /// Initialize the handler + fn initialize(&self, _io: &NetworkContext) {} + /// Called when new network packet received. + fn read(&self, io: &NetworkContext, peer: &NodeIndex, packet_id: u8, data: &[u8]); + /// Called when new peer is connected. Only called when peer supports the same protocol. + fn connected(&self, io: &NetworkContext, peer: &NodeIndex); + /// Called when a previously connected peer disconnects. + fn disconnected(&self, io: &NetworkContext, peer: &NodeIndex); + /// Timer function called after a timeout created with `NetworkContext::timeout`. + fn timeout(&self, _io: &NetworkContext, _timer: TimerToken) {} +} + +/// Non-reserved peer modes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum NonReservedPeerMode { + /// Accept them. This is the default. + Accept, + /// Deny them. + Deny, +} + +impl NonReservedPeerMode { + /// Attempt to parse the peer mode from a string. + pub fn parse(s: &str) -> Option { + match s { + "accept" => Some(NonReservedPeerMode::Accept), + "deny" => Some(NonReservedPeerMode::Deny), + _ => None, + } + } +} diff --git a/core/network-libp2p/src/transport.rs b/core/network-libp2p/src/transport.rs new file mode 100644 index 00000000000..8acb9ecab08 --- /dev/null +++ b/core/network-libp2p/src/transport.rs @@ -0,0 +1,50 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use libp2p::{self, PeerId, Transport, mplex, secio, yamux}; +use libp2p::core::{either, upgrade, transport::BoxedMuxed}; +use libp2p::transport_timeout::TransportTimeout; +use std::time::Duration; +use std::usize; +use tokio_io::{AsyncRead, AsyncWrite}; + +/// Builds the transport that serves as a common ground for all connections. +pub fn build_transport( + local_private_key: secio::SecioKeyPair +) -> BoxedMuxed<(PeerId, impl AsyncRead + AsyncWrite)> { + let mut mplex_config = mplex::MplexConfig::new(); + mplex_config.max_buffer_len_behaviour(mplex::MaxBufferBehaviour::Block); + mplex_config.max_buffer_len(usize::MAX); + + let base = libp2p::CommonTransport::new() + .with_upgrade(secio::SecioConfig { + key: local_private_key, + }) + .and_then(move |out, endpoint, client_addr| { + let upgrade = upgrade::or( + upgrade::map(mplex_config, either::EitherOutput::First), + upgrade::map(yamux::Config::default(), either::EitherOutput::Second), + ); + let key = out.remote_key; + let upgrade = upgrade::map(upgrade, move |muxer| (key, muxer)); + upgrade::apply(out.stream, upgrade, endpoint, client_addr) + }) + .into_connection_reuse() + .map(|(key, substream), _| (key.into_peer_id(), substream)); + + TransportTimeout::new(base, Duration::from_secs(20)) + .boxed_muxed() +} diff --git a/core/network-libp2p/tests/tests.rs b/core/network-libp2p/tests/tests.rs new file mode 100644 index 00000000000..a7477217195 --- /dev/null +++ b/core/network-libp2p/tests/tests.rs @@ -0,0 +1,130 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +extern crate parking_lot; +extern crate parity_bytes; +extern crate ethcore_io as io; +extern crate ethcore_logger; +extern crate substrate_network_libp2p; +extern crate ethkey; + +use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; +use std::sync::Arc; +use std::thread; +use std::time::*; +use parking_lot::Mutex; +use parity_bytes::Bytes; +use substrate_network_libp2p::*; +use ethkey::{Random, Generator}; +use io::TimerToken; + +pub struct TestProtocol { + drop_session: bool, + pub packet: Mutex, + pub got_timeout: AtomicBool, + pub got_disconnect: AtomicBool, +} + +impl TestProtocol { + pub fn new(drop_session: bool) -> Self { + TestProtocol { + packet: Mutex::new(Vec::new()), + got_timeout: AtomicBool::new(false), + got_disconnect: AtomicBool::new(false), + drop_session: drop_session, + } + } + + pub fn got_packet(&self) -> bool { + self.packet.lock()[..] == b"hello"[..] + } + + pub fn got_timeout(&self) -> bool { + self.got_timeout.load(AtomicOrdering::Relaxed) + } + + pub fn got_disconnect(&self) -> bool { + self.got_disconnect.load(AtomicOrdering::Relaxed) + } +} + +impl NetworkProtocolHandler for TestProtocol { + fn initialize(&self, io: &NetworkContext) { + io.register_timer(0, Duration::from_millis(10)).unwrap(); + } + + fn read(&self, _io: &NetworkContext, _peer: &NodeIndex, packet_id: u8, data: &[u8]) { + assert_eq!(packet_id, 33); + self.packet.lock().extend(data); + } + + fn connected(&self, io: &NetworkContext, peer: &NodeIndex) { + if self.drop_session { + io.report_peer(*peer, Severity::Bad("We are evil and just want to drop")) + } else { + io.respond(33, "hello".to_owned().into_bytes()); + } + } + + fn disconnected(&self, _io: &NetworkContext, _peer: &NodeIndex) { + self.got_disconnect.store(true, AtomicOrdering::Relaxed); + } + + /// Timer function called after a timeout created with `NetworkContext::timeout`. + fn timeout(&self, _io: &NetworkContext, timer: TimerToken) { + assert_eq!(timer, 0); + self.got_timeout.store(true, AtomicOrdering::Relaxed); + } +} + + +#[test] +fn net_service() { + let _service = NetworkService::new( + NetworkConfiguration::new_local(), + vec![(Arc::new(TestProtocol::new(false)), *b"myp", &[(1u8, 1)])] + ).expect("Error creating network service"); +} + +#[test] +#[ignore] // TODO: how is this test even supposed to work? +fn net_disconnect() { + let key1 = Random.generate().unwrap(); + let mut config1 = NetworkConfiguration::new_local(); + config1.use_secret = Some(key1.secret().clone()); + config1.boot_nodes = vec![ ]; + let handler1 = Arc::new(TestProtocol::new(false)); + let service1 = NetworkService::new(config1, vec![(handler1.clone(), *b"tst", &[(42u8, 1), (43u8, 1)])]).unwrap(); + let mut config2 = NetworkConfiguration::new_local(); + config2.boot_nodes = vec![ service1.external_url().unwrap() ]; + let handler2 = Arc::new(TestProtocol::new(true)); + let _service2 = NetworkService::new(config2, vec![(handler2.clone(), *b"tst", &[(42u8, 1), (43u8, 1)])]).unwrap(); + while !(handler1.got_disconnect() && handler2.got_disconnect()) { + thread::sleep(Duration::from_millis(50)); + } + assert!(handler1.got_disconnect()); + assert!(handler2.got_disconnect()); +} + +#[test] +fn net_timeout() { + let config = NetworkConfiguration::new_local(); + let handler = Arc::new(TestProtocol::new(false)); + let _service = NetworkService::new(config, vec![(handler.clone(), *b"tst", &[(42u8, 1), (43u8, 1)])]).unwrap(); + while !handler.got_timeout() { + thread::sleep(Duration::from_millis(50)); + } +} diff --git a/core/network/Cargo.toml b/core/network/Cargo.toml new file mode 100644 index 00000000000..fdd31aa7d02 --- /dev/null +++ b/core/network/Cargo.toml @@ -0,0 +1,30 @@ +[package] +description = "Polkadot network protocol" +name = "substrate-network" +version = "0.1.0" +license = "GPL-3.0" +authors = ["Parity Technologies "] + +[lib] + +[dependencies] +log = "0.3" +parking_lot = "0.4" +error-chain = "0.12" +bitflags = "1.0" +futures = "0.1.17" +linked-hash-map = "0.5" +rustc-hex = "1.0" +ethcore-io = { git = "https://github.com/paritytech/parity.git" } + +substrate-primitives = { path = "../primitives" } +substrate-client = { path = "../client" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-codec = { path = "../codec" } +substrate-codec-derive = { path = "../codec/derive" } +substrate-network-libp2p = { path = "../network-libp2p" } + +[dev-dependencies] +env_logger = "0.4" +substrate-keyring = { path = "../keyring" } +substrate-test-client = { path = "../test-client" } diff --git a/core/network/README.adoc b/core/network/README.adoc new file mode 100644 index 00000000000..ac29b0cd0bf --- /dev/null +++ b/core/network/README.adoc @@ -0,0 +1,13 @@ + += Network + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/network/src/blocks.rs b/core/network/src/blocks.rs new file mode 100644 index 00000000000..b078b9200b3 --- /dev/null +++ b/core/network/src/blocks.rs @@ -0,0 +1,282 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::mem; +use std::cmp; +use std::ops::Range; +use std::collections::{HashMap, BTreeMap}; +use std::collections::hash_map::Entry; +use network_libp2p::NodeIndex; +use runtime_primitives::traits::{Block as BlockT, NumberFor, As}; +use message; + +const MAX_PARALLEL_DOWNLOADS: u32 = 1; + +/// Block data with origin. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockData { + pub block: message::BlockData, + pub origin: NodeIndex, +} + +#[derive(Debug)] +enum BlockRangeState { + Downloading { + len: NumberFor, + downloading: u32, + }, + Complete(Vec>), +} + +impl BlockRangeState { + pub fn len(&self) -> NumberFor { + match *self { + BlockRangeState::Downloading { len, .. } => len, + BlockRangeState::Complete(ref blocks) => As::sa(blocks.len() as u64), + } + } +} + +/// A collection of blocks being downloaded. +#[derive(Default)] +pub struct BlockCollection { + /// Downloaded blocks. + blocks: BTreeMap, BlockRangeState>, + peer_requests: HashMap>, +} + +impl BlockCollection { + /// Create a new instance. + pub fn new() -> Self { + BlockCollection { + blocks: BTreeMap::new(), + peer_requests: HashMap::new(), + } + } + + /// Clear everything. + pub fn clear(&mut self) { + self.blocks.clear(); + self.peer_requests.clear(); + } + + /// Insert a set of blocks into collection. + pub fn insert(&mut self, start: NumberFor, blocks: Vec>, who: NodeIndex) { + if blocks.is_empty() { + return; + } + + match self.blocks.get(&start) { + Some(&BlockRangeState::Downloading { .. }) => { + trace!(target: "sync", "Ignored block data still marked as being downloaded: {}", start); + debug_assert!(false); + return; + }, + Some(&BlockRangeState::Complete(ref existing)) if existing.len() >= blocks.len() => { + trace!(target: "sync", "Ignored block data already downloaded: {}", start); + return; + }, + _ => (), + } + + self.blocks.insert(start, BlockRangeState::Complete(blocks.into_iter().map(|b| BlockData { origin: who, block: b }).collect())); + } + + /// Returns a set of block hashes that require a header download. The returned set is marked as being downloaded. + pub fn needed_blocks(&mut self, who: NodeIndex, count: usize, peer_best: NumberFor, common: NumberFor) -> Option>> { + // First block number that we need to download + let first_different = common + As::sa(1); + let count = As::sa(count as u64); + let (mut range, downloading) = { + let mut downloading_iter = self.blocks.iter().peekable(); + let mut prev: Option<(&NumberFor, &BlockRangeState)> = None; + loop { + let next = downloading_iter.next(); + break match &(prev, next) { + &(Some((start, &BlockRangeState::Downloading { ref len, downloading })), _) if downloading < MAX_PARALLEL_DOWNLOADS => + (*start .. *start + *len, downloading), + &(Some((start, r)), Some((next_start, _))) if *start + r.len() < *next_start => + (*start + r.len() .. cmp::min(*next_start, *start + r.len() + count), 0), // gap + &(Some((start, r)), None) => + (*start + r.len() .. *start + r.len() + count, 0), // last range + &(None, None) => + (first_different .. first_different + count, 0), // empty + &(None, Some((start, _))) if *start > first_different => + (first_different .. cmp::min(first_different + count, *start), 0), // gap at the start + _ => { + prev = next; + continue + }, + } + } + }; + // crop to peers best + if range.start > peer_best { + trace!(target: "sync", "Out of range for peer {} ({} vs {})", who, range.start, peer_best); + return None; + } + range.end = cmp::min(peer_best + As::sa(1), range.end); + self.peer_requests.insert(who, range.start); + self.blocks.insert(range.start, BlockRangeState::Downloading { len: range.end - range.start, downloading: downloading + 1 }); + if range.end <= range.start { + panic!("Empty range {:?}, count={}, peer_best={}, common={}, blocks={:?}", range, count, peer_best, common, self.blocks); + } + Some(range) + } + + /// Get a valid chain of blocks ordered in descending order and ready for importing into blockchain. + pub fn drain(&mut self, from: NumberFor) -> Vec> { + let mut drained = Vec::new(); + let mut ranges = Vec::new(); + { + let mut prev = from; + for (start, range_data) in &mut self.blocks { + match range_data { + &mut BlockRangeState::Complete(ref mut blocks) if *start <= prev => { + prev = *start + As::sa(blocks.len() as u64); + let mut blocks = mem::replace(blocks, Vec::new()); + drained.append(&mut blocks); + ranges.push(*start); + }, + _ => break, + } + } + } + for r in ranges { + self.blocks.remove(&r); + } + trace!(target: "sync", "Drained {} blocks", drained.len()); + drained + } + + pub fn clear_peer_download(&mut self, who: NodeIndex) { + match self.peer_requests.entry(who) { + Entry::Occupied(entry) => { + let start = entry.remove(); + let remove = match self.blocks.get_mut(&start) { + Some(&mut BlockRangeState::Downloading { ref mut downloading, .. }) if *downloading > 1 => { + *downloading = *downloading - 1; + false + }, + Some(&mut BlockRangeState::Downloading { .. }) => { + true + }, + _ => { + debug_assert!(false); + false + } + }; + if remove { + self.blocks.remove(&start); + } + }, + _ => (), + } + } +} + +#[cfg(test)] +mod test { + use super::{BlockCollection, BlockData, BlockRangeState}; + use message; + use runtime_primitives::testing::Block as RawBlock; + use primitives::H256; + + type Block = RawBlock; + + fn is_empty(bc: &BlockCollection) -> bool { + bc.blocks.is_empty() && + bc.peer_requests.is_empty() + } + + fn generate_blocks(n: usize) -> Vec> { + (0 .. n).map(|_| message::generic::BlockData { + hash: H256::random(), + header: None, + body: None, + message_queue: None, + receipt: None, + justification: None, + }).collect() + } + + #[test] + fn create_clear() { + let mut bc = BlockCollection::new(); + assert!(is_empty(&bc)); + bc.insert(1, generate_blocks(100), 0); + assert!(!is_empty(&bc)); + bc.clear(); + assert!(is_empty(&bc)); + } + + #[test] + fn insert_blocks() { + let mut bc = BlockCollection::new(); + assert!(is_empty(&bc)); + let peer0 = 0; + let peer1 = 1; + let peer2 = 2; + + let blocks = generate_blocks(150); + assert_eq!(bc.needed_blocks(peer0, 40, 150, 0), Some(1 .. 41)); + assert_eq!(bc.needed_blocks(peer1, 40, 150, 0), Some(41 .. 81)); + assert_eq!(bc.needed_blocks(peer2, 40, 150, 0), Some(81 .. 121)); + + bc.clear_peer_download(peer1); + bc.insert(41, blocks[41..81].to_vec(), peer1); + assert_eq!(bc.drain(1), vec![]); + assert_eq!(bc.needed_blocks(peer1, 40, 150, 0), Some(121 .. 151)); + bc.clear_peer_download(peer0); + bc.insert(1, blocks[1..11].to_vec(), peer0); + + assert_eq!(bc.needed_blocks(peer0, 40, 150, 0), Some(11 .. 41)); + assert_eq!(bc.drain(1), blocks[1..11].iter().map(|b| BlockData { block: b.clone(), origin: 0 }).collect::>()); + + bc.clear_peer_download(peer0); + bc.insert(11, blocks[11..41].to_vec(), peer0); + + let drained = bc.drain(12); + assert_eq!(drained[..30], blocks[11..41].iter().map(|b| BlockData { block: b.clone(), origin: 0 }).collect::>()[..]); + assert_eq!(drained[30..], blocks[41..81].iter().map(|b| BlockData { block: b.clone(), origin: 1 }).collect::>()[..]); + + bc.clear_peer_download(peer2); + assert_eq!(bc.needed_blocks(peer2, 40, 150, 80), Some(81 .. 121)); + bc.clear_peer_download(peer2); + bc.insert(81, blocks[81..121].to_vec(), peer2); + bc.clear_peer_download(peer1); + bc.insert(121, blocks[121..150].to_vec(), peer1); + + assert_eq!(bc.drain(80), vec![]); + let drained = bc.drain(81); + assert_eq!(drained[..40], blocks[81..121].iter().map(|b| BlockData { block: b.clone(), origin: 2 }).collect::>()[..]); + assert_eq!(drained[40..], blocks[121..150].iter().map(|b| BlockData { block: b.clone(), origin: 1 }).collect::>()[..]); + } + + #[test] + fn large_gap() { + let mut bc: BlockCollection = BlockCollection::new(); + bc.blocks.insert(100, BlockRangeState::Downloading { + len: 128, + downloading: 1, + }); + let blocks = generate_blocks(10).into_iter().map(|b| BlockData { block: b, origin: 0 }).collect(); + bc.blocks.insert(114305, BlockRangeState::Complete(blocks)); + + assert_eq!(bc.needed_blocks(0, 128, 10000, 000), Some(1 .. 100)); + assert_eq!(bc.needed_blocks(0, 128, 10000, 600), Some(100 + 128 .. 100 + 128 + 128)); + } +} diff --git a/core/network/src/chain.rs b/core/network/src/chain.rs new file mode 100644 index 00000000000..704e32d55d1 --- /dev/null +++ b/core/network/src/chain.rs @@ -0,0 +1,105 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Blockchain access trait + +use client::{self, Client as SubstrateClient, ImportResult, ClientInfo, BlockStatus, BlockOrigin, CallExecutor}; +use client::error::Error; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT}; +use runtime_primitives::generic::BlockId; +use runtime_primitives::bft::Justification; +use primitives::{Blake2Hasher, RlpCodec}; + +/// Local client abstraction for the network. +pub trait Client: Send + Sync { + /// Import a new block. Parent is supposed to be existing in the blockchain. + fn import(&self, origin: BlockOrigin, header: Block::Header, justification: Justification, body: Option>) -> Result; + + /// Get blockchain info. + fn info(&self) -> Result, Error>; + + /// Get block status. + fn block_status(&self, id: &BlockId) -> Result; + + /// Get block hash by number. + fn block_hash(&self, block_number: ::Number) -> Result, Error>; + + /// Get block header. + fn header(&self, id: &BlockId) -> Result, Error>; + + /// Get block body. + fn body(&self, id: &BlockId) -> Result>, Error>; + + /// Get block justification. + fn justification(&self, id: &BlockId) -> Result>, Error>; + + /// Get block header proof. + fn header_proof(&self, block_number: ::Number) -> Result<(Block::Header, Vec>), Error>; + + /// Get storage read execution proof. + fn read_proof(&self, block: &Block::Hash, key: &[u8]) -> Result>, Error>; + + /// Get method execution proof. + fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec, Vec>), Error>; +} + +impl Client for SubstrateClient where + B: client::backend::Backend + Send + Sync + 'static, + E: CallExecutor + Send + Sync + 'static, + Block: BlockT, +{ + fn import(&self, origin: BlockOrigin, header: Block::Header, justification: Justification, body: Option>) -> Result { + // TODO: defer justification check. + let justified_header = self.check_justification(header, justification.into())?; + (self as &SubstrateClient).import_block(origin, justified_header, body) + } + + fn info(&self) -> Result, Error> { + (self as &SubstrateClient).info() + } + + fn block_status(&self, id: &BlockId) -> Result { + (self as &SubstrateClient).block_status(id) + } + + fn block_hash(&self, block_number: ::Number) -> Result, Error> { + (self as &SubstrateClient).block_hash(block_number) + } + + fn header(&self, id: &BlockId) -> Result, Error> { + (self as &SubstrateClient).header(id) + } + + fn body(&self, id: &BlockId) -> Result>, Error> { + (self as &SubstrateClient).body(id) + } + + fn justification(&self, id: &BlockId) -> Result>, Error> { + (self as &SubstrateClient).justification(id) + } + + fn header_proof(&self, block_number: ::Number) -> Result<(Block::Header, Vec>), Error> { + (self as &SubstrateClient).header_proof(&BlockId::Number(block_number)) + } + + fn read_proof(&self, block: &Block::Hash, key: &[u8]) -> Result>, Error> { + (self as &SubstrateClient).read_proof(&BlockId::Hash(block.clone()), key) + } + + fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec, Vec>), Error> { + (self as &SubstrateClient).execution_proof(&BlockId::Hash(block.clone()), method, data) + } +} diff --git a/core/network/src/config.rs b/core/network/src/config.rs new file mode 100644 index 00000000000..ad5de3957de --- /dev/null +++ b/core/network/src/config.rs @@ -0,0 +1,32 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +pub use service::Roles; + +/// Protocol configuration +#[derive(Clone)] +pub struct ProtocolConfig { + /// Assigned roles. + pub roles: Roles, +} + +impl Default for ProtocolConfig { + fn default() -> ProtocolConfig { + ProtocolConfig { + roles: Roles::FULL, + } + } +} diff --git a/core/network/src/consensus_gossip.rs b/core/network/src/consensus_gossip.rs new file mode 100644 index 00000000000..7b2f72bd669 --- /dev/null +++ b/core/network/src/consensus_gossip.rs @@ -0,0 +1,382 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Utility for gossip of network messages between authorities. +//! Handles chain-specific and standard BFT messages. + +use std::collections::{HashMap, HashSet}; +use futures::sync::mpsc; +use std::time::{Instant, Duration}; +use network_libp2p::NodeIndex; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT}; +use runtime_primitives::generic::BlockId; +use message::{self, generic::Message as GenericMessage}; +use protocol::Context; +use service::Roles; + +// TODO: Add additional spam/DoS attack protection. +const MESSAGE_LIFETIME: Duration = Duration::from_secs(600); + +struct PeerConsensus { + known_messages: HashSet, +} + +/// Consensus messages. +#[derive(Debug, Clone, PartialEq)] +pub enum ConsensusMessage { + /// A message concerning BFT agreement + Bft(message::LocalizedBftMessage), + /// A message concerning some chain-specific aspect of consensus + ChainSpecific(Vec, B::Hash), +} + +struct MessageEntry { + hash: B::Hash, + message: ConsensusMessage, + instant: Instant, +} + +/// Consensus network protocol handler. Manages statements and candidate requests. +pub struct ConsensusGossip { + peers: HashMap>, + message_sink: Option<(mpsc::UnboundedSender>, B::Hash)>, + messages: Vec>, + message_hashes: HashSet, +} + +impl ConsensusGossip where B::Header: HeaderT { + /// Create a new instance. + pub fn new() -> Self { + ConsensusGossip { + peers: HashMap::new(), + message_sink: None, + messages: Default::default(), + message_hashes: Default::default(), + } + } + + /// Closes all notification streams. + pub fn abort(&mut self) { + self.message_sink = None; + } + + /// Handle new connected peer. + pub fn new_peer(&mut self, protocol: &mut Context, who: NodeIndex, roles: Roles) { + if roles.intersects(Roles::AUTHORITY | Roles::FULL) { + trace!(target:"gossip", "Registering {:?} {}", roles, who); + // Send out all known messages. + // TODO: limit by size + let mut known_messages = HashSet::new(); + for entry in self.messages.iter() { + known_messages.insert(entry.hash); + let message = match entry.message { + ConsensusMessage::Bft(ref bft) => GenericMessage::BftMessage(bft.clone()), + ConsensusMessage::ChainSpecific(ref msg, _) => GenericMessage::ChainSpecific(msg.clone()), + }; + + protocol.send_message(who, message); + } + self.peers.insert(who, PeerConsensus { + known_messages, + }); + } + } + + fn propagate(&mut self, protocol: &mut Context, message: message::Message, hash: B::Hash) { + for (id, ref mut peer) in self.peers.iter_mut() { + if peer.known_messages.insert(hash.clone()) { + trace!(target:"gossip", "Propagating to {}: {:?}", id, message); + protocol.send_message(*id, message.clone()); + } + } + } + + fn register_message(&mut self, hash: B::Hash, message: ConsensusMessage) { + if self.message_hashes.insert(hash) { + self.messages.push(MessageEntry { + hash, + instant: Instant::now(), + message, + }); + } + } + + /// Handles incoming BFT message, passing to stream and repropagating. + pub fn on_bft_message(&mut self, protocol: &mut Context, who: NodeIndex, message: message::LocalizedBftMessage) { + if let Some((hash, message)) = self.handle_incoming(protocol, who, ConsensusMessage::Bft(message)) { + // propagate to other peers. + self.multicast(protocol, message, Some(hash)); + } + } + + /// Handles incoming chain-specific message and repropagates + pub fn on_chain_specific(&mut self, protocol: &mut Context, who: NodeIndex, message: Vec, parent_hash: B::Hash) { + debug!(target: "gossip", "received chain-specific gossip message"); + if let Some((hash, message)) = self.handle_incoming(protocol, who, ConsensusMessage::ChainSpecific(message, parent_hash)) { + debug!(target: "gossip", "handled incoming chain-specific message"); + // propagate to other peers. + self.multicast(protocol, message, Some(hash)); + } + } + + /// Get a stream of messages relevant to consensus on top of a given parent hash. + pub fn messages_for(&mut self, parent_hash: B::Hash) -> mpsc::UnboundedReceiver> { + let (sink, stream) = mpsc::unbounded(); + + for entry in self.messages.iter() { + let message_matches = match entry.message { + ConsensusMessage::Bft(ref msg) => msg.parent_hash == parent_hash, + ConsensusMessage::ChainSpecific(_, ref h) => h == &parent_hash, + }; + + if message_matches { + sink.unbounded_send(entry.message.clone()).expect("receiving end known to be open; qed"); + } + } + + self.message_sink = Some((sink, parent_hash)); + stream + } + + /// Multicast a chain-specific message to other authorities. + pub fn multicast_chain_specific(&mut self, protocol: &mut Context, message: Vec, parent_hash: B::Hash) { + trace!(target:"gossip", "sending chain-specific message"); + self.multicast(protocol, ConsensusMessage::ChainSpecific(message, parent_hash), None); + } + + /// Multicast a BFT message to other authorities + pub fn multicast_bft_message(&mut self, protocol: &mut Context, message: message::LocalizedBftMessage) { + // Broadcast message to all authorities. + trace!(target:"gossip", "Broadcasting BFT message {:?}", message); + self.multicast(protocol, ConsensusMessage::Bft(message), None); + } + + /// Call when a peer has been disconnected to stop tracking gossip status. + pub fn peer_disconnected(&mut self, _protocol: &mut Context, who: NodeIndex) { + self.peers.remove(&who); + } + + /// Prune old or no longer relevant consensus messages. + /// Supply an optional block hash where consensus is known to have concluded. + pub fn collect_garbage(&mut self, best_hash: Option<&B::Hash>) { + let hashes = &mut self.message_hashes; + let before = self.messages.len(); + let now = Instant::now(); + self.messages.retain(|entry| { + if entry.instant + MESSAGE_LIFETIME >= now && + best_hash.map_or(true, |parent_hash| + match entry.message { + ConsensusMessage::Bft(ref msg) => &msg.parent_hash != parent_hash, + ConsensusMessage::ChainSpecific(_, ref h) => h != parent_hash, + }) + { + true + } else { + hashes.remove(&entry.hash); + false + } + }); + if self.messages.len() != before { + trace!(target:"gossip", "Cleaned up {} stale messages", before - self.messages.len()); + } + for (_, ref mut peer) in self.peers.iter_mut() { + peer.known_messages.retain(|h| hashes.contains(h)); + } + } + + fn handle_incoming(&mut self, protocol: &mut Context, who: NodeIndex, message: ConsensusMessage) -> Option<(B::Hash, ConsensusMessage)> { + let (hash, parent, message) = match message { + ConsensusMessage::Bft(msg) => { + let parent = msg.parent_hash; + let generic = GenericMessage::BftMessage(msg); + ( + ::protocol::hash_message(&generic), + parent, + match generic { + GenericMessage::BftMessage(msg) => ConsensusMessage::Bft(msg), + _ => panic!("`generic` is known to be the `BftMessage` variant; qed"), + } + ) + } + ConsensusMessage::ChainSpecific(msg, parent) => { + let generic = GenericMessage::ChainSpecific(msg); + ( + ::protocol::hash_message::(&generic), + parent, + match generic { + GenericMessage::ChainSpecific(msg) => ConsensusMessage::ChainSpecific(msg, parent), + _ => panic!("`generic` is known to be the `ChainSpecific` variant; qed"), + } + ) + } + }; + + if self.message_hashes.contains(&hash) { + trace!(target:"gossip", "Ignored already known message from {}", who); + return None; + } + + match (protocol.client().info(), protocol.client().header(&BlockId::Hash(parent))) { + (_, Err(e)) | (Err(e), _) => { + debug!(target:"gossip", "Error reading blockchain: {:?}", e); + return None; + }, + (Ok(info), Ok(Some(header))) => { + if header.number() < &info.chain.best_number { + trace!(target:"gossip", "Ignored ancient message from {}, hash={}", who, parent); + return None; + } + }, + (Ok(_), Ok(None)) => {}, + } + + if let Some(ref mut peer) = self.peers.get_mut(&who) { + peer.known_messages.insert(hash); + if let Some((sink, parent_hash)) = self.message_sink.take() { + if parent == parent_hash { + debug!(target: "gossip", "Pushing relevant consensus message to sink."); + if let Err(e) = sink.unbounded_send(message.clone()) { + trace!(target:"gossip", "Error broadcasting message notification: {:?}", e); + } + } + + self.message_sink = Some((sink, parent_hash)); + } + } else { + trace!(target:"gossip", "Ignored statement from unregistered peer {}", who); + return None; + } + + Some((hash, message)) + } + + fn multicast(&mut self, protocol: &mut Context, message: ConsensusMessage, hash: Option) { + let generic = match message { + ConsensusMessage::Bft(ref message) => GenericMessage::BftMessage(message.clone()), + ConsensusMessage::ChainSpecific(ref message, _) => GenericMessage::ChainSpecific(message.clone()), + }; + + let hash = hash.unwrap_or_else(|| ::protocol::hash_message(&generic)); + self.register_message(hash, message); + self.propagate(protocol, generic, hash); + } +} + +#[cfg(test)] +mod tests { + use runtime_primitives::bft::Justification; + use runtime_primitives::testing::{H256, Header, Block as RawBlock}; + use std::time::Instant; + use message::{self, generic}; + use super::*; + + type Block = RawBlock; + + #[test] + fn collects_garbage() { + let prev_hash = H256::random(); + let best_hash = H256::random(); + let mut consensus = ConsensusGossip::::new(); + let now = Instant::now(); + let m1_hash = H256::random(); + let m2_hash = H256::random(); + let m1 = ConsensusMessage::Bft(message::LocalizedBftMessage { + parent_hash: prev_hash, + message: message::generic::BftMessage::Auxiliary(Justification { + round_number: 0, + hash: Default::default(), + signatures: Default::default(), + }), + }); + let m2 = ConsensusMessage::ChainSpecific(vec![1, 2, 3], best_hash); + + macro_rules! push_msg { + ($hash:expr, $now: expr, $m:expr) => { + consensus.messages.push(MessageEntry { + hash: $hash, + instant: $now, + message: $m, + }) + } + } + + push_msg!(m1_hash, now, m1); + push_msg!(m2_hash, now, m2.clone()); + consensus.message_hashes.insert(m1_hash); + consensus.message_hashes.insert(m2_hash); + + // nothing to collect + consensus.collect_garbage(None); + assert_eq!(consensus.messages.len(), 2); + assert_eq!(consensus.message_hashes.len(), 2); + + // random header, nothing should be cleared + let mut header = Header { + parent_hash: H256::default(), + number: 0, + state_root: H256::default(), + extrinsics_root: H256::default(), + digest: Default::default(), + }; + + consensus.collect_garbage(Some(&H256::default())); + assert_eq!(consensus.messages.len(), 2); + assert_eq!(consensus.message_hashes.len(), 2); + + // header that matches one of the messages + header.parent_hash = prev_hash; + consensus.collect_garbage(Some(&prev_hash)); + assert_eq!(consensus.messages.len(), 1); + assert_eq!(consensus.message_hashes.len(), 1); + assert!(consensus.message_hashes.contains(&m2_hash)); + + // make timestamp expired + consensus.messages.clear(); + push_msg!(m2_hash, now - MESSAGE_LIFETIME, m2); + consensus.collect_garbage(None); + assert!(consensus.messages.is_empty()); + assert!(consensus.message_hashes.is_empty()); + } + + #[test] + fn message_stream_include_those_sent_before_asking_for_stream() { + use futures::Stream; + + let mut consensus = ConsensusGossip::new(); + + let bft_message = generic::BftMessage::Consensus(generic::SignedConsensusMessage::Vote(generic::SignedConsensusVote { + vote: generic::ConsensusVote::AdvanceRound(0), + sender: [0; 32].into(), + signature: Default::default(), + })); + + let parent_hash = [1; 32].into(); + + let localized = ::message::LocalizedBftMessage:: { + message: bft_message, + parent_hash: parent_hash, + }; + + let message = generic::Message::BftMessage(localized.clone()); + let message_hash = ::protocol::hash_message::(&message); + + let message = ConsensusMessage::Bft(localized); + consensus.register_message(message_hash, message.clone()); + let stream = consensus.messages_for(parent_hash); + + assert_eq!(stream.wait().next(), Some(Ok(message))); + } +} diff --git a/core/network/src/error.rs b/core/network/src/error.rs new file mode 100644 index 00000000000..5ad888eb573 --- /dev/null +++ b/core/network/src/error.rs @@ -0,0 +1,35 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot service possible errors. + +use std::io::Error as IoError; +use network_libp2p::Error as NetworkError; +use client; + +error_chain! { + foreign_links { + Network(NetworkError) #[doc = "Devp2p error."]; + Io(IoError) #[doc = "IO error."]; + } + + links { + Client(client::error::Error, client::error::ErrorKind) #[doc="Client error"]; + } + + errors { + } +} diff --git a/core/network/src/import_queue.rs b/core/network/src/import_queue.rs new file mode 100644 index 00000000000..647070383b4 --- /dev/null +++ b/core/network/src/import_queue.rs @@ -0,0 +1,593 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Blocks import queue. + +use std::collections::{HashSet, VecDeque}; +use std::sync::{Arc, Weak}; +use std::sync::atomic::{AtomicBool, Ordering}; +use parking_lot::{Condvar, Mutex, RwLock}; + +use client::{BlockOrigin, ImportResult}; +use network_libp2p::{NodeIndex, Severity}; + +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}; + +use blocks::BlockData; +use chain::Client; +use error::{ErrorKind, Error}; +use protocol::Context; +use service::ExecuteInContext; +use sync::ChainSync; + +/// Blocks import queue API. +pub trait ImportQueue: Send + Sync { + /// Clear the queue when sync is restarting. + fn clear(&self); + /// Clears the import queue and stops importing. + fn stop(&self); + /// Get queue status. + fn status(&self) -> ImportQueueStatus; + /// Is block with given hash currently in the queue. + fn is_importing(&self, hash: &B::Hash) -> bool; + /// Import bunch of blocks. + fn import_blocks(&self, sync: &mut ChainSync, protocol: &mut Context, blocks: (BlockOrigin, Vec>)); +} + +/// Import queue status. It isn't completely accurate. +pub struct ImportQueueStatus { + /// Number of blocks that are currently in the queue. + pub importing_count: usize, + /// The number of the best block that was ever in the queue since start/last failure. + pub best_importing_number: <::Header as HeaderT>::Number, +} + +/// Blocks import queue that is importing blocks in the separate thread. +pub struct AsyncImportQueue { + handle: Mutex>>, + data: Arc>, +} + +/// Locks order: queue, queue_blocks, best_importing_number +struct AsyncImportQueueData { + signal: Condvar, + queue: Mutex>)>>, + queue_blocks: RwLock>, + best_importing_number: RwLock<<::Header as HeaderT>::Number>, + is_stopping: AtomicBool, +} + +impl AsyncImportQueue { + pub fn new() -> Self { + Self { + handle: Mutex::new(None), + data: Arc::new(AsyncImportQueueData::new()), + } + } + + pub fn start>(&self, sync: Weak>>, service: Weak, chain: Weak>) -> Result<(), Error> { + debug_assert!(self.handle.lock().is_none()); + + let qdata = self.data.clone(); + *self.handle.lock() = Some(::std::thread::Builder::new().name("ImportQueue".into()).spawn(move || { + import_thread(sync, service, chain, qdata) + }).map_err(|err| Error::from(ErrorKind::Io(err)))?); + Ok(()) + } +} + +impl AsyncImportQueueData { + pub fn new() -> Self { + Self { + signal: Default::default(), + queue: Mutex::new(VecDeque::new()), + queue_blocks: RwLock::new(HashSet::new()), + best_importing_number: RwLock::new(Zero::zero()), + is_stopping: Default::default(), + } + } +} + +impl ImportQueue for AsyncImportQueue { + fn clear(&self) { + let mut queue = self.data.queue.lock(); + let mut queue_blocks = self.data.queue_blocks.write(); + let mut best_importing_number = self.data.best_importing_number.write(); + queue_blocks.clear(); + queue.clear(); + *best_importing_number = Zero::zero(); + } + + fn stop(&self) { + self.clear(); + if let Some(handle) = self.handle.lock().take() { + self.data.is_stopping.store(true, Ordering::SeqCst); + self.data.signal.notify_one(); + + let _ = handle.join(); + } + } + + fn status(&self) -> ImportQueueStatus { + ImportQueueStatus { + importing_count: self.data.queue_blocks.read().len(), + best_importing_number: *self.data.best_importing_number.read(), + } + } + + fn is_importing(&self, hash: &B::Hash) -> bool { + self.data.queue_blocks.read().contains(hash) + } + + fn import_blocks(&self, _sync: &mut ChainSync, _protocol: &mut Context, blocks: (BlockOrigin, Vec>)) { + if blocks.1.is_empty() { + return; + } + + trace!(target:"sync", "Scheduling {} blocks for import", blocks.1.len()); + + let mut queue = self.data.queue.lock(); + let mut queue_blocks = self.data.queue_blocks.write(); + let mut best_importing_number = self.data.best_importing_number.write(); + let new_best_importing_number = blocks.1.last().and_then(|b| b.block.header.as_ref().map(|h| h.number().clone())).unwrap_or_else(|| Zero::zero()); + queue_blocks.extend(blocks.1.iter().map(|b| b.block.hash.clone())); + if new_best_importing_number > *best_importing_number { + *best_importing_number = new_best_importing_number; + } + queue.push_back(blocks); + self.data.signal.notify_one(); + } +} + +impl Drop for AsyncImportQueue { + fn drop(&mut self) { + self.stop(); + } +} + +/// Blocks import thread. +fn import_thread>(sync: Weak>>, service: Weak, chain: Weak>, qdata: Arc>) { + trace!(target: "sync", "Starting import thread"); + loop { + if qdata.is_stopping.load(Ordering::SeqCst) { + break; + } + + let new_blocks = { + let mut queue_lock = qdata.queue.lock(); + if queue_lock.is_empty() { + qdata.signal.wait(&mut queue_lock); + } + + match queue_lock.pop_front() { + Some(new_blocks) => new_blocks, + None => break, + } + }; + + match (sync.upgrade(), service.upgrade(), chain.upgrade()) { + (Some(sync), Some(service), Some(chain)) => { + let blocks_hashes: Vec = new_blocks.1.iter().map(|b| b.block.hash.clone()).collect(); + if !import_many_blocks(&mut SyncLink::Indirect(&sync, &*chain, &*service), Some(&*qdata), new_blocks) { + break; + } + + let mut queue_blocks = qdata.queue_blocks.write(); + for blocks_hash in blocks_hashes { + queue_blocks.remove(&blocks_hash); + } + }, + _ => break, + } + } + + trace!(target: "sync", "Stopping import thread"); +} + +/// ChainSync link trait. +trait SyncLinkApi { + /// Get chain reference. + fn chain(&self) -> &Client; + /// Block imported. + fn block_imported(&mut self, hash: &B::Hash, number: NumberFor); + /// Maintain sync. + fn maintain_sync(&mut self); + /// Disconnect from peer. + fn useless_peer(&mut self, who: NodeIndex, reason: &str); + /// Disconnect from peer and restart sync. + fn note_useless_and_restart_sync(&mut self, who: NodeIndex, reason: &str); + /// Restart sync. + fn restart(&mut self); +} + +/// Link with the ChainSync service. +enum SyncLink<'a, B: 'a + BlockT, E: 'a + ExecuteInContext> { + /// Indirect link (through service). + Indirect(&'a RwLock>, &'a Client, &'a E), + /// Direct references are given. + #[cfg(test)] + Direct(&'a mut ChainSync, &'a mut Context), +} + +/// Block import successful result. +#[derive(Debug, PartialEq)] +enum BlockImportResult { + /// Imported known block. + ImportedKnown(H, N), + /// Imported unknown block. + ImportedUnknown(H, N), +} + +/// Block import error. +#[derive(Debug, PartialEq)] +enum BlockImportError { + /// Disconnect from peer and continue import of next bunch of blocks. + Disconnect(NodeIndex), + /// Disconnect from peer and restart sync. + DisconnectAndRestart(NodeIndex), + /// Restart sync. + Restart, +} + +/// Import a bunch of blocks. +fn import_many_blocks<'a, B: BlockT>( + link: &mut SyncLinkApi, + qdata: Option<&AsyncImportQueueData>, + blocks: (BlockOrigin, Vec>) +) -> bool +{ + let (blocks_origin, blocks) = blocks; + let count = blocks.len(); + let mut imported = 0; + + let blocks_range = match ( + blocks.first().and_then(|b| b.block.header.as_ref().map(|h| h.number())), + blocks.last().and_then(|b| b.block.header.as_ref().map(|h| h.number())), + ) { + (Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last), + (Some(first), Some(_)) => format!(" ({})", first), + _ => Default::default(), + }; + trace!(target:"sync", "Starting import of {} blocks{}", count, blocks_range); + + // Blocks in the response/drain should be in ascending order. + for block in blocks { + let import_result = import_single_block(link.chain(), blocks_origin.clone(), block); + let is_import_failed = import_result.is_err(); + imported += process_import_result(link, import_result); + if is_import_failed { + qdata.map(|qdata| *qdata.best_importing_number.write() = Zero::zero()); + return true; + } + + if qdata.map(|qdata| qdata.is_stopping.load(Ordering::SeqCst)).unwrap_or_default() { + return false; + } + } + + trace!(target: "sync", "Imported {} of {}", imported, count); + link.maintain_sync(); + true +} + +/// Single block import function. +fn import_single_block( + chain: &Client, + block_origin: BlockOrigin, + block: BlockData +) -> Result::Header as HeaderT>::Number>, BlockImportError> +{ + let origin = block.origin; + let block = block.block; + match (block.header, block.justification) { + (Some(header), Some(justification)) => { + let number = header.number().clone(); + let hash = header.hash(); + let parent = header.parent_hash().clone(); + + let result = chain.import( + block_origin, + header, + justification, + block.body, + ); + match result { + Ok(ImportResult::AlreadyInChain) => { + trace!(target: "sync", "Block already in chain {}: {:?}", number, hash); + Ok(BlockImportResult::ImportedKnown(hash, number)) + }, + Ok(ImportResult::AlreadyQueued) => { + trace!(target: "sync", "Block already queued {}: {:?}", number, hash); + Ok(BlockImportResult::ImportedKnown(hash, number)) + }, + Ok(ImportResult::Queued) => { + trace!(target: "sync", "Block queued {}: {:?}", number, hash); + Ok(BlockImportResult::ImportedUnknown(hash, number)) + }, + Ok(ImportResult::UnknownParent) => { + debug!(target: "sync", "Block with unknown parent {}: {:?}, parent: {:?}", number, hash, parent); + Err(BlockImportError::Restart) + }, + Ok(ImportResult::KnownBad) => { + debug!(target: "sync", "Peer gave us a bad block {}: {:?}", number, hash); + Err(BlockImportError::DisconnectAndRestart(origin)) //TODO: use persistent ID + } + Err(e) => { + debug!(target: "sync", "Error importing block {}: {:?}: {:?}", number, hash, e); + Err(BlockImportError::Restart) + } + } + }, + (None, _) => { + debug!(target: "sync", "Header {} was not provided by {} ", block.hash, origin); + Err(BlockImportError::Disconnect(origin)) //TODO: use persistent ID + }, + (_, None) => { + debug!(target: "sync", "Justification set for block {} was not provided by {} ", block.hash, origin); + Err(BlockImportError::Disconnect(origin)) //TODO: use persistent ID + } + } +} + +/// Process single block import result. +fn process_import_result<'a, B: BlockT>( + link: &mut SyncLinkApi, + result: Result::Header as HeaderT>::Number>, BlockImportError> +) -> usize +{ + match result { + Ok(BlockImportResult::ImportedKnown(hash, number)) => { + link.block_imported(&hash, number); + 1 + }, + Ok(BlockImportResult::ImportedUnknown(hash, number)) => { + link.block_imported(&hash, number); + 1 + }, + Err(BlockImportError::Disconnect(who)) => { + // TODO: FIXME: @arkpar BlockImport shouldn't be trying to manage the peer set. + // This should contain an actual reason. + link.useless_peer(who, "Import result was stated Disconnect"); + 0 + }, + Err(BlockImportError::DisconnectAndRestart(who)) => { + // TODO: FIXME: @arkpar BlockImport shouldn't be trying to manage the peer set. + // This should contain an actual reason. + link.note_useless_and_restart_sync(who, "Import result was stated DisconnectAndRestart"); + 0 + }, + Err(BlockImportError::Restart) => { + link.restart(); + 0 + }, + } +} + +impl<'a, B: 'static + BlockT, E: 'a + ExecuteInContext> SyncLink<'a, B, E> { + /// Execute closure with locked ChainSync. + fn with_sync, &mut Context)>(&mut self, closure: F) { + match *self { + #[cfg(test)] + SyncLink::Direct(ref mut sync, ref mut protocol) => + closure(*sync, *protocol), + SyncLink::Indirect(ref sync, _, ref service) => + service.execute_in_context(move |protocol| { + let mut sync = sync.write(); + closure(&mut *sync, protocol) + }), + } + } +} + +impl<'a, B: 'static + BlockT, E: ExecuteInContext> SyncLinkApi for SyncLink<'a, B, E> { + fn chain(&self) -> &Client { + match *self { + #[cfg(test)] + SyncLink::Direct(_, ref protocol) => protocol.client(), + SyncLink::Indirect(_, ref chain, _) => *chain, + } + } + + fn block_imported(&mut self, hash: &B::Hash, number: NumberFor) { + self.with_sync(|sync, _| sync.block_imported(&hash, number)) + } + + fn maintain_sync(&mut self) { + self.with_sync(|sync, protocol| sync.maintain_sync(protocol)) + } + + fn useless_peer(&mut self, who: NodeIndex, reason: &str) { + self.with_sync(|_, protocol| protocol.report_peer(who, Severity::Useless(reason))) + } + + fn note_useless_and_restart_sync(&mut self, who: NodeIndex, reason: &str) { + self.with_sync(|sync, protocol| { + protocol.report_peer(who, Severity::Useless(reason)); // is this actually malign or just useless? + sync.restart(protocol); + }) + } + + fn restart(&mut self) { + self.with_sync(|sync, protocol| sync.restart(protocol)) + } +} + +#[cfg(test)] +pub mod tests { + use client; + use message; + use test_client::{self, TestClient}; + use test_client::runtime::{Block, Hash}; + use on_demand::tests::DummyExecutor; + use runtime_primitives::generic::BlockId; + use super::*; + + /// Blocks import queue that is importing blocks in the same thread. + pub struct SyncImportQueue; + struct DummyExecuteInContext; + + impl ExecuteInContext for DummyExecuteInContext { + fn execute_in_context)>(&self, _closure: F) { } + } + + impl ImportQueue for SyncImportQueue { + fn clear(&self) { } + + fn stop(&self) { } + + fn status(&self) -> ImportQueueStatus { + ImportQueueStatus { + importing_count: 0, + best_importing_number: Zero::zero(), + } + } + + fn is_importing(&self, _hash: &B::Hash) -> bool { + false + } + + fn import_blocks(&self, sync: &mut ChainSync, protocol: &mut Context, blocks: (BlockOrigin, Vec>)) { + import_many_blocks(&mut SyncLink::Direct::<_, DummyExecuteInContext>(sync, protocol), None, blocks); + } + } + + struct TestLink { + chain: Arc>, + imported: usize, + maintains: usize, + disconnects: usize, + restarts: usize, + } + + impl TestLink { + fn new() -> TestLink { + TestLink { + chain: Arc::new(test_client::new()), + imported: 0, + maintains: 0, + disconnects: 0, + restarts: 0, + } + } + + fn total(&self) -> usize { + self.imported + self.maintains + self.disconnects + self.restarts + } + } + + impl SyncLinkApi for TestLink { + fn chain(&self) -> &Client { &*self.chain } + fn block_imported(&mut self, _hash: &Hash, _number: NumberFor) { self.imported += 1; } + fn maintain_sync(&mut self) { self.maintains += 1; } + fn useless_peer(&mut self, _: NodeIndex, _: &str) { self.disconnects += 1; } + fn note_useless_and_restart_sync(&mut self, _: NodeIndex, _: &str) { self.disconnects += 1; self.restarts += 1; } + fn restart(&mut self) { self.restarts += 1; } + } + + fn prepare_good_block() -> (client::Client, Hash, u64, BlockData) { + let client = test_client::new(); + let block = client.new_block().unwrap().bake().unwrap(); + client.justify_and_import(BlockOrigin::File, block).unwrap(); + + let (hash, number) = (client.block_hash(1).unwrap().unwrap(), 1); + let block = message::BlockData:: { + hash: client.block_hash(1).unwrap().unwrap(), + header: client.header(&BlockId::Number(1)).unwrap(), + body: None, + receipt: None, + message_queue: None, + justification: client.justification(&BlockId::Number(1)).unwrap(), + }; + + (client, hash, number, BlockData { block, origin: 0 }) + } + + #[test] + fn import_single_good_block_works() { + let (_, hash, number, block) = prepare_good_block(); + assert_eq!(import_single_block(&test_client::new(), BlockOrigin::File, block), Ok(BlockImportResult::ImportedUnknown(hash, number))); + } + + #[test] + fn import_single_good_known_block_is_ignored() { + let (client, hash, number, block) = prepare_good_block(); + assert_eq!(import_single_block(&client, BlockOrigin::File, block), Ok(BlockImportResult::ImportedKnown(hash, number))); + } + + #[test] + fn import_single_good_block_without_header_fails() { + let (_, _, _, mut block) = prepare_good_block(); + block.block.header = None; + assert_eq!(import_single_block(&test_client::new(), BlockOrigin::File, block), Err(BlockImportError::Disconnect(0))); + } + + #[test] + fn import_single_good_block_without_justification_fails() { + let (_, _, _, mut block) = prepare_good_block(); + block.block.justification = None; + assert_eq!(import_single_block(&test_client::new(), BlockOrigin::File, block), Err(BlockImportError::Disconnect(0))); + } + + #[test] + fn process_import_result_works() { + let mut link = TestLink::new(); + assert_eq!(process_import_result::(&mut link, Ok(BlockImportResult::ImportedKnown(Default::default(), 0))), 1); + assert_eq!(link.total(), 1); + + let mut link = TestLink::new(); + assert_eq!(process_import_result::(&mut link, Ok(BlockImportResult::ImportedKnown(Default::default(), 0))), 1); + assert_eq!(link.total(), 1); + assert_eq!(link.imported, 1); + + let mut link = TestLink::new(); + assert_eq!(process_import_result::(&mut link, Ok(BlockImportResult::ImportedUnknown(Default::default(), 0))), 1); + assert_eq!(link.total(), 1); + assert_eq!(link.imported, 1); + + let mut link = TestLink::new(); + assert_eq!(process_import_result::(&mut link, Err(BlockImportError::Disconnect(0))), 0); + assert_eq!(link.total(), 1); + assert_eq!(link.disconnects, 1); + + let mut link = TestLink::new(); + assert_eq!(process_import_result::(&mut link, Err(BlockImportError::DisconnectAndRestart(0))), 0); + assert_eq!(link.total(), 2); + assert_eq!(link.disconnects, 1); + assert_eq!(link.restarts, 1); + + let mut link = TestLink::new(); + assert_eq!(process_import_result::(&mut link, Err(BlockImportError::Restart)), 0); + assert_eq!(link.total(), 1); + assert_eq!(link.restarts, 1); + } + + #[test] + fn import_many_blocks_stops_when_stopping() { + let (_, _, _, block) = prepare_good_block(); + let qdata = AsyncImportQueueData::new(); + qdata.is_stopping.store(true, Ordering::SeqCst); + assert!(!import_many_blocks(&mut TestLink::new(), Some(&qdata), (BlockOrigin::File, vec![block.clone(), block]))); + } + + #[test] + fn async_import_queue_drops() { + let queue = AsyncImportQueue::new(); + let service = Arc::new(DummyExecutor); + let chain = Arc::new(test_client::new()); + queue.start(Weak::new(), Arc::downgrade(&service), Arc::downgrade(&chain) as Weak>).unwrap(); + drop(queue); + } +} diff --git a/core/network/src/io.rs b/core/network/src/io.rs new file mode 100644 index 00000000000..4a7d57b1589 --- /dev/null +++ b/core/network/src/io.rs @@ -0,0 +1,72 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use network_libp2p::{NetworkContext, Severity, NodeIndex, SessionInfo}; + +/// IO interface for the syncing handler. +/// Provides peer connection management and an interface to the blockchain client. +pub trait SyncIo { + /// Report a peer for misbehaviour. + fn report_peer(&mut self, who: NodeIndex, reason: Severity); + /// Send a packet to a peer. + fn send(&mut self, who: NodeIndex, data: Vec); + /// Returns peer identifier string + fn peer_info(&self, who: NodeIndex) -> String { + who.to_string() + } + /// Returns information on p2p session + fn peer_session_info(&self, who: NodeIndex) -> Option; + /// Check if the session is expired + fn is_expired(&self) -> bool; +} + +/// Wraps `NetworkContext` and the blockchain client +pub struct NetSyncIo<'s> { + network: &'s NetworkContext, +} + +impl<'s> NetSyncIo<'s> { + /// Creates a new instance from the `NetworkContext` and the blockchain client reference. + pub fn new(network: &'s NetworkContext) -> NetSyncIo<'s> { + NetSyncIo { + network: network, + } + } +} + +impl<'s> SyncIo for NetSyncIo<'s> { + fn report_peer(&mut self, who: NodeIndex, reason: Severity) { + self.network.report_peer(who, reason); + } + + fn send(&mut self, who: NodeIndex, data: Vec) { + self.network.send(who, 0, data) + } + + fn peer_session_info(&self, who: NodeIndex) -> Option { + self.network.session_info(who) + } + + fn is_expired(&self) -> bool { + self.network.is_expired() + } + + fn peer_info(&self, who: NodeIndex) -> String { + self.network.peer_client_version(who) + } +} + + diff --git a/core/network/src/lib.rs b/core/network/src/lib.rs new file mode 100644 index 00000000000..7f0817653eb --- /dev/null +++ b/core/network/src/lib.rs @@ -0,0 +1,69 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![warn(unused_extern_crates)] +#![warn(missing_docs)] + +// tag::description[] +//! Substrate-specific P2P networking: synchronizing blocks, propagating BFT messages. +//! Allows attachment of an optional subprotocol for chain-specific requests. +// end::description[] + +extern crate ethcore_io as core_io; +extern crate linked_hash_map; +extern crate parking_lot; +extern crate substrate_primitives as primitives; +extern crate substrate_client as client; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_network_libp2p as network_libp2p; +extern crate substrate_codec as codec; +extern crate futures; +extern crate rustc_hex; +#[macro_use] extern crate log; +#[macro_use] extern crate bitflags; +#[macro_use] extern crate error_chain; +#[macro_use] extern crate substrate_codec_derive; + +#[cfg(test)] extern crate env_logger; +#[cfg(test)] extern crate substrate_keyring as keyring; +#[cfg(test)] extern crate substrate_test_client as test_client; + +mod service; +mod sync; +mod protocol; +mod io; +mod config; +mod chain; +mod blocks; +mod on_demand; +mod import_queue; +pub mod consensus_gossip; +pub mod error; +pub mod message; +pub mod specialization; + +#[cfg(test)] mod test; + +pub use chain::Client as ClientHandle; +pub use service::{Service, FetchFuture, ConsensusService, BftMessageStream, + TransactionPool, Params, ManageNetwork, SyncProvider}; +pub use protocol::{ProtocolStatus, PeerInfo, Context}; +pub use sync::{Status as SyncStatus, SyncState}; +pub use network_libp2p::{NonReservedPeerMode, NetworkConfiguration, NodeIndex, ProtocolId, ConnectionFilter, ConnectionDirection, Severity}; +pub use message::{generic as generic_message, RequestId, BftMessage, LocalizedBftMessage, ConsensusVote, SignedConsensusVote, SignedConsensusMessage, SignedConsensusProposal, Status as StatusMessage}; +pub use error::Error; +pub use config::{Roles, ProtocolConfig}; +pub use on_demand::{OnDemand, OnDemandService, RemoteResponse}; diff --git a/core/network/src/message.rs b/core/network/src/message.rs new file mode 100644 index 00000000000..3bb080e1335 --- /dev/null +++ b/core/network/src/message.rs @@ -0,0 +1,375 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Network packet message types. These get serialized and put into the lower level protocol payload. + +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT}; +use codec::{Encode, Decode, Input, Output}; +pub use self::generic::{ + BlockAnnounce, RemoteCallRequest, RemoteReadRequest, + RemoteHeaderRequest, RemoteHeaderResponse, ConsensusVote, + SignedConsensusVote, FromBlock +}; + +/// A unique ID of a request. +pub type RequestId = u64; + +/// Type alias for using the message type using block type parameters. +pub type Message = generic::Message< + B, + ::Header, + ::Hash, + <::Header as HeaderT>::Number, + ::Extrinsic, +>; + +/// Type alias for using the status type using block type parameters. +pub type Status = generic::Status< + ::Hash, + <::Header as HeaderT>::Number, +>; + +/// Type alias for using the block request type using block type parameters. +pub type BlockRequest = generic::BlockRequest< + ::Hash, + <::Header as HeaderT>::Number, +>; + +/// Type alias for using the localized bft message type using block type parameters. +pub type LocalizedBftMessage = generic::LocalizedBftMessage< + B, + ::Hash, +>; + +/// Type alias for using the BlockData type using block type parameters. +pub type BlockData = generic::BlockData< + ::Header, + ::Hash, + ::Extrinsic, +>; + +/// Type alias for using the BlockResponse type using block type parameters. +pub type BlockResponse = generic::BlockResponse< + ::Header, + ::Hash, + ::Extrinsic, +>; + +/// Type alias for using the BftMessage type using block type parameters. +pub type BftMessage = generic::BftMessage< + B, + ::Hash, +>; + +/// Type alias for using the SignedConsensusProposal type using block type parameters. +pub type SignedConsensusProposal = generic::SignedConsensusProposal< + B, + ::Hash, +>; + +/// Type alias for using the SignedConsensusProposal type using block type parameters. +pub type SignedConsensusMessage = generic::SignedConsensusProposal< + B, + ::Hash, +>; + +/// A set of transactions. +pub type Transactions = Vec; + +/// Bits of block data and associated artefacts to request. +bitflags! { + /// Node roles bitmask. + pub struct BlockAttributes: u8 { + /// Include block header. + const HEADER = 0b00000001; + /// Include block body. + const BODY = 0b00000010; + /// Include block receipt. + const RECEIPT = 0b00000100; + /// Include block message queue. + const MESSAGE_QUEUE = 0b00001000; + /// Include a justification for the block. + const JUSTIFICATION = 0b00010000; + } +} + +impl Encode for BlockAttributes { + fn encode_to(&self, dest: &mut T) { + dest.push_byte(self.bits()) + } +} + +impl Decode for BlockAttributes { + fn decode(input: &mut I) -> Option { + Self::from_bits(input.read_byte()?) + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Encode, Decode)] +/// Block enumeration direction. +pub enum Direction { + /// Enumerate in ascending order (from child to parent). + Ascending = 0, + /// Enumerate in descendfing order (from parent to canonical child). + Descending = 1, +} + +/// Remote call response. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +pub struct RemoteCallResponse { + /// Id of a request this response was made for. + pub id: RequestId, + /// Execution proof. + pub proof: Vec>, +} + +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +/// Remote read response. +pub struct RemoteReadResponse { + /// Id of a request this response was made for. + pub id: RequestId, + /// Read proof. + pub proof: Vec>, +} + +/// Generic types. +pub mod generic { + use primitives::{AuthorityId, ed25519}; + use runtime_primitives::bft::Justification; + use service::Roles; + use super::{ + BlockAttributes, RemoteCallResponse, RemoteReadResponse, + RequestId, Transactions, Direction + }; + + /// Block data sent in the response. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct BlockData { + /// Block header hash. + pub hash: Hash, + /// Block header if requested. + pub header: Option

, + /// Block body if requested. + pub body: Option>, + /// Block receipt if requested. + pub receipt: Option>, + /// Block message queue if requested. + pub message_queue: Option>, + /// Justification if requested. + pub justification: Option>, + } + + /// Identifies starting point of a block sequence. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub enum FromBlock { + /// Start with given hash. + Hash(Hash), + /// Start with given block number. + Number(Number), + } + + /// Communication that can occur between participants in consensus. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub enum BftMessage { + /// A consensus message (proposal or vote) + Consensus(SignedConsensusMessage), + /// Auxiliary communication (just proof-of-lock for now). + Auxiliary(Justification), + } + + /// BFT Consensus message with parent header hash attached to it. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct LocalizedBftMessage { + /// Consensus message. + pub message: BftMessage, + /// Parent header hash. + pub parent_hash: Hash, + } + + /// A localized proposal message. Contains two signed pieces of data. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct SignedConsensusProposal { + /// The round number. + pub round_number: u32, + /// The proposal sent. + pub proposal: Block, + /// The digest of the proposal. + pub digest: Hash, + /// The sender of the proposal + pub sender: AuthorityId, + /// The signature on the message (propose, round number, digest) + pub digest_signature: ed25519::Signature, + /// The signature on the message (propose, round number, proposal) + pub full_signature: ed25519::Signature, + } + + /// A localized vote message, including the sender. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct SignedConsensusVote { + /// The message sent. + pub vote: ConsensusVote, + /// The sender of the message + pub sender: AuthorityId, + /// The signature of the message. + pub signature: ed25519::Signature, + } + + /// Votes during a consensus round. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub enum ConsensusVote { + /// Prepare to vote for proposal with digest D. + Prepare(u32, H), + /// Commit to proposal with digest D.. + Commit(u32, H), + /// Propose advancement to a new round. + AdvanceRound(u32), + } + + /// A localized message. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub enum SignedConsensusMessage { + /// A proposal. + Propose(SignedConsensusProposal), + /// A vote. + Vote(SignedConsensusVote), + } + + /// A network message. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub enum Message { + /// Status packet. + Status(Status), + /// Block request. + BlockRequest(BlockRequest), + /// Block response. + BlockResponse(BlockResponse), + /// Block announce. + BlockAnnounce(BlockAnnounce
), + /// Transactions. + Transactions(Transactions), + /// BFT Consensus statement. + BftMessage(LocalizedBftMessage), + /// Remote method call request. + RemoteCallRequest(RemoteCallRequest), + /// Remote method call response. + RemoteCallResponse(RemoteCallResponse), + /// Remote storage read request. + RemoteReadRequest(RemoteReadRequest), + /// Remote storage read response. + RemoteReadResponse(RemoteReadResponse), + /// Remote header request. + RemoteHeaderRequest(RemoteHeaderRequest), + /// Remote header response. + RemoteHeaderResponse(RemoteHeaderResponse
), + /// Chain-specific message + #[codec(index = "255")] + ChainSpecific(Vec), + } + + /// Status sent on connection. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct Status { + /// Protocol version. + pub version: u32, + /// Supported roles. + pub roles: Roles, + /// Best block number. + pub best_number: Number, + /// Best block hash. + pub best_hash: Hash, + /// Genesis block hash. + pub genesis_hash: Hash, + /// Chain-specific status. + pub chain_status: Vec, + } + + /// Request block data from a peer. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct BlockRequest { + /// Unique request id. + pub id: RequestId, + /// Bits of block data to request. + pub fields: BlockAttributes, + /// Start from this block. + pub from: FromBlock, + /// End at this block. An implementation defined maximum is used when unspecified. + pub to: Option, + /// Sequence direction. + pub direction: Direction, + /// Maximum number of blocks to return. An implementation defined maximum is used when unspecified. + pub max: Option, + } + + /// Response to `BlockRequest` + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct BlockResponse { + /// Id of a request this response was made for. + pub id: RequestId, + /// Block data for the requested sequence. + pub blocks: Vec>, + } + + /// Announce a new complete relay chain block on the network. + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + pub struct BlockAnnounce { + /// New block header. + pub header: H, + } + + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + /// Remote call request. + pub struct RemoteCallRequest { + /// Unique request id. + pub id: RequestId, + /// Block at which to perform call. + pub block: H, + /// Method name. + pub method: String, + /// Call data. + pub data: Vec, + } + + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + /// Remote storage read request. + pub struct RemoteReadRequest { + /// Unique request id. + pub id: RequestId, + /// Block at which to perform call. + pub block: H, + /// Storage key. + pub key: Vec, + } + + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + /// Remote header request. + pub struct RemoteHeaderRequest { + /// Unique request id. + pub id: RequestId, + /// Block number to request header for. + pub block: N, + } + + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] + /// Remote header response. + pub struct RemoteHeaderResponse
{ + /// Id of a request this response was made for. + pub id: RequestId, + /// Header. None if proof generation has failed (e.g. header is unknown). + pub header: Option
, + /// Header proof. + pub proof: Vec>, + } +} diff --git a/core/network/src/on_demand.rs b/core/network/src/on_demand.rs new file mode 100644 index 00000000000..7883b5fa143 --- /dev/null +++ b/core/network/src/on_demand.rs @@ -0,0 +1,736 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! On-demand requests service. + +use std::collections::VecDeque; +use std::sync::{Arc, Weak}; +use std::time::{Instant, Duration}; +use futures::{Async, Future, Poll}; +use futures::sync::oneshot::{channel, Receiver, Sender}; +use linked_hash_map::LinkedHashMap; +use linked_hash_map::Entry; +use parking_lot::Mutex; +use client; +use client::light::fetcher::{Fetcher, FetchChecker, RemoteHeaderRequest, + RemoteCallRequest, RemoteReadRequest}; +use io::SyncIo; +use message; +use network_libp2p::{Severity, NodeIndex}; +use service; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT}; + +/// Remote request timeout. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(15); +/// Default request retry count. +const RETRY_COUNT: usize = 1; + +/// On-demand service API. +pub trait OnDemandService: Send + Sync { + /// When new node is connected. + fn on_connect(&self, peer: NodeIndex, role: service::Roles); + + /// When node is disconnected. + fn on_disconnect(&self, peer: NodeIndex); + + /// Maintain peers requests. + fn maintain_peers(&self, io: &mut SyncIo); + + /// When header response is received from remote node. + fn on_remote_header_response( + &self, + io: &mut SyncIo, + peer: NodeIndex, + response: message::RemoteHeaderResponse + ); + + /// When read response is received from remote node. + fn on_remote_read_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteReadResponse); + + /// When call response is received from remote node. + fn on_remote_call_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteCallResponse); +} + +/// On-demand requests service. Dispatches requests to appropriate peers. +pub struct OnDemand> { + core: Mutex>, + checker: Arc>, +} + +/// On-demand remote call response. +pub struct RemoteResponse { + receiver: Receiver>, +} + +#[derive(Default)] +struct OnDemandCore> { + service: Weak, + next_request_id: u64, + pending_requests: VecDeque>, + active_peers: LinkedHashMap>, + idle_peers: VecDeque, +} + +struct Request { + id: u64, + timestamp: Instant, + retry_count: usize, + data: RequestData, +} + +enum RequestData { + RemoteHeader(RemoteHeaderRequest, Sender>), + RemoteRead(RemoteReadRequest, Sender>, client::error::Error>>), + RemoteCall(RemoteCallRequest, Sender>), +} + +enum Accept { + Ok, + CheckFailed(client::error::Error, RequestData), + Unexpected(RequestData), +} + +impl Future for RemoteResponse { + type Item = T; + type Error = client::error::Error; + + fn poll(&mut self) -> Poll { + self.receiver.poll() + .map_err(|_| client::error::ErrorKind::RemoteFetchCancelled.into()) + .and_then(|r| match r { + Async::Ready(Ok(ready)) => Ok(Async::Ready(ready)), + Async::Ready(Err(error)) => Err(error), + Async::NotReady => Ok(Async::NotReady), + }) + } +} + +impl OnDemand where + E: service::ExecuteInContext, + B::Header: HeaderT, +{ + /// Creates new on-demand service. + pub fn new(checker: Arc>) -> Self { + OnDemand { + checker, + core: Mutex::new(OnDemandCore { + service: Weak::new(), + next_request_id: 0, + pending_requests: VecDeque::new(), + active_peers: LinkedHashMap::new(), + idle_peers: VecDeque::new(), + }) + } + } + + /// Sets weak reference to network service. + pub fn set_service_link(&self, service: Weak) { + self.core.lock().service = service; + } + + /// Schedule && dispatch all scheduled requests. + fn schedule_request(&self, retry_count: Option, data: RequestData, result: R) -> R { + let mut core = self.core.lock(); + core.insert(retry_count.unwrap_or(RETRY_COUNT), data); + core.dispatch(); + result + } + + /// Try to accept response from given peer. + fn accept_response) -> Accept>(&self, rtype: &str, io: &mut SyncIo, peer: NodeIndex, request_id: u64, try_accept: F) { + let mut core = self.core.lock(); + let request = match core.remove(peer, request_id) { + Some(request) => request, + None => { + io.report_peer(peer, Severity::Bad(&format!("Invalid remote {} response from peer", rtype))); + core.remove_peer(peer); + return; + }, + }; + + let retry_count = request.retry_count; + let (retry_count, retry_request_data) = match try_accept(request) { + Accept::Ok => (retry_count, None), + Accept::CheckFailed(error, retry_request_data) => { + io.report_peer(peer, Severity::Bad(&format!("Failed to check remote {} response from peer: {}", rtype, error))); + core.remove_peer(peer); + + if retry_count > 0 { + (retry_count - 1, Some(retry_request_data)) + } else { + trace!(target: "sync", "Failed to get remote {} response for given number of retries", rtype); + retry_request_data.fail(client::error::ErrorKind::RemoteFetchFailed.into()); + (0, None) + } + }, + Accept::Unexpected(retry_request_data) => { + io.report_peer(peer, Severity::Bad(&format!("Unexpected response to remote {} from peer", rtype))); + core.remove_peer(peer); + + (retry_count, Some(retry_request_data)) + }, + }; + + if let Some(request_data) = retry_request_data { + core.insert(retry_count, request_data); + } + + core.dispatch(); + } +} + +impl OnDemandService for OnDemand where + B: BlockT, + E: service::ExecuteInContext, + B::Header: HeaderT, +{ + fn on_connect(&self, peer: NodeIndex, role: service::Roles) { + if !role.intersects(service::Roles::FULL | service::Roles::AUTHORITY) { // TODO: correct? + return; + } + + let mut core = self.core.lock(); + core.add_peer(peer); + core.dispatch(); + } + + fn on_disconnect(&self, peer: NodeIndex) { + let mut core = self.core.lock(); + core.remove_peer(peer); + core.dispatch(); + } + + fn maintain_peers(&self, io: &mut SyncIo) { + let mut core = self.core.lock(); + for bad_peer in core.maintain_peers() { + io.report_peer(bad_peer, Severity::Timeout); + } + core.dispatch(); + } + + fn on_remote_header_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteHeaderResponse) { + self.accept_response("header", io, peer, response.id, |request| match request.data { + RequestData::RemoteHeader(request, sender) => match self.checker.check_header_proof(&request, response.header, response.proof) { + Ok(response) => { + // we do not bother if receiver has been dropped already + let _ = sender.send(Ok(response)); + Accept::Ok + }, + Err(error) => Accept::CheckFailed(error, RequestData::RemoteHeader(request, sender)), + }, + data @ _ => Accept::Unexpected(data), + }) + } + + fn on_remote_read_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteReadResponse) { + self.accept_response("read", io, peer, response.id, |request| match request.data { + RequestData::RemoteRead(request, sender) => match self.checker.check_read_proof(&request, response.proof) { + Ok(response) => { + // we do not bother if receiver has been dropped already + let _ = sender.send(Ok(response)); + Accept::Ok + }, + Err(error) => Accept::CheckFailed(error, RequestData::RemoteRead(request, sender)), + }, + data @ _ => Accept::Unexpected(data), + }) + } + + fn on_remote_call_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteCallResponse) { + self.accept_response("call", io, peer, response.id, |request| match request.data { + RequestData::RemoteCall(request, sender) => match self.checker.check_execution_proof(&request, response.proof) { + Ok(response) => { + // we do not bother if receiver has been dropped already + let _ = sender.send(Ok(response)); + Accept::Ok + }, + Err(error) => Accept::CheckFailed(error, RequestData::RemoteCall(request, sender)), + }, + data @ _ => Accept::Unexpected(data), + }) + } +} + +impl Fetcher for OnDemand where + B: BlockT, + E: service::ExecuteInContext, + B::Header: HeaderT, +{ + type RemoteHeaderResult = RemoteResponse; + type RemoteReadResult = RemoteResponse>>; + type RemoteCallResult = RemoteResponse; + + fn remote_header(&self, request: RemoteHeaderRequest) -> Self::RemoteHeaderResult { + let (sender, receiver) = channel(); + self.schedule_request(request.retry_count.clone(), RequestData::RemoteHeader(request, sender), + RemoteResponse { receiver }) + } + + fn remote_read(&self, request: RemoteReadRequest) -> Self::RemoteReadResult { + let (sender, receiver) = channel(); + self.schedule_request(request.retry_count.clone(), RequestData::RemoteRead(request, sender), + RemoteResponse { receiver }) + } + + fn remote_call(&self, request: RemoteCallRequest) -> Self::RemoteCallResult { + let (sender, receiver) = channel(); + self.schedule_request(request.retry_count.clone(), RequestData::RemoteCall(request, sender), + RemoteResponse { receiver }) + } +} + +impl OnDemandCore where + B: BlockT, + E: service::ExecuteInContext, + B::Header: HeaderT, +{ + pub fn add_peer(&mut self, peer: NodeIndex) { + self.idle_peers.push_back(peer); + } + + pub fn remove_peer(&mut self, peer: NodeIndex) { + if let Some(request) = self.active_peers.remove(&peer) { + self.pending_requests.push_front(request); + return; + } + + if let Some(idle_index) = self.idle_peers.iter().position(|i| *i == peer) { + self.idle_peers.swap_remove_back(idle_index); + } + } + + pub fn maintain_peers(&mut self) -> Vec { + let now = Instant::now(); + let mut bad_peers = Vec::new(); + loop { + match self.active_peers.front() { + Some((_, request)) if now - request.timestamp >= REQUEST_TIMEOUT => (), + _ => return bad_peers, + } + + let (bad_peer, request) = self.active_peers.pop_front().expect("front() is Some as checked above"); + self.pending_requests.push_front(request); + bad_peers.push(bad_peer); + } + } + + pub fn insert(&mut self, retry_count: usize, data: RequestData) { + let request_id = self.next_request_id; + self.next_request_id += 1; + + self.pending_requests.push_back(Request { + id: request_id, + timestamp: Instant::now(), + retry_count, + data, + }); + } + + pub fn remove(&mut self, peer: NodeIndex, id: u64) -> Option> { + match self.active_peers.entry(peer) { + Entry::Occupied(entry) => match entry.get().id == id { + true => { + self.idle_peers.push_back(peer); + Some(entry.remove()) + }, + false => None, + }, + Entry::Vacant(_) => None, + } + } + + pub fn dispatch(&mut self) { + let service = match self.service.upgrade() { + Some(service) => service, + None => return, + }; + + while !self.pending_requests.is_empty() { + let peer = match self.idle_peers.pop_front() { + Some(peer) => peer, + None => return, + }; + + let mut request = self.pending_requests.pop_front().expect("checked in loop condition; qed"); + request.timestamp = Instant::now(); + trace!(target: "sync", "Dispatching remote request {} to peer {}", request.id, peer); + + service.execute_in_context(|ctx| ctx.send_message(peer, request.message())); + self.active_peers.insert(peer, request); + } + } +} + +impl Request { + pub fn message(&self) -> message::Message { + match self.data { + RequestData::RemoteHeader(ref data, _) => message::generic::Message::RemoteHeaderRequest( + message::RemoteHeaderRequest { + id: self.id, + block: data.block, + }), + RequestData::RemoteRead(ref data, _) => message::generic::Message::RemoteReadRequest( + message::RemoteReadRequest { + id: self.id, + block: data.block, + key: data.key.clone(), + }), + RequestData::RemoteCall(ref data, _) => message::generic::Message::RemoteCallRequest( + message::RemoteCallRequest { + id: self.id, + block: data.block, + method: data.method.clone(), + data: data.call_data.clone(), + }), + } + } +} + +impl RequestData { + pub fn fail(self, error: client::error::Error) { + // don't care if anyone is listening + match self { + RequestData::RemoteHeader(_, sender) => { let _ = sender.send(Err(error)); }, + RequestData::RemoteCall(_, sender) => { let _ = sender.send(Err(error)); }, + RequestData::RemoteRead(_, sender) => { let _ = sender.send(Err(error)); }, + } + } +} + +#[cfg(test)] +pub mod tests { + use std::collections::VecDeque; + use std::sync::Arc; + use std::time::Instant; + use futures::Future; + use parking_lot::RwLock; + use client; + use client::light::fetcher::{Fetcher, FetchChecker, RemoteHeaderRequest, + RemoteCallRequest, RemoteReadRequest}; + use message; + use network_libp2p::NodeIndex; + use service::{Roles, ExecuteInContext}; + use test::TestIo; + use super::{REQUEST_TIMEOUT, OnDemand, OnDemandService}; + use test_client::runtime::{Block, Header}; + + pub struct DummyExecutor; + struct DummyFetchChecker { ok: bool } + + impl ExecuteInContext for DummyExecutor { + fn execute_in_context)>(&self, _closure: F) {} + } + + impl FetchChecker for DummyFetchChecker { + fn check_header_proof( + &self, + _request: &RemoteHeaderRequest
, + header: Option
, + _remote_proof: Vec> + ) -> client::error::Result
{ + match self.ok { + true if header.is_some() => Ok(header.unwrap()), + _ => Err(client::error::ErrorKind::Backend("Test error".into()).into()), + } + } + + fn check_read_proof(&self, _request: &RemoteReadRequest
, _remote_proof: Vec>) -> client::error::Result>> { + match self.ok { + true => Ok(Some(vec![42])), + false => Err(client::error::ErrorKind::Backend("Test error".into()).into()), + } + } + + fn check_execution_proof(&self, _request: &RemoteCallRequest
, _remote_proof: Vec>) -> client::error::Result { + match self.ok { + true => Ok(client::CallResult { + return_data: vec![42], + changes: Default::default(), + }), + false => Err(client::error::ErrorKind::Backend("Test error".into()).into()), + } + } + } + + fn dummy(ok: bool) -> (Arc, Arc>) { + let executor = Arc::new(DummyExecutor); + let service = Arc::new(OnDemand::new(Arc::new(DummyFetchChecker { ok }))); + service.set_service_link(Arc::downgrade(&executor)); + (executor, service) + } + + fn total_peers(on_demand: &OnDemand) -> usize { + let core = on_demand.core.lock(); + core.idle_peers.len() + core.active_peers.len() + } + + fn receive_call_response(on_demand: &OnDemand, network: &mut TestIo, peer: NodeIndex, id: message::RequestId) { + on_demand.on_remote_call_response(network, peer, message::RemoteCallResponse { + id: id, + proof: vec![vec![2]], + }); + } + + fn dummy_header() -> Header { + Header { + parent_hash: Default::default(), + number: 0, + state_root: Default::default(), + extrinsics_root: Default::default(), + digest: Default::default(), + } + } + + #[test] + fn knows_about_peers_roles() { + let (_, on_demand) = dummy(true); + on_demand.on_connect(0, Roles::LIGHT); + on_demand.on_connect(1, Roles::FULL); + on_demand.on_connect(2, Roles::AUTHORITY); + assert_eq!(vec![1, 2], on_demand.core.lock().idle_peers.iter().cloned().collect::>()); + } + + #[test] + fn disconnects_from_idle_peer() { + let (_, on_demand) = dummy(true); + on_demand.on_connect(0, Roles::FULL); + assert_eq!(1, total_peers(&*on_demand)); + on_demand.on_disconnect(0); + assert_eq!(0, total_peers(&*on_demand)); + } + + #[test] + fn disconnects_from_timeouted_peer() { + let (_x, on_demand) = dummy(true); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + + on_demand.on_connect(0, Roles::FULL); + on_demand.on_connect(1, Roles::FULL); + assert_eq!(vec![0, 1], on_demand.core.lock().idle_peers.iter().cloned().collect::>()); + assert!(on_demand.core.lock().active_peers.is_empty()); + + on_demand.remote_call(RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: None, + }); + assert_eq!(vec![1], on_demand.core.lock().idle_peers.iter().cloned().collect::>()); + assert_eq!(vec![0], on_demand.core.lock().active_peers.keys().cloned().collect::>()); + + on_demand.core.lock().active_peers[&0].timestamp = Instant::now() - REQUEST_TIMEOUT - REQUEST_TIMEOUT; + on_demand.maintain_peers(&mut network); + assert!(on_demand.core.lock().idle_peers.is_empty()); + assert_eq!(vec![1], on_demand.core.lock().active_peers.keys().cloned().collect::>()); + assert!(network.to_disconnect.contains(&0)); + } + + #[test] + fn disconnects_from_peer_on_response_with_wrong_id() { + let (_x, on_demand) = dummy(true); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + on_demand.on_connect(0, Roles::FULL); + + on_demand.remote_call(RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: None, + }); + receive_call_response(&*on_demand, &mut network, 0, 1); + assert!(network.to_disconnect.contains(&0)); + assert_eq!(on_demand.core.lock().pending_requests.len(), 1); + } + + #[test] + fn disconnects_from_peer_on_incorrect_response() { + let (_x, on_demand) = dummy(false); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + on_demand.remote_call(RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }); + + on_demand.on_connect(0, Roles::FULL); + receive_call_response(&*on_demand, &mut network, 0, 0); + assert!(network.to_disconnect.contains(&0)); + assert_eq!(on_demand.core.lock().pending_requests.len(), 1); + } + + #[test] + fn disconnects_from_peer_on_unexpected_response() { + let (_x, on_demand) = dummy(true); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + on_demand.on_connect(0, Roles::FULL); + + receive_call_response(&*on_demand, &mut network, 0, 0); + assert!(network.to_disconnect.contains(&0)); + } + + #[test] + fn disconnects_from_peer_on_wrong_response_type() { + let (_x, on_demand) = dummy(false); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + on_demand.on_connect(0, Roles::FULL); + + on_demand.remote_call(RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }); + + on_demand.on_remote_read_response(&mut network, 0, message::RemoteReadResponse { + id: 0, + proof: vec![vec![2]], + }); + assert!(network.to_disconnect.contains(&0)); + assert_eq!(on_demand.core.lock().pending_requests.len(), 1); + } + + #[test] + fn receives_remote_failure_after_retry_count_failures() { + use parking_lot::{Condvar, Mutex}; + + let retry_count = 2; + let (_x, on_demand) = dummy(false); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + for i in 0..retry_count+1 { + on_demand.on_connect(i, Roles::FULL); + } + + let sync = Arc::new((Mutex::new(0), Mutex::new(0), Condvar::new())); + let thread_sync = sync.clone(); + + let response = on_demand.remote_call(RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(retry_count) + }); + let thread = ::std::thread::spawn(move || { + let &(ref current, ref finished_at, ref finished) = &*thread_sync; + let _ = response.wait().unwrap_err(); + *finished_at.lock() = *current.lock(); + finished.notify_one(); + }); + + let &(ref current, ref finished_at, ref finished) = &*sync; + for i in 0..retry_count+1 { + let mut current = current.lock(); + *current = *current + 1; + receive_call_response(&*on_demand, &mut network, i, i as u64); + } + + let mut finished_at = finished_at.lock(); + assert!(!finished.wait_for(&mut finished_at, ::std::time::Duration::from_millis(1000)).timed_out()); + assert_eq!(*finished_at, retry_count + 1); + + thread.join().unwrap(); + } + + #[test] + fn receives_remote_call_response() { + let (_x, on_demand) = dummy(true); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + on_demand.on_connect(0, Roles::FULL); + + let response = on_demand.remote_call(RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: None, + }); + let thread = ::std::thread::spawn(move || { + let result = response.wait().unwrap(); + assert_eq!(result.return_data, vec![42]); + }); + + receive_call_response(&*on_demand, &mut network, 0, 0); + thread.join().unwrap(); + } + + #[test] + fn receives_remote_read_response() { + let (_x, on_demand) = dummy(true); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + on_demand.on_connect(0, Roles::FULL); + + let response = on_demand.remote_read(RemoteReadRequest { + header: dummy_header(), + block: Default::default(), + key: b":key".to_vec(), + retry_count: None, + }); + let thread = ::std::thread::spawn(move || { + let result = response.wait().unwrap(); + assert_eq!(result, Some(vec![42])); + }); + + on_demand.on_remote_read_response(&mut network, 0, message::RemoteReadResponse { + id: 0, + proof: vec![vec![2]], + }); + thread.join().unwrap(); + } + + #[test] + fn receives_remote_header_response() { + let (_x, on_demand) = dummy(true); + let queue = RwLock::new(VecDeque::new()); + let mut network = TestIo::new(&queue, None); + on_demand.on_connect(0, Roles::FULL); + + let response = on_demand.remote_header(RemoteHeaderRequest { + cht_root: Default::default(), + block: 1, + retry_count: None, + }); + let thread = ::std::thread::spawn(move || { + let result = response.wait().unwrap(); + assert_eq!(result.hash(), "80729accb7bb10ff9c637a10e8bb59f21c52571aa7b46544c5885ca89ed190f4".into()); + }); + + on_demand.on_remote_header_response(&mut network, 0, message::RemoteHeaderResponse { + id: 0, + header: Some(Header { + parent_hash: Default::default(), + number: 1, + state_root: Default::default(), + extrinsics_root: Default::default(), + digest: Default::default(), + }), + proof: vec![vec![2]], + }); + thread.join().unwrap(); + } +} diff --git a/core/network/src/protocol.rs b/core/network/src/protocol.rs new file mode 100644 index 00000000000..8866a167a28 --- /dev/null +++ b/core/network/src/protocol.rs @@ -0,0 +1,679 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::collections::{HashMap, HashSet}; +use std::{mem, cmp}; +use std::sync::Arc; +use std::time; +use parking_lot::RwLock; +use rustc_hex::ToHex; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor, As}; +use runtime_primitives::generic::BlockId; +use network_libp2p::{NodeIndex, Severity}; +use codec::{Encode, Decode}; + +use message::{self, Message}; +use message::generic::Message as GenericMessage; +use specialization::Specialization; +use sync::{ChainSync, Status as SyncStatus, SyncState}; +use service::{Roles, TransactionPool, ExHashT}; +use import_queue::ImportQueue; +use config::ProtocolConfig; +use chain::Client; +use on_demand::OnDemandService; +use io::SyncIo; +use error; + +const REQUEST_TIMEOUT_SEC: u64 = 40; + +/// Current protocol version. +pub (crate) const CURRENT_VERSION: u32 = 1; +/// Current packet count. +pub (crate) const CURRENT_PACKET_COUNT: u8 = 1; + +// Maximum allowed entries in `BlockResponse` +const MAX_BLOCK_DATA_RESPONSE: u32 = 128; + +// Lock must always be taken in order declared here. +pub struct Protocol, H: ExHashT> { + config: ProtocolConfig, + on_demand: Option>>, + genesis_hash: B::Hash, + sync: Arc>>, + specialization: RwLock, + context_data: ContextData, + // Connected peers pending Status message. + handshaking_peers: RwLock>, + transaction_pool: Arc>, +} +/// Syncing status and statistics +#[derive(Clone)] +pub struct ProtocolStatus { + /// Sync status. + pub sync: SyncStatus, + /// Total number of connected peers + pub num_peers: usize, + /// Total number of active peers. + pub num_active_peers: usize, +} + +/// Peer information +struct Peer { + /// Protocol version + protocol_version: u32, + /// Roles + roles: Roles, + /// Peer best block hash + best_hash: B::Hash, + /// Peer best block number + best_number: ::Number, + /// Pending block request if any + block_request: Option>, + /// Request timestamp + request_timestamp: Option, + /// Holds a set of transactions known to this peer. + known_extrinsics: HashSet, + /// Holds a set of blocks known to this peer. + known_blocks: HashSet, + /// Request counter, + next_request_id: message::RequestId, +} + +/// Info about a peer's known state. +#[derive(Debug)] +pub struct PeerInfo { + /// Roles + pub roles: Roles, + /// Protocol version + pub protocol_version: u32, + /// Peer best block hash + pub best_hash: B::Hash, + /// Peer best block number + pub best_number: ::Number, +} + +/// Context for a network-specific handler. +pub trait Context { + /// Get a reference to the client. + fn client(&self) -> &::chain::Client; + + /// Point out that a peer has been malign or irresponsible or appeared lazy. + fn report_peer(&mut self, who: NodeIndex, reason: Severity); + + /// Get peer info. + fn peer_info(&self, peer: NodeIndex) -> Option>; + + /// Send a message to a peer. + fn send_message(&mut self, who: NodeIndex, data: ::message::Message); +} + +/// Protocol context. +pub(crate) struct ProtocolContext<'a, B: 'a + BlockT, H: 'a + ExHashT> { + io: &'a mut SyncIo, + context_data: &'a ContextData, +} + +impl<'a, B: BlockT + 'a, H: 'a + ExHashT> ProtocolContext<'a, B, H> { + pub(crate) fn new(context_data: &'a ContextData, io: &'a mut SyncIo) -> Self { + ProtocolContext { + io, + context_data, + } + } + + /// Send a message to a peer. + pub fn send_message(&mut self, who: NodeIndex, message: Message) { + send_message(&self.context_data.peers, self.io, who, message) + } + + /// Point out that a peer has been malign or irresponsible or appeared lazy. + pub fn report_peer(&mut self, who: NodeIndex, reason: Severity) { + self.io.report_peer(who, reason); + } + + /// Get peer info. + pub fn peer_info(&self, peer: NodeIndex) -> Option> { + self.context_data.peers.read().get(&peer).map(|p| { + PeerInfo { + roles: p.roles, + protocol_version: p.protocol_version, + best_hash: p.best_hash, + best_number: p.best_number, + } + }) + } +} + +impl<'a, B: BlockT + 'a, H: ExHashT + 'a> Context for ProtocolContext<'a, B, H> { + fn send_message(&mut self, who: NodeIndex, message: Message) { + ProtocolContext::send_message(self, who, message); + } + + fn report_peer(&mut self, who: NodeIndex, reason: Severity) { + ProtocolContext::report_peer(self, who, reason); + } + + fn peer_info(&self, who: NodeIndex) -> Option> { + ProtocolContext::peer_info(self, who) + } + + fn client(&self) -> &Client { + &*self.context_data.chain + } +} + +/// Data necessary to create a context. +pub(crate) struct ContextData { + // All connected peers + peers: RwLock>>, + chain: Arc>, +} + +impl, H: ExHashT> Protocol { + /// Create a new instance. + pub fn new( + config: ProtocolConfig, + chain: Arc>, + import_queue: Arc>, + on_demand: Option>>, + transaction_pool: Arc>, + specialization: S, + ) -> error::Result { + let info = chain.info()?; + let sync = ChainSync::new(config.roles, &info, import_queue); + let protocol = Protocol { + config: config, + context_data: ContextData { + peers: RwLock::new(HashMap::new()), + chain, + }, + on_demand, + genesis_hash: info.chain.genesis_hash, + sync: Arc::new(RwLock::new(sync)), + specialization: RwLock::new(specialization), + handshaking_peers: RwLock::new(HashMap::new()), + transaction_pool: transaction_pool, + }; + Ok(protocol) + } + + pub(crate) fn context_data(&self) -> &ContextData { + &self.context_data + } + + pub(crate) fn sync(&self) -> &Arc>> { + &self.sync + } + + /// Returns protocol status + pub fn status(&self) -> ProtocolStatus { + let sync = self.sync.read(); + let peers = self.context_data.peers.read(); + ProtocolStatus { + sync: sync.status(), + num_peers: peers.values().count(), + num_active_peers: peers.values().filter(|p| p.block_request.is_some()).count(), + } + } + + pub fn handle_packet(&self, io: &mut SyncIo, who: NodeIndex, mut data: &[u8]) { + let message: Message = match Decode::decode(&mut data) { + Some(m) => m, + None => { + trace!(target: "sync", "Invalid packet from {}", who); + io.report_peer(who, Severity::Bad("Peer sent us a packet with invalid format")); + return; + } + }; + + match message { + GenericMessage::Status(s) => self.on_status_message(io, who, s), + GenericMessage::BlockRequest(r) => self.on_block_request(io, who, r), + GenericMessage::BlockResponse(r) => { + let request = { + let mut peers = self.context_data.peers.write(); + if let Some(ref mut peer) = peers.get_mut(&who) { + peer.request_timestamp = None; + match mem::replace(&mut peer.block_request, None) { + Some(r) => r, + None => { + io.report_peer(who, Severity::Bad("Unexpected response packet received from peer")); + return; + } + } + } else { + io.report_peer(who, Severity::Bad("Unexpected packet received from peer")); + return; + } + }; + if request.id != r.id { + trace!(target: "sync", "Ignoring mismatched response packet from {} (expected {} got {})", who, request.id, r.id); + return; + } + self.on_block_response(io, who, request, r); + }, + GenericMessage::BlockAnnounce(announce) => self.on_block_announce(io, who, announce), + GenericMessage::Transactions(m) => self.on_extrinsics(io, who, m), + GenericMessage::RemoteCallRequest(request) => self.on_remote_call_request(io, who, request), + GenericMessage::RemoteCallResponse(response) => self.on_remote_call_response(io, who, response), + GenericMessage::RemoteReadRequest(request) => self.on_remote_read_request(io, who, request), + GenericMessage::RemoteReadResponse(response) => self.on_remote_read_response(io, who, response), + GenericMessage::RemoteHeaderRequest(request) => self.on_remote_header_request(io, who, request), + GenericMessage::RemoteHeaderResponse(response) => self.on_remote_header_response(io, who, response), + other => self.specialization.write().on_message(&mut ProtocolContext::new(&self.context_data, io), who, other), + } + } + + pub fn send_message(&self, io: &mut SyncIo, who: NodeIndex, message: Message) { + send_message::(&self.context_data.peers, io, who, message) + } + + /// Called when a new peer is connected + pub fn on_peer_connected(&self, io: &mut SyncIo, who: NodeIndex) { + trace!(target: "sync", "Connected {}: {}", who, io.peer_info(who)); + self.handshaking_peers.write().insert(who, time::Instant::now()); + self.send_status(io, who); + } + + /// Called by peer when it is disconnecting + pub fn on_peer_disconnected(&self, io: &mut SyncIo, peer: NodeIndex) { + trace!(target: "sync", "Disconnecting {}: {}", peer, io.peer_info(peer)); + + // lock all the the peer lists so that add/remove peer events are in order + let mut sync = self.sync.write(); + let mut spec = self.specialization.write(); + + let removed = { + let mut peers = self.context_data.peers.write(); + let mut handshaking_peers = self.handshaking_peers.write(); + handshaking_peers.remove(&peer); + peers.remove(&peer).is_some() + }; + if removed { + let mut context = ProtocolContext::new(&self.context_data, io); + sync.peer_disconnected(&mut context, peer); + spec.on_disconnect(&mut context, peer); + self.on_demand.as_ref().map(|s| s.on_disconnect(peer)); + } + } + + fn on_block_request(&self, io: &mut SyncIo, peer: NodeIndex, request: message::BlockRequest) { + trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?}", request.id, peer, request.from, request.to, request.max); + let mut blocks = Vec::new(); + let mut id = match request.from { + message::FromBlock::Hash(h) => BlockId::Hash(h), + message::FromBlock::Number(n) => BlockId::Number(n), + }; + let max = cmp::min(request.max.unwrap_or(u32::max_value()), MAX_BLOCK_DATA_RESPONSE) as usize; + // TODO: receipts, etc. + let get_header = request.fields.contains(message::BlockAttributes::HEADER); + let get_body = request.fields.contains(message::BlockAttributes::BODY); + let get_justification = request.fields.contains(message::BlockAttributes::JUSTIFICATION); + while let Some(header) = self.context_data.chain.header(&id).unwrap_or(None) { + if blocks.len() >= max { + break; + } + let number = header.number().clone(); + let hash = header.hash(); + let justification = if get_justification { self.context_data.chain.justification(&BlockId::Hash(hash)).unwrap_or(None) } else { None }; + let block_data = message::generic::BlockData { + hash: hash, + header: if get_header { Some(header) } else { None }, + body: if get_body { self.context_data.chain.body(&BlockId::Hash(hash)).unwrap_or(None) } else { None }, + receipt: None, + message_queue: None, + justification, + }; + blocks.push(block_data); + match request.direction { + message::Direction::Ascending => id = BlockId::Number(number + As::sa(1)), + message::Direction::Descending => { + if number == As::sa(0) { + break; + } + id = BlockId::Number(number - As::sa(1)) + } + } + } + let response = message::generic::BlockResponse { + id: request.id, + blocks: blocks, + }; + trace!(target: "sync", "Sending BlockResponse with {} blocks", response.blocks.len()); + self.send_message(io, peer, GenericMessage::BlockResponse(response)) + } + + fn on_block_response(&self, io: &mut SyncIo, peer: NodeIndex, request: message::BlockRequest, response: message::BlockResponse) { + // TODO: validate response + let blocks_range = match ( + response.blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())), + response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), + ) { + (Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last), + (Some(first), Some(_)) => format!(" ({})", first), + _ => Default::default(), + }; + trace!(target: "sync", "BlockResponse {} from {} with {} blocks{}", + response.id, peer, response.blocks.len(), blocks_range); + + self.sync.write().on_block_data(&mut ProtocolContext::new(&self.context_data, io), peer, request, response); + } + + /// Perform time based maintenance. + pub fn tick(&self, io: &mut SyncIo) { + self.maintain_peers(io); + self.on_demand.as_ref().map(|s| s.maintain_peers(io)); + } + + fn maintain_peers(&self, io: &mut SyncIo) { + let tick = time::Instant::now(); + let mut aborting = Vec::new(); + { + let peers = self.context_data.peers.read(); + let handshaking_peers = self.handshaking_peers.read(); + for (who, timestamp) in peers.iter() + .filter_map(|(id, peer)| peer.request_timestamp.as_ref().map(|r| (id, r))) + .chain(handshaking_peers.iter()) { + if (tick - *timestamp).as_secs() > REQUEST_TIMEOUT_SEC { + trace!(target: "sync", "Timeout {}", who); + aborting.push(*who); + } + } + } + + self.specialization.write().maintain_peers(&mut ProtocolContext::new(&self.context_data, io)); + for p in aborting { + io.report_peer(p, Severity::Timeout); + } + } + + pub fn peer_info(&self, peer: NodeIndex) -> Option> { + self.context_data.peers.read().get(&peer).map(|p| { + PeerInfo { + roles: p.roles, + protocol_version: p.protocol_version, + best_hash: p.best_hash, + best_number: p.best_number, + } + }) + } + + /// Called by peer to report status + fn on_status_message(&self, io: &mut SyncIo, who: NodeIndex, status: message::Status) { + trace!(target: "sync", "New peer {} {:?}", who, status); + if io.is_expired() { + trace!(target: "sync", "Status packet from expired session {}:{}", who, io.peer_info(who)); + return; + } + + { + let mut peers = self.context_data.peers.write(); + let mut handshaking_peers = self.handshaking_peers.write(); + if peers.contains_key(&who) { + debug!(target: "sync", "Unexpected status packet from {}:{}", who, io.peer_info(who)); + return; + } + if status.genesis_hash != self.genesis_hash { + io.report_peer(who, Severity::Bad(&format!("Peer is on different chain (our genesis: {} theirs: {})", self.genesis_hash, status.genesis_hash))); + return; + } + if status.version != CURRENT_VERSION { + io.report_peer(who, Severity::Bad(&format!("Peer using unsupported protocol version {}", status.version))); + return; + } + + let peer = Peer { + protocol_version: status.version, + roles: status.roles, + best_hash: status.best_hash, + best_number: status.best_number, + block_request: None, + request_timestamp: None, + known_extrinsics: HashSet::new(), + known_blocks: HashSet::new(), + next_request_id: 0, + }; + peers.insert(who.clone(), peer); + handshaking_peers.remove(&who); + debug!(target: "sync", "Connected {} {}", who, io.peer_info(who)); + } + + let mut context = ProtocolContext::new(&self.context_data, io); + self.sync.write().new_peer(&mut context, who); + self.specialization.write().on_connect(&mut context, who, status.clone()); + self.on_demand.as_ref().map(|s| s.on_connect(who, status.roles)); + } + + /// Called when peer sends us new extrinsics + fn on_extrinsics(&self, _io: &mut SyncIo, who: NodeIndex, extrinsics: message::Transactions) { + // Accept extrinsics only when fully synced + if self.sync.read().status().state != SyncState::Idle { + trace!(target: "sync", "{} Ignoring extrinsics while syncing", who); + return; + } + trace!(target: "sync", "Received {} extrinsics from {}", extrinsics.len(), who); + let mut peers = self.context_data.peers.write(); + if let Some(ref mut peer) = peers.get_mut(&who) { + for t in extrinsics { + if let Some(hash) = self.transaction_pool.import(&t) { + peer.known_extrinsics.insert(hash); + } + } + } + } + + /// Called when we propagate ready extrinsics to peers. + pub fn propagate_extrinsics(&self, io: &mut SyncIo) { + debug!(target: "sync", "Propagating extrinsics"); + + // Accept transactions only when fully synced + if self.sync.read().status().state != SyncState::Idle { + return; + } + + let extrinsics = self.transaction_pool.transactions(); + + let mut propagated_to = HashMap::new(); + let mut peers = self.context_data.peers.write(); + for (who, ref mut peer) in peers.iter_mut() { + let (hashes, to_send): (Vec<_>, Vec<_>) = extrinsics + .iter() + .cloned() + .filter(|&(ref hash, _)| peer.known_extrinsics.insert(hash.clone())) + .unzip(); + + if !to_send.is_empty() { + let node_id = io.peer_session_info(*who).map(|info| match info.id { + Some(id) => format!("{}@{:x}", info.remote_address, id), + None => info.remote_address.clone(), + }); + + if let Some(id) = node_id { + for hash in hashes { + propagated_to.entry(hash).or_insert_with(Vec::new).push(id.clone()); + } + } + trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); + self.send_message(io, *who, GenericMessage::Transactions(to_send)); + } + } + self.transaction_pool.on_broadcasted(propagated_to); + } + + /// Send Status message + fn send_status(&self, io: &mut SyncIo, who: NodeIndex) { + if let Ok(info) = self.context_data.chain.info() { + let status = message::generic::Status { + version: CURRENT_VERSION, + genesis_hash: info.chain.genesis_hash, + roles: self.config.roles.into(), + best_number: info.chain.best_number, + best_hash: info.chain.best_hash, + chain_status: self.specialization.read().status(), + }; + self.send_message(io, who, GenericMessage::Status(status)) + } + } + + pub fn abort(&self) { + let mut sync = self.sync.write(); + let mut spec = self.specialization.write(); + let mut peers = self.context_data.peers.write(); + let mut handshaking_peers = self.handshaking_peers.write(); + sync.clear(); + spec.on_abort(); + peers.clear(); + handshaking_peers.clear(); + } + + pub fn stop(&self) { + // stop processing import requests first (without holding a sync lock) + let import_queue = self.sync.read().import_queue(); + import_queue.stop(); + + // and then clear all the sync data + self.abort(); + } + + pub fn on_block_announce(&self, io: &mut SyncIo, who: NodeIndex, announce: message::BlockAnnounce) { + let header = announce.header; + let hash = header.hash(); + { + let mut peers = self.context_data.peers.write(); + if let Some(ref mut peer) = peers.get_mut(&who) { + peer.known_blocks.insert(hash.clone()); + } + } + self.sync.write().on_block_announce(&mut ProtocolContext::new(&self.context_data, io), who, hash, &header); + } + + pub fn on_block_imported(&self, io: &mut SyncIo, hash: B::Hash, header: &B::Header) { + self.sync.write().update_chain_info(&header); + self.specialization.write().on_block_imported( + &mut ProtocolContext::new(&self.context_data, io), + hash.clone(), + header + ); + + // blocks are not announced by light clients + if self.config.roles & Roles::LIGHT == Roles::LIGHT { + return; + } + + // send out block announcements + let mut peers = self.context_data.peers.write(); + + for (who, ref mut peer) in peers.iter_mut() { + if peer.known_blocks.insert(hash.clone()) { + trace!(target: "sync", "Announcing block {:?} to {}", hash, who); + self.send_message(io, *who, GenericMessage::BlockAnnounce(message::BlockAnnounce { + header: header.clone() + })); + } + } + } + + fn on_remote_call_request(&self, io: &mut SyncIo, who: NodeIndex, request: message::RemoteCallRequest) { + trace!(target: "sync", "Remote call request {} from {} ({} at {})", request.id, who, request.method, request.block); + let proof = match self.context_data.chain.execution_proof(&request.block, &request.method, &request.data) { + Ok((_, proof)) => proof, + Err(error) => { + trace!(target: "sync", "Remote call request {} from {} ({} at {}) failed with: {}", + request.id, who, request.method, request.block, error); + Default::default() + }, + }; + + self.send_message(io, who, GenericMessage::RemoteCallResponse(message::RemoteCallResponse { + id: request.id, proof, + })); + } + + fn on_remote_call_response(&self, io: &mut SyncIo, who: NodeIndex, response: message::RemoteCallResponse) { + trace!(target: "sync", "Remote call response {} from {}", response.id, who); + self.on_demand.as_ref().map(|s| s.on_remote_call_response(io, who, response)); + } + + fn on_remote_read_request(&self, io: &mut SyncIo, who: NodeIndex, request: message::RemoteReadRequest) { + trace!(target: "sync", "Remote read request {} from {} ({} at {})", + request.id, who, request.key.to_hex(), request.block); + let proof = match self.context_data.chain.read_proof(&request.block, &request.key) { + Ok(proof) => proof, + Err(error) => { + trace!(target: "sync", "Remote read request {} from {} ({} at {}) failed with: {}", + request.id, who, request.key.to_hex(), request.block, error); + Default::default() + }, + }; + self.send_message(io, who, GenericMessage::RemoteReadResponse(message::RemoteReadResponse { + id: request.id, proof, + })); + } + fn on_remote_read_response(&self, io: &mut SyncIo, who: NodeIndex, response: message::RemoteReadResponse) { + trace!(target: "sync", "Remote read response {} from {}", response.id, who); + self.on_demand.as_ref().map(|s| s.on_remote_read_response(io, who, response)); + } + + fn on_remote_header_request(&self, io: &mut SyncIo, who: NodeIndex, request: message::RemoteHeaderRequest>) { + trace!(target: "sync", "Remote header proof request {} from {} ({})", + request.id, who, request.block); + let (header, proof) = match self.context_data.chain.header_proof(request.block) { + Ok((header, proof)) => (Some(header), proof), + Err(error) => { + trace!(target: "sync", "Remote header proof request {} from {} ({}) failed with: {}", + request.id, who, request.block, error); + (Default::default(), Default::default()) + }, + }; + self.send_message(io, who, GenericMessage::RemoteHeaderResponse(message::RemoteHeaderResponse { + id: request.id, header, proof, + })); + } + + fn on_remote_header_response(&self, io: &mut SyncIo, who: NodeIndex, response: message::RemoteHeaderResponse) { + trace!(target: "sync", "Remote header proof response {} from {}", response.id, who); + self.on_demand.as_ref().map(|s| s.on_remote_header_response(io, who, response)); + } + + /// Execute a closure with access to a network context and specialization. + pub fn with_spec(&self, io: &mut SyncIo, f: F) -> U + where F: FnOnce(&mut S, &mut Context) -> U + { + f(&mut* self.specialization.write(), &mut ProtocolContext::new(&self.context_data, io)) + } +} + +fn send_message(peers: &RwLock>>, io: &mut SyncIo, who: NodeIndex, mut message: Message) { + match &mut message { + &mut GenericMessage::BlockRequest(ref mut r) => { + let mut peers = peers.write(); + if let Some(ref mut peer) = peers.get_mut(&who) { + r.id = peer.next_request_id; + peer.next_request_id = peer.next_request_id + 1; + peer.block_request = Some(r.clone()); + peer.request_timestamp = Some(time::Instant::now()); + } + }, + _ => (), + } + io.send(who, message.encode()); +} + +/// Hash a message. +pub(crate) fn hash_message(message: &Message) -> B::Hash { + let data = message.encode(); + HashFor::::hash(&data) +} diff --git a/core/network/src/service.rs b/core/network/src/service.rs new file mode 100644 index 00000000000..2260b86fb6d --- /dev/null +++ b/core/network/src/service.rs @@ -0,0 +1,340 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::collections::HashMap; +use std::sync::Arc; +use std::io; +use std::time::Duration; +use futures::sync::{oneshot, mpsc}; +use network_libp2p::{NetworkProtocolHandler, NetworkContext, NodeIndex, ProtocolId, +NetworkConfiguration , NonReservedPeerMode, ErrorKind}; +use network_libp2p::{NetworkService}; +use core_io::{TimerToken}; +use io::NetSyncIo; +use protocol::{Protocol, ProtocolContext, Context, ProtocolStatus, PeerInfo as ProtocolPeerInfo}; +use config::{ProtocolConfig}; +use error::Error; +use chain::Client; +use message::LocalizedBftMessage; +use specialization::Specialization; +use on_demand::OnDemandService; +use import_queue::AsyncImportQueue; +use runtime_primitives::traits::{Block as BlockT}; + +/// Type that represents fetch completion future. +pub type FetchFuture = oneshot::Receiver>; +/// Type that represents bft messages stream. +pub type BftMessageStream = mpsc::UnboundedReceiver>; + +const TICK_TOKEN: TimerToken = 0; +const TICK_TIMEOUT: Duration = Duration::from_millis(1000); + +const PROPAGATE_TOKEN: TimerToken = 1; +const PROPAGATE_TIMEOUT: Duration = Duration::from_millis(5000); + +bitflags! { + /// Node roles bitmask. + pub struct Roles: u8 { + /// No network. + const NONE = 0b00000000; + /// Full node, does not participate in consensus. + const FULL = 0b00000001; + /// Light client node. + const LIGHT = 0b00000010; + /// Act as an authority + const AUTHORITY = 0b00000100; + } +} + +impl ::codec::Encode for Roles { + fn encode_to(&self, dest: &mut T) { + dest.push_byte(self.bits()) + } +} + +impl ::codec::Decode for Roles { + fn decode(input: &mut I) -> Option { + Self::from_bits(input.read_byte()?) + } +} + +/// Sync status +pub trait SyncProvider: Send + Sync { + /// Get sync status + fn status(&self) -> ProtocolStatus; + /// Get peers information + fn peers(&self) -> Vec>; + /// Get this node id if available. + fn node_id(&self) -> Option; +} + +pub trait ExHashT: ::std::hash::Hash + Eq + ::std::fmt::Debug + Clone + Send + Sync + 'static {} +impl ExHashT for T where T: ::std::hash::Hash + Eq + ::std::fmt::Debug + Clone + Send + Sync + 'static {} + +/// Transaction pool interface +pub trait TransactionPool: Send + Sync { + /// Get transactions from the pool that are ready to be propagated. + fn transactions(&self) -> Vec<(H, B::Extrinsic)>; + /// Import a transaction into the pool. + fn import(&self, transaction: &B::Extrinsic) -> Option; + /// Notify the pool about transactions broadcast. + fn on_broadcasted(&self, propagations: HashMap>); +} + +/// ConsensusService +pub trait ConsensusService: Send + Sync { + /// Maintain connectivity to given addresses. + fn connect_to_authorities(&self, addresses: &[String]); + + /// Get BFT message stream for messages corresponding to consensus on given + /// parent hash. + fn bft_messages(&self, parent_hash: B::Hash) -> BftMessageStream; + /// Send out a BFT message. + fn send_bft_message(&self, message: LocalizedBftMessage); +} + +/// Service able to execute closure in the network context. +pub trait ExecuteInContext: Send + Sync { + /// Execute closure in network context. + fn execute_in_context)>(&self, closure: F); +} + +/// Network protocol handler +struct ProtocolHandler, H: ExHashT> { + protocol: Protocol, +} + +/// Peer connection information +#[derive(Debug)] +pub struct PeerInfo { + /// Public node id + pub id: Option, + /// Node client ID + pub client_version: String, + /// Capabilities + pub capabilities: Vec, + /// Remote endpoint address + pub remote_address: String, + /// Local endpoint address + pub local_address: String, + /// Dot protocol info. + pub dot_info: Option>, +} + +/// Service initialization parameters. +pub struct Params { + /// Configuration. + pub config: ProtocolConfig, + /// Network layer configuration. + pub network_config: NetworkConfiguration, + /// Polkadot relay chain access point. + pub chain: Arc>, + /// On-demand service reference. + pub on_demand: Option>>, + /// Transaction pool. + pub transaction_pool: Arc>, + /// Protocol specialization. + pub specialization: S, +} + +/// Polkadot network service. Handles network IO and manages connectivity. +pub struct Service, H: ExHashT> { + /// Network service + network: NetworkService, + /// Devp2p protocol handler + handler: Arc>, + /// Devp2p protocol ID. + protocol_id: ProtocolId, +} + +impl, H: ExHashT> Service { + /// Creates and register protocol with the network service + pub fn new(params: Params, protocol_id: ProtocolId) -> Result>, Error> { + let chain = params.chain.clone(); + let import_queue = Arc::new(AsyncImportQueue::new()); + let handler = Arc::new(ProtocolHandler { + protocol: Protocol::new( + params.config, + params.chain, + import_queue.clone(), + params.on_demand, + params.transaction_pool, + params.specialization, + )?, + }); + let versions = [(::protocol::CURRENT_VERSION as u8, ::protocol::CURRENT_PACKET_COUNT)]; + let protocols = vec![(handler.clone() as Arc<_>, protocol_id, &versions[..])]; + let service = match NetworkService::new(params.network_config.clone(), protocols) { + Ok(service) => service, + Err(err) => { + match err.kind() { + ErrorKind::Io(ref e) if e.kind() == io::ErrorKind::AddrInUse => + warn!("Network port is already in use, make sure that another instance of Polkadot client is not running or change the port using the --port option."), + _ => warn!("Error starting network: {}", err), + }; + return Err(err.into()) + }, + }; + let sync = Arc::new(Service { + network: service, + protocol_id, + handler, + }); + + import_queue.start( + Arc::downgrade(sync.handler.protocol.sync()), + Arc::downgrade(&sync), + Arc::downgrade(&chain) + )?; + + Ok(sync) + } + + /// Called when a new block is imported by the client. + pub fn on_block_imported(&self, hash: B::Hash, header: &B::Header) { + self.network.with_context(self.protocol_id, |context| { + self.handler.protocol.on_block_imported(&mut NetSyncIo::new(context), hash, header) + }); + } + + /// Called when new transactons are imported by the client. + pub fn trigger_repropagate(&self) { + self.network.with_context(self.protocol_id, |context| { + self.handler.protocol.propagate_extrinsics(&mut NetSyncIo::new(context)); + }); + } + + /// Execute a closure with the chain-specific network specialization. + /// If the network is unavailable, this will return `None`. + pub fn with_spec(&self, f: F) -> Option + where F: FnOnce(&mut S, &mut Context) -> U + { + let mut res = None; + self.network.with_context(self.protocol_id, |context| { + res = Some(self.handler.protocol.with_spec(&mut NetSyncIo::new(context), f)) + }); + + res + } +} + +impl, H:ExHashT> Drop for Service { + fn drop(&mut self) { + self.handler.protocol.stop(); + } +} +impl, H: ExHashT> ExecuteInContext for Service { + fn execute_in_context)>(&self, closure: F) { + self.network.with_context(self.protocol_id, |context| { + closure(&mut ProtocolContext::new(self.handler.protocol.context_data(), &mut NetSyncIo::new(context))) + }); + } +} + +impl, H: ExHashT> SyncProvider for Service { + /// Get sync status + fn status(&self) -> ProtocolStatus { + self.handler.protocol.status() + } + + /// Get sync peers + fn peers(&self) -> Vec> { + self.network.with_context_eval(self.protocol_id, |ctx| { + let peer_ids = self.network.connected_peers(); + + peer_ids.into_iter().filter_map(|who| { + let session_info = match ctx.session_info(who) { + None => return None, + Some(info) => info, + }; + + Some(PeerInfo { + id: session_info.id.map(|id| format!("{:x}", id)), + client_version: session_info.client_version, + capabilities: session_info.peer_capabilities.into_iter().map(|c| c.to_string()).collect(), + remote_address: session_info.remote_address, + local_address: session_info.local_address, + dot_info: self.handler.protocol.peer_info(who), + }) + }).collect() + }).unwrap_or_else(Vec::new) + } + + fn node_id(&self) -> Option { + self.network.external_url() + } +} + +impl, H: ExHashT> NetworkProtocolHandler for ProtocolHandler { + fn initialize(&self, io: &NetworkContext) { + io.register_timer(TICK_TOKEN, TICK_TIMEOUT) + .expect("Error registering sync timer"); + + io.register_timer(PROPAGATE_TOKEN, PROPAGATE_TIMEOUT) + .expect("Error registering transaction propagation timer"); + } + + fn read(&self, io: &NetworkContext, peer: &NodeIndex, _packet_id: u8, data: &[u8]) { + self.protocol.handle_packet(&mut NetSyncIo::new(io), *peer, data); + } + + fn connected(&self, io: &NetworkContext, peer: &NodeIndex) { + self.protocol.on_peer_connected(&mut NetSyncIo::new(io), *peer); + } + + fn disconnected(&self, io: &NetworkContext, peer: &NodeIndex) { + self.protocol.on_peer_disconnected(&mut NetSyncIo::new(io), *peer); + } + + fn timeout(&self, io: &NetworkContext, timer: TimerToken) { + match timer { + TICK_TOKEN => self.protocol.tick(&mut NetSyncIo::new(io)), + PROPAGATE_TOKEN => self.protocol.propagate_extrinsics(&mut NetSyncIo::new(io)), + _ => {} + } + } +} + +/// Trait for managing network +pub trait ManageNetwork: Send + Sync { + /// Set to allow unreserved peers to connect + fn accept_unreserved_peers(&self); + /// Set to deny unreserved peers to connect + fn deny_unreserved_peers(&self); + /// Remove reservation for the peer + fn remove_reserved_peer(&self, peer: String) -> Result<(), String>; + /// Add reserved peer + fn add_reserved_peer(&self, peer: String) -> Result<(), String>; +} + + +impl, H: ExHashT> ManageNetwork for Service { + fn accept_unreserved_peers(&self) { + self.network.set_non_reserved_mode(NonReservedPeerMode::Accept); + } + + fn deny_unreserved_peers(&self) { + self.network.set_non_reserved_mode(NonReservedPeerMode::Deny); + } + + fn remove_reserved_peer(&self, peer: String) -> Result<(), String> { + self.network.remove_reserved_peer(&peer).map_err(|e| format!("{:?}", e)) + } + + fn add_reserved_peer(&self, peer: String) -> Result<(), String> { + self.network.add_reserved_peer(&peer).map_err(|e| format!("{:?}", e)) + } +} diff --git a/core/network/src/specialization.rs b/core/network/src/specialization.rs new file mode 100644 index 00000000000..3c04a367059 --- /dev/null +++ b/core/network/src/specialization.rs @@ -0,0 +1,48 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Specializations of the substrate network protocol to allow more complex forms of communication. + +use ::NodeIndex; +use runtime_primitives::traits::Block as BlockT; +use protocol::Context; + +/// A specialization of the substrate network protocol. Handles events and sends messages. +pub trait Specialization: Send + Sync + 'static { + /// Get the current specialization-status. + fn status(&self) -> Vec; + + /// Called on start-up. + fn on_start(&mut self) { } + + /// Called when a peer successfully handshakes. + fn on_connect(&mut self, ctx: &mut Context, who: NodeIndex, status: ::message::Status); + + /// Called when a peer is disconnected. If the peer ID is unknown, it should be ignored. + fn on_disconnect(&mut self, ctx: &mut Context, who: NodeIndex); + + /// Called when a network-specific message arrives. + fn on_message(&mut self, ctx: &mut Context, who: NodeIndex, message: ::message::Message); + + /// Called on abort. + fn on_abort(&mut self) { } + + /// Called periodically to maintain peers and handle timeouts. + fn maintain_peers(&mut self, _ctx: &mut Context) { } + + /// Called when a block is _imported_ at the head of the chain (not during major sync). + fn on_block_imported(&mut self, _ctx: &mut Context, _hash: B::Hash, _header: &B::Header) { } +} diff --git a/core/network/src/sync.rs b/core/network/src/sync.rs new file mode 100644 index 00000000000..33c3e57d784 --- /dev/null +++ b/core/network/src/sync.rs @@ -0,0 +1,428 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::collections::HashMap; +use std::sync::Arc; +use protocol::Context; +use network_libp2p::{Severity, NodeIndex}; +use client::{BlockStatus, BlockOrigin, ClientInfo}; +use client::error::Error as ClientError; +use blocks::{self, BlockCollection}; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, NumberFor}; +use runtime_primitives::generic::BlockId; +use message::{self, generic::Message as GenericMessage}; +use service::Roles; +use import_queue::ImportQueue; + +// Maximum blocks to request in a single packet. +const MAX_BLOCKS_TO_REQUEST: usize = 128; +// Maximum blocks to store in the import queue. +const MAX_IMPORTING_BLOCKS: usize = 2048; + +struct PeerSync { + pub common_hash: B::Hash, + pub common_number: NumberFor, + pub best_hash: B::Hash, + pub best_number: NumberFor, + pub state: PeerSyncState, +} + +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +enum PeerSyncState { + AncestorSearch(NumberFor), + Available, + DownloadingNew(NumberFor), + DownloadingStale(B::Hash), +} + +/// Relay chain sync strategy. +pub struct ChainSync { + genesis_hash: B::Hash, + peers: HashMap>, + blocks: BlockCollection, + best_queued_number: NumberFor, + best_queued_hash: B::Hash, + required_block_attributes: message::BlockAttributes, + import_queue: Arc>, +} + +/// Reported sync state. +#[derive(Clone, Eq, PartialEq, Debug)] +pub enum SyncState { + /// Initial sync is complete, keep-up sync is active. + Idle, + /// Actively catching up with the chain. + Downloading +} + +/// Syncing status and statistics +#[derive(Clone)] +pub struct Status { + /// Current global sync state. + pub state: SyncState, + /// Target sync block number. + pub best_seen_block: Option>, +} + +impl ChainSync { + /// Create a new instance. + pub(crate) fn new(role: Roles, info: &ClientInfo, import_queue: Arc>) -> Self { + let mut required_block_attributes = message::BlockAttributes::HEADER | message::BlockAttributes::JUSTIFICATION; + if role.intersects(Roles::FULL | Roles::AUTHORITY) { + required_block_attributes |= message::BlockAttributes::BODY; + } + + ChainSync { + genesis_hash: info.chain.genesis_hash, + peers: HashMap::new(), + blocks: BlockCollection::new(), + best_queued_hash: info.best_queued_hash.unwrap_or(info.chain.best_hash), + best_queued_number: info.best_queued_number.unwrap_or(info.chain.best_number), + required_block_attributes, + import_queue, + } + } + + fn best_seen_block(&self) -> Option> { + self.peers.values().max_by_key(|p| p.best_number).map(|p| p.best_number) + } + + /// Returns import queue reference. + pub(crate) fn import_queue(&self) -> Arc> { + self.import_queue.clone() + } + + /// Returns sync status. + pub(crate) fn status(&self) -> Status { + let best_seen = self.best_seen_block(); + let state = match &best_seen { + &Some(n) if n > self.best_queued_number && n - self.best_queued_number > As::sa(5) => SyncState::Downloading, + _ => SyncState::Idle, + }; + Status { + state: state, + best_seen_block: best_seen, + } + } + + /// Handle new connected peer. + pub(crate) fn new_peer(&mut self, protocol: &mut Context, who: NodeIndex) { + if let Some(info) = protocol.peer_info(who) { + match (block_status(&*protocol.client(), &*self.import_queue, info.best_hash), info.best_number) { + (Err(e), _) => { + debug!(target:"sync", "Error reading blockchain: {:?}", e); + protocol.report_peer(who, Severity::Useless(&format!("Error legimimately reading blockchain status: {:?}", e))); + }, + (Ok(BlockStatus::KnownBad), _) => { + protocol.report_peer(who, Severity::Bad(&format!("New peer with known bad best block {} ({}).", info.best_hash, info.best_number))); + }, + (Ok(BlockStatus::Unknown), b) if b == As::sa(0) => { + protocol.report_peer(who, Severity::Bad(&format!("New peer with unknown genesis hash {} ({}).", info.best_hash, info.best_number))); + }, + (Ok(BlockStatus::Unknown), _) => { + let our_best = self.best_queued_number; + if our_best > As::sa(0) { + debug!(target:"sync", "New peer with unknown best hash {} ({}), searching for common ancestor.", info.best_hash, info.best_number); + self.peers.insert(who, PeerSync { + common_hash: self.genesis_hash, + common_number: As::sa(0), + best_hash: info.best_hash, + best_number: info.best_number, + state: PeerSyncState::AncestorSearch(our_best), + }); + Self::request_ancestry(protocol, who, our_best) + } else { + // We are at genesis, just start downloading + debug!(target:"sync", "New peer with best hash {} ({}).", info.best_hash, info.best_number); + self.peers.insert(who, PeerSync { + common_hash: self.genesis_hash, + common_number: As::sa(0), + best_hash: info.best_hash, + best_number: info.best_number, + state: PeerSyncState::Available, + }); + self.download_new(protocol, who) + } + }, + (Ok(BlockStatus::Queued), _) | (Ok(BlockStatus::InChain), _) => { + debug!(target:"sync", "New peer with known best hash {} ({}).", info.best_hash, info.best_number); + self.peers.insert(who, PeerSync { + common_hash: info.best_hash, + common_number: info.best_number, + best_hash: info.best_hash, + best_number: info.best_number, + state: PeerSyncState::Available, + }); + } + } + } + } + + pub(crate) fn on_block_data(&mut self, protocol: &mut Context, who: NodeIndex, _request: message::BlockRequest, response: message::BlockResponse) { + let new_blocks = if let Some(ref mut peer) = self.peers.get_mut(&who) { + match peer.state { + PeerSyncState::DownloadingNew(start_block) => { + self.blocks.clear_peer_download(who); + peer.state = PeerSyncState::Available; + + self.blocks.insert(start_block, response.blocks, who); + self.blocks.drain(self.best_queued_number + As::sa(1)) + }, + PeerSyncState::DownloadingStale(_) => { + peer.state = PeerSyncState::Available; + response.blocks.into_iter().map(|b| blocks::BlockData { + origin: who, + block: b + }).collect() + }, + PeerSyncState::AncestorSearch(n) => { + match response.blocks.get(0) { + Some(ref block) => { + trace!(target: "sync", "Got ancestry block #{} ({}) from peer {}", n, block.hash, who); + match protocol.client().block_hash(n) { + Ok(Some(block_hash)) if block_hash == block.hash => { + if peer.common_number < n { + peer.common_hash = block.hash; + peer.common_number = n; + } + peer.state = PeerSyncState::Available; + trace!(target:"sync", "Found common ancestor for peer {}: {} ({})", who, block.hash, n); + vec![] + }, + Ok(our_best) if n > As::sa(0) => { + trace!(target:"sync", "Ancestry block mismatch for peer {}: theirs: {} ({}), ours: {:?}", who, block.hash, n, our_best); + let n = n - As::sa(1); + peer.state = PeerSyncState::AncestorSearch(n); + Self::request_ancestry(protocol, who, n); + return; + }, + Ok(_) => { // genesis mismatch + trace!(target:"sync", "Ancestry search: genesis mismatch for peer {}", who); + protocol.report_peer(who, Severity::Bad("Ancestry search: genesis mismatch for peer")); + return; + }, + Err(e) => { + protocol.report_peer(who, Severity::Useless(&format!("Error answering legitimate blockchain query: {:?}", e))); + return; + } + } + }, + None => { + trace!(target:"sync", "Invalid response when searching for ancestor from {}", who); + protocol.report_peer(who, Severity::Bad("Invalid response when searching for ancestor")); + return; + } + } + }, + PeerSyncState::Available => Vec::new(), + } + } else { + vec![] + }; + + let best_seen = self.best_seen_block(); + let is_best = new_blocks.first().and_then(|b| b.block.header.as_ref()).map(|h| best_seen.as_ref().map_or(false, |n| h.number() >= n)); + let origin = if is_best.unwrap_or_default() { BlockOrigin::NetworkBroadcast } else { BlockOrigin::NetworkInitialSync }; + let import_queue = self.import_queue.clone(); + if let Some((hash, number)) = new_blocks.last() + .and_then(|b| b.block.header.as_ref().map(|h|(b.block.hash.clone(), *h.number()))) + { + if number > self.best_queued_number { + self.best_queued_number = number; + self.best_queued_hash = hash; + } + } + import_queue.import_blocks(self, protocol, (origin, new_blocks)); + self.maintain_sync(protocol); + } + + pub fn maintain_sync(&mut self, protocol: &mut Context) { + let peers: Vec = self.peers.keys().map(|p| *p).collect(); + for peer in peers { + self.download_new(protocol, peer); + } + } + + pub fn block_imported(&mut self, hash: &B::Hash, number: NumberFor) { + if number > self.best_queued_number { + self.best_queued_number = number; + self.best_queued_hash = *hash; + } + // Update common blocks + for (_, peer) in self.peers.iter_mut() { + trace!("Updating peer info ours={}, theirs={}", number, peer.best_number); + if peer.best_number >= number { + peer.common_number = number; + peer.common_hash = *hash; + } + } + } + + pub(crate) fn update_chain_info(&mut self, best_header: &B::Header) { + let hash = best_header.hash(); + self.block_imported(&hash, best_header.number().clone()) + } + + pub(crate) fn on_block_announce(&mut self, protocol: &mut Context, who: NodeIndex, hash: B::Hash, header: &B::Header) { + let number = *header.number(); + if let Some(ref mut peer) = self.peers.get_mut(&who) { + if number > peer.best_number { + peer.best_number = number; + peer.best_hash = hash; + } + if number <= self.best_queued_number && number > peer.common_number { + peer.common_number = number + } + } else { + return; + } + + if !self.is_known_or_already_downloading(protocol, &hash) { + let stale = number <= self.best_queued_number; + if stale { + if !self.is_known_or_already_downloading(protocol, header.parent_hash()) { + trace!(target: "sync", "Ignoring unknown stale block announce from {}: {} {:?}", who, hash, header); + } else { + trace!(target: "sync", "Downloading new stale block announced from {}: {} {:?}", who, hash, header); + self.download_stale(protocol, who, &hash); + } + } else { + trace!(target: "sync", "Downloading new block announced from {}: {} {:?}", who, hash, header); + self.download_new(protocol, who); + } + } else { + trace!(target: "sync", "Known block announce from {}: {}", who, hash); + } + } + + fn is_known_or_already_downloading(&self, protocol: &mut Context, hash: &B::Hash) -> bool { + self.peers.iter().any(|(_, p)| p.state == PeerSyncState::DownloadingStale(*hash)) + || block_status(&*protocol.client(), &*self.import_queue, *hash).ok().map_or(false, |s| s != BlockStatus::Unknown) + } + + pub(crate) fn peer_disconnected(&mut self, protocol: &mut Context, who: NodeIndex) { + self.blocks.clear_peer_download(who); + self.peers.remove(&who); + self.maintain_sync(protocol); + } + + pub(crate) fn restart(&mut self, protocol: &mut Context) { + self.import_queue.clear(); + self.blocks.clear(); + let ids: Vec = self.peers.keys().map(|p| *p).collect(); + for id in ids { + self.new_peer(protocol, id); + } + match protocol.client().info() { + Ok(info) => { + self.best_queued_hash = info.best_queued_hash.unwrap_or(info.chain.best_hash); + self.best_queued_number = info.best_queued_number.unwrap_or(info.chain.best_number); + }, + Err(e) => { + debug!(target:"sync", "Error reading blockchain: {:?}", e); + self.best_queued_hash = self.genesis_hash; + self.best_queued_number = As::sa(0); + } + } + } + + pub(crate) fn clear(&mut self) { + self.blocks.clear(); + self.peers.clear(); + } + + // Download old block. + fn download_stale(&mut self, protocol: &mut Context, who: NodeIndex, hash: &B::Hash) { + if let Some(ref mut peer) = self.peers.get_mut(&who) { + match peer.state { + PeerSyncState::Available => { + let request = message::generic::BlockRequest { + id: 0, + fields: self.required_block_attributes.clone(), + from: message::FromBlock::Hash(*hash), + to: None, + direction: message::Direction::Ascending, + max: Some(1), + }; + peer.state = PeerSyncState::DownloadingStale(*hash); + protocol.send_message(who, GenericMessage::BlockRequest(request)); + }, + _ => (), + } + } + } + + // Issue a request for a peer to download new blocks, if any are available + fn download_new(&mut self, protocol: &mut Context, who: NodeIndex) { + if let Some(ref mut peer) = self.peers.get_mut(&who) { + let import_status = self.import_queue.status(); + // when there are too many blocks in the queue => do not try to download new blocks + if import_status.importing_count > MAX_IMPORTING_BLOCKS { + return; + } + // we should not download already queued blocks + let common_number = ::std::cmp::max(peer.common_number, import_status.best_importing_number); + + trace!(target: "sync", "Considering new block download from {}, common block is {}, best is {:?}", who, common_number, peer.best_number); + match peer.state { + PeerSyncState::Available => { + if let Some(range) = self.blocks.needed_blocks(who, MAX_BLOCKS_TO_REQUEST, peer.best_number, common_number) { + trace!(target: "sync", "Requesting blocks from {}, ({} to {})", who, range.start, range.end); + let request = message::generic::BlockRequest { + id: 0, + fields: self.required_block_attributes.clone(), + from: message::FromBlock::Number(range.start), + to: None, + direction: message::Direction::Ascending, + max: Some((range.end - range.start).as_() as u32), + }; + peer.state = PeerSyncState::DownloadingNew(range.start); + protocol.send_message(who, GenericMessage::BlockRequest(request)); + } else { + trace!(target: "sync", "Nothing to request"); + } + }, + _ => (), + } + } + } + + fn request_ancestry(protocol: &mut Context, who: NodeIndex, block: NumberFor) { + trace!(target: "sync", "Requesting ancestry block #{} from {}", block, who); + let request = message::generic::BlockRequest { + id: 0, + fields: message::BlockAttributes::HEADER | message::BlockAttributes::JUSTIFICATION, + from: message::FromBlock::Number(block), + to: None, + direction: message::Direction::Ascending, + max: Some(1), + }; + protocol.send_message(who, GenericMessage::BlockRequest(request)); + } +} + +/// Get block status, taking into account import queue. +fn block_status( + chain: &::chain::Client, + queue: &ImportQueue, + hash: B::Hash) -> Result +{ + if queue.is_importing(&hash) { + return Ok(BlockStatus::Queued); + } + + chain.block_status(&BlockId::Hash(hash)) +} diff --git a/core/network/src/test/mod.rs b/core/network/src/test/mod.rs new file mode 100644 index 00000000000..7a78dd28fc0 --- /dev/null +++ b/core/network/src/test/mod.rs @@ -0,0 +1,330 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +mod sync; + +use std::collections::{VecDeque, HashSet, HashMap}; +use std::sync::Arc; + +use parking_lot::RwLock; +use client; +use client::block_builder::BlockBuilder; +use runtime_primitives::traits::Block as BlockT; +use runtime_primitives::generic::BlockId; +use io::SyncIo; +use protocol::{Context, Protocol}; +use primitives::{Blake2Hasher, RlpCodec}; +use config::ProtocolConfig; +use service::TransactionPool; +use network_libp2p::{NodeIndex, SessionInfo, Severity}; +use keyring::Keyring; +use codec::Encode; +use import_queue::tests::SyncImportQueue; +use test_client::{self, TestClient}; +use test_client::runtime::{Block, Hash, Transfer, Extrinsic}; +use specialization::Specialization; + +pub struct DummySpecialization; + +impl Specialization for DummySpecialization { + fn status(&self) -> Vec { vec![] } + + fn on_connect(&mut self, _ctx: &mut Context, _peer_id: NodeIndex, _status: ::message::Status) { + + } + + fn on_disconnect(&mut self, _ctx: &mut Context, _peer_id: NodeIndex) { + + } + + fn on_message(&mut self, _ctx: &mut Context, _peer_id: NodeIndex, _message: ::message::Message) { + + } +} + +pub struct TestIo<'p> { + queue: &'p RwLock>, + pub to_disconnect: HashSet, + packets: Vec, + peers_info: HashMap, + _sender: Option, +} + +impl<'p> TestIo<'p> where { + pub fn new(queue: &'p RwLock>, sender: Option) -> TestIo<'p> { + TestIo { + queue: queue, + _sender: sender, + to_disconnect: HashSet::new(), + packets: Vec::new(), + peers_info: HashMap::new(), + } + } +} + +impl<'p> Drop for TestIo<'p> { + fn drop(&mut self) { + self.queue.write().extend(self.packets.drain(..)); + } +} + +impl<'p> SyncIo for TestIo<'p> { + fn report_peer(&mut self, who: NodeIndex, _reason: Severity) { + self.to_disconnect.insert(who); + } + + fn is_expired(&self) -> bool { + false + } + + fn send(&mut self, who: NodeIndex, data: Vec) { + self.packets.push(TestPacket { + data: data, + recipient: who, + }); + } + + fn peer_info(&self, who: NodeIndex) -> String { + self.peers_info.get(&who) + .cloned() + .unwrap_or_else(|| who.to_string()) + } + + fn peer_session_info(&self, _peer_id: NodeIndex) -> Option { + None + } +} + +/// Mocked subprotocol packet +pub struct TestPacket { + data: Vec, + recipient: NodeIndex, +} + +pub struct Peer { + client: Arc>, + pub sync: Protocol, + pub queue: RwLock>, +} + +impl Peer { + /// Called after blockchain has been populated to updated current state. + fn start(&self) { + // Update the sync state to the latest chain state. + let info = self.client.info().expect("In-mem client does not fail"); + let header = self.client.header(&BlockId::Hash(info.chain.best_hash)).unwrap().unwrap(); + self.sync.on_block_imported(&mut TestIo::new(&self.queue, None), info.chain.best_hash, &header); + } + + /// Called on connection to other indicated peer. + fn on_connect(&self, other: NodeIndex) { + self.sync.on_peer_connected(&mut TestIo::new(&self.queue, Some(other)), other); + } + + /// Called on disconnect from other indicated peer. + fn on_disconnect(&self, other: NodeIndex) { + let mut io = TestIo::new(&self.queue, Some(other)); + self.sync.on_peer_disconnected(&mut io, other); + } + + /// Receive a message from another peer. Return a set of peers to disconnect. + fn receive_message(&self, from: NodeIndex, msg: TestPacket) -> HashSet { + let mut io = TestIo::new(&self.queue, Some(from)); + self.sync.handle_packet(&mut io, from, &msg.data); + self.flush(); + io.to_disconnect.clone() + } + + /// Produce the next pending message to send to another peer. + fn pending_message(&self) -> Option { + self.flush(); + self.queue.write().pop_front() + } + + /// Whether this peer is done syncing (has no messages to send). + fn is_done(&self) -> bool { + self.queue.read().is_empty() + } + + /// Execute a "sync step". This is called for each peer after it sends a packet. + fn sync_step(&self) { + self.flush(); + self.sync.tick(&mut TestIo::new(&self.queue, None)); + } + + /// Restart sync for a peer. + fn restart_sync(&self) { + self.sync.abort(); + } + + fn flush(&self) { + } + + fn generate_blocks(&self, count: usize, mut edit_block: F) + where F: FnMut(&mut BlockBuilder) + { + for _ in 0 .. count { + let mut builder = self.client.new_block().unwrap(); + edit_block(&mut builder); + let block = builder.bake().unwrap(); + trace!("Generating {}, (#{})", block.hash(), block.header.number); + self.client.justify_and_import(client::BlockOrigin::File, block).unwrap(); + } + } + + fn push_blocks(&self, count: usize, with_tx: bool) { + let mut nonce = 0; + if with_tx { + self.generate_blocks(count, |builder| { + let transfer = Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Alice.to_raw_public().into(), + amount: 1, + nonce, + }; + let signature = Keyring::from_raw_public(transfer.from.0).unwrap().sign(&transfer.encode()).into(); + builder.push(Extrinsic { transfer, signature }).unwrap(); + nonce = nonce + 1; + }); + } else { + self.generate_blocks(count, |_| ()); + } + } +} + +pub struct EmptyTransactionPool; + +impl TransactionPool for EmptyTransactionPool { + fn transactions(&self) -> Vec<(Hash, Extrinsic)> { + Vec::new() + } + + fn import(&self, _transaction: &Extrinsic) -> Option { + None + } + + fn on_broadcasted(&self, _: HashMap>) {} +} + +pub struct TestNet { + peers: Vec>, + started: bool, + disconnect_events: Vec<(NodeIndex, NodeIndex)>, //disconnected (initiated by, to) +} + +impl TestNet { + fn new(n: usize) -> Self { + Self::new_with_config(n, ProtocolConfig::default()) + } + + fn new_with_config(n: usize, config: ProtocolConfig) -> Self { + let mut net = TestNet { + peers: Vec::new(), + started: false, + disconnect_events: Vec::new(), + }; + + for _ in 0..n { + net.add_peer(&config); + } + net + } + + pub fn add_peer(&mut self, config: &ProtocolConfig) { + let client = Arc::new(test_client::new()); + let tx_pool = Arc::new(EmptyTransactionPool); + let import_queue = Arc::new(SyncImportQueue); + let sync = Protocol::new(config.clone(), client.clone(), import_queue, None, tx_pool, DummySpecialization).unwrap(); + self.peers.push(Arc::new(Peer { + sync: sync, + client: client, + queue: RwLock::new(VecDeque::new()), + })); + } + + pub fn peer(&self, i: usize) -> &Peer { + &self.peers[i] + } + + fn start(&mut self) { + if self.started { + return; + } + for peer in 0..self.peers.len() { + self.peers[peer].start(); + for client in 0..self.peers.len() { + if peer != client { + self.peers[peer].on_connect(client as NodeIndex); + } + } + } + self.started = true; + } + + fn sync_step(&mut self) { + for peer in 0..self.peers.len() { + let packet = self.peers[peer].pending_message(); + if let Some(packet) = packet { + let disconnecting = { + let recipient = packet.recipient; + trace!("--- {} -> {} ---", peer, recipient); + let to_disconnect = self.peers[recipient].receive_message(peer as NodeIndex, packet); + for d in &to_disconnect { + // notify this that disconnecting peers are disconnecting + self.peers[recipient].on_disconnect(*d as NodeIndex); + self.disconnect_events.push((peer, *d)); + } + to_disconnect + }; + for d in &disconnecting { + // notify other peers that this peer is disconnecting + self.peers[*d].on_disconnect(peer as NodeIndex); + } + } + + self.sync_step_peer(peer); + } + } + + fn sync_step_peer(&mut self, peer_num: usize) { + self.peers[peer_num].sync_step(); + } + + fn restart_peer(&mut self, i: usize) { + self.peers[i].restart_sync(); + } + + fn sync(&mut self) -> u32 { + self.start(); + let mut total_steps = 0; + while !self.done() { + self.sync_step(); + total_steps += 1; + } + total_steps + } + + fn sync_steps(&mut self, count: usize) { + self.start(); + for _ in 0..count { + self.sync_step(); + } + } + + fn done(&self) -> bool { + self.peers.iter().all(|p| p.is_done()) + } +} diff --git a/core/network/src/test/sync.rs b/core/network/src/test/sync.rs new file mode 100644 index 00000000000..77367a3df13 --- /dev/null +++ b/core/network/src/test/sync.rs @@ -0,0 +1,121 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use client::backend::Backend; +use client::blockchain::HeaderBackend as BlockchainHeaderBackend; +use sync::SyncState; +use Roles; +use super::*; + +#[test] +fn sync_from_two_peers_works() { + ::env_logger::init().ok(); + let mut net = TestNet::new(3); + net.peer(1).push_blocks(100, false); + net.peer(2).push_blocks(100, false); + net.sync(); + assert!(net.peer(0).client.backend().blockchain().equals_to(net.peer(1).client.backend().blockchain())); + let status = net.peer(0).sync.status(); + assert_eq!(status.sync.state, SyncState::Idle); +} + +#[test] +fn sync_from_two_peers_with_ancestry_search_works() { + ::env_logger::init().ok(); + let mut net = TestNet::new(3); + net.peer(0).push_blocks(10, true); + net.peer(1).push_blocks(100, false); + net.peer(2).push_blocks(100, false); + net.restart_peer(0); + net.sync(); + assert!(net.peer(0).client.backend().blockchain().canon_equals_to(net.peer(1).client.backend().blockchain())); +} + +#[test] +fn sync_long_chain_works() { + let mut net = TestNet::new(2); + net.peer(1).push_blocks(500, false); + net.sync_steps(3); + assert_eq!(net.peer(0).sync.status().sync.state, SyncState::Downloading); + net.sync(); + assert!(net.peer(0).client.backend().blockchain().equals_to(net.peer(1).client.backend().blockchain())); +} + +#[test] +fn sync_no_common_longer_chain_fails() { + ::env_logger::init().ok(); + let mut net = TestNet::new(3); + net.peer(0).push_blocks(20, true); + net.peer(1).push_blocks(20, false); + net.sync(); + assert!(!net.peer(0).client.backend().blockchain().canon_equals_to(net.peer(1).client.backend().blockchain())); +} + +#[test] +fn sync_after_fork_works() { + ::env_logger::init().ok(); + let mut net = TestNet::new(3); + net.peer(0).push_blocks(30, false); + net.peer(1).push_blocks(30, false); + net.peer(2).push_blocks(30, false); + + net.peer(0).push_blocks(10, true); + net.peer(1).push_blocks(20, false); + net.peer(2).push_blocks(20, false); + + net.peer(1).push_blocks(10, true); + net.peer(2).push_blocks(1, false); + + // peer 1 has the best chain + let peer1_chain = net.peer(1).client.backend().blockchain().clone(); + net.sync(); + assert!(net.peer(0).client.backend().blockchain().canon_equals_to(&peer1_chain)); + assert!(net.peer(1).client.backend().blockchain().canon_equals_to(&peer1_chain)); + assert!(net.peer(2).client.backend().blockchain().canon_equals_to(&peer1_chain)); +} + +#[test] +fn blocks_are_not_announced_by_light_nodes() { + ::env_logger::init().ok(); + let mut net = TestNet::new(0); + + // full peer0 is connected to light peer + // light peer1 is connected to full peer2 + let mut light_config = ProtocolConfig::default(); + light_config.roles = Roles::LIGHT; + net.add_peer(&ProtocolConfig::default()); + net.add_peer(&light_config); + net.add_peer(&ProtocolConfig::default()); + + net.peer(0).push_blocks(1, false); + net.peer(0).start(); + net.peer(1).start(); + net.peer(2).start(); + net.peer(0).on_connect(1); + net.peer(1).on_connect(2); + + // generate block at peer0 && run sync + while !net.done() { + net.sync_step(); + } + + // peer 0 has the best chain + // peer 1 has the best chain + // peer 2 has genesis-chain only + assert_eq!(net.peer(0).client.backend().blockchain().info().unwrap().best_number, 1); + assert_eq!(net.peer(1).client.backend().blockchain().info().unwrap().best_number, 1); + assert_eq!(net.peer(2).client.backend().blockchain().info().unwrap().best_number, 0); +} diff --git a/core/primitives/Cargo.toml b/core/primitives/Cargo.toml new file mode 100644 index 00000000000..d14c3a17286 --- /dev/null +++ b/core/primitives/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "substrate-primitives" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +crunchy = "0.1" +substrate-runtime-std = { path = "../runtime-std", default_features = false } +substrate-codec = { path = "../codec", default_features = false } +substrate-codec-derive = { path = "../codec/derive", default_features = false } +elastic-array = {version = "0.10", optional = true } +fixed-hash = { version = "0.2.2", default_features = false } +rustc-hex = { version = "2.0", default_features = false } +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +uint = { version = "0.4.1", default_features = false } +rlp = { version = "0.2.4", optional = true } +twox-hash = { version = "1.1.0", optional = true } +byteorder = { version = "1.1", default_features = false } +wasmi = { version = "0.4", optional = true } +hashdb = { version = "0.2.1", default_features = false } +patricia-trie = { version = "0.2.1", optional = true } +plain_hasher = { version = "0.2", default_features = false } +blake2-rfc = { version = "0.2.18", optional = true } +ring = { version = "0.12", optional = true } +untrusted = { version = "0.5", optional = true } +hex-literal = { version = "0.1", optional = true } +base58 = { version = "0.1", optional = true } + +[dev-dependencies] +substrate-serializer = { path = "../serializer" } +pretty_assertions = "0.4" +heapsize = "0.4" + +[features] +default = ["std"] +std = [ + "wasmi", + "uint/std", + "fixed-hash/std", + "fixed-hash/heapsizeof", + "fixed-hash/libc", + "substrate-codec/std", + "substrate-runtime-std/std", + "serde/std", + "rustc-hex/std", + "twox-hash", + "blake2-rfc", + "ring", + "untrusted", + "hex-literal", + "base58", + "serde_derive", + "byteorder/std", + "patricia-trie", + "rlp", + "elastic-array", +] diff --git a/core/primitives/README.adoc b/core/primitives/README.adoc new file mode 100644 index 00000000000..ed98cf12adf --- /dev/null +++ b/core/primitives/README.adoc @@ -0,0 +1,13 @@ + += Primitives + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/primitives/src/authority_id.rs b/core/primitives/src/authority_id.rs new file mode 100644 index 00000000000..c82261bce58 --- /dev/null +++ b/core/primitives/src/authority_id.rs @@ -0,0 +1,106 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + + +#[cfg(feature = "std")] +use serde::{Serialize, Serializer, Deserialize, Deserializer}; +use H256; + +/// An identifier for an authority in the consensus algorithm. The same size as ed25519::Public. +#[derive(Clone, Copy, PartialEq, Eq, Default, Encode, Decode)] +pub struct AuthorityId(pub [u8; 32]); + +impl AuthorityId { + /// Create an id from a 32-byte slice. Panics with other lengths. + #[cfg(feature = "std")] + fn from_slice(data: &[u8]) -> Self { + let mut r = [0u8; 32]; + r.copy_from_slice(data); + AuthorityId(r) + } +} + +#[cfg(feature = "std")] +impl ::std::fmt::Display for AuthorityId { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + write!(f, "{}", ::hexdisplay::HexDisplay::from(&self.0)) + } +} + +#[cfg(feature = "std")] +impl ::std::fmt::Debug for AuthorityId { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + write!(f, "{}", ::hexdisplay::HexDisplay::from(&self.0)) + } +} + +#[cfg(feature = "std")] +impl ::std::hash::Hash for AuthorityId { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + +impl AsRef<[u8; 32]> for AuthorityId { + fn as_ref(&self) -> &[u8; 32] { + &self.0 + } +} + +impl AsRef<[u8]> for AuthorityId { + fn as_ref(&self) -> &[u8] { + &self.0[..] + } +} + +impl Into<[u8; 32]> for AuthorityId { + fn into(self) -> [u8; 32] { + self.0 + } +} + +impl From<[u8; 32]> for AuthorityId { + fn from(a: [u8; 32]) -> Self { + AuthorityId(a) + } +} + +impl AsRef for AuthorityId { + fn as_ref(&self) -> &AuthorityId { + &self + } +} + +impl Into for AuthorityId { + fn into(self) -> H256 { + self.0.into() + } +} + +#[cfg(feature = "std")] +impl Serialize for AuthorityId { + fn serialize(&self, serializer: S) -> Result where S: Serializer { + ::bytes::serialize(&self.0, serializer) + } +} + +#[cfg(feature = "std")] +impl<'de> Deserialize<'de> for AuthorityId { + fn deserialize(deserializer: D) -> Result where D: Deserializer<'de> { + ::bytes::deserialize_check_len(deserializer, ::bytes::ExpectedLen::Exact(32)) + .map(|x| AuthorityId::from_slice(&x)) + } +} diff --git a/core/primitives/src/bytes.rs b/core/primitives/src/bytes.rs new file mode 100644 index 00000000000..04605bedf8b --- /dev/null +++ b/core/primitives/src/bytes.rs @@ -0,0 +1,158 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Simple type for representing Vec with regards to serde. + +use core::fmt; + +use serde::{de, Serializer, Deserializer}; + +#[cfg(not(feature = "std"))] +mod alloc_types { + pub use ::alloc::string::String; + pub use ::alloc::vec::Vec; +} + +#[cfg(feature = "std")] +mod alloc_types { + pub use ::std::vec::Vec; + pub use ::std::string::String; +} + +pub use self::alloc_types::*; + +/// Serializes a slice of bytes. +pub fn serialize(bytes: &[u8], serializer: S) -> Result where + S: Serializer, +{ + let hex: String = ::rustc_hex::ToHex::to_hex(bytes); + serializer.serialize_str(&format!("0x{}", hex)) +} + +/// Serialize a slice of bytes as uint. +/// +/// The representation will have all leading zeros trimmed. +pub fn serialize_uint(bytes: &[u8], serializer: S) -> Result where + S: Serializer, +{ + let non_zero = bytes.iter().take_while(|b| **b == 0).count(); + let bytes = &bytes[non_zero..]; + if bytes.is_empty() { + return serializer.serialize_str("0x0"); + } + + let hex: String = ::rustc_hex::ToHex::to_hex(bytes); + let has_leading_zero = !hex.is_empty() && &hex[0..1] == "0"; + serializer.serialize_str( + &format!("0x{}", if has_leading_zero { &hex[1..] } else { &hex }) + ) +} + +/// Expected length of bytes vector. +#[derive(PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum ExpectedLen { + /// Any length in bytes. + #[cfg_attr(not(feature = "std"), allow(unused))] + Any, + /// Exact length in bytes. + Exact(usize), + /// A bytes length between (min; max]. + Between(usize, usize), +} + +impl fmt::Display for ExpectedLen { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + match *self { + ExpectedLen::Any => write!(fmt, "even length"), + ExpectedLen::Exact(v) => write!(fmt, "length of {}", v * 2), + ExpectedLen::Between(min, max) => write!(fmt, "length between ({}; {}]", min * 2, max * 2), + } + } +} + +/// Deserialize into vector of bytes. +#[cfg(feature = "std")] +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> where + D: Deserializer<'de>, +{ + deserialize_check_len(deserializer, ExpectedLen::Any) +} + +/// Deserialize into vector of bytes with additional size check. +pub fn deserialize_check_len<'de, D>(deserializer: D, len: ExpectedLen) -> Result, D::Error> where + D: Deserializer<'de>, +{ + struct Visitor { + len: ExpectedLen, + } + + impl<'a> de::Visitor<'a> for Visitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a 0x-prefixed hex string with {}", self.len) + } + + fn visit_str(self, v: &str) -> Result { + if v.len() < 2 || &v[0..2] != "0x" { + return Err(E::custom("prefix is missing")) + } + + let is_len_valid = match self.len { + // just make sure that we have all nibbles + ExpectedLen::Any => v.len() % 2 == 0, + ExpectedLen::Exact(len) => v.len() == 2 * len + 2, + ExpectedLen::Between(min, max) => v.len() <= 2 * max + 2 && v.len() > 2 * min + 2, + }; + + if !is_len_valid { + return Err(E::invalid_length(v.len() - 2, &self)) + } + + let bytes = match self.len { + ExpectedLen::Between(..) if v.len() % 2 != 0 => { + ::rustc_hex::FromHex::from_hex(&*format!("0{}", &v[2..])) + }, + _ => ::rustc_hex::FromHex::from_hex(&v[2..]) + }; + + #[cfg(feature = "std")] + fn format_err(e: ::rustc_hex::FromHexError) -> String { + format!("invalid hex value: {:?}", e) + } + + #[cfg(not(feature = "std"))] + fn format_err(e: ::rustc_hex::FromHexError) -> String { + match e { + ::rustc_hex::InvalidHexLength => format!("invalid hex value: invalid length"), + ::rustc_hex::InvalidHexCharacter(c, p) => + format!("invalid hex value: invalid character {} at position {}", c, p), + } + } + + bytes.map_err(|e| E::custom(format_err(e))) + } + + #[cfg(feature = "std")] + fn visit_string(self, v: String) -> Result { + self.visit_str(&v) + } + } + // TODO [ToDr] Use raw bytes if we switch to RLP / binencoding + // (visit_bytes, visit_bytes_buf) + deserializer.deserialize_str(Visitor { len }) +} diff --git a/core/primitives/src/ed25519.rs b/core/primitives/src/ed25519.rs new file mode 100644 index 00000000000..bc5b691d4d2 --- /dev/null +++ b/core/primitives/src/ed25519.rs @@ -0,0 +1,357 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Simple Ed25519 API. +// end::description[] + +use untrusted; +use blake2_rfc; +use ring::{rand, signature}; +use hash::H512; +use AuthorityId; +use base58::{ToBase58, FromBase58}; + +#[cfg(test)] +#[macro_use] +extern crate hex_literal; + +/// Alias to 512-bit hash when used in the context of a signature on the relay chain. +pub type Signature = H512; + +/// Length of the PKCS#8 encoding of the key. +pub const PKCS_LEN: usize = 85; + +/// A localized signature also contains sender information. +#[derive(PartialEq, Eq, Clone, Debug)] +pub struct LocalizedSignature { + /// The signer of the signature. + pub signer: Public, + /// The signature itself. + pub signature: Signature, +} + +/// Verify a message without type checking the parameters' types for the right size. +pub fn verify>(sig: &[u8], message: &[u8], public: P) -> bool { + let public_key = untrusted::Input::from(public.as_ref()); + let msg = untrusted::Input::from(message); + let sig = untrusted::Input::from(sig); + + match signature::verify(&signature::ED25519, public_key, msg, sig) { + Ok(_) => true, + _ => false, + } +} + +/// A public key. +#[derive(PartialEq, Eq, Clone)] +pub struct Public(pub [u8; 32]); + +/// A key pair. +pub struct Pair(signature::Ed25519KeyPair); + +impl ::std::hash::Hash for Public { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + +/// Error with the SS58 encoding. +#[derive(Clone, Copy, Eq, PartialEq, Debug)] +pub enum PublicError { + /// Bad base-58 encoding. + BadBase58, + /// Bad length of key. + BadLength, + /// Unknown version identifier. + UnknownVersion, + /// Checksum is invalid. + InvalidChecksum, +} + +impl Public { + /// A new instance from the given 32-byte `data`. + pub fn from_raw(data: [u8; 32]) -> Self { + Public(data) + } + + /// A new instance from the given slice that should be 32 bytes long. + pub fn from_slice(data: &[u8]) -> Self { + let mut r = [0u8; 32]; + r.copy_from_slice(data); + Public(r) + } + + /// Some if the string is a properly encoded SS58Check address. + pub fn from_ss58check(s: &str) -> Result { + let d = s.from_base58().map_err(|_| PublicError::BadBase58)?; // failure here would be invalid encoding. + if d.len() != 35 { + // Invalid length. + return Err(PublicError::BadLength); + } + if d[0] != 42 { + // Invalid version. + return Err(PublicError::UnknownVersion); + } + if d[33..35] != blake2_rfc::blake2b::blake2b(64, &[], &d[0..33]).as_bytes()[0..2] { + // Invalid checksum. + return Err(PublicError::InvalidChecksum); + } + Ok(Self::from_slice(&d[1..33])) + } + + /// Return a `Vec` filled with raw data. + pub fn to_raw_vec(self) -> Vec { + let r: &[u8; 32] = self.as_ref(); + r.to_vec() + } + + /// Return a slice filled with raw data. + pub fn as_slice(&self) -> &[u8] { + let r: &[u8; 32] = self.as_ref(); + &r[..] + } + + /// Return a slice filled with raw data. + pub fn as_array_ref(&self) -> &[u8; 32] { + self.as_ref() + } + + /// Return the ss58-check string for this key. + pub fn to_ss58check(&self) -> String { + let mut v = vec![42u8]; + v.extend(self.as_slice()); + let r = blake2_rfc::blake2b::blake2b(64, &[], &v); + v.extend(&r.as_bytes()[0..2]); + v.to_base58() + } +} + +impl AsRef<[u8; 32]> for Public { + fn as_ref(&self) -> &[u8; 32] { + &self.0 + } +} + +impl AsRef<[u8]> for Public { + fn as_ref(&self) -> &[u8] { + &self.0[..] + } +} + +impl Into<[u8; 32]> for Public { + fn into(self) -> [u8; 32] { + self.0 + } +} + +impl AsRef for Public { + fn as_ref(&self) -> &Public { + &self + } +} + +impl AsRef for Pair { + fn as_ref(&self) -> &Pair { + &self + } +} + +impl Into for Public { + fn into(self) -> AuthorityId { + AuthorityId(self.0) + } +} + +impl From for Public { + fn from(id: AuthorityId) -> Self { + Public(id.0) + } +} + +impl ::std::fmt::Display for Public { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + write!(f, "{}", self.to_ss58check()) + } +} + +impl ::std::fmt::Debug for Public { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + let s = self.to_ss58check(); + write!(f, "{} ({}...)", ::hexdisplay::HexDisplay::from(&self.0), &s[0..8]) + } +} + +impl Pair { + /// Generate new secure (random) key pair, yielding it and the corresponding pkcs#8 bytes. + pub fn generate_with_pkcs8() -> (Self, [u8; PKCS_LEN]) { + let rng = rand::SystemRandom::new(); + let pkcs8_bytes = signature::Ed25519KeyPair::generate_pkcs8(&rng).expect("system randomness is available; qed"); + let pair = Self::from_pkcs8(&pkcs8_bytes).expect("just-generated pkcs#8 data is valid; qed"); + + (pair, pkcs8_bytes) + } + + /// Generate new secure (random) key pair. + pub fn generate() -> Pair { + let (pair, _) = Self::generate_with_pkcs8(); + pair + } + + /// Generate from pkcs#8 bytes. + pub fn from_pkcs8(pkcs8_bytes: &[u8]) -> Result { + signature::Ed25519KeyPair::from_pkcs8(untrusted::Input::from(&pkcs8_bytes)).map(Pair) + } + + /// Make a new key pair from a seed phrase. + /// NOTE: prefer pkcs#8 unless security doesn't matter -- this is used primarily for tests. + pub fn from_seed(seed: &[u8; 32]) -> Pair { + let key = signature::Ed25519KeyPair::from_seed_unchecked(untrusted::Input::from(&seed[..])) + .expect("seed has valid length; qed"); + + Pair(key) + } + + /// Sign a message. + pub fn sign(&self, message: &[u8]) -> Signature { + let mut r = [0u8; 64]; + r.copy_from_slice(self.0.sign(message).as_ref()); + Signature::from(r) + } + + /// Get the public key. + pub fn public(&self) -> Public { + let mut r = [0u8; 32]; + let pk = self.0.public_key_bytes(); + r.copy_from_slice(pk); + Public(r) + } + + /// Derive a child key. Probably unsafe and broken. + // TODO: proper HD derivation https://cardanolaunch.com/assets/Ed25519_BIP.pdf + pub fn derive_child_probably_bad(&self, chain_data: &[u8]) -> Pair { + let sig = self.sign(chain_data); + let mut seed = [0u8; 32]; + seed.copy_from_slice(&sig.0[..32]); + + Pair::from_seed(&seed) + } +} + +/// Verify a signature on a message. +pub fn verify_strong>(sig: &Signature, message: &[u8], pubkey: P) -> bool { + let public_key = untrusted::Input::from(&pubkey.as_ref().0[..]); + let msg = untrusted::Input::from(message); + let sig = untrusted::Input::from(&sig.0[..]); + + match signature::verify(&signature::ED25519, public_key, msg, sig) { + Ok(_) => true, + _ => false, + } +} + +/// Something that can verify a message against a given public key. +pub trait Verifiable { + /// Verify something that acts like a signature. + fn verify>(&self, message: &[u8], pubkey: P) -> bool; +} + +impl Verifiable for Signature { + /// Verify something that acts like a signature. + fn verify>(&self, message: &[u8], pubkey: P) -> bool { + verify_strong(&self, message, pubkey) + } +} + +impl Verifiable for LocalizedSignature { + fn verify>(&self, message: &[u8], pubkey: P) -> bool { + pubkey.as_ref() == &self.signer && self.signature.verify(message, pubkey) + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn _test_primitives_signature_and_local_the_same() { + fn takes_two(_: T, _: T) { } + takes_two(Signature::default(), primitives::Signature::default()) + } + + #[test] + fn test_vector_should_work() { + let pair: Pair = Pair::from_seed(&hex!("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60")); + let public = pair.public(); + assert_eq!(public, Public::from_raw(hex!("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a"))); + let message = b""; + let signature: Signature = hex!("e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e065224901555fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b").into(); + assert!(&pair.sign(&message[..]) == &signature); + assert!(verify_strong(&signature, &message[..], &public)); + } + + #[test] + fn generated_pair_should_work() { + let pair = Pair::generate(); + let public = pair.public(); + let message = b"Something important"; + let signature = pair.sign(&message[..]); + assert!(verify_strong(&signature, &message[..], &public)); + } + + #[test] + fn seeded_pair_should_work() { + use primitives::hexdisplay::HexDisplay; + + let pair = Pair::from_seed(b"12345678901234567890123456789012"); + let public = pair.public(); + assert_eq!(public, Public::from_raw(hex!("2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee"))); + let message = hex!("2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee00000000000000000200d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a4500000000000000"); + let signature = pair.sign(&message[..]); + println!("Correct signature: {}", HexDisplay::from(&signature.0)); + assert!(verify_strong(&signature, &message[..], &public)); + } + + #[test] + fn generate_with_pkcs8_recovery_possible() { + let (pair1, pkcs8) = Pair::generate_with_pkcs8(); + let pair2 = Pair::from_pkcs8(&pkcs8).unwrap(); + + assert_eq!(pair1.public(), pair2.public()); + } + + #[test] + fn derive_child() { + let pair = Pair::generate(); + let _pair2 = pair.derive_child_probably_bad(b"session_1234"); + } + + #[test] + fn ss58check_roundtrip_works() { + let pair = Pair::from_seed(b"12345678901234567890123456789012"); + let public = pair.public(); + let s = public.to_ss58check(); + println!("Correct: {}", s); + let cmp = Public::from_ss58check(&s).unwrap(); + assert_eq!(cmp, public); + } + + #[test] + fn ss58check_known_works() { + let k = "5CGavy93sZgPPjHyziRohwVumxiHXMGmQLyuqQP4ZFx5vRU9"; + let enc = hex!["090fa15cb5b1666222fff584b4cc2b1761fe1e238346b340491b37e25ea183ff"]; + assert_eq!(Public::from_ss58check(k).unwrap(), Public::from_raw(enc)); + } +} diff --git a/core/primitives/src/hash.rs b/core/primitives/src/hash.rs new file mode 100644 index 00000000000..c8883da42d3 --- /dev/null +++ b/core/primitives/src/hash.rs @@ -0,0 +1,166 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! A fixed hash type. + +#[cfg(feature = "std")] +use serde::{Serialize, Serializer, Deserialize, Deserializer}; + +#[cfg(feature = "std")] +use bytes; +#[cfg(feature = "std")] +use core::cmp; +#[cfg(feature = "std")] +use rlp::{Rlp, RlpStream, DecoderError}; + +macro_rules! impl_rest { + ($name: ident, $len: expr) => { + #[cfg(feature = "std")] + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result where S: Serializer { + bytes::serialize(&self.0, serializer) + } + } + + #[cfg(feature = "std")] + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result where D: Deserializer<'de> { + bytes::deserialize_check_len(deserializer, bytes::ExpectedLen::Exact($len)) + .map(|x| (&*x).into()) + } + } + + impl ::codec::Encode for $name { + fn using_encoded R>(&self, f: F) -> R { + self.0.using_encoded(f) + } + } + impl ::codec::Decode for $name { + fn decode(input: &mut I) -> Option { + <[u8; $len] as ::codec::Decode>::decode(input).map($name) + } + } + + #[cfg(feature = "std")] + impl ::rlp::Encodable for $name { + fn rlp_append(&self, s: &mut RlpStream) { + s.encoder().encode_value(self); + } + } + + #[cfg(feature = "std")] + impl ::rlp::Decodable for $name { + fn decode(rlp: &Rlp) -> Result { + rlp.decoder().decode_value(|bytes| match bytes.len().cmp(&$len) { + cmp::Ordering::Less => Err(DecoderError::RlpIsTooShort), + cmp::Ordering::Greater => Err(DecoderError::RlpIsTooBig), + cmp::Ordering::Equal => { + let mut t = [0u8; $len]; + t.copy_from_slice(bytes); + Ok($name(t)) + } + }) + } + } + + } +} + +construct_hash!(H160, 20); +construct_hash!(H256, 32); +construct_hash!(H512, 64); +impl_rest!(H160, 20); +impl_rest!(H256, 32); +impl_rest!(H512, 64); + +#[cfg(test)] +mod tests { + use super::*; + use substrate_serializer as ser; + use rlp::{Encodable, RlpStream}; + + #[test] + fn test_hash_is_encodable() { + let h = H160::from(21); + let mut s = RlpStream::new(); + h.rlp_append(&mut s); + assert_eq!(s.drain().into_vec(), &[148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21]); + } + + #[test] + fn test_hash_is_decodable() { + let data = vec![148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123]; + let res = ::rlp::decode::(&data); + assert!(res.is_ok()); + assert_eq!(res.unwrap(), H160::from(123)); + + let res = ::rlp::decode::(&data); + assert!(res.is_err()); + } + + #[test] + fn test_h160() { + let tests = vec![ + (Default::default(), "0x0000000000000000000000000000000000000000"), + (H160::from(2), "0x0000000000000000000000000000000000000002"), + (H160::from(15), "0x000000000000000000000000000000000000000f"), + (H160::from(16), "0x0000000000000000000000000000000000000010"), + (H160::from(1_000), "0x00000000000000000000000000000000000003e8"), + (H160::from(100_000), "0x00000000000000000000000000000000000186a0"), + (H160::from(u64::max_value()), "0x000000000000000000000000ffffffffffffffff"), + ]; + + for (number, expected) in tests { + assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number)); + assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap()); + } + } + + #[test] + fn test_h256() { + let tests = vec![ + (Default::default(), "0x0000000000000000000000000000000000000000000000000000000000000000"), + (H256::from(2), "0x0000000000000000000000000000000000000000000000000000000000000002"), + (H256::from(15), "0x000000000000000000000000000000000000000000000000000000000000000f"), + (H256::from(16), "0x0000000000000000000000000000000000000000000000000000000000000010"), + (H256::from(1_000), "0x00000000000000000000000000000000000000000000000000000000000003e8"), + (H256::from(100_000), "0x00000000000000000000000000000000000000000000000000000000000186a0"), + (H256::from(u64::max_value()), "0x000000000000000000000000000000000000000000000000ffffffffffffffff"), + ]; + + for (number, expected) in tests { + assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number)); + assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap()); + } + } + + #[test] + fn test_invalid() { + assert!(ser::from_str::("\"0x000000000000000000000000000000000000000000000000000000000000000\"").unwrap_err().is_data()); + assert!(ser::from_str::("\"0x000000000000000000000000000000000000000000000000000000000000000g\"").unwrap_err().is_data()); + assert!(ser::from_str::("\"0x00000000000000000000000000000000000000000000000000000000000000000\"").unwrap_err().is_data()); + assert!(ser::from_str::("\"\"").unwrap_err().is_data()); + assert!(ser::from_str::("\"0\"").unwrap_err().is_data()); + assert!(ser::from_str::("\"10\"").unwrap_err().is_data()); + } + + #[test] + fn test_heapsizeof() { + use heapsize::HeapSizeOf; + let h = H256::new(); + assert_eq!(h.heap_size_of_children(), 0); + } +} diff --git a/core/primitives/src/hasher.rs b/core/primitives/src/hasher.rs new file mode 100644 index 00000000000..d4dd39b8554 --- /dev/null +++ b/core/primitives/src/hasher.rs @@ -0,0 +1,53 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot Blake2b Hasher implementation + +use hashdb::Hasher; +use plain_hasher::PlainHasher; +use hash::H256; + +pub mod blake2 { + use super::{Hasher, PlainHasher, H256}; + #[cfg(feature = "std")] + use hashing::blake2_256; + + #[cfg(not(feature = "std"))] + extern "C" { + fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8); + } + #[cfg(not(feature = "std"))] + fn blake2_256(data: &[u8]) -> [u8; 32] { + let mut result: [u8; 32] = Default::default(); + unsafe { + ext_blake2_256(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); + } + result + } + + /// Concrete implementation of Hasher using Blake2b 256-bit hashes + #[derive(Debug)] + pub struct Blake2Hasher; + + impl Hasher for Blake2Hasher { + type Out = H256; + type StdHasher = PlainHasher; + const LENGTH:usize = 32; + fn hash(x: &[u8]) -> Self::Out { + blake2_256(x).into() + } + } +} diff --git a/core/primitives/src/hashing.rs b/core/primitives/src/hashing.rs new file mode 100644 index 00000000000..379ea4a56f4 --- /dev/null +++ b/core/primitives/src/hashing.rs @@ -0,0 +1,106 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Hashing functions. + +use blake2_rfc; +use twox_hash; + +/// Do a Blake2 512-bit hash and place result in `dest`. +pub fn blake2_512_into(data: &[u8], dest: &mut [u8; 64]) { + dest.copy_from_slice(blake2_rfc::blake2b::blake2b(64, &[], data).as_bytes()); +} + +/// Do a Blake2 512-bit hash and return result. +pub fn blake2_512(data: &[u8]) -> [u8; 64] { + let mut r = [0; 64]; + blake2_512_into(data, &mut r); + r +} + +/// Do a Blake2 256-bit hash and place result in `dest`. +pub fn blake2_256_into(data: &[u8], dest: &mut [u8; 32]) { + dest.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], data).as_bytes()); +} + +/// Do a Blake2 256-bit hash and return result. +pub fn blake2_256(data: &[u8]) -> [u8; 32] { + let mut r = [0; 32]; + blake2_256_into(data, &mut r); + r +} + +/// Do a Blake2 128-bit hash and place result in `dest`. +pub fn blake2_128_into(data: &[u8], dest: &mut [u8; 16]) { + dest.copy_from_slice(blake2_rfc::blake2b::blake2b(16, &[], data).as_bytes()); +} + +/// Do a Blake2 128-bit hash and return result. +pub fn blake2_128(data: &[u8]) -> [u8; 16] { + let mut r = [0; 16]; + blake2_128_into(data, &mut r); + r +} + +/// Do a XX 128-bit hash and place result in `dest`. +pub fn twox_128_into(data: &[u8], dest: &mut [u8; 16]) { + use ::core::hash::Hasher; + let mut h0 = twox_hash::XxHash::with_seed(0); + let mut h1 = twox_hash::XxHash::with_seed(1); + h0.write(data); + h1.write(data); + let r0 = h0.finish(); + let r1 = h1.finish(); + use byteorder::{ByteOrder, LittleEndian}; + LittleEndian::write_u64(&mut dest[0..8], r0); + LittleEndian::write_u64(&mut dest[8..16], r1); +} + +/// Do a XX 128-bit hash and return result. +pub fn twox_128(data: &[u8]) -> [u8; 16] { + let mut r: [u8; 16] = [0; 16]; + twox_128_into(data, &mut r); + r +} + +/// Do a XX 256-bit hash and place result in `dest`. +pub fn twox_256_into(data: &[u8], dest: &mut [u8; 32]) { + use ::core::hash::Hasher; + use byteorder::{ByteOrder, LittleEndian}; + let mut h0 = twox_hash::XxHash::with_seed(0); + let mut h1 = twox_hash::XxHash::with_seed(1); + let mut h2 = twox_hash::XxHash::with_seed(2); + let mut h3 = twox_hash::XxHash::with_seed(3); + h0.write(data); + h1.write(data); + h2.write(data); + h3.write(data); + let r0 = h0.finish(); + let r1 = h1.finish(); + let r2 = h2.finish(); + let r3 = h3.finish(); + LittleEndian::write_u64(&mut dest[0..8], r0); + LittleEndian::write_u64(&mut dest[8..16], r1); + LittleEndian::write_u64(&mut dest[16..24], r2); + LittleEndian::write_u64(&mut dest[24..32], r3); +} + +/// Do a XX 256-bit hash and return result. +pub fn twox_256(data: &[u8]) -> [u8; 32] { + let mut r: [u8; 32] = [0; 32]; + twox_256_into(data, &mut r); + r +} diff --git a/core/primitives/src/hexdisplay.rs b/core/primitives/src/hexdisplay.rs new file mode 100644 index 00000000000..938d3945c5a --- /dev/null +++ b/core/primitives/src/hexdisplay.rs @@ -0,0 +1,94 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Wrapper type for byte collections that outputs hex. + +/// Simple wrapper to display hex representation of bytes. +pub struct HexDisplay<'a>(&'a [u8]); + +impl<'a> HexDisplay<'a> { + /// Create new instance that will display `d` as a hex string when displayed. + pub fn from(d: &'a AsBytesRef) -> Self { HexDisplay(d.as_bytes_ref()) } +} + +impl<'a> ::core::fmt::Display for HexDisplay<'a> { + fn fmt(&self, fmtr: &mut ::core::fmt::Formatter) -> Result<(), ::core::fmt::Error> { + if self.0.len() < 1027 { + for byte in self.0 { + fmtr.write_fmt(format_args!("{:02x}", byte))?; + } + } else { + for byte in &self.0[0..512] { + fmtr.write_fmt(format_args!("{:02x}", byte))?; + } + fmtr.write_str("...")?; + for byte in &self.0[self.0.len() - 512..] { + fmtr.write_fmt(format_args!("{:02x}", byte))?; + } + } + Ok(()) + } +} + +/// Simple trait to transform various types to `&[u8]` +pub trait AsBytesRef { + /// Transform `self` into `&[u8]`. + fn as_bytes_ref(&self) -> &[u8]; +} + +impl<'a> AsBytesRef for &'a [u8] { + fn as_bytes_ref(&self) -> &[u8] { self } +} + +impl AsBytesRef for [u8] { + fn as_bytes_ref(&self) -> &[u8] { &self } +} + +impl AsBytesRef for ::bytes::Vec { + fn as_bytes_ref(&self) -> &[u8] { &self } +} + +macro_rules! impl_non_endians { + ( $( $t:ty ),* ) => { $( + impl AsBytesRef for $t { + fn as_bytes_ref(&self) -> &[u8] { &self[..] } + } + )* } +} + +impl_non_endians!([u8; 1], [u8; 2], [u8; 3], [u8; 4], [u8; 5], [u8; 6], [u8; 7], [u8; 8], + [u8; 10], [u8; 12], [u8; 14], [u8; 16], [u8; 20], [u8; 24], [u8; 28], [u8; 32], [u8; 40], + [u8; 48], [u8; 56], [u8; 64], [u8; 80], [u8; 96], [u8; 112], [u8; 128]); + +/// Format into ASCII + # + hex, suitable for storage key preimages. +pub fn ascii_format(asciish: &[u8]) -> String { + let mut r = String::new(); + let mut latch = false; + for c in asciish { + match (latch, *c) { + (false, 32...127) => r.push(*c as char), + _ => { + if !latch { + r.push('#'); + latch = true; + } + r.push_str(&format!("{:02x}", *c)); + } + } + } + r +} + diff --git a/core/primitives/src/lib.rs b/core/primitives/src/lib.rs new file mode 100644 index 00000000000..e61a9627041 --- /dev/null +++ b/core/primitives/src/lib.rs @@ -0,0 +1,142 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Shareable Polkadot types. +// end::description[] + +#![warn(missing_docs)] + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +#[macro_use] +extern crate crunchy; +#[macro_use] +extern crate fixed_hash; +#[macro_use] +extern crate uint as uint_crate; +#[macro_use] +extern crate substrate_codec_derive; + +extern crate rustc_hex; +extern crate byteorder; +extern crate substrate_codec as codec; +#[cfg(feature = "std")] +extern crate rlp; + +#[cfg(feature = "std")] +extern crate serde; +#[cfg(feature = "std")] +extern crate twox_hash; + +#[cfg(feature = "std")] +extern crate ring; +#[cfg(feature = "std")] +extern crate base58; +#[cfg(feature = "std")] +extern crate untrusted; +#[cfg(feature = "std")] +extern crate blake2_rfc; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; +#[cfg(feature = "std")] +extern crate core; +#[cfg(feature = "std")] +extern crate wasmi; +extern crate hashdb; +extern crate plain_hasher; +#[cfg(feature = "std")] +extern crate patricia_trie; +#[cfg(feature = "std")] +extern crate elastic_array; + +extern crate substrate_runtime_std as rstd; + +#[cfg(test)] +extern crate substrate_serializer; + +#[cfg(test)] +extern crate heapsize; + +#[cfg(test)] +#[macro_use] +extern crate pretty_assertions; + +#[macro_export] +macro_rules! map { + ($( $name:expr => $value:expr ),*) => ( + vec![ $( ( $name, $value ) ),* ].into_iter().collect() + ) +} + +use rstd::prelude::*; +use rstd::ops::Deref; + +#[cfg(feature = "std")] +pub mod bytes; +#[cfg(feature = "std")] +pub mod hashing; +#[cfg(feature = "std")] +pub use hashing::{blake2_256, twox_128, twox_256}; +#[cfg(feature = "std")] +pub mod hexdisplay; + +#[cfg(feature = "std")] +pub mod ed25519; +pub mod u32_trait; + +pub mod hash; +mod hasher; +pub mod sandbox; +pub mod storage; +pub mod uint; +mod authority_id; +#[cfg(feature = "std")] +mod rlp_codec; + +#[cfg(test)] +mod tests; + +pub use self::hash::{H160, H256, H512}; +pub use self::uint::U256; +pub use authority_id::AuthorityId; + +// Switch back to Blake after PoC-3 is out +// pub use self::hasher::blake::BlakeHasher; +pub use self::hasher::blake2::Blake2Hasher; + +#[cfg(feature = "std")] +pub use self::rlp_codec::RlpCodec; + +/// A 512-bit value interpreted as a signature. +pub type Signature = hash::H512; + +/// Hex-serialised shim for `Vec`. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Hash, PartialOrd, Ord))] +pub struct Bytes(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec); + +impl From> for Bytes { + fn from(s: Vec) -> Self { Bytes(s) } +} + +impl Deref for Bytes { + type Target = [u8]; + fn deref(&self) -> &[u8] { &self.0[..] } +} diff --git a/core/primitives/src/rlp_codec.rs b/core/primitives/src/rlp_codec.rs new file mode 100644 index 00000000000..bf1ae978b3d --- /dev/null +++ b/core/primitives/src/rlp_codec.rs @@ -0,0 +1,123 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot Blake2b (trie) NodeCodec implementation + +use elastic_array::{ElasticArray1024, ElasticArray128}; +use hashdb::Hasher; +use rlp::{DecoderError, RlpStream, Rlp, Prototype}; +use core::marker::PhantomData; +use patricia_trie::{NibbleSlice, NodeCodec, node::Node, ChildReference}; + +use hash::H256; +use Blake2Hasher; + +/// Concrete implementation of a `NodeCodec` with Rlp encoding, generic over the `Hasher` +pub struct RlpNodeCodec {mark: PhantomData} + +/// Convenience type for a Blake2_256/Rlp flavoured NodeCodec +pub type RlpCodec = RlpNodeCodec; + +impl NodeCodec for RlpCodec { + type Error = DecoderError; + const HASHED_NULL_NODE : H256 = H256( [0x45, 0xb0, 0xcf, 0xc2, 0x20, 0xce, 0xec, 0x5b, 0x7c, 0x1c, 0x62, 0xc4, 0xd4, 0x19, 0x3d, 0x38, 0xe4, 0xeb, 0xa4, 0x8e, 0x88, 0x15, 0x72, 0x9c, 0xe7, 0x5f, 0x9c, 0xa, 0xb0, 0xe4, 0xc1, 0xc0] ); + fn decode(data: &[u8]) -> ::std::result::Result { + let r = Rlp::new(data); + match r.prototype()? { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { + (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), + (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes = [&[] as &[u8]; 16]; + for i in 0..16 { + nodes[i] = r.at(i)?.as_raw(); + } + Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) + }, + // an empty branch index. + Prototype::Data(0) => Ok(Node::Empty), + // something went wrong. + _ => Err(DecoderError::Custom("Rlp is not valid.")) + } + } + fn try_decode_hash(data: &[u8]) -> Option<::Out> { + let r = Rlp::new(data); + if r.is_data() && r.size() == Blake2Hasher::LENGTH { + Some(r.as_val().expect("Hash is the correct size; qed")) + } else { + None + } + } + fn is_empty_node(data: &[u8]) -> bool { + Rlp::new(data).is_empty() + } + fn empty_node() -> ElasticArray1024 { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.drain() + } + + fn leaf_node(partial: &[u8], value: &[u8]) -> ElasticArray1024 { + let mut stream = RlpStream::new_list(2); + stream.append(&partial); + stream.append(&value); + stream.drain() + } + + fn ext_node(partial: &[u8], child_ref: ChildReference<::Out>) -> ElasticArray1024 { + let mut stream = RlpStream::new_list(2); + stream.append(&partial); + match child_ref { + ChildReference::Hash(h) => stream.append(&h), + ChildReference::Inline(inline_data, len) => { + let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; + stream.append_raw(bytes, 1) + }, + }; + stream.drain() + } + + fn branch_node(children: I, value: Option>) -> ElasticArray1024 + where I: IntoIterator::Out>>> + { + let mut stream = RlpStream::new_list(17); + for child_ref in children { + match child_ref { + Some(c) => match c { + ChildReference::Hash(h) => stream.append(&h), + ChildReference::Inline(inline_data, len) => { + let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; + stream.append_raw(bytes, 1) + }, + }, + None => stream.append_empty_data() + }; + } + if let Some(value) = value { + stream.append(&&*value); + } else { + stream.append_empty_data(); + } + stream.drain() + } +} diff --git a/core/primitives/src/sandbox.rs b/core/primitives/src/sandbox.rs new file mode 100644 index 00000000000..2e3144b24ff --- /dev/null +++ b/core/primitives/src/sandbox.rs @@ -0,0 +1,221 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Definition of a sandbox environment. + +#[cfg(test)] +use codec::Encode; +use rstd::vec::Vec; + +/// Error error that can be returned from host function. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct HostError; + +/// Representation of a typed wasm value. +#[derive(Clone, Copy, PartialEq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum TypedValue { + /// Value of 32-bit signed or unsigned integer. + #[codec(index = "1")] + I32(i32), + + /// Value of 64-bit signed or unsigned integer. + #[codec(index = "2")] + I64(i64), + + /// Value of 32-bit IEEE 754-2008 floating point number represented as a bit pattern. + #[codec(index = "3")] + F32(i32), + + /// Value of 64-bit IEEE 754-2008 floating point number represented as a bit pattern. + #[codec(index = "4")] + F64(i64), +} + +impl TypedValue { + /// Returns `Some` if this value of type `I32`. + pub fn as_i32(&self) -> Option { + match *self { + TypedValue::I32(v) => Some(v), + _ => None, + } + } +} + +#[cfg(feature = "std")] +impl From<::wasmi::RuntimeValue> for TypedValue { + fn from(val: ::wasmi::RuntimeValue) -> TypedValue { + use ::wasmi::RuntimeValue; + match val { + RuntimeValue::I32(v) => TypedValue::I32(v), + RuntimeValue::I64(v) => TypedValue::I64(v), + RuntimeValue::F32(v) => TypedValue::F32(v.to_bits() as i32), + RuntimeValue::F64(v) => TypedValue::F64(v.to_bits() as i64), + } + } +} + +#[cfg(feature = "std")] +impl From for ::wasmi::RuntimeValue { + fn from(val: TypedValue) -> ::wasmi::RuntimeValue { + use ::wasmi::RuntimeValue; + use ::wasmi::nan_preserving_float::{F32, F64}; + match val { + TypedValue::I32(v) => RuntimeValue::I32(v), + TypedValue::I64(v) => RuntimeValue::I64(v), + TypedValue::F32(v_bits) => RuntimeValue::F32(F32::from_bits(v_bits as u32)), + TypedValue::F64(v_bits) => RuntimeValue::F64(F64::from_bits(v_bits as u64)), + } + } +} + +/// Typed value that can be returned from a function. +/// +/// Basically a `TypedValue` plus `Unit`, for functions which return nothing. +#[derive(Clone, Copy, PartialEq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum ReturnValue { + /// For returning nothing. + Unit, + /// For returning some concrete value. + Value(TypedValue), +} + +impl From for ReturnValue { + fn from(v: TypedValue) -> ReturnValue { + ReturnValue::Value(v) + } +} + +impl ReturnValue { + /// Maximum number of bytes `ReturnValue` might occupy when serialized with + /// `Codec`. + /// + /// Breakdown: + /// 1 byte for encoding unit/value variant + /// 1 byte for encoding value type + /// 8 bytes for encoding the biggest value types available in wasm: f64, i64. + pub const ENCODED_MAX_SIZE: usize = 10; +} + +#[test] +fn return_value_encoded_max_size() { + let encoded = ReturnValue::Value(TypedValue::I64(-1)).encode(); + assert_eq!(encoded.len(), ReturnValue::ENCODED_MAX_SIZE); +} + +/// Describes an entity to define or import into the environment. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum ExternEntity { + /// Function that is specified by an index in a default table of + /// a module that creates the sandbox. + #[codec(index = "1")] + Function(u32), + + /// Linear memory that is specified by some identifier returned by sandbox + /// module upon creation new sandboxed memory. + #[codec(index = "2")] + Memory(u32), +} + +/// An entry in a environment definition table. +/// +/// Each entry has a two-level name and description of an entity +/// being defined. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Entry { + /// Module name of which corresponding entity being defined. + pub module_name: Vec, + /// Field name in which corresponding entity being defined. + pub field_name: Vec, + /// External entity being defined. + pub entity: ExternEntity, +} + +/// Definition of runtime that could be used by sandboxed code. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct EnvironmentDefinition { + /// Vector of all entries in the environment definition. + pub entries: Vec, +} + +/// Constant for specifying no limit when creating a sandboxed +/// memory instance. For FFI purposes. +pub const MEM_UNLIMITED: u32 = -1i32 as u32; + +/// No error happened. +/// +/// For FFI purposes. +pub const ERR_OK: u32 = 0; + +/// Validation or instantiation error occured when creating new +/// sandboxed module instance. +/// +/// For FFI purposes. +pub const ERR_MODULE: u32 = -1i32 as u32; + +/// Out-of-bounds access attempted with memory or table. +/// +/// For FFI purposes. +pub const ERR_OUT_OF_BOUNDS: u32 = -2i32 as u32; + +/// Execution error occurred (typically trap). +/// +/// For FFI purposes. +pub const ERR_EXECUTION: u32 = -3i32 as u32; + +#[cfg(test)] +mod tests { + use super::*; + use std::fmt; + use codec::Codec; + + fn roundtrip(s: S) { + let encoded = s.encode(); + assert_eq!(S::decode(&mut &encoded[..]).unwrap(), s); + } + + #[test] + fn env_def_roundtrip() { + roundtrip(EnvironmentDefinition { + entries: vec![], + }); + + roundtrip(EnvironmentDefinition { + entries: vec![ + Entry { + module_name: b"kernel"[..].into(), + field_name: b"memory"[..].into(), + entity: ExternEntity::Memory(1337), + }, + ], + }); + + roundtrip(EnvironmentDefinition { + entries: vec![ + Entry { + module_name: b"env"[..].into(), + field_name: b"abort"[..].into(), + entity: ExternEntity::Function(228), + }, + ], + }); + } +} diff --git a/core/primitives/src/storage.rs b/core/primitives/src/storage.rs new file mode 100644 index 00000000000..55d62f6a813 --- /dev/null +++ b/core/primitives/src/storage.rs @@ -0,0 +1,44 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Contract execution data. + +#[cfg(feature = "std")] +use bytes; +use rstd::vec::Vec; + +/// Contract storage key. +#[derive(PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Hash, PartialOrd, Ord, Clone))] +pub struct StorageKey(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec); + +/// Contract storage entry data. +#[derive(PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Hash, PartialOrd, Ord, Clone))] +pub struct StorageData(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec); + +/// Storage change set +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, PartialEq, Eq))] +pub struct StorageChangeSet { + /// Block hash + pub block: Hash, + /// A list of changes + pub changes: Vec<( + StorageKey, + Option, + )>, +} + diff --git a/core/primitives/src/tests.rs b/core/primitives/src/tests.rs new file mode 100644 index 00000000000..c6501043945 --- /dev/null +++ b/core/primitives/src/tests.rs @@ -0,0 +1,17 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tests. diff --git a/core/primitives/src/u32_trait.rs b/core/primitives/src/u32_trait.rs new file mode 100644 index 00000000000..5fba6f8e04d --- /dev/null +++ b/core/primitives/src/u32_trait.rs @@ -0,0 +1,89 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! An u32 trait with "values" as impl'd types. + +/// A u32 value, wrapped in a trait because we don't yet have const generics. +pub trait Value { + /// The actual value represented by the impl'ing type. + const VALUE: u32; +} +/// Type representing the value 0 for the `Value` trait. +pub struct _0; impl Value for _0 { const VALUE: u32 = 0; } +/// Type representing the value 1 for the `Value` trait. +pub struct _1; impl Value for _1 { const VALUE: u32 = 1; } +/// Type representing the value 2 for the `Value` trait. +pub struct _2; impl Value for _2 { const VALUE: u32 = 2; } +/// Type representing the value 3 for the `Value` trait. +pub struct _3; impl Value for _3 { const VALUE: u32 = 3; } +/// Type representing the value 4 for the `Value` trait. +pub struct _4; impl Value for _4 { const VALUE: u32 = 4; } +/// Type representing the value 5 for the `Value` trait. +pub struct _5; impl Value for _5 { const VALUE: u32 = 5; } +/// Type representing the value 6 for the `Value` trait. +pub struct _6; impl Value for _6 { const VALUE: u32 = 6; } +/// Type representing the value 7 for the `Value` trait. +pub struct _7; impl Value for _7 { const VALUE: u32 = 7; } +/// Type representing the value 8 for the `Value` trait. +pub struct _8; impl Value for _8 { const VALUE: u32 = 8; } +/// Type representing the value 9 for the `Value` trait. +pub struct _9; impl Value for _9 { const VALUE: u32 = 9; } +/// Type representing the value 10 for the `Value` trait. +pub struct _10; impl Value for _10 { const VALUE: u32 = 10; } +/// Type representing the value 11 for the `Value` trait. +pub struct _11; impl Value for _11 { const VALUE: u32 = 11; } +/// Type representing the value 12 for the `Value` trait. +pub struct _12; impl Value for _12 { const VALUE: u32 = 12; } +/// Type representing the value 13 for the `Value` trait. +pub struct _13; impl Value for _13 { const VALUE: u32 = 13; } +/// Type representing the value 14 for the `Value` trait. +pub struct _14; impl Value for _14 { const VALUE: u32 = 14; } +/// Type representing the value 15 for the `Value` trait. +pub struct _15; impl Value for _15 { const VALUE: u32 = 15; } +/// Type representing the value 16 for the `Value` trait. +pub struct _16; impl Value for _16 { const VALUE: u32 = 16; } +/// Type representing the value 24 for the `Value` trait. +pub struct _24; impl Value for _24 { const VALUE: u32 = 24; } +/// Type representing the value 32 for the `Value` trait. +pub struct _32; impl Value for _32 { const VALUE: u32 = 32; } +/// Type representing the value 40 for the `Value` trait. +pub struct _40; impl Value for _40 { const VALUE: u32 = 40; } +/// Type representing the value 48 for the `Value` trait. +pub struct _48; impl Value for _48 { const VALUE: u32 = 48; } +/// Type representing the value 56 for the `Value` trait. +pub struct _56; impl Value for _56 { const VALUE: u32 = 56; } +/// Type representing the value 64 for the `Value` trait. +pub struct _64; impl Value for _64 { const VALUE: u32 = 64; } +/// Type representing the value 80 for the `Value` trait. +pub struct _80; impl Value for _80 { const VALUE: u32 = 80; } +/// Type representing the value 96 for the `Value` trait. +pub struct _96; impl Value for _96 { const VALUE: u32 = 96; } +/// Type representing the value 112 for the `Value` trait. +pub struct _112; impl Value for _112 { const VALUE: u32 = 112; } +/// Type representing the value 128 for the `Value` trait. +pub struct _128; impl Value for _128 { const VALUE: u32 = 128; } +/// Type representing the value 160 for the `Value` trait. +pub struct _160; impl Value for _160 { const VALUE: u32 = 160; } +/// Type representing the value 192 for the `Value` trait. +pub struct _192; impl Value for _192 { const VALUE: u32 = 192; } +/// Type representing the value 224 for the `Value` trait. +pub struct _224; impl Value for _224 { const VALUE: u32 = 224; } +/// Type representing the value 256 for the `Value` trait. +pub struct _256; impl Value for _256 { const VALUE: u32 = 256; } +/// Type representing the value 384 for the `Value` trait. +pub struct _384; impl Value for _384 { const VALUE: u32 = 384; } +/// Type representing the value 512 for the `Value` trait. +pub struct _512; impl Value for _512 { const VALUE: u32 = 512; } diff --git a/core/primitives/src/uint.rs b/core/primitives/src/uint.rs new file mode 100644 index 00000000000..af3278ef018 --- /dev/null +++ b/core/primitives/src/uint.rs @@ -0,0 +1,99 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! An unsigned fixed-size integer. + +#[cfg(feature = "std")] +use serde::{Serialize, Serializer, Deserialize, Deserializer}; + +#[cfg(feature = "std")] +use bytes; + +macro_rules! impl_serde { + ($name: ident, $len: expr) => { + #[cfg(feature = "std")] + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result where S: Serializer { + let mut bytes = [0u8; $len * 8]; + self.to_big_endian(&mut bytes); + bytes::serialize_uint(&bytes, serializer) + } + } + + #[cfg(feature = "std")] + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result where D: Deserializer<'de> { + bytes::deserialize_check_len(deserializer, bytes::ExpectedLen::Between(0, $len * 8)) + .map(|x| (&*x).into()) + } + } + } +} + +construct_uint!(U256, 4); +impl_serde!(U256, 4); + +#[cfg(test)] +mod tests { + use super::*; + use substrate_serializer as ser; + + macro_rules! test { + ($name: ident, $test_name: ident) => { + #[test] + fn $test_name() { + let tests = vec![ + ($name::from(0), "0x0"), + ($name::from(1), "0x1"), + ($name::from(2), "0x2"), + ($name::from(10), "0xa"), + ($name::from(15), "0xf"), + ($name::from(15), "0xf"), + ($name::from(16), "0x10"), + ($name::from(1_000), "0x3e8"), + ($name::from(100_000), "0x186a0"), + ($name::from(u64::max_value()), "0xffffffffffffffff"), + ($name::from(u64::max_value()) + $name::from(1), "0x10000000000000000"), + ]; + + for (number, expected) in tests { + assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number)); + assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap()); + } + + // Invalid examples + assert!(ser::from_str::<$name>("\"0x\"").unwrap_err().is_data()); + assert!(ser::from_str::<$name>("\"0xg\"").unwrap_err().is_data()); + assert!(ser::from_str::<$name>("\"\"").unwrap_err().is_data()); + assert!(ser::from_str::<$name>("\"10\"").unwrap_err().is_data()); + assert!(ser::from_str::<$name>("\"0\"").unwrap_err().is_data()); + } + } + } + + test!(U256, test_u256); + + #[test] + fn test_large_values() { + assert_eq!( + ser::to_string_pretty(&!U256::zero()), + "\"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"" + ); + assert!( + ser::from_str::("\"0x1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"").unwrap_err().is_data() + ); + } +} diff --git a/core/pwasm-alloc/Cargo.lock b/core/pwasm-alloc/Cargo.lock new file mode 100644 index 00000000000..56aacd92e42 --- /dev/null +++ b/core/pwasm-alloc/Cargo.lock @@ -0,0 +1,37 @@ +[[package]] +name = "pwasm-alloc" +version = "0.1.0" +dependencies = [ + "pwasm-libc 0.1.0", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pwasm-libc" +version = "0.1.0" + +[[package]] +name = "rustc_version" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a54aa04a10c68c1c4eacb4337fd883b435997ede17a9385784b990777686b09a" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" diff --git a/core/pwasm-alloc/Cargo.toml b/core/pwasm-alloc/Cargo.toml new file mode 100644 index 00000000000..d5dbe87e6ad --- /dev/null +++ b/core/pwasm-alloc/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "pwasm-alloc" +version = "0.1.0" +authors = ["Nikolay Volf "] +license = "MIT/Apache-2.0" +readme = "README.md" +repository = "https://github.com/paritytech/pwasm-std" +homepage = "https://github.com/paritytech/pwasm-std" +documentation = "https://paritytech.github.io/pwasm-std/pwasm_std/" +description = "Parity WebAssembly standard library internal allocator" +keywords = ["wasm", "parity", "webassembly", "blockchain"] +categories = ["no-std", "embedded"] +build = "build.rs" + +[dependencies] +pwasm-libc = { path = "../pwasm-libc", version = "0.1" } + +[build-dependencies] +rustc_version = "0.2" + +[features] +strict = [] +nightly = [] diff --git a/core/pwasm-alloc/README.adoc b/core/pwasm-alloc/README.adoc new file mode 100644 index 00000000000..5ddc97ea1b9 --- /dev/null +++ b/core/pwasm-alloc/README.adoc @@ -0,0 +1,25 @@ + += Pwasm-alloc + +Parity WASM contracts standard library libc bindings. + +See https://paritytech.github.io/pwasm-std/pwasm_alloc/ for the documentation. + +== License + +`pwasm_alloc` is primarily distributed under the terms of both the MIT +license and the Apache License (Version 2.0), at your choice. + +See LICENSE-APACHE, and LICENSE-MIT for details. + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- + diff --git a/core/pwasm-alloc/build.rs b/core/pwasm-alloc/build.rs new file mode 100644 index 00000000000..35eb154f3a6 --- /dev/null +++ b/core/pwasm-alloc/build.rs @@ -0,0 +1,14 @@ +//! Set a nightly feature + +extern crate rustc_version; +use rustc_version::{version, version_meta, Channel}; + +fn main() { + // Assert we haven't travelled back in time + assert!(version().unwrap().major >= 1); + + // Set cfg flags depending on release channel + if let Channel::Nightly = version_meta().unwrap().channel { + println!("cargo:rustc-cfg=feature=\"nightly\""); + } +} diff --git a/core/pwasm-alloc/src/lib.rs b/core/pwasm-alloc/src/lib.rs new file mode 100644 index 00000000000..b08eef68548 --- /dev/null +++ b/core/pwasm-alloc/src/lib.rs @@ -0,0 +1,34 @@ +#![warn(missing_docs)] +#![cfg_attr(feature = "strict", deny(warnings))] +#![no_std] +#![crate_type = "rlib"] + +// tag::description[] +//! Custom allocator crate for wasm +// end::description[] + +/// Wasm allocator +pub struct WasmAllocator; + +#[cfg(feature = "nightly")] +#[global_allocator] +static ALLOCATOR: WasmAllocator = WasmAllocator; + +#[cfg(feature = "nightly")] +mod __impl { + extern crate pwasm_libc; + + use core::alloc::{GlobalAlloc, Layout}; + + use super::WasmAllocator; + + unsafe impl GlobalAlloc for WasmAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + pwasm_libc::malloc(layout.size()) as *mut u8 + } + + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + pwasm_libc::free(ptr as *mut u8) + } + } +} diff --git a/core/pwasm-libc/Cargo.lock b/core/pwasm-libc/Cargo.lock new file mode 100644 index 00000000000..6cc9d84c092 --- /dev/null +++ b/core/pwasm-libc/Cargo.lock @@ -0,0 +1,4 @@ +[[package]] +name = "pwasm-libc" +version = "0.1.0" + diff --git a/core/pwasm-libc/Cargo.toml b/core/pwasm-libc/Cargo.toml new file mode 100644 index 00000000000..d3ff1f1f320 --- /dev/null +++ b/core/pwasm-libc/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "pwasm-libc" +version = "0.1.0" +authors = ["Sergey Pepyakin "] +license = "MIT/Apache-2.0" +readme = "README.md" +repository = "https://github.com/paritytech/pwasm-std" +homepage = "https://github.com/paritytech/pwasm-std" +documentation = "https://paritytech.github.io/pwasm-std/pwasm_std/" +description = "Parity WebAssembly standard library libc bindings" +keywords = ["wasm", "parity", "webassembly", "blockchain"] +categories = ["no-std", "embedded"] + +[features] +strict = [] diff --git a/core/pwasm-libc/README.adoc b/core/pwasm-libc/README.adoc new file mode 100644 index 00000000000..e1cba6632f1 --- /dev/null +++ b/core/pwasm-libc/README.adoc @@ -0,0 +1,24 @@ + += Pwasm-libc + +Parity WASM contracts standard library libc bindings + +https://paritytech.github.io/pwasm-std/pwasm_libc/[Documentation] + +== License + +`pwasm-libc` is primarily distributed under the terms of both the MIT +license and the Apache License (Version 2.0), at your choice. + +See LICENSE-APACHE, and LICENSE-MIT for details. + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/pwasm-libc/src/lib.rs b/core/pwasm-libc/src/lib.rs new file mode 100644 index 00000000000..6b97ff37889 --- /dev/null +++ b/core/pwasm-libc/src/lib.rs @@ -0,0 +1,55 @@ +#![warn(missing_docs)] +#![cfg_attr(feature = "strict", deny(warnings))] +#![no_std] + +// tag::description[] +//! libc externs crate +// end::description[] + +extern "C" { + fn ext_memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8; + fn ext_memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8; + fn ext_memset(dest: *mut u8, c: i32, n: usize) -> *mut u8; + fn ext_memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32; + fn ext_malloc(size: usize) -> *mut u8; + fn ext_free(ptr: *mut u8); +} + +// Declaring these function here prevents Emscripten from including it's own verisons +// into final binary. + +/// memcpy extern +#[no_mangle] +pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + ext_memcpy(dest, src, n) +} + +/// memcmp extern +#[no_mangle] +pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + ext_memcmp(s1, s2, n) +} + +/// memmove extern +#[no_mangle] +pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + ext_memmove(dest, src, n) +} + +/// memset extern +#[no_mangle] +pub unsafe extern "C" fn memset(dest: *mut u8, c: i32, n: usize) -> *mut u8 { + ext_memset(dest, c, n) +} + +/// malloc extern +#[no_mangle] +pub unsafe extern "C" fn malloc(size: usize) -> *mut u8 { + ext_malloc(size) +} + +/// free extern +#[no_mangle] +pub unsafe extern "C" fn free(ptr: *mut u8) { + ext_free(ptr); +} diff --git a/core/rpc-servers/Cargo.toml b/core/rpc-servers/Cargo.toml new file mode 100644 index 00000000000..ffbd900357b --- /dev/null +++ b/core/rpc-servers/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "substrate-rpc-servers" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc.git" } +jsonrpc-http-server = { git = "https://github.com/paritytech/jsonrpc.git" } +jsonrpc-pubsub = { git = "https://github.com/paritytech/jsonrpc.git" } +jsonrpc-ws-server = { git = "https://github.com/paritytech/jsonrpc.git" } +log = "0.3" +serde = "1.0" +substrate-rpc = { path = "../rpc", version = "0.1" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } diff --git a/core/rpc-servers/README.adoc b/core/rpc-servers/README.adoc new file mode 100644 index 00000000000..18840be421f --- /dev/null +++ b/core/rpc-servers/README.adoc @@ -0,0 +1,14 @@ + += RPC Server + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- + diff --git a/core/rpc-servers/src/lib.rs b/core/rpc-servers/src/lib.rs new file mode 100644 index 00000000000..6139d32aec9 --- /dev/null +++ b/core/rpc-servers/src/lib.rs @@ -0,0 +1,93 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Substrate RPC servers. +// end::description[] + +#[warn(missing_docs)] + +pub extern crate substrate_rpc as apis; + +extern crate jsonrpc_core as rpc; +extern crate jsonrpc_http_server as http; +extern crate jsonrpc_pubsub as pubsub; +extern crate jsonrpc_ws_server as ws; +extern crate serde; +extern crate substrate_runtime_primitives; + +#[macro_use] +extern crate log; + +use std::io; +use substrate_runtime_primitives::traits::{Block as BlockT, NumberFor}; + +type Metadata = apis::metadata::Metadata; +type RpcHandler = pubsub::PubSubHandler; +pub type HttpServer = http::Server; +pub type WsServer = ws::Server; + +/// Construct rpc `IoHandler` +pub fn rpc_handler( + state: S, + chain: C, + author: A, + system: Y, +) -> RpcHandler where + Block: BlockT + 'static, + ExHash: Send + Sync + 'static + substrate_runtime_primitives::Serialize + substrate_runtime_primitives::DeserializeOwned, + PendingExtrinsics: serde::Serialize + serde::de::DeserializeOwned + Send + Sync + 'static, + S: apis::state::StateApi, + C: apis::chain::ChainApi, Block::Extrinsic, Metadata=Metadata>, + A: apis::author::AuthorApi, + Y: apis::system::SystemApi, +{ + let mut io = pubsub::PubSubHandler::default(); + io.extend_with(state.to_delegate()); + io.extend_with(chain.to_delegate()); + io.extend_with(author.to_delegate()); + io.extend_with(system.to_delegate()); + io +} + +/// Start HTTP server listening on given address. +pub fn start_http( + addr: &std::net::SocketAddr, + io: RpcHandler, +) -> io::Result { + http::ServerBuilder::new(io) + .threads(4) + .rest_api(http::RestApi::Unsecure) + .cors(http::DomainsValidation::Disabled) + .start_http(addr) +} + +/// Start WS server listening on given address. +pub fn start_ws( + addr: &std::net::SocketAddr, + io: RpcHandler, +) -> io::Result { + ws::ServerBuilder::with_meta_extractor(io, |context: &ws::RequestContext| Metadata::new(context.sender())) + .start(addr) + .map_err(|err| match err { + ws::Error(ws::ErrorKind::Io(io), _) => io, + ws::Error(ws::ErrorKind::ConnectionClosed, _) => io::ErrorKind::BrokenPipe.into(), + ws::Error(e, _) => { + error!("{}", e); + io::ErrorKind::Other.into() + } + }) +} diff --git a/core/rpc/Cargo.toml b/core/rpc/Cargo.toml new file mode 100644 index 00000000000..5460137dc75 --- /dev/null +++ b/core/rpc/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "substrate-rpc" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +error-chain = "0.12" +jsonrpc-core = { git="https://github.com/paritytech/jsonrpc.git" } +jsonrpc-macros = { git="https://github.com/paritytech/jsonrpc.git" } +jsonrpc-pubsub = { git="https://github.com/paritytech/jsonrpc.git" } +log = "0.3" +parking_lot = "0.4" +substrate-codec = { path = "../codec" } +substrate-client = { path = "../client" } +substrate-executor = { path = "../executor" } +substrate-extrinsic-pool = { path = "../extrinsic-pool" } +substrate-primitives = { path = "../primitives" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-runtime-version = { path = "../../runtime/version" } +substrate-state-machine = { path = "../state-machine" } +tokio = "0.1.7" +serde_json = "1.0" + +[dev-dependencies] +assert_matches = "1.1" +substrate-test-client = { path = "../test-client" } +rustc-hex = "2.0" diff --git a/core/rpc/README.adoc b/core/rpc/README.adoc new file mode 100644 index 00000000000..5e4db490997 --- /dev/null +++ b/core/rpc/README.adoc @@ -0,0 +1,13 @@ + += RPC + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/rpc/src/author/error.rs b/core/rpc/src/author/error.rs new file mode 100644 index 00000000000..e8317700562 --- /dev/null +++ b/core/rpc/src/author/error.rs @@ -0,0 +1,68 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Authoring RPC module errors. + +use client; +use extrinsic_pool; +use rpc; + +use errors; + +error_chain! { + links { + Pool(extrinsic_pool::Error, extrinsic_pool::ErrorKind) #[doc = "Pool error"]; + Client(client::error::Error, client::error::ErrorKind) #[doc = "Client error"]; + } + errors { + /// Not implemented yet + Unimplemented { + description("not yet implemented"), + display("Method Not Implemented"), + } + /// Incorrect extrinsic format. + BadFormat { + description("bad format"), + display("Invalid extrinsic format"), + } + /// Verification error + Verification(e: Box<::std::error::Error + Send>) { + description("extrinsic verification error"), + display("Extrinsic verification error: {}", e.description()), + } + } +} + +const ERROR: i64 = 1000; + +impl From for rpc::Error { + fn from(e: Error) -> Self { + match e { + Error(ErrorKind::Unimplemented, _) => errors::unimplemented(), + Error(ErrorKind::BadFormat, _) => rpc::Error { + code: rpc::ErrorCode::ServerError(ERROR + 1), + message: "Extrinsic has invalid format.".into(), + data: None, + }, + Error(ErrorKind::Verification(e), _) => rpc::Error { + code: rpc::ErrorCode::ServerError(ERROR + 2), + message: e.description().into(), + data: Some(format!("{:?}", e).into()), + }, + e => errors::internal(e), + } + } +} diff --git a/core/rpc/src/author/mod.rs b/core/rpc/src/author/mod.rs new file mode 100644 index 00000000000..5ba5140c5ee --- /dev/null +++ b/core/rpc/src/author/mod.rs @@ -0,0 +1,162 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate block-author/full-node API. + +use std::sync::Arc; + +use client::{self, Client}; +use codec::Decode; +use extrinsic_pool::{ + Pool, + IntoPoolError, + ChainApi as PoolChainApi, + watcher::Status, + VerifiedTransaction, + AllExtrinsics, + ExHash, + ExtrinsicFor, +}; +use jsonrpc_macros::pubsub; +use jsonrpc_pubsub::SubscriptionId; +use primitives::{Bytes, Blake2Hasher, RlpCodec}; +use rpc::futures::{Sink, Stream, Future}; +use runtime_primitives::{generic, traits}; +use subscriptions::Subscriptions; +use tokio::runtime::TaskExecutor; + +pub mod error; + +#[cfg(test)] +mod tests; + +use self::error::Result; + +build_rpc_trait! { + /// Substrate authoring RPC API + pub trait AuthorApi { + type Metadata; + + /// Submit extrinsic for inclusion in block. + #[rpc(name = "author_submitRichExtrinsic")] + fn submit_rich_extrinsic(&self, Extrinsic) -> Result; + /// Submit hex-encoded extrinsic for inclusion in block. + #[rpc(name = "author_submitExtrinsic")] + fn submit_extrinsic(&self, Bytes) -> Result; + + /// Returns all pending extrinsics, potentially grouped by sender. + #[rpc(name = "author_pendingExtrinsics")] + fn pending_extrinsics(&self) -> Result; + + #[pubsub(name = "author_extrinsicUpdate")] { + /// Submit an extrinsic to watch. + #[rpc(name = "author_submitAndWatchExtrinsic")] + fn watch_extrinsic(&self, Self::Metadata, pubsub::Subscriber>, Bytes); + + /// Unsubscribe from extrinsic watching. + #[rpc(name = "author_unwatchExtrinsic")] + fn unwatch_extrinsic(&self, SubscriptionId) -> Result; + } + + } +} + +/// Authoring API +pub struct Author where + P: PoolChainApi + Sync + Send + 'static, +{ + /// Substrate client + client: Arc::Block>>, + /// Extrinsic pool + pool: Arc>, + /// Subscriptions manager + subscriptions: Subscriptions, +} + +impl Author where + P: PoolChainApi + Sync + Send + 'static, +{ + /// Create new instance of Authoring API. + pub fn new(client: Arc::Block>>, pool: Arc>, executor: TaskExecutor) -> Self { + Author { + client, + pool, + subscriptions: Subscriptions::new(executor), + } + } +} + +impl AuthorApi, ExtrinsicFor

, AllExtrinsics

> for Author where + B: client::backend::Backend<

::Block, Blake2Hasher, RlpCodec> + Send + Sync + 'static, + E: client::CallExecutor<

::Block, Blake2Hasher, RlpCodec> + Send + Sync + 'static, + P: PoolChainApi + Sync + Send + 'static, + P::Error: 'static, +{ + type Metadata = ::metadata::Metadata; + + fn submit_extrinsic(&self, xt: Bytes) -> Result> { + let dxt = Decode::decode(&mut &xt[..]).ok_or(error::Error::from(error::ErrorKind::BadFormat))?; + self.submit_rich_extrinsic(dxt) + } + + fn submit_rich_extrinsic(&self, xt: <

::Block as traits::Block>::Extrinsic) -> Result> { + let best_block_hash = self.client.info()?.chain.best_hash; + self.pool + .submit_one(&generic::BlockId::hash(best_block_hash), xt) + .map_err(|e| e.into_pool_error() + .map(Into::into) + .unwrap_or_else(|e| error::ErrorKind::Verification(Box::new(e)).into()) + ) + .map(|ex| ex.hash().clone()) + } + + fn pending_extrinsics(&self) -> Result> { + Ok(self.pool.all()) + } + + fn watch_extrinsic(&self, _metadata: Self::Metadata, subscriber: pubsub::Subscriber>>, xt: Bytes) { + let submit = || -> Result<_> { + let best_block_hash = self.client.info()?.chain.best_hash; + let dxt = <

::Block as traits::Block>::Extrinsic::decode(&mut &xt[..]).ok_or(error::Error::from(error::ErrorKind::BadFormat))?; + self.pool + .submit_and_watch(&generic::BlockId::hash(best_block_hash), dxt) + .map_err(|e| e.into_pool_error() + .map(Into::into) + .unwrap_or_else(|e| error::ErrorKind::Verification(Box::new(e)).into()) + ) + }; + + let watcher = match submit() { + Ok(watcher) => watcher, + Err(err) => { + // reject the subscriber (ignore errors - we don't care if subscriber is no longer there). + let _ = subscriber.reject(err.into()); + return; + }, + }; + + self.subscriptions.add(subscriber, move |sink| { + sink + .sink_map_err(|e| warn!("Error sending notifications: {:?}", e)) + .send_all(watcher.into_stream().map(Ok)) + .map(|_| ()) + }) + } + + fn unwatch_extrinsic(&self, id: SubscriptionId) -> Result { + Ok(self.subscriptions.cancel(id)) + } +} diff --git a/core/rpc/src/author/tests.rs b/core/rpc/src/author/tests.rs new file mode 100644 index 00000000000..e5cdf759da4 --- /dev/null +++ b/core/rpc/src/author/tests.rs @@ -0,0 +1,178 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use super::*; + +use std::{sync::Arc, result::Result}; +use codec::Encode; +use extrinsic_pool::{VerifiedTransaction, scoring, Transaction, ChainApi, Error as PoolError, + Readiness, ExtrinsicFor, VerifiedFor}; +use test_client::runtime::{Block, Extrinsic, Transfer}; +use test_client; +use tokio::runtime; +use runtime_primitives::generic::BlockId; + +#[derive(Clone, Debug)] +pub struct Verified +{ + sender: u64, + hash: u64, +} + +impl VerifiedTransaction for Verified { + type Hash = u64; + type Sender = u64; + + fn hash(&self) -> &Self::Hash { &self.hash } + fn sender(&self) -> &Self::Sender { &self.sender } + fn mem_usage(&self) -> usize { 256 } +} + +struct TestApi; + +impl ChainApi for TestApi { + type Block = Block; + type Hash = u64; + type Sender = u64; + type Error = PoolError; + type VEx = Verified; + type Score = u64; + type Event = (); + type Ready = (); + + fn verify_transaction(&self, _at: &BlockId, uxt: &ExtrinsicFor) -> Result { + Ok(Verified { + sender: uxt.transfer.from[31] as u64, + hash: uxt.transfer.nonce, + }) + } + + fn is_ready(&self, _at: &BlockId, _c: &mut Self::Ready, _xt: &VerifiedFor) -> Readiness { + Readiness::Ready + } + + fn ready(&self) -> Self::Ready { } + + fn compare(old: &VerifiedFor, other: &VerifiedFor) -> ::std::cmp::Ordering { + old.verified.hash().cmp(&other.verified.hash()) + } + + fn choose(_old: &VerifiedFor, _new: &VerifiedFor) -> scoring::Choice { + scoring::Choice::ReplaceOld + } + + fn update_scores(xts: &[Transaction>], scores: &mut [Self::Score], _change: scoring::Change<()>) { + for i in 0..xts.len() { + scores[i] = xts[i].verified.sender + } + } + + fn should_replace(_old: &VerifiedFor, _new: &VerifiedFor) -> scoring::Choice { + scoring::Choice::ReplaceOld + } +} + +type DummyTxPool = Pool; + +fn uxt(sender: u64, hash: u64) -> Extrinsic { + Extrinsic { + signature: Default::default(), + transfer: Transfer { + amount: Default::default(), + nonce: hash, + from: From::from(sender), + to: Default::default(), + } + } +} + +#[test] +fn submit_transaction_should_not_cause_error() { + let runtime = runtime::Runtime::new().unwrap(); + let p = Author { + client: Arc::new(test_client::new()), + pool: Arc::new(DummyTxPool::new(Default::default(), TestApi)), + subscriptions: Subscriptions::new(runtime.executor()), + }; + + assert_matches!( + AuthorApi::submit_extrinsic(&p, uxt(5, 1).encode().into()), + Ok(1) + ); + assert!( + AuthorApi::submit_extrinsic(&p, uxt(5, 1).encode().into()).is_err() + ); +} + +#[test] +fn submit_rich_transaction_should_not_cause_error() { + let runtime = runtime::Runtime::new().unwrap(); + let p = Author { + client: Arc::new(test_client::new()), + pool: Arc::new(DummyTxPool::new(Default::default(), TestApi)), + subscriptions: Subscriptions::new(runtime.executor()), + }; + + assert_matches!( + AuthorApi::submit_rich_extrinsic(&p, uxt(5, 0)), + Ok(0) + ); + assert!( + AuthorApi::submit_rich_extrinsic(&p, uxt(5, 0)).is_err() + ); +} + +#[test] +fn should_watch_extrinsic() { + //given + let mut runtime = runtime::Runtime::new().unwrap(); + let pool = Arc::new(DummyTxPool::new(Default::default(), TestApi)); + let p = Author { + client: Arc::new(test_client::new()), + pool: pool.clone(), + subscriptions: Subscriptions::new(runtime.executor()), + }; + let (subscriber, id_rx, data) = ::jsonrpc_macros::pubsub::Subscriber::new_test("test"); + + // when + p.watch_extrinsic(Default::default(), subscriber, uxt(5, 5).encode().into()); + + // then + assert_eq!(runtime.block_on(id_rx), Ok(Ok(0.into()))); + // check notifications + AuthorApi::submit_rich_extrinsic(&p, uxt(5, 1)).unwrap(); + assert_eq!( + runtime.block_on(data.into_future()).unwrap().0, + Some(r#"{"jsonrpc":"2.0","method":"test","params":{"result":{"usurped":1},"subscription":0}}"#.into()) + ); +} + +#[test] +fn should_return_pending_extrinsics() { + let runtime = runtime::Runtime::new().unwrap(); + let pool = Arc::new(DummyTxPool::new(Default::default(), TestApi)); + let p = Author { + client: Arc::new(test_client::new()), + pool: pool.clone(), + subscriptions: Subscriptions::new(runtime.executor()), + }; + let ex = uxt(5, 1); + AuthorApi::submit_rich_extrinsic(&p, ex.clone()).unwrap(); + assert_matches!( + p.pending_extrinsics(), + Ok(ref expected) if expected.get(&5) == Some(&vec![ex]) + ); +} diff --git a/core/rpc/src/chain/error.rs b/core/rpc/src/chain/error.rs new file mode 100644 index 00000000000..2c42e8c9895 --- /dev/null +++ b/core/rpc/src/chain/error.rs @@ -0,0 +1,42 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use client; +use rpc; + +use errors; + +error_chain! { + links { + Client(client::error::Error, client::error::ErrorKind) #[doc = "Client error"]; + } + errors { + /// Not implemented yet + Unimplemented { + description("not yet implemented"), + display("Method Not Implemented"), + } + } +} + +impl From for rpc::Error { + fn from(e: Error) -> Self { + match e { + Error(ErrorKind::Unimplemented, _) => errors::unimplemented(), + e => errors::internal(e), + } + } +} diff --git a/core/rpc/src/chain/mod.rs b/core/rpc/src/chain/mod.rs new file mode 100644 index 00000000000..86551e04847 --- /dev/null +++ b/core/rpc/src/chain/mod.rs @@ -0,0 +1,165 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate blockchain API. + +use std::sync::Arc; + +use client::{self, Client, BlockchainEvents}; +use jsonrpc_macros::{pubsub, Trailing}; +use jsonrpc_pubsub::SubscriptionId; +use rpc::Result as RpcResult; +use rpc::futures::{stream, Future, Sink, Stream}; +use runtime_primitives::generic::{BlockId, SignedBlock}; +use runtime_primitives::traits::{Block as BlockT, Header, NumberFor}; +use runtime_version::RuntimeVersion; +use tokio::runtime::TaskExecutor; +use primitives::{Blake2Hasher, RlpCodec}; + +use subscriptions::Subscriptions; + +mod error; +#[cfg(test)] +mod tests; + +use self::error::Result; + +build_rpc_trait! { + /// Polkadot blockchain API + pub trait ChainApi { + type Metadata; + + /// Get header of a relay chain block. + #[rpc(name = "chain_getHeader")] + fn header(&self, Trailing) -> Result>; + + /// Get header and body of a relay chain block. + #[rpc(name = "chain_getBlock")] + fn block(&self, Trailing) -> Result>>; + + /// Get hash of the n-th block in the canon chain. + /// + /// By default returns latest block hash. + #[rpc(name = "chain_getBlockHash", alias = ["chain_getHead", ])] + fn block_hash(&self, Trailing) -> Result>; + + /// Get the runtime version. + #[rpc(name = "chain_getRuntimeVersion")] + fn runtime_version(&self, Trailing) -> Result; + + #[pubsub(name = "chain_newHead")] { + /// New head subscription + #[rpc(name = "chain_subscribeNewHead", alias = ["subscribe_newHead", ])] + fn subscribe_new_head(&self, Self::Metadata, pubsub::Subscriber

); + + /// Unsubscribe from new head subscription. + #[rpc(name = "chain_unsubscribeNewHead", alias = ["unsubscribe_newHead", ])] + fn unsubscribe_new_head(&self, SubscriptionId) -> RpcResult; + } + } +} + +/// Chain API with subscriptions support. +pub struct Chain { + /// Substrate client. + client: Arc>, + /// Current subscriptions. + subscriptions: Subscriptions, +} + +impl Chain { + /// Create new Chain API RPC handler. + pub fn new(client: Arc>, executor: TaskExecutor) -> Self { + Self { + client, + subscriptions: Subscriptions::new(executor), + } + } +} + +impl Chain where + Block: BlockT + 'static, + B: client::backend::Backend + Send + Sync + 'static, + E: client::CallExecutor + Send + Sync + 'static, +{ + fn unwrap_or_best(&self, hash: Trailing) -> Result { + Ok(match hash.into() { + None => self.client.info()?.chain.best_hash, + Some(hash) => hash, + }) + } +} + +impl ChainApi, Block::Extrinsic> for Chain where + Block: BlockT + 'static, + B: client::backend::Backend + Send + Sync + 'static, + E: client::CallExecutor + Send + Sync + 'static, +{ + type Metadata = ::metadata::Metadata; + + fn header(&self, hash: Trailing) -> Result> { + let hash = self.unwrap_or_best(hash)?; + Ok(self.client.header(&BlockId::Hash(hash))?) + } + + fn block(&self, hash: Trailing) -> Result>> { + let hash = self.unwrap_or_best(hash)?; + Ok(self.client.block(&BlockId::Hash(hash))?) + } + + fn block_hash(&self, number: Trailing>) -> Result> { + Ok(match number.into() { + None => Some(self.client.info()?.chain.best_hash), + Some(number) => self.client.header(&BlockId::number(number))?.map(|h| h.hash()), + }) + } + + fn runtime_version(&self, at: Trailing) -> Result { + let at = self.unwrap_or_best(at)?; + Ok(self.client.runtime_version_at(&BlockId::Hash(at))?) + } + + fn subscribe_new_head(&self, _metadata: Self::Metadata, subscriber: pubsub::Subscriber) { + self.subscriptions.add(subscriber, |sink| { + // send current head right at the start. + let header = self.block_hash(None.into()) + .and_then(|hash| self.header(hash.into())) + .and_then(|header| { + header.ok_or_else(|| self::error::ErrorKind::Unimplemented.into()) + }) + .map_err(Into::into); + + // send further subscriptions + let stream = self.client.import_notification_stream() + .filter(|notification| notification.is_new_best) + .map(|notification| Ok(notification.header)) + .map_err(|e| warn!("Block notification stream error: {:?}", e)); + + sink + .sink_map_err(|e| warn!("Error sending notifications: {:?}", e)) + .send_all( + stream::iter_result(vec![Ok(header)]) + .chain(stream) + ) + // we ignore the resulting Stream (if the first stream is over we are unsubscribed) + .map(|_| ()) + }); + } + + fn unsubscribe_new_head(&self, id: SubscriptionId) -> RpcResult { + Ok(self.subscriptions.cancel(id)) + } +} diff --git a/core/rpc/src/chain/tests.rs b/core/rpc/src/chain/tests.rs new file mode 100644 index 00000000000..def410532cd --- /dev/null +++ b/core/rpc/src/chain/tests.rs @@ -0,0 +1,207 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use super::*; +use jsonrpc_macros::pubsub; +use client::BlockOrigin; +use test_client::{self, TestClient}; +use test_client::runtime::{Block, Header}; + +#[test] +fn should_return_header() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + + let client = Chain { + client: Arc::new(test_client::new()), + subscriptions: Subscriptions::new(remote), + }; + + assert_matches!( + client.header(Some(client.client.genesis_hash()).into()), + Ok(Some(ref x)) if x == &Header { + parent_hash: 0.into(), + number: 0, + state_root: x.state_root.clone(), + extrinsics_root: "45b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c0".into(), + digest: Default::default(), + } + ); + + assert_matches!( + client.header(None.into()), + Ok(Some(ref x)) if x == &Header { + parent_hash: 0.into(), + number: 0, + state_root: x.state_root.clone(), + extrinsics_root: "45b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c0".into(), + digest: Default::default(), + } + ); + + assert_matches!( + client.header(Some(5.into()).into()), + Ok(None) + ); +} + +#[test] +fn should_return_a_block() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + + let api = Chain { + client: Arc::new(test_client::new()), + subscriptions: Subscriptions::new(remote), + }; + + let block = api.client.new_block().unwrap().bake().unwrap(); + let block_hash = block.hash(); + api.client.justify_and_import(BlockOrigin::Own, block).unwrap(); + + + // Genesis block is not justified, so we can't query it? + assert_matches!( + api.block(Some(api.client.genesis_hash()).into()), + Ok(None) + ); + + assert_matches!( + api.block(Some(block_hash).into()), + Ok(Some(ref x)) if x.block == Block { + header: Header { + parent_hash: api.client.genesis_hash(), + number: 1, + state_root: x.block.header.state_root.clone(), + extrinsics_root: "45b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c0".into(), + digest: Default::default(), + }, + extrinsics: vec![], + } + ); + + assert_matches!( + api.block(None.into()), + Ok(Some(ref x)) if x.block == Block { + header: Header { + parent_hash: api.client.genesis_hash(), + number: 1, + state_root: x.block.header.state_root.clone(), + extrinsics_root: "45b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c0".into(), + digest: Default::default(), + }, + extrinsics: vec![], + } + ); + + assert_matches!( + api.block(Some(5.into()).into()), + Ok(None) + ); +} + +#[test] +fn should_return_block_hash() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + + let client = Chain { + client: Arc::new(test_client::new()), + subscriptions: Subscriptions::new(remote), + }; + + assert_matches!( + client.block_hash(None.into()), + Ok(Some(ref x)) if x == &client.client.genesis_hash() + ); + + + assert_matches!( + client.block_hash(Some(0u64).into()), + Ok(Some(ref x)) if x == &client.client.genesis_hash() + ); + + assert_matches!( + client.block_hash(Some(1u64).into()), + Ok(None) + ); + + let block = client.client.new_block().unwrap().bake().unwrap(); + client.client.justify_and_import(BlockOrigin::Own, block.clone()).unwrap(); + + assert_matches!( + client.block_hash(Some(0u64).into()), + Ok(Some(ref x)) if x == &client.client.genesis_hash() + ); + assert_matches!( + client.block_hash(Some(1u64).into()), + Ok(Some(ref x)) if x == &block.hash() + ); + +} + +#[test] +fn should_notify_about_latest_block() { + let mut core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + let (subscriber, id, transport) = pubsub::Subscriber::new_test("test"); + + { + let api = Chain { + client: Arc::new(test_client::new()), + subscriptions: Subscriptions::new(remote), + }; + + api.subscribe_new_head(Default::default(), subscriber); + + // assert id assigned + assert_eq!(core.block_on(id), Ok(Ok(SubscriptionId::Number(0)))); + + let builder = api.client.new_block().unwrap(); + api.client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + } + + // assert initial head sent. + let (notification, next) = core.block_on(transport.into_future()).unwrap(); + assert!(notification.is_some()); + // assert notification sent to transport + let (notification, next) = core.block_on(next.into_future()).unwrap(); + assert!(notification.is_some()); + // no more notifications on this channel + assert_eq!(core.block_on(next.into_future()).unwrap().0, None); +} + +#[test] +fn should_return_runtime_version() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + + let client = Chain { + client: Arc::new(test_client::new()), + subscriptions: Subscriptions::new(remote), + }; + + assert_matches!( + client.runtime_version(None.into()), + Ok(ref ver) if ver == &RuntimeVersion { + spec_name: "test".into(), + impl_name: "parity-test".into(), + authoring_version: 1, + spec_version: 1, + impl_version: 1, + } + ); +} diff --git a/core/rpc/src/errors.rs b/core/rpc/src/errors.rs new file mode 100644 index 00000000000..a9b9e27a9cb --- /dev/null +++ b/core/rpc/src/errors.rs @@ -0,0 +1,34 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use rpc; + +pub fn unimplemented() -> rpc::Error { + rpc::Error { + code: rpc::ErrorCode::ServerError(1), + message: "Not implemented yet".into(), + data: None, + } +} + +pub fn internal(e: E) -> rpc::Error { + warn!("Unknown error: {:?}", e); + rpc::Error { + code: rpc::ErrorCode::InternalError, + message: "Unknown error occured".into(), + data: Some(format!("{:?}", e).into()), + } +} diff --git a/core/rpc/src/helpers.rs b/core/rpc/src/helpers.rs new file mode 100644 index 00000000000..dca1a45db56 --- /dev/null +++ b/core/rpc/src/helpers.rs @@ -0,0 +1,25 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +/// Unwraps the trailing parameter or falls back with the closure result. +pub fn unwrap_or_else(or_else: F, optional: ::jsonrpc_macros::Trailing) -> Result where + F: FnOnce() -> Result, +{ + match optional.into() { + None => or_else(), + Some(x) => Ok(x), + } +} diff --git a/core/rpc/src/lib.rs b/core/rpc/src/lib.rs new file mode 100644 index 00000000000..b4e07fc823e --- /dev/null +++ b/core/rpc/src/lib.rs @@ -0,0 +1,59 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Substrate RPC interfaces. +// end::description[] + +#![warn(missing_docs)] + +extern crate jsonrpc_core as rpc; +extern crate jsonrpc_pubsub; +extern crate parking_lot; +extern crate substrate_codec as codec; +extern crate substrate_client as client; +extern crate substrate_extrinsic_pool as extrinsic_pool; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_state_machine as state_machine; +extern crate substrate_runtime_version as runtime_version; +extern crate tokio; +extern crate serde_json; + +#[macro_use] +extern crate error_chain; +#[macro_use] +extern crate jsonrpc_macros; +#[macro_use] +extern crate log; + +#[cfg(test)] +#[macro_use] +extern crate assert_matches; +#[cfg(test)] +extern crate substrate_test_client as test_client; +#[cfg(test)] +extern crate rustc_hex; + +mod errors; +mod helpers; +mod subscriptions; + +pub mod author; +pub mod chain; +pub mod metadata; +pub mod state; +pub mod system; diff --git a/core/rpc/src/metadata.rs b/core/rpc/src/metadata.rs new file mode 100644 index 00000000000..06d19d11d8c --- /dev/null +++ b/core/rpc/src/metadata.rs @@ -0,0 +1,54 @@ +// Copyright 2017-2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! RPC Metadata +use std::sync::Arc; + +use jsonrpc_pubsub::{Session, PubSubMetadata}; +use rpc::futures::sync::mpsc; + +/// RPC Metadata. +/// +/// Manages persistent session for transports that support it +/// and may contain some additional info extracted from specific transports +/// (like remote client IP address, request headers, etc) +#[derive(Default, Clone)] +pub struct Metadata { + session: Option>, +} + +impl ::rpc::Metadata for Metadata {} +impl PubSubMetadata for Metadata { + fn session(&self) -> Option> { + self.session.clone() + } +} + +impl Metadata { + /// Create new `Metadata` with session (Pub/Sub) support. + pub fn new(transport: mpsc::Sender) -> Self { + Metadata { + session: Some(Arc::new(Session::new(transport))), + } + } + + /// Create new `Metadata` for tests. + #[cfg(test)] + pub fn new_test() -> (mpsc::Receiver, Self) { + let (tx, rx) = mpsc::channel(1); + (rx, Self::new(tx)) + } +} diff --git a/core/rpc/src/state/error.rs b/core/rpc/src/state/error.rs new file mode 100644 index 00000000000..de65e466c08 --- /dev/null +++ b/core/rpc/src/state/error.rs @@ -0,0 +1,54 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use client; +use rpc; + +use errors; + +use serde_json; + +error_chain! { + links { + Client(client::error::Error, client::error::ErrorKind) #[doc = "Client error"]; + } + + foreign_links { + Json(serde_json::Error); + } + + errors { + /// Provided block range couldn't be resolved to a list of blocks. + InvalidBlockRange(from: String, to: String, details: String) { + description("Invalid block range"), + display("Cannot resolve a block range ['{:?}' ... '{:?}]. {}", from, to, details), + } + /// Not implemented yet + Unimplemented { + description("not implemented yet"), + display("Method Not Implemented"), + } + } +} + +impl From for rpc::Error { + fn from(e: Error) -> Self { + match e { + Error(ErrorKind::Unimplemented, _) => errors::unimplemented(), + e => errors::internal(e), + } + } +} diff --git a/core/rpc/src/state/mod.rs b/core/rpc/src/state/mod.rs new file mode 100644 index 00000000000..e335310af74 --- /dev/null +++ b/core/rpc/src/state/mod.rs @@ -0,0 +1,277 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot state API. + +use std::{ + collections::HashMap, + sync::Arc, +}; + +use client::{self, Client, CallExecutor, BlockchainEvents}; +use jsonrpc_macros::Trailing; +use jsonrpc_macros::pubsub; +use jsonrpc_pubsub::SubscriptionId; +use primitives::hexdisplay::HexDisplay; +use primitives::storage::{StorageKey, StorageData, StorageChangeSet}; +use primitives::{Blake2Hasher, RlpCodec}; +use rpc::Result as RpcResult; +use rpc::futures::{stream, Future, Sink, Stream}; +use runtime_primitives::generic::BlockId; +use runtime_primitives::traits::{Block as BlockT, Header}; +use tokio::runtime::TaskExecutor; +use serde_json; + +use subscriptions::Subscriptions; + +mod error; +#[cfg(test)] +mod tests; + +use self::error::Result; + +build_rpc_trait! { + /// Polkadot state API + pub trait StateApi { + type Metadata; + + /// Call a contract at a block's state. + #[rpc(name = "state_call", alias = ["state_callAt", ])] + fn call(&self, String, Vec, Trailing) -> Result>; + + /// Returns a storage entry at a specific block's state. + #[rpc(name = "state_getStorage", alias = ["state_getStorageAt", ])] + fn storage(&self, StorageKey, Trailing) -> Result>; + + /// Returns the hash of a storage entry at a block's state. + #[rpc(name = "state_getStorageHash", alias = ["state_getStorageHashAt", ])] + fn storage_hash(&self, StorageKey, Trailing) -> Result>; + + /// Returns the size of a storage entry at a block's state. + #[rpc(name = "state_getStorageSize", alias = ["state_getStorageSizeAt", ])] + fn storage_size(&self, StorageKey, Trailing) -> Result>; + + /// Returns the runtime metadata as JSON. + #[rpc(name = "state_getMetadata")] + fn metadata(&self, Trailing) -> Result; + + /// Query historical storage entries (by key) starting from a block given as the second parameter. + /// + /// NOTE This first returned result contains the initial state of storage for all keys. + /// Subsequent values in the vector represent changes to the previous state (diffs). + #[rpc(name = "state_queryStorage")] + fn query_storage(&self, Vec, Hash, Trailing) -> Result>>; + + #[pubsub(name = "state_storage")] { + /// New storage subscription + #[rpc(name = "state_subscribeStorage")] + fn subscribe_storage(&self, Self::Metadata, pubsub::Subscriber>, Trailing>); + + /// Unsubscribe from storage subscription + #[rpc(name = "state_unsubscribeStorage")] + fn unsubscribe_storage(&self, SubscriptionId) -> RpcResult; + } + } +} + +/// State API with subscriptions support. +pub struct State { + /// Substrate client. + client: Arc>, + /// Current subscriptions. + subscriptions: Subscriptions, +} + +impl State { + /// Create new State API RPC handler. + pub fn new(client: Arc>, executor: TaskExecutor) -> Self { + Self { + client, + subscriptions: Subscriptions::new(executor), + } + } +} + +impl State where + Block: BlockT, + B: client::backend::Backend, + E: CallExecutor, +{ + fn unwrap_or_best(&self, hash: Trailing) -> Result { + ::helpers::unwrap_or_else(|| Ok(self.client.info()?.chain.best_hash), hash) + } +} + +impl StateApi for State where + Block: BlockT + 'static, + B: client::backend::Backend + Send + Sync + 'static, + E: CallExecutor + Send + Sync + 'static, +{ + type Metadata = ::metadata::Metadata; + + fn call(&self, method: String, data: Vec, block: Trailing) -> Result> { + let block = self.unwrap_or_best(block)?; + trace!(target: "rpc", "Calling runtime at {:?} for method {} ({})", block, method, HexDisplay::from(&data)); + Ok(self.client.executor().call(&BlockId::Hash(block), &method, &data)?.return_data) + } + + fn storage(&self, key: StorageKey, block: Trailing) -> Result> { + let block = self.unwrap_or_best(block)?; + trace!(target: "rpc", "Querying storage at {:?} for key {}", block, HexDisplay::from(&key.0)); + Ok(self.client.storage(&BlockId::Hash(block), &key)?) + } + + fn storage_hash(&self, key: StorageKey, block: Trailing) -> Result> { + use runtime_primitives::traits::{Hash, Header as HeaderT}; + Ok(self.storage(key, block)?.map(|x| ::Hashing::hash(&x.0))) + } + + fn storage_size(&self, key: StorageKey, block: Trailing) -> Result> { + Ok(self.storage(key, block)?.map(|x| x.0.len() as u64)) + } + + fn metadata(&self, block: Trailing) -> Result { + let block = self.unwrap_or_best(block)?; + let metadata = self.client.json_metadata(&BlockId::Hash(block))?; + serde_json::from_str(&metadata).map_err(Into::into) + } + + fn query_storage(&self, keys: Vec, from: Block::Hash, to: Trailing) -> Result>> { + let to = self.unwrap_or_best(to)?; + + let from_hdr = self.client.header(&BlockId::hash(from))?; + let to_hdr = self.client.header(&BlockId::hash(to))?; + + match (from_hdr, to_hdr) { + (Some(ref from), Some(ref to)) if from.number() <= to.number() => { + let from = from.clone(); + let to = to.clone(); + // check if we can get from `to` to `from` by going through parent_hashes. + let blocks = { + let mut blocks = vec![to.hash()]; + let mut last = to.clone(); + while last.number() > from.number() { + if let Some(hdr) = self.client.header(&BlockId::hash(*last.parent_hash()))? { + blocks.push(hdr.hash()); + last = hdr; + } else { + bail!(invalid_block_range( + Some(from), + Some(to), + format!("Parent of {} ({}) not found", last.number(), last.hash()), + )) + } + } + if last.hash() != from.hash() { + bail!(invalid_block_range( + Some(from), + Some(to), + format!("Expected to reach `from`, got {} ({})", last.number(), last.hash()), + )) + } + blocks.reverse(); + blocks + }; + let mut result = Vec::new(); + let mut last_state: HashMap<_, Option<_>> = Default::default(); + for block in blocks { + let mut changes = vec![]; + let id = BlockId::hash(block.clone()); + + for key in &keys { + let (has_changed, data) = { + let curr_data = self.client.storage(&id, key)?; + let prev_data = last_state.get(key).and_then(|x| x.as_ref()); + + (curr_data.as_ref() != prev_data, curr_data) + }; + + if has_changed { + changes.push((key.clone(), data.clone())); + } + + last_state.insert(key.clone(), data); + } + + result.push(StorageChangeSet { + block, + changes, + }); + } + Ok(result) + }, + (from, to) => bail!(invalid_block_range(from, to, "Invalid range or unknown block".into())), + } + } + + fn subscribe_storage( + &self, + _meta: Self::Metadata, + subscriber: pubsub::Subscriber>, + keys: Trailing> + ) { + let keys = Into::>>::into(keys); + let stream = match self.client.storage_changes_notification_stream(keys.as_ref().map(|x| &**x)) { + Ok(stream) => stream, + Err(err) => { + let _ = subscriber.reject(error::Error::from(err).into()); + return; + }, + }; + + // initial values + let initial = stream::iter_result(keys + .map(|keys| { + let block = self.client.info().map(|info| info.chain.best_hash).unwrap_or_default(); + let changes = keys + .into_iter() + .map(|key| self.storage(key.clone(), Some(block.clone()).into()) + .map(|val| (key.clone(), val)) + .unwrap_or_else(|_| (key, None)) + ) + .collect(); + vec![Ok(Ok(StorageChangeSet { block, changes }))] + }).unwrap_or_default()); + + self.subscriptions.add(subscriber, |sink| { + let stream = stream + .map_err(|e| warn!("Error creating storage notification stream: {:?}", e)) + .map(|(block, changes)| Ok(StorageChangeSet { + block, + changes: changes.iter().cloned().collect(), + })); + + sink + .sink_map_err(|e| warn!("Error sending notifications: {:?}", e)) + .send_all(initial.chain(stream)) + // we ignore the resulting Stream (if the first stream is over we are unsubscribed) + .map(|_| ()) + }) + } + + fn unsubscribe_storage(&self, id: SubscriptionId) -> RpcResult { + Ok(self.subscriptions.cancel(id)) + } +} + +fn invalid_block_range(from: Option, to: Option, reason: String) -> error::ErrorKind { + let to_string = |x: Option| match x { + None => "unknown hash".into(), + Some(h) => format!("{} ({})", h.number(), h.hash()), + }; + + error::ErrorKind::InvalidBlockRange(to_string(from), to_string(to), reason) +} diff --git a/core/rpc/src/state/tests.rs b/core/rpc/src/state/tests.rs new file mode 100644 index 00000000000..bad934b8405 --- /dev/null +++ b/core/rpc/src/state/tests.rs @@ -0,0 +1,186 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use super::*; +use self::error::{Error, ErrorKind}; + +use client::BlockOrigin; +use jsonrpc_macros::pubsub; +use rustc_hex::FromHex; +use test_client::{self, runtime, keyring::Keyring, TestClient, BlockBuilderExt}; + +#[test] +fn should_return_storage() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let client = Arc::new(test_client::new()); + let genesis_hash = client.genesis_hash(); + let client = State::new(client, core.executor()); + + assert_matches!( + client.storage(StorageKey(vec![10]), Some(genesis_hash).into()), + Ok(None) + ) +} + +#[test] +fn should_call_contract() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let client = Arc::new(test_client::new()); + let genesis_hash = client.genesis_hash(); + let client = State::new(client, core.executor()); + + assert_matches!( + client.call("balanceOf".into(), vec![1,2,3], Some(genesis_hash).into()), + Err(Error(ErrorKind::Client(client::error::ErrorKind::Execution(_)), _)) + ) +} + +#[test] +fn should_notify_about_storage_changes() { + let mut core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + let (subscriber, id, transport) = pubsub::Subscriber::new_test("test"); + + { + let api = State { + client: Arc::new(test_client::new()), + subscriptions: Subscriptions::new(remote), + }; + + api.subscribe_storage(Default::default(), subscriber, None.into()); + + // assert id assigned + assert_eq!(core.block_on(id), Ok(Ok(SubscriptionId::Number(0)))); + + let mut builder = api.client.new_block().unwrap(); + builder.push_transfer(runtime::Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Ferdie.to_raw_public().into(), + amount: 42, + nonce: 0, + }).unwrap(); + api.client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + } + + // assert notification sent to transport + let (notification, next) = core.block_on(transport.into_future()).unwrap(); + assert!(notification.is_some()); + // no more notifications on this channel + assert_eq!(core.block_on(next.into_future()).unwrap().0, None); +} + +#[test] +fn should_send_initial_storage_changes_and_notifications() { + let mut core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + let (subscriber, id, transport) = pubsub::Subscriber::new_test("test"); + + { + let api = State { + client: Arc::new(test_client::new()), + subscriptions: Subscriptions::new(remote), + }; + + api.subscribe_storage(Default::default(), subscriber, Some(vec![ + StorageKey("a52da2b7c269da1366b3ed1cdb7299ce".from_hex().unwrap()), + ]).into()); + + // assert id assigned + assert_eq!(core.block_on(id), Ok(Ok(SubscriptionId::Number(0)))); + + let mut builder = api.client.new_block().unwrap(); + builder.push_transfer(runtime::Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Ferdie.to_raw_public().into(), + amount: 42, + nonce: 0, + }).unwrap(); + api.client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap(); + } + + // assert initial values sent to transport + let (notification, next) = core.block_on(transport.into_future()).unwrap(); + assert!(notification.is_some()); + // assert notification sent to transport + let (notification, next) = core.block_on(next.into_future()).unwrap(); + assert!(notification.is_some()); + // no more notifications on this channel + assert_eq!(core.block_on(next.into_future()).unwrap().0, None); +} + +#[test] +fn should_query_storage() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let client = Arc::new(test_client::new()); + let api = State::new(client.clone(), core.executor()); + + let add_block = |nonce| { + let mut builder = client.new_block().unwrap(); + builder.push_transfer(runtime::Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Ferdie.to_raw_public().into(), + amount: 42, + nonce, + }).unwrap(); + let block = builder.bake().unwrap(); + let hash = block.header.hash(); + client.justify_and_import(BlockOrigin::Own, block).unwrap(); + hash + }; + let block1_hash = add_block(0); + let block2_hash = add_block(1); + let genesis_hash = client.genesis_hash(); + + + let mut expected = vec![ + StorageChangeSet { + block: genesis_hash, + changes: vec![ + (StorageKey("a52da2b7c269da1366b3ed1cdb7299ce".from_hex().unwrap()), Some(StorageData(vec![232, 3, 0, 0, 0, 0, 0, 0]))), + ], + }, + StorageChangeSet { + block: block1_hash, + changes: vec![ + (StorageKey("a52da2b7c269da1366b3ed1cdb7299ce".from_hex().unwrap()), Some(StorageData(vec![190, 3, 0, 0, 0, 0, 0, 0]))), + ], + }, + ]; + + // Query changes only up to block1 + let result = api.query_storage( + vec![StorageKey("a52da2b7c269da1366b3ed1cdb7299ce".from_hex().unwrap())], + genesis_hash, + Some(block1_hash).into(), + ); + + assert_eq!(result.unwrap(), expected); + + // Query all changes + let result = api.query_storage( + vec![StorageKey("a52da2b7c269da1366b3ed1cdb7299ce".from_hex().unwrap())], + genesis_hash, + None.into(), + ); + + expected.push(StorageChangeSet { + block: block2_hash, + changes: vec![ + (StorageKey("a52da2b7c269da1366b3ed1cdb7299ce".from_hex().unwrap()), Some(StorageData(vec![148, 3, 0, 0, 0, 0, 0, 0]))), + ], + }); + assert_eq!(result.unwrap(), expected); +} diff --git a/core/rpc/src/subscriptions.rs b/core/rpc/src/subscriptions.rs new file mode 100644 index 00000000000..9013edf742e --- /dev/null +++ b/core/rpc/src/subscriptions.rs @@ -0,0 +1,85 @@ +// Copyright 2017-2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::collections::HashMap; +use std::sync::atomic::{self, AtomicUsize}; + +use jsonrpc_macros::pubsub; +use jsonrpc_pubsub::SubscriptionId; +use parking_lot::Mutex; +use rpc::futures::sync::oneshot; +use rpc::futures::{Future, future}; +use tokio::runtime::TaskExecutor; + +type Id = u64; + +/// Subscriptions manager. +/// +/// Takes care of assigning unique subscription ids and +/// driving the sinks into completion. +#[derive(Debug)] +pub struct Subscriptions { + next_id: AtomicUsize, + active_subscriptions: Mutex>>, + executor: TaskExecutor, +} + +impl Subscriptions { + /// Creates new `Subscriptions` object. + pub fn new(executor: TaskExecutor) -> Self { + Subscriptions { + next_id: Default::default(), + active_subscriptions: Default::default(), + executor, + } + } + + /// Creates new subscription for given subscriber. + /// + /// Second parameter is a function that converts Subscriber sink into a future. + /// This future will be driven to completion bu underlying event loop + /// or will be cancelled in case #cancel is invoked. + pub fn add(&self, subscriber: pubsub::Subscriber, into_future: G) where + G: FnOnce(pubsub::Sink) -> R, + R: future::IntoFuture, + F: future::Future + Send + 'static, + { + let id = self.next_id.fetch_add(1, atomic::Ordering::AcqRel) as u64; + if let Ok(sink) = subscriber.assign_id(id.into()) { + let (tx, rx) = oneshot::channel(); + let future = into_future(sink) + .into_future() + .select(rx.map_err(|e| warn!("Error timeing out: {:?}", e))) + .then(|_| Ok(())); + + self.active_subscriptions.lock().insert(id, tx); + self.executor.spawn(future); + } + } + + /// Cancel subscription. + /// + /// Returns true if subscription existed or false otherwise. + pub fn cancel(&self, id: SubscriptionId) -> bool { + if let SubscriptionId::Number(id) = id { + if let Some(tx) = self.active_subscriptions.lock().remove(&id) { + let _ = tx.send(()); + return true; + } + } + false + } +} diff --git a/core/rpc/src/system/error.rs b/core/rpc/src/system/error.rs new file mode 100644 index 00000000000..42d215caefd --- /dev/null +++ b/core/rpc/src/system/error.rs @@ -0,0 +1,40 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! System RPC module errors. + +use rpc; + +use errors; + +error_chain! { + errors { + /// Not implemented yet + Unimplemented { + description("not yet implemented"), + display("Method Not Implemented"), + } + } +} + +impl From for rpc::Error { + fn from(e: Error) -> Self { + match e { + Error(ErrorKind::Unimplemented, _) => errors::unimplemented(), + e => errors::internal(e), + } + } +} diff --git a/core/rpc/src/system/mod.rs b/core/rpc/src/system/mod.rs new file mode 100644 index 00000000000..56d06be9e5c --- /dev/null +++ b/core/rpc/src/system/mod.rs @@ -0,0 +1,41 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate system API. + +pub mod error; + +#[cfg(test)] +mod tests; + +use self::error::Result; + +build_rpc_trait! { + /// Substrate system RPC API + pub trait SystemApi { + /// Get the node's implementation name. Plain old string. + #[rpc(name = "system_name")] + fn system_name(&self) -> Result; + + /// Get the node implementation's version. Should be a semver string. + #[rpc(name = "system_version")] + fn system_version(&self) -> Result; + + /// Get the chain's type. Given as a string identifier. + #[rpc(name = "system_chain")] + fn system_chain(&self) -> Result; + } +} diff --git a/core/rpc/src/system/tests.rs b/core/rpc/src/system/tests.rs new file mode 100644 index 00000000000..f22cd5a1577 --- /dev/null +++ b/core/rpc/src/system/tests.rs @@ -0,0 +1,54 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use super::*; +use super::error::*; + +impl SystemApi for () { + fn system_name(&self) -> Result { + Ok("testclient".into()) + } + fn system_version(&self) -> Result { + Ok("0.2.0".into()) + } + fn system_chain(&self) -> Result { + Ok("testchain".into()) + } +} + +#[test] +fn system_name_works() { + assert_eq!( + SystemApi::system_name(&()).unwrap(), + "testclient".to_owned() + ); +} + +#[test] +fn system_version_works() { + assert_eq!( + SystemApi::system_version(&()).unwrap(), + "0.2.0".to_owned() + ); +} + +#[test] +fn system_chain_works() { + assert_eq!( + SystemApi::system_chain(&()).unwrap(), + "testchain".to_owned() + ); +} diff --git a/core/runtime-io/Cargo.toml b/core/runtime-io/Cargo.toml new file mode 100644 index 00000000000..8bf3c3406f6 --- /dev/null +++ b/core/runtime-io/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-runtime-io" +version = "0.1.0" +authors = ["Parity Technologies "] +build = "build.rs" + +[build-dependencies] +rustc_version = "0.2" + +[dependencies] +substrate-runtime-std = { path = "../runtime-std", default_features = false } +environmental = { path = "../environmental", optional = true } +substrate-state-machine = { path = "../state-machine", optional = true } +substrate-primitives = { path = "../primitives", default_features = false } +substrate-codec = { path = "../codec", default_features = false } +triehash = { version = "0.2", optional = true } +hashdb = { version = "0.2", default_features = false } +rlp = { version = "0.2", optional = true, default_features = false } + +[features] +default = ["std"] +std = [ + "environmental", + "substrate-state-machine", + "triehash", + "substrate-primitives/std", + "substrate-codec/std", + "substrate-runtime-std/std", + "rlp" +] +nightly = [] +strict = [] diff --git a/core/runtime-io/README.adoc b/core/runtime-io/README.adoc new file mode 100644 index 00000000000..c035be14611 --- /dev/null +++ b/core/runtime-io/README.adoc @@ -0,0 +1,13 @@ + += Runtime io + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/runtime-io/build.rs b/core/runtime-io/build.rs new file mode 100644 index 00000000000..35eb154f3a6 --- /dev/null +++ b/core/runtime-io/build.rs @@ -0,0 +1,14 @@ +//! Set a nightly feature + +extern crate rustc_version; +use rustc_version::{version, version_meta, Channel}; + +fn main() { + // Assert we haven't travelled back in time + assert!(version().unwrap().major >= 1); + + // Set cfg flags depending on release channel + if let Channel::Nightly = version_meta().unwrap().channel { + println!("cargo:rustc-cfg=feature=\"nightly\""); + } +} diff --git a/core/runtime-io/src/lib.rs b/core/runtime-io/src/lib.rs new file mode 100644 index 00000000000..9214a7aa37b --- /dev/null +++ b/core/runtime-io/src/lib.rs @@ -0,0 +1,35 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! This is part of the Substrate runtime. +// end::description[] + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(lang_items))] +#![cfg_attr(not(feature = "std"), feature(panic_handler))] +#![cfg_attr(not(feature = "std"), feature(alloc_error_handler))] +#![cfg_attr(not(feature = "std"), feature(core_intrinsics))] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +#![cfg_attr(feature = "std", doc = "Substrate runtime standard library as compiled when linked with Rust's standard library.")] +#![cfg_attr(not(feature = "std"), doc = "Substrate's runtime standard library as compiled without Rust's standard library.")] + +#[cfg(feature = "std")] +include!("../with_std.rs"); + +#[cfg(not(feature = "std"))] +include!("../without_std.rs"); diff --git a/core/runtime-io/with_std.rs b/core/runtime-io/with_std.rs new file mode 100644 index 00000000000..3790f925bd4 --- /dev/null +++ b/core/runtime-io/with_std.rs @@ -0,0 +1,266 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[macro_use] +extern crate environmental; + +#[cfg_attr(test, macro_use)] +extern crate substrate_primitives as primitives; + +extern crate substrate_state_machine; +extern crate triehash; +extern crate hashdb; +extern crate rlp; + +#[doc(hidden)] +pub extern crate substrate_codec as codec; +// re-export hashing functions. +pub use primitives::{blake2_256, twox_128, twox_256, ed25519}; + +pub use primitives::Blake2Hasher; +// Switch to this after PoC-3 +// pub use primitives::BlakeHasher; +pub use substrate_state_machine::{Externalities, TestExternalities}; +use primitives::hexdisplay::HexDisplay; +use primitives::H256; +use hashdb::Hasher; +use rlp::Encodable; + +// TODO: use the real error, not NoError. + +environmental!(ext: trait Externalities); + +/// Get `key` from storage and return a `Vec`, empty if there's a problem. +pub fn storage(key: &[u8]) -> Option> { + ext::with(|ext| ext.storage(key).map(|s| s.to_vec())) + .expect("storage cannot be called outside of an Externalities-provided environment.") +} + +/// Get `key` from storage, placing the value into `value_out` (as much of it as possible) and return +/// the number of bytes that the entry in storage had beyond the offset or None if the storage entry +/// doesn't exist at all. Note that if the buffer is smaller than the storage entry length, the returned +/// number of bytes is not equal to the number of bytes written to the `value_out`. +pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { + ext::with(|ext| ext.storage(key).map(|value| { + let value = &value[value_offset..]; + let written = ::std::cmp::min(value.len(), value_out.len()); + value_out[0..written].copy_from_slice(&value[0..written]); + value.len() + })).expect("read_storage cannot be called outside of an Externalities-provided environment.") +} + +/// Set the storage of a key to some value. +pub fn set_storage(key: &[u8], value: &[u8]) { + ext::with(|ext| + ext.set_storage(key.to_vec(), value.to_vec()) + ); +} + +/// Clear the storage of a key. +pub fn clear_storage(key: &[u8]) { + ext::with(|ext| + ext.clear_storage(key) + ); +} + +/// Check whether a given `key` exists in storage. +pub fn exists_storage(key: &[u8]) -> bool { + ext::with(|ext| + ext.exists_storage(key) + ).unwrap_or(false) +} + +/// Clear the storage entries with a key that starts with the given prefix. +pub fn clear_prefix(prefix: &[u8]) { + ext::with(|ext| + ext.clear_prefix(prefix) + ); +} + +/// The current relay chain identifier. +pub fn chain_id() -> u64 { + ext::with(|ext| + ext.chain_id() + ).unwrap_or(0) +} + +/// "Commit" all existing operations and compute the resultant storage root. +pub fn storage_root() -> H256 { + ext::with(|ext| + ext.storage_root() + ).unwrap_or(H256::new()) +} + +/// A trie root formed from the enumerated items. +pub fn enumerated_trie_root(serialised_values: &[&[u8]]) -> H::Out +where + H: Hasher, + H::Out: Encodable + Ord, +{ + triehash::ordered_trie_root::(serialised_values.iter().map(|s| s.to_vec())) +} + +/// A trie root formed from the iterated items. +pub fn trie_root(input: I) -> H::Out +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + H: Hasher, + ::Out: Encodable + Ord, +{ + triehash::trie_root::(input) +} + +/// A trie root formed from the enumerated items. +pub fn ordered_trie_root(input: I) -> H::Out +where + I: IntoIterator, + A: AsRef<[u8]>, + H: Hasher, + ::Out: Encodable + Ord, +{ + triehash::ordered_trie_root::(input) +} + +/// Verify a ed25519 signature. +pub fn ed25519_verify>(sig: &[u8; 64], msg: &[u8], pubkey: P) -> bool { + ed25519::verify(sig, msg, pubkey) +} + +/// Execute the given closure with global function available whose functionality routes into the +/// externalities `ext`. Forwards the value that the closure returns. +// NOTE: need a concrete hasher here due to limitations of the `environmental!` macro, otherwise a type param would have been fine I think. +pub fn with_externalities R>(ext: &mut Externalities, f: F) -> R { + ext::using(ext, f) +} + +/// Trait for things which can be printed. +pub trait Printable { + fn print(self); +} + +impl<'a> Printable for &'a [u8] { + fn print(self) { + println!("Runtime: {}", HexDisplay::from(&self)); + } +} + +impl<'a> Printable for &'a str { + fn print(self) { + println!("Runtime: {}", self); + } +} + +impl Printable for u64 { + fn print(self) { + println!("Runtime: {}", self); + } +} + +/// Print a printable value. +pub fn print(value: T) { + value.print(); +} + +#[macro_export] +macro_rules! impl_stubs { + ( $( $new_name:ident $($nodecode:ident)* => $invoke: expr ),*) => { + /// Dispatch logic for the native runtime. + pub fn dispatch(method: &str, data: &[u8]) -> Option> { + match method { + $( + stringify!($new_name) => { impl_stubs!(@METHOD data $new_name $($nodecode)* => $invoke) } + )* + _ => None, + } + } + }; + (@METHOD $data: ident $new_name: ident NO_DECODE => $invoke:expr) => { + Some($invoke($data)) + }; + (@METHOD $data: ident $new_name: ident => $invoke:expr) => {{ + let mut data = $data; + let input = match $crate::codec::Decode::decode(&mut data) { + Some(input) => input, + None => panic!("Bad input data provided to {}", stringify!($new_name)), + }; + + let output = $invoke(input); + Some($crate::codec::Encode::encode(&output)) + }} +} + +#[cfg(test)] +mod std_tests { + use super::*; + + #[test] + fn storage_works() { + let mut t = TestExternalities::::new(); + assert!(with_externalities(&mut t, || { + assert_eq!(storage(b"hello"), None); + set_storage(b"hello", b"world"); + assert_eq!(storage(b"hello"), Some(b"world".to_vec())); + assert_eq!(storage(b"foo"), None); + set_storage(b"foo", &[1, 2, 3][..]); + true + })); + + t = map![b"foo".to_vec() => b"bar".to_vec()]; + + assert!(!with_externalities(&mut t, || { + assert_eq!(storage(b"hello"), None); + assert_eq!(storage(b"foo"), Some(b"bar".to_vec())); + false + })); + } + + #[test] + fn read_storage_works() { + let mut t: TestExternalities = map![ + b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec() + ]; + + with_externalities(&mut t, || { + let mut v = [0u8; 4]; + assert!(read_storage(b":test", &mut v[..], 0).unwrap() >= 4); + assert_eq!(v, [11u8, 0, 0, 0]); + let mut w = [0u8; 11]; + assert!(read_storage(b":test", &mut w[..], 4).unwrap() >= 11); + assert_eq!(&w, b"Hello world"); + }); + } + + #[test] + fn clear_prefix_works() { + let mut t: TestExternalities = map![ + b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abcd".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() + ]; + + with_externalities(&mut t, || { + clear_prefix(b":abc"); + + assert!(storage(b":a").is_some()); + assert!(storage(b":abdd").is_some()); + assert!(storage(b":abcd").is_none()); + assert!(storage(b":abc").is_none()); + }); + } +} diff --git a/core/runtime-io/without_std.rs b/core/runtime-io/without_std.rs new file mode 100644 index 00000000000..34c4191e97e --- /dev/null +++ b/core/runtime-io/without_std.rs @@ -0,0 +1,329 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + + +extern crate substrate_primitives as primitives; +extern crate hashdb; + +#[doc(hidden)] +pub extern crate substrate_runtime_std as rstd; + +#[doc(hidden)] +pub extern crate substrate_codec as codec; + +use core::intrinsics; +use rstd::vec::Vec; +use hashdb::Hasher; +use primitives::Blake2Hasher; +pub use rstd::{mem, slice}; + +#[panic_handler] +#[no_mangle] +pub fn panic(info: &::core::panic::PanicInfo) -> ! { + unsafe { + if let Some(loc) = info.location() { + ext_print_utf8(loc.file().as_ptr() as *const u8, loc.file().len() as u32); + ext_print_num(loc.line() as u64); + ext_print_num(loc.column() as u64); + } + intrinsics::abort() + } +} + +#[alloc_error_handler] +pub extern fn oom(_: ::core::alloc::Layout) -> ! { + static OOM_MSG: &str = "Runtime memory exhausted. Aborting"; + + unsafe { + ext_print_utf8(OOM_MSG.as_ptr(), OOM_MSG.len() as u32); + intrinsics::abort(); + } +} + +extern "C" { + fn ext_print_utf8(utf8_data: *const u8, utf8_len: u32); + fn ext_print_hex(data: *const u8, len: u32); + fn ext_print_num(value: u64); + fn ext_set_storage(key_data: *const u8, key_len: u32, value_data: *const u8, value_len: u32); + fn ext_clear_storage(key_data: *const u8, key_len: u32); + fn ext_exists_storage(key_data: *const u8, key_len: u32) -> u32; + fn ext_clear_prefix(prefix_data: *const u8, prefix_len: u32); + fn ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8; + fn ext_get_storage_into(key_data: *const u8, key_len: u32, value_data: *mut u8, value_len: u32, value_offset: u32) -> u32; + fn ext_storage_root(result: *mut u8); + fn ext_blake2_256_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8); + fn ext_chain_id() -> u64; + fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8); + fn ext_twox_128(data: *const u8, len: u32, out: *mut u8); + fn ext_twox_256(data: *const u8, len: u32, out: *mut u8); + fn ext_ed25519_verify(msg_data: *const u8, msg_len: u32, sig_data: *const u8, pubkey_data: *const u8) -> u32; +} + +/// Ensures we use the right crypto when calling into native +pub trait ExternTrieCrypto { + fn enumerated_trie_root(values: &[&[u8]]) -> [u8; 32]; +} + +// Ensures we use a Blake2_256-flavoured Hasher when calling into native +impl ExternTrieCrypto for Blake2Hasher { + fn enumerated_trie_root(values: &[&[u8]]) -> [u8; 32] { + let lengths = values.iter().map(|v| (v.len() as u32).to_le()).collect::>(); + let values = values.iter().fold(Vec::new(), |mut acc, sl| { acc.extend_from_slice(sl); acc }); + let mut result: [u8; 32] = Default::default(); + unsafe { + ext_blake2_256_enumerated_trie_root( + values.as_ptr(), + lengths.as_ptr(), + lengths.len() as u32, + result.as_mut_ptr() + ); + } + result + } +} + +/// Get `key` from storage and return a `Vec`, empty if there's a problem. +pub fn storage(key: &[u8]) -> Option> { + let mut length: u32 = 0; + unsafe { + let ptr = ext_get_allocated_storage(key.as_ptr(), key.len() as u32, &mut length); + if length == u32::max_value() { + None + } else { + Some(Vec::from_raw_parts(ptr, length as usize, length as usize)) + } + } +} + +/// Set the storage of some particular key to Some value. +pub fn set_storage(key: &[u8], value: &[u8]) { + unsafe { + ext_set_storage( + key.as_ptr(), key.len() as u32, + value.as_ptr(), value.len() as u32 + ); + } +} + +/// Clear the storage of some particular key. +pub fn clear_storage(key: &[u8]) { + unsafe { + ext_clear_storage( + key.as_ptr(), key.len() as u32 + ); + } +} + +/// Determine whether a particular key exists in storage. +pub fn exists_storage(key: &[u8]) -> bool { + unsafe { + ext_exists_storage( + key.as_ptr(), key.len() as u32 + ) != 0 + } +} + +/// Clear the storage entries key of which starts with the given prefix. +pub fn clear_prefix(prefix: &[u8]) { + unsafe { + ext_clear_prefix( + prefix.as_ptr(), + prefix.len() as u32 + ); + } +} + +/// Get `key` from storage, placing the value into `value_out` (as much as possible) and return +/// the number of bytes that the key in storage was beyond the offset. +pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { + unsafe { + match ext_get_storage_into( + key.as_ptr(), key.len() as u32, + value_out.as_mut_ptr(), value_out.len() as u32, + value_offset as u32 + ) { + none if none == u32::max_value() => None, + length => Some(length as usize), + } + } +} + +/// The current storage's root. +pub fn storage_root() -> [u8; 32] { + let mut result: [u8; 32] = Default::default(); + unsafe { + ext_storage_root(result.as_mut_ptr()); + } + result +} + +/// A trie root calculated from enumerated values. +pub fn enumerated_trie_root(values: &[&[u8]]) -> [u8; 32] { + H::enumerated_trie_root(values) +} + +/// A trie root formed from the iterated items. +pub fn trie_root< + H: Hasher + ExternTrieCrypto, + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, +>(_input: I) -> [u8; 32] { + unimplemented!() + // TODO Maybe implement (though probably easier/cleaner to have blake2 be the only thing + // implemneted natively and compile the trie logic as wasm). +} + +/// A trie root formed from the enumerated items. +pub fn ordered_trie_root< + H: Hasher + ExternTrieCrypto, + I: IntoIterator, + A: AsRef<[u8]> +>(_input: I) -> [u8; 32] { + unimplemented!() + // TODO Maybe implement (though probably easier/cleaner to have blake2 be the only thing + // implemneted natively and compile the trie logic as wasm). +} + +/// The current relay chain identifier. +pub fn chain_id() -> u64 { + unsafe { + ext_chain_id() + } +} + +/// Conduct a 256-bit Blake2 hash. +pub fn blake2_256(data: &[u8]) -> [u8; 32] { + let mut result: [u8; 32] = Default::default(); + unsafe { + ext_blake2_256(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); + } + result +} + +/// Conduct four XX hashes to give a 256-bit result. +pub fn twox_256(data: &[u8]) -> [u8; 32] { + let mut result: [u8; 32] = Default::default(); + unsafe { + ext_twox_256(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); + } + result +} + +/// Conduct two XX hashes to give a 128-bit result. +pub fn twox_128(data: &[u8]) -> [u8; 16] { + let mut result: [u8; 16] = Default::default(); + unsafe { + ext_twox_128(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); + } + result +} + +/// Verify a ed25519 signature. +pub fn ed25519_verify>(sig: &[u8; 64], msg: &[u8], pubkey: P) -> bool { + unsafe { + ext_ed25519_verify(msg.as_ptr(), msg.len() as u32, sig.as_ptr(), pubkey.as_ref().as_ptr()) == 0 + } +} + +/// Trait for things which can be printed. +pub trait Printable { + fn print(self); +} + +impl<'a> Printable for &'a [u8] { + fn print(self) { + unsafe { + ext_print_hex(self.as_ptr(), self.len() as u32); + } + } +} + +impl<'a> Printable for &'a str { + fn print(self) { + unsafe { + ext_print_utf8(self.as_ptr() as *const u8, self.len() as u32); + } + } +} + +impl Printable for u64 { + fn print(self) { + unsafe { ext_print_num(self); } + } +} + +/// Print a printable value. +pub fn print(value: T) { + value.print(); +} + +#[macro_export] +macro_rules! impl_stubs { + ( $( $new_name:ident $($nodecode:ident)* => $invoke:expr ),* ) => { + $( + impl_stubs!(@METHOD $new_name $($nodecode)* => $invoke); + )* + }; + ( @METHOD $new_name:ident NO_DECODE => $invoke:expr ) => { + #[no_mangle] + pub fn $new_name(input_data: *mut u8, input_len: usize) -> u64 { + let input: &[u8] = if input_len == 0 { + &[0u8; 0] + } else { + unsafe { + $crate::slice::from_raw_parts(input_data, input_len) + } + }; + + let output: $crate::rstd::vec::Vec = $invoke(input); + let res = output.as_ptr() as u64 + ((output.len() as u64) << 32); + + // Leak the output vector to avoid it being freed. + // This is fine in a WASM context since the heap + // will be discarded after the call. + ::core::mem::forget(output); + res + } + }; + ( @METHOD $new_name:ident => $invoke:expr ) => { + #[no_mangle] + pub fn $new_name(input_data: *mut u8, input_len: usize) -> u64 { + let mut input = if input_len == 0 { + &[0u8; 0] + } else { + unsafe { + $crate::slice::from_raw_parts(input_data, input_len) + } + }; + + let input = match $crate::codec::Decode::decode(&mut input) { + Some(input) => input, + None => panic!("Bad input data provided to {}", stringify!($name)), + }; + + let output = ($invoke)(input); + let output = $crate::codec::Encode::encode(&output); + let res = output.as_ptr() as u64 + ((output.len() as u64) << 32); + + // Leak the output vector to avoid it being freed. + // This is fine in a WASM context since the heap + // will be discarded after the call. + ::core::mem::forget(output); + res + } + } +} diff --git a/core/runtime-sandbox/Cargo.toml b/core/runtime-sandbox/Cargo.toml new file mode 100755 index 00000000000..b80cb4443f8 --- /dev/null +++ b/core/runtime-sandbox/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "substrate-runtime-sandbox" +version = "0.1.0" +authors = ["Parity Technologies "] +build = "build.rs" + +[build-dependencies] +rustc_version = "0.2" + +[dependencies] +wasmi = { version = "0.4", optional = true } +substrate-primitives = { path = "../primitives", default_features = false } +substrate-runtime-std = { path = "../runtime-std", default_features = false } +substrate-runtime-io = { path = "../runtime-io", default_features = false } +substrate-codec = { path = "../codec", default_features = false } + +[dev-dependencies] +wabt = "0.4" +assert_matches = "1.1" + +[features] +default = ["std"] +std = [ + "wasmi", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-codec/std", + "substrate-runtime-io/std", +] +nightly = [] +strict = [] diff --git a/core/runtime-sandbox/README.adoc b/core/runtime-sandbox/README.adoc new file mode 100644 index 00000000000..61a203708c8 --- /dev/null +++ b/core/runtime-sandbox/README.adoc @@ -0,0 +1,13 @@ + += Runtime Sandbox + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/runtime-sandbox/build.rs b/core/runtime-sandbox/build.rs new file mode 100755 index 00000000000..35eb154f3a6 --- /dev/null +++ b/core/runtime-sandbox/build.rs @@ -0,0 +1,14 @@ +//! Set a nightly feature + +extern crate rustc_version; +use rustc_version::{version, version_meta, Channel}; + +fn main() { + // Assert we haven't travelled back in time + assert!(version().unwrap().major >= 1); + + // Set cfg flags depending on release channel + if let Channel::Nightly = version_meta().unwrap().channel { + println!("cargo:rustc-cfg=feature=\"nightly\""); + } +} diff --git a/core/runtime-sandbox/src/lib.rs b/core/runtime-sandbox/src/lib.rs new file mode 100755 index 00000000000..16066af160f --- /dev/null +++ b/core/runtime-sandbox/src/lib.rs @@ -0,0 +1,221 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! This crate provides means of instantiation and execution of wasm modules. +//! +//! It works even when the user of this library is itself executes +//! inside the wasm VM. In this case same VM is used for execution +//! of both the sandbox owner and the sandboxed module, without compromising security +//! and without performance penalty of full wasm emulation inside wasm. +//! +//! This is achieved by using bindings to wasm VM which are published by the host API. +//! This API is thin and consists of only handful functions. It contains functions for instantiating +//! modules and executing them and for example doesn't contain functions for inspecting the module +//! structure. The user of this library is supposed to read wasm module by it's own means. +//! +//! When this crate is used in `std` environment all these functions are implemented by directly +//! calling wasm VM. +//! +//! Example of possible use-cases for this library are following: +//! +//! - implementing smart-contract runtimes which uses wasm for contract code +//! - executing wasm substrate runtime inside of a wasm parachain +//! - etc +// end::description[] + +#![warn(missing_docs)] +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(panic_handler))] +#![cfg_attr(not(feature = "std"), feature(core_intrinsics))] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +extern crate substrate_codec as codec; +extern crate substrate_runtime_io as runtime_io; +#[cfg_attr(not(feature = "std"), macro_use)] +extern crate substrate_runtime_std as rstd; +extern crate substrate_primitives as primitives; + +#[cfg(test)] +extern crate wabt; + +#[cfg(test)] +#[macro_use] +extern crate assert_matches; + +use rstd::prelude::*; + +pub use primitives::sandbox::{TypedValue, ReturnValue, HostError}; + +mod imp { + #[cfg(feature = "std")] + include!("../with_std.rs"); + + #[cfg(not(feature = "std"))] + include!("../without_std.rs"); +} + +/// Error that can occur while using this crate. +#[cfg_attr(feature = "std", derive(Debug))] +pub enum Error { + /// Module is not valid, couldn't be instantiated or it's `start` function trapped + /// when executed. + Module, + + /// Access to a memory or table was made with an address or an index which is out of bounds. + /// + /// Note that if wasm module makes an out-of-bounds access then trap will occur. + OutOfBounds, + + /// Failed to invoke an exported function for some reason. + Execution, +} + +impl From for HostError { + fn from(_e: Error) -> HostError { + HostError + } +} + +/// Function pointer for specifying functions by the +/// supervisor in [`EnvironmentDefinitionBuilder`]. +/// +/// [`EnvironmentDefinitionBuilder`]: struct.EnvironmentDefinitionBuilder.html +pub type HostFuncType = fn(&mut T, &[TypedValue]) -> Result; + +/// Reference to a sandboxed linear memory, that +/// will be used by the guest module. +/// +/// The memory can't be directly accessed by supervisor, but only +/// through designated functions [`get`] and [`set`]. +/// +/// [`get`]: #method.get +/// [`set`]: #method.set +#[derive(Clone)] +pub struct Memory { + inner: imp::Memory, +} + +impl Memory { + /// Construct a new linear memory instance. + /// + /// The memory allocated with initial number of pages specified by `initial`. + /// Minimal possible value for `initial` is 0 and maximum possible is `65536`. + /// (Since maximum addressible memory is 232 = 4GiB = 65536 * 64KiB). + /// + /// It is possible to limit maximum number of pages this memory instance can have by specifying + /// `maximum`. If not specified, this memory instance would be able to allocate up to 4GiB. + /// + /// Allocated memory is always zeroed. + pub fn new(initial: u32, maximum: Option) -> Result { + Ok(Memory { + inner: imp::Memory::new(initial, maximum)?, + }) + } + + /// Read a memory area at the address `ptr` with the size of the provided slice `buf`. + /// + /// Returns `Err` if the range is out-of-bounds. + pub fn get(&self, ptr: u32, buf: &mut [u8]) -> Result<(), Error> { + self.inner.get(ptr, buf) + } + + /// Write a memory area at the address `ptr` with contents of the provided slice `buf`. + /// + /// Returns `Err` if the range is out-of-bounds. + pub fn set(&self, ptr: u32, value: &[u8]) -> Result<(), Error> { + self.inner.set(ptr, value) + } +} + +/// Struct that can be used for defining an environment for a sandboxed module. +/// +/// The sandboxed module can access only the entities which were defined and passed +/// to the module at the instantiation time. +pub struct EnvironmentDefinitionBuilder { + inner: imp::EnvironmentDefinitionBuilder, +} + +impl EnvironmentDefinitionBuilder { + /// Construct a new `EnvironmentDefinitionBuilder`. + pub fn new() -> EnvironmentDefinitionBuilder { + EnvironmentDefinitionBuilder { + inner: imp::EnvironmentDefinitionBuilder::new(), + } + } + + /// Register a host function in this environment defintion. + /// + /// NOTE that there is no constraints on type of this function. An instance + /// can import function passed here with any signature it wants. It can even import + /// the same function (i.e. with same `module` and `field`) several times. It's up to + /// the user code to check or constrain the types of signatures. + pub fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) + where + N1: Into>, + N2: Into>, + { + self.inner.add_host_func(module, field, f); + } + + /// Register a memory in this environment definition. + pub fn add_memory(&mut self, module: N1, field: N2, mem: Memory) + where + N1: Into>, + N2: Into>, + { + self.inner.add_memory(module, field, mem.inner); + } +} + +/// Sandboxed instance of a wasm module. +/// +/// This instance can be used for invoking exported functions. +pub struct Instance { + inner: imp::Instance, + +} + +impl Instance { + /// Instantiate a module with the given [`EnvironmentDefinitionBuilder`]. + /// + /// [`EnvironmentDefinitionBuilder`]: struct.EnvironmentDefinitionBuilder.html + pub fn new(code: &[u8], env_def_builder: &EnvironmentDefinitionBuilder, state: &mut T) -> Result, Error> { + Ok(Instance { + inner: imp::Instance::new(code, &env_def_builder.inner, state)?, + }) + } + + /// Invoke an exported function with the given name. + /// + /// # Errors + /// + /// Returns `Err(Error::Execution)` if: + /// + /// - An export function name isn't a proper utf8 byte sequence, + /// - This module doesn't have an exported function with the given name, + /// - If types of the arguments passed to the function doesn't match function signature + /// then trap occurs (as if the exported function was called via call_indirect), + /// - Trap occured at the execution time. + pub fn invoke( + &mut self, + name: &[u8], + args: &[TypedValue], + state: &mut T, + ) -> Result { + self.inner.invoke(name, args, state) + } +} diff --git a/core/runtime-sandbox/with_std.rs b/core/runtime-sandbox/with_std.rs new file mode 100755 index 00000000000..e4e4aa131ea --- /dev/null +++ b/core/runtime-sandbox/with_std.rs @@ -0,0 +1,482 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +extern crate wasmi; + +use rstd::collections::btree_map::BTreeMap; +use rstd::fmt; + +use self::wasmi::{ + Externals, FuncInstance, FuncRef, GlobalDescriptor, GlobalRef, ImportResolver, + MemoryDescriptor, MemoryInstance, MemoryRef, Module, ModuleInstance, ModuleRef, + RuntimeArgs, RuntimeValue, Signature, TableDescriptor, TableRef, Trap, TrapKind +}; +use self::wasmi::memory_units::Pages; +use super::{Error, TypedValue, ReturnValue, HostFuncType, HostError}; + +#[derive(Clone)] +pub struct Memory { + memref: MemoryRef, +} + +impl Memory { + pub fn new(initial: u32, maximum: Option) -> Result { + Ok(Memory { + memref: MemoryInstance::alloc( + Pages(initial as usize), + maximum.map(|m| Pages(m as usize)), + ).map_err(|_| Error::Module)?, + }) + } + + pub fn get(&self, ptr: u32, buf: &mut [u8]) -> Result<(), Error> { + self.memref.get_into(ptr, buf).map_err(|_| Error::OutOfBounds)?; + Ok(()) + } + + pub fn set(&self, ptr: u32, value: &[u8]) -> Result<(), Error> { + self.memref.set(ptr, value).map_err(|_| Error::OutOfBounds)?; + Ok(()) + } +} + +struct HostFuncIndex(usize); + +struct DefinedHostFunctions { + funcs: Vec>, +} + +impl Clone for DefinedHostFunctions { + fn clone(&self) -> DefinedHostFunctions { + DefinedHostFunctions { + funcs: self.funcs.clone(), + } + } +} + +impl DefinedHostFunctions { + fn new() -> DefinedHostFunctions { + DefinedHostFunctions { + funcs: Vec::new(), + } + } + + fn define(&mut self, f: HostFuncType) -> HostFuncIndex { + let idx = self.funcs.len(); + self.funcs.push(f); + HostFuncIndex(idx) + } +} + +#[derive(Debug)] +struct DummyHostError; + +impl fmt::Display for DummyHostError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "DummyHostError") + } +} + +impl self::wasmi::HostError for DummyHostError { +} + +fn from_runtime_value(v: RuntimeValue) -> TypedValue { + match v { + RuntimeValue::I32(v) => TypedValue::I32(v), + RuntimeValue::I64(v) => TypedValue::I64(v), + RuntimeValue::F32(v) => TypedValue::F32(v.to_bits() as i32), + RuntimeValue::F64(v) => TypedValue::F64(v.to_bits() as i64), + } +} + +fn to_runtime_value(v: TypedValue) -> RuntimeValue { + use self::wasmi::nan_preserving_float::{F32, F64}; + match v { + TypedValue::I32(v) => RuntimeValue::I32(v as i32), + TypedValue::I64(v) => RuntimeValue::I64(v as i64), + TypedValue::F32(v_bits) => RuntimeValue::F32(F32::from_bits(v_bits as u32)), + TypedValue::F64(v_bits) => RuntimeValue::F64(F64::from_bits(v_bits as u64)), + } +} + +struct GuestExternals<'a, T: 'a> { + state: &'a mut T, + defined_host_functions: &'a DefinedHostFunctions, +} + +impl<'a, T> Externals for GuestExternals<'a, T> { + fn invoke_index( + &mut self, + index: usize, + args: RuntimeArgs, + ) -> Result, Trap> { + let args = args.as_ref() + .iter() + .cloned() + .map(from_runtime_value) + .collect::>(); + + let result = (self.defined_host_functions.funcs[index])(self.state, &args); + match result { + Ok(value) => Ok(match value { + ReturnValue::Value(v) => Some(to_runtime_value(v)), + ReturnValue::Unit => None, + }), + Err(HostError) => Err(TrapKind::Host(Box::new(DummyHostError)).into()), + } + } +} + +enum ExternVal { + HostFunc(HostFuncIndex), + Memory(Memory), +} + +pub struct EnvironmentDefinitionBuilder { + map: BTreeMap<(Vec, Vec), ExternVal>, + defined_host_functions: DefinedHostFunctions, +} + +impl EnvironmentDefinitionBuilder { + pub fn new() -> EnvironmentDefinitionBuilder { + EnvironmentDefinitionBuilder { + map: BTreeMap::new(), + defined_host_functions: DefinedHostFunctions::new(), + } + } + + pub fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) + where + N1: Into>, + N2: Into>, + { + let idx = self.defined_host_functions.define(f); + self.map + .insert((module.into(), field.into()), ExternVal::HostFunc(idx)); + } + + pub fn add_memory(&mut self, module: N1, field: N2, mem: Memory) + where + N1: Into>, + N2: Into>, + { + self.map + .insert((module.into(), field.into()), ExternVal::Memory(mem)); + } +} + +impl ImportResolver for EnvironmentDefinitionBuilder { + fn resolve_func( + &self, + module_name: &str, + field_name: &str, + signature: &Signature, + ) -> Result { + let key = ( + module_name.as_bytes().to_owned(), + field_name.as_bytes().to_owned(), + ); + let externval = self.map.get(&key).ok_or_else(|| { + wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name)) + })?; + let host_func_idx = match *externval { + ExternVal::HostFunc(ref idx) => idx, + _ => { + return Err(wasmi::Error::Instantiation(format!( + "Export {}:{} is not a host func", + module_name, field_name + ))) + } + }; + Ok(FuncInstance::alloc_host(signature.clone(), host_func_idx.0)) + } + + fn resolve_global( + &self, + _module_name: &str, + _field_name: &str, + _global_type: &GlobalDescriptor, + ) -> Result { + Err(wasmi::Error::Instantiation(format!( + "Importing globals is not supported yet" + ))) + } + + fn resolve_memory( + &self, + module_name: &str, + field_name: &str, + _memory_type: &MemoryDescriptor, + ) -> Result { + let key = ( + module_name.as_bytes().to_owned(), + field_name.as_bytes().to_owned(), + ); + let externval = self.map.get(&key).ok_or_else(|| { + wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name)) + })?; + let memory = match *externval { + ExternVal::Memory(ref m) => m, + _ => { + return Err(wasmi::Error::Instantiation(format!( + "Export {}:{} is not a memory", + module_name, field_name + ))) + } + }; + Ok(memory.memref.clone()) + } + + fn resolve_table( + &self, + _module_name: &str, + _field_name: &str, + _table_type: &TableDescriptor, + ) -> Result { + Err(wasmi::Error::Instantiation(format!( + "Importing tables is not supported yet" + ))) + } +} + +pub struct Instance { + instance: ModuleRef, + defined_host_functions: DefinedHostFunctions, + _marker: ::std::marker::PhantomData, +} + +impl Instance { + pub fn new(code: &[u8], env_def_builder: &EnvironmentDefinitionBuilder, state: &mut T) -> Result, Error> { + let module = Module::from_buffer(code).map_err(|_| Error::Module)?; + let not_started_instance = ModuleInstance::new(&module, env_def_builder) + .map_err(|_| Error::Module)?; + + + let defined_host_functions = env_def_builder.defined_host_functions.clone(); + let instance = { + let mut externals = GuestExternals { + state, + defined_host_functions: &defined_host_functions, + }; + let instance = not_started_instance.run_start(&mut externals).map_err(|_| Error::Module)?; + instance + }; + + Ok(Instance { + instance, + defined_host_functions, + _marker: ::std::marker::PhantomData::, + }) + } + + pub fn invoke( + &mut self, + name: &[u8], + args: &[TypedValue], + state: &mut T, + ) -> Result { + let args = args.iter().cloned().map(Into::into).collect::>(); + + let name = ::std::str::from_utf8(name).map_err(|_| Error::Execution)?; + let mut externals = GuestExternals { + state, + defined_host_functions: &self.defined_host_functions, + }; + let result = self.instance + .invoke_export(&name, &args, &mut externals); + + match result { + Ok(None) => Ok(ReturnValue::Unit), + Ok(Some(val)) => Ok(ReturnValue::Value(val.into())), + Err(_err) => Err(Error::Execution), + } + } +} + +#[cfg(test)] +mod tests { + use wabt; + use ::{Error, TypedValue, ReturnValue, HostError, EnvironmentDefinitionBuilder, Instance}; + + fn execute_sandboxed(code: &[u8], args: &[TypedValue]) -> Result { + struct State { + counter: u32, + } + + fn env_assert(_e: &mut State, args: &[TypedValue]) -> Result { + if args.len() != 1 { + return Err(HostError); + } + let condition = args[0].as_i32().ok_or_else(|| HostError)?; + if condition != 0 { + Ok(ReturnValue::Unit) + } else { + Err(HostError) + } + } + fn env_inc_counter(e: &mut State, args: &[TypedValue]) -> Result { + if args.len() != 1 { + return Err(HostError); + } + let inc_by = args[0].as_i32().ok_or_else(|| HostError)?; + e.counter += inc_by as u32; + Ok(ReturnValue::Value(TypedValue::I32(e.counter as i32))) + } + /// Function that takes one argument of any type and returns that value. + fn env_polymorphic_id(_e: &mut State, args: &[TypedValue]) -> Result { + if args.len() != 1 { + return Err(HostError); + } + Ok(ReturnValue::Value(args[0])) + } + + let mut state = State { counter: 0 }; + + let mut env_builder = EnvironmentDefinitionBuilder::new(); + env_builder.add_host_func("env", "assert", env_assert); + env_builder.add_host_func("env", "inc_counter", env_inc_counter); + env_builder.add_host_func("env", "polymorphic_id", env_polymorphic_id); + + let mut instance = Instance::new(code, &env_builder, &mut state)?; + let result = instance.invoke(b"call", args, &mut state); + + result.map_err(|_| HostError) + } + + #[test] + fn invoke_args() { + let code = wabt::wat2wasm(r#" + (module + (import "env" "assert" (func $assert (param i32))) + + (func (export "call") (param $x i32) (param $y i64) + ;; assert that $x = 0x12345678 + (call $assert + (i32.eq + (get_local $x) + (i32.const 0x12345678) + ) + ) + + (call $assert + (i64.eq + (get_local $y) + (i64.const 0x1234567887654321) + ) + ) + ) + ) + "#).unwrap(); + + let result = execute_sandboxed( + &code, + &[ + TypedValue::I32(0x12345678), + TypedValue::I64(0x1234567887654321), + ] + ); + assert!(result.is_ok()); + } + + #[test] + fn return_value() { + let code = wabt::wat2wasm(r#" + (module + (func (export "call") (param $x i32) (result i32) + (i32.add + (get_local $x) + (i32.const 1) + ) + ) + ) + "#).unwrap(); + + let return_val = execute_sandboxed( + &code, + &[ + TypedValue::I32(0x1336), + ] + ).unwrap(); + assert_eq!(return_val, ReturnValue::Value(TypedValue::I32(0x1337))); + } + + #[test] + fn signatures_dont_matter() { + let code = wabt::wat2wasm(r#" + (module + (import "env" "polymorphic_id" (func $id_i32 (param i32) (result i32))) + (import "env" "polymorphic_id" (func $id_i64 (param i64) (result i64))) + (import "env" "assert" (func $assert (param i32))) + + (func (export "call") + ;; assert that we can actually call the "same" function with different + ;; signatures. + (call $assert + (i32.eq + (call $id_i32 + (i32.const 0x012345678) + ) + (i32.const 0x012345678) + ) + ) + (call $assert + (i64.eq + (call $id_i64 + (i64.const 0x0123456789abcdef) + ) + (i64.const 0x0123456789abcdef) + ) + ) + ) + ) + "#).unwrap(); + + let return_val = execute_sandboxed(&code, &[]).unwrap(); + assert_eq!(return_val, ReturnValue::Unit); + } + + #[test] + fn cant_return_unmatching_type() { + fn env_returns_i32(_e: &mut (), _args: &[TypedValue]) -> Result { + Ok(ReturnValue::Value(TypedValue::I32(42))) + } + + let mut env_builder = EnvironmentDefinitionBuilder::new(); + env_builder.add_host_func("env", "returns_i32", env_returns_i32); + + let code = wabt::wat2wasm(r#" + (module + ;; It's actually returns i32, but imported as if it returned i64 + (import "env" "returns_i32" (func $returns_i32 (result i64))) + + (func (export "call") + (drop + (call $returns_i32) + ) + ) + ) + "#).unwrap(); + + // It succeeds since we are able to import functions with types we want. + let mut instance = Instance::new(&code, &env_builder, &mut ()).unwrap(); + + // But this fails since we imported a function that returns i32 as if it returned i64. + assert_matches!( + instance.invoke(b"call", &[], &mut ()), + Err(Error::Execution) + ); + } +} diff --git a/core/runtime-sandbox/without_std.rs b/core/runtime-sandbox/without_std.rs new file mode 100755 index 00000000000..f7feb466a09 --- /dev/null +++ b/core/runtime-sandbox/without_std.rs @@ -0,0 +1,303 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use rstd::prelude::*; +use rstd::{slice, marker, mem}; +use codec::{Decode, Encode}; +use primitives::sandbox as sandbox_primitives; +use super::{Error, TypedValue, ReturnValue, HostFuncType}; + +mod ffi { + use rstd::mem; + use super::HostFuncType; + + /// Index into the default table that points to a `HostFuncType`. + pub type HostFuncIndex = usize; + + /// Coerce `HostFuncIndex` to a callable host function pointer. + /// + /// # Safety + /// + /// This function should be only called with a `HostFuncIndex` that was previously registered + /// in the environment defintion. Typically this should only + /// be called with an argument received in `dispatch_thunk`. + pub unsafe fn coerce_host_index_to_func(idx: HostFuncIndex) -> HostFuncType { + // We need to ensure that sizes of a callable function pointer and host function index is + // indeed equal. + // We can't use `static_assertions` create because it makes compiler panic, fallback to runtime assert. + // const_assert!(mem::size_of::() == mem::size_of::>(),); + assert!(mem::size_of::() == mem::size_of::>()); + mem::transmute::>(idx) + } + + extern "C" { + pub fn ext_sandbox_instantiate( + dispatch_thunk: extern "C" fn( + serialized_args_ptr: *const u8, + serialized_args_len: usize, + state: usize, + f: HostFuncIndex, + ) -> u64, + wasm_ptr: *const u8, + wasm_len: usize, + imports_ptr: *const u8, + imports_len: usize, + state: usize, + ) -> u32; + pub fn ext_sandbox_invoke( + instance_idx: u32, + export_ptr: *const u8, + export_len: usize, + args_ptr: *const u8, + args_len: usize, + return_val_ptr: *mut u8, + return_val_len: usize, + state: usize, + ) -> u32; + pub fn ext_sandbox_memory_new(initial: u32, maximum: u32) -> u32; + pub fn ext_sandbox_memory_get( + memory_idx: u32, + offset: u32, + buf_ptr: *mut u8, + buf_len: usize, + ) -> u32; + pub fn ext_sandbox_memory_set( + memory_idx: u32, + offset: u32, + val_ptr: *const u8, + val_len: usize, + ) -> u32; + pub fn ext_sandbox_memory_teardown( + memory_idx: u32, + ); + pub fn ext_sandbox_instance_teardown( + instance_idx: u32, + ); + } +} + +#[derive(Clone)] +pub struct Memory { + memory_idx: u32, +} + +impl Memory { + pub fn new(initial: u32, maximum: Option) -> Result { + let result = unsafe { + let maximum = if let Some(maximum) = maximum { + maximum + } else { + sandbox_primitives::MEM_UNLIMITED + }; + ffi::ext_sandbox_memory_new(initial, maximum) + }; + match result { + sandbox_primitives::ERR_MODULE => Err(Error::Module), + memory_idx => Ok(Memory { memory_idx }), + } + } + + pub fn get(&self, offset: u32, buf: &mut [u8]) -> Result<(), Error> { + let result = unsafe { ffi::ext_sandbox_memory_get(self.memory_idx, offset, buf.as_mut_ptr(), buf.len()) }; + match result { + sandbox_primitives::ERR_OK => Ok(()), + sandbox_primitives::ERR_OUT_OF_BOUNDS => Err(Error::OutOfBounds), + _ => unreachable!(), + } + } + + pub fn set(&self, offset: u32, val: &[u8]) -> Result<(), Error> { + let result = unsafe { ffi::ext_sandbox_memory_set(self.memory_idx, offset, val.as_ptr(), val.len()) }; + match result { + sandbox_primitives::ERR_OK => Ok(()), + sandbox_primitives::ERR_OUT_OF_BOUNDS => Err(Error::OutOfBounds), + _ => unreachable!(), + } + } +} + +impl Drop for Memory { + fn drop(&mut self) { + unsafe { + ffi::ext_sandbox_memory_teardown(self.memory_idx); + } + } +} + +pub struct EnvironmentDefinitionBuilder { + env_def: sandbox_primitives::EnvironmentDefinition, + _marker: marker::PhantomData, +} + +impl EnvironmentDefinitionBuilder { + pub fn new() -> EnvironmentDefinitionBuilder { + EnvironmentDefinitionBuilder { + env_def: sandbox_primitives::EnvironmentDefinition { + entries: Vec::new(), + }, + _marker: marker::PhantomData::, + } + } + + fn add_entry( + &mut self, + module: N1, + field: N2, + extern_entity: sandbox_primitives::ExternEntity, + ) where + N1: Into>, + N2: Into>, + { + let entry = sandbox_primitives::Entry { + module_name: module.into(), + field_name: field.into(), + entity: extern_entity, + }; + self.env_def.entries.push(entry); + } + + pub fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) + where + N1: Into>, + N2: Into>, + { + let f = sandbox_primitives::ExternEntity::Function(f as u32); + self.add_entry(module, field, f); + } + + pub fn add_memory(&mut self, module: N1, field: N2, mem: Memory) + where + N1: Into>, + N2: Into>, + { + let mem = sandbox_primitives::ExternEntity::Memory(mem.memory_idx as u32); + self.add_entry(module, field, mem); + } +} + +pub struct Instance { + instance_idx: u32, + _marker: marker::PhantomData, +} + +/// The primary responsibility of this thunk is to deserialize arguments and +/// call the original function, specified by the index. +extern "C" fn dispatch_thunk( + serialized_args_ptr: *const u8, + serialized_args_len: usize, + state: usize, + f: ffi::HostFuncIndex, +) -> u64 { + let serialized_args = unsafe { + if serialized_args_len == 0 { + &[] + } else { + slice::from_raw_parts(serialized_args_ptr, serialized_args_len) + } + }; + let args = Vec::::decode(&mut &serialized_args[..]).expect( + "serialized args should be provided by the runtime; + correctly serialized data should be deserializable; + qed", + ); + + unsafe { + // This should be safe since `coerce_host_index_to_func` is called with an argument + // received in an `dispatch_thunk` implementation, so `f` should point + // on a valid host function. + let f = ffi::coerce_host_index_to_func(f); + + // This should be safe since mutable reference to T is passed upon the invocation. + let state = &mut *(state as *mut T); + + // Pass control flow to the designated function. + let result = f(state, &args).encode(); + + // Leak the result vector and return the pointer to return data. + let result_ptr = result.as_ptr() as u64; + let result_len = result.len() as u64; + mem::forget(result); + + (result_ptr << 32) | result_len + } +} + +impl Instance { + pub fn new(code: &[u8], env_def_builder: &EnvironmentDefinitionBuilder, state: &mut T) -> Result, Error> { + let serialized_env_def: Vec = env_def_builder.env_def.encode(); + let result = unsafe { + // It's very important to instantiate thunk with the right type. + let dispatch_thunk = dispatch_thunk::; + + ffi::ext_sandbox_instantiate( + dispatch_thunk, + code.as_ptr(), + code.len(), + serialized_env_def.as_ptr(), + serialized_env_def.len(), + state as *const T as usize, + ) + }; + let instance_idx = match result { + sandbox_primitives::ERR_MODULE => return Err(Error::Module), + instance_idx => instance_idx, + }; + Ok(Instance { + instance_idx, + _marker: marker::PhantomData::, + }) + } + + pub fn invoke( + &mut self, + name: &[u8], + args: &[TypedValue], + state: &mut T, + ) -> Result { + let serialized_args = args.to_vec().encode(); + let mut return_val = vec![0u8; sandbox_primitives::ReturnValue::ENCODED_MAX_SIZE]; + + let result = unsafe { + ffi::ext_sandbox_invoke( + self.instance_idx, + name.as_ptr(), + name.len(), + serialized_args.as_ptr(), + serialized_args.len(), + return_val.as_mut_ptr(), + return_val.len(), + state as *const T as usize, + ) + }; + match result { + sandbox_primitives::ERR_OK => { + let return_val = sandbox_primitives::ReturnValue::decode(&mut &return_val[..]) + .ok_or(Error::Execution)?; + Ok(return_val) + } + sandbox_primitives::ERR_EXECUTION => Err(Error::Execution), + _ => unreachable!(), + } + } +} + +impl Drop for Instance { + fn drop(&mut self) { + unsafe { + ffi::ext_sandbox_instance_teardown(self.instance_idx); + } + } +} diff --git a/core/runtime-std/Cargo.toml b/core/runtime-std/Cargo.toml new file mode 100644 index 00000000000..42745a0152a --- /dev/null +++ b/core/runtime-std/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substrate-runtime-std" +version = "0.1.0" +authors = ["Parity Technologies "] +build = "build.rs" + +[build-dependencies] +rustc_version = "0.2" + +[dependencies] +pwasm-alloc = { path = "../pwasm-alloc" } +pwasm-libc = { path = "../pwasm-libc" } + +[features] +default = ["std"] +std = [] +nightly = [] +strict = [] diff --git a/core/runtime-std/README.adoc b/core/runtime-std/README.adoc new file mode 100644 index 00000000000..36ea8d99c22 --- /dev/null +++ b/core/runtime-std/README.adoc @@ -0,0 +1,14 @@ + += Runtime std + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- + diff --git a/core/runtime-std/build.rs b/core/runtime-std/build.rs new file mode 100644 index 00000000000..55688bad9cc --- /dev/null +++ b/core/runtime-std/build.rs @@ -0,0 +1,14 @@ +//! Set a nightly feature + +extern crate rustc_version; +use rustc_version::{version, version_meta, Channel}; + +fn main() { + // Assert we haven't travelled back in time + assert!(version().unwrap().major >= 1); + + // Set cfg flags depending on release channel + if let Channel::Nightly = version_meta().unwrap().channel { + println!("cargo:rustc-cfg=feature=\"nightly\""); + } +} diff --git a/core/runtime-std/src/lib.rs b/core/runtime-std/src/lib.rs new file mode 100644 index 00000000000..d0fddddd298 --- /dev/null +++ b/core/runtime-std/src/lib.rs @@ -0,0 +1,51 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Lowest-abstraction level for the Substrate runtime: just exports useful primitives from std +//! or core/alloc to be used with any code that depends on the runtime. +// end::description[] + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(panic_handler))] +#![cfg_attr(not(feature = "std"), feature(core_intrinsics))] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +#![cfg_attr(feature = "std", doc = "Polkadot runtime standard library as compiled when linked with Rust's standard library.")] +#![cfg_attr(not(feature = "std"), doc = "Polkadot's runtime standard library as compiled without Rust's standard library.")] + +#[macro_export] +macro_rules! map { + ($( $name:expr => $value:expr ),*) => ( + vec![ $( ( $name, $value ) ),* ].into_iter().collect() + ) +} + +#[cfg(feature = "std")] +include!("../with_std.rs"); + +#[cfg(not(feature = "std"))] +include!("../without_std.rs"); + +/// Prelude of common useful imports. +/// +/// This should include only things which are in the normal std prelude. +pub mod prelude { + pub use ::vec::Vec; + pub use ::boxed::Box; + pub use ::cmp::{Eq, PartialEq}; + pub use ::clone::Clone; +} diff --git a/core/runtime-std/with_std.rs b/core/runtime-std/with_std.rs new file mode 100644 index 00000000000..443fea1a617 --- /dev/null +++ b/core/runtime-std/with_std.rs @@ -0,0 +1,38 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +pub use std::borrow; +pub use std::boxed; +pub use std::cell; +pub use std::clone; +pub use std::cmp; +pub use std::fmt; +pub use std::hash; +pub use std::iter; +pub use std::marker; +pub use std::mem; +pub use std::num; +pub use std::ops; +pub use std::ptr; +pub use std::rc; +pub use std::slice; +pub use std::string; +pub use std::vec; +pub use std::result; + +pub mod collections { + pub use std::collections::btree_map; +} diff --git a/core/runtime-std/without_std.rs b/core/runtime-std/without_std.rs new file mode 100644 index 00000000000..5219cba0627 --- /dev/null +++ b/core/runtime-std/without_std.rs @@ -0,0 +1,46 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[cfg(feature = "nightly")] +extern crate alloc; +#[cfg(feature = "nightly")] +extern crate pwasm_libc; +#[cfg(feature = "nightly")] +extern crate pwasm_alloc; + +pub use alloc::boxed; +pub use alloc::rc; +pub use alloc::vec; +pub use alloc::string; +pub use core::borrow; +pub use core::cell; +pub use core::clone; +pub use core::cmp; +pub use core::fmt; +pub use core::hash; +pub use core::intrinsics; +pub use core::iter; +pub use core::marker; +pub use core::mem; +pub use core::num; +pub use core::ops; +pub use core::ptr; +pub use core::slice; +pub use core::result; + +pub mod collections { + pub use alloc::collections::btree_map; +} diff --git a/core/serializer/Cargo.toml b/core/serializer/Cargo.toml new file mode 100644 index 00000000000..412586599e6 --- /dev/null +++ b/core/serializer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "substrate-serializer" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +serde = { version = "1.0", default_features = false } +serde_json = "1.0" diff --git a/core/serializer/README.adoc b/core/serializer/README.adoc new file mode 100644 index 00000000000..7b780bae436 --- /dev/null +++ b/core/serializer/README.adoc @@ -0,0 +1,14 @@ + += Serializer + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- + diff --git a/core/serializer/src/lib.rs b/core/serializer/src/lib.rs new file mode 100644 index 00000000000..48afc574973 --- /dev/null +++ b/core/serializer/src/lib.rs @@ -0,0 +1,46 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Substrate customizable serde serializer. +//! +//! The idea is that we can later change the implementation +//! to something more compact, but for now we're using JSON. +// end::description[] + +#![warn(missing_docs)] + +extern crate serde; +extern crate serde_json; + +pub use serde_json::{from_str, from_slice, from_reader, Result, Error}; + +const PROOF: &str = "Serializers are infallible; qed"; + +/// Serialize the given data structure as a pretty-printed String of JSON. +pub fn to_string_pretty(value: &T) -> String { + serde_json::to_string_pretty(value).expect(PROOF) +} + +/// Serialize the given data structure as a JSON byte vector. +pub fn encode(value: &T) -> Vec { + serde_json::to_vec(value).expect(PROOF) +} + +/// Serialize the given data structure as JSON into the IO stream. +pub fn to_writer(writer: W, value: &T) -> Result<()> { + serde_json::to_writer(writer, value) +} diff --git a/core/service/Cargo.toml b/core/service/Cargo.toml new file mode 100644 index 00000000000..e2ddcb0704b --- /dev/null +++ b/core/service/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "substrate-service" +version = "0.3.0" +authors = ["Parity Technologies "] + +[dependencies] +futures = "0.1.17" +parking_lot = "0.4" +error-chain = "0.12" +lazy_static = "1.0" +log = "0.3" +slog = "^2" +tokio = "0.1.7" +exit-future = "0.1" +serde = "1.0" +serde_json = "1.0" +serde_derive = "1.0" +target_info = "0.1" +substrate-keystore = { path = "../keystore" } +substrate-runtime-io = { path = "../runtime-io" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-primitives = { path = "../primitives" } +substrate-network = { path = "../network" } +substrate-client = { path = "../client" } +substrate-client-db = { path = "../client/db" } +substrate-codec = { path = "../codec" } +substrate-executor = { path = "../executor" } +substrate-extrinsic-pool = { path = "../extrinsic-pool" } +substrate-rpc = { path = "../rpc" } +substrate-rpc-servers = { path = "../rpc-servers" } +substrate-telemetry = { path = "../telemetry" } diff --git a/core/service/README.adoc b/core/service/README.adoc new file mode 100644 index 00000000000..4d74c098b2e --- /dev/null +++ b/core/service/README.adoc @@ -0,0 +1,14 @@ + += Service + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- + diff --git a/core/service/src/chain_ops.rs b/core/service/src/chain_ops.rs new file mode 100644 index 00000000000..aeefda6a676 --- /dev/null +++ b/core/service/src/chain_ops.rs @@ -0,0 +1,139 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Chain utilities. + +use std::{self, io::{Read, Write}}; +use futures::Future; +use serde_json; + +use client::BlockOrigin; +use runtime_primitives::generic::{SignedBlock, BlockId}; +use runtime_primitives::traits::{As}; +use components::{ServiceFactory, FactoryFullConfiguration, FactoryBlockNumber, RuntimeGenesis}; +use new_client; +use codec::{Decode, Encode}; +use error; +use chain_spec::ChainSpec; + +/// Export a range of blocks to a binary stream. +pub fn export_blocks(config: FactoryFullConfiguration, exit: E, mut output: W, from: FactoryBlockNumber, to: Option>, json: bool) -> error::Result<()> + where F: ServiceFactory, E: Future + Send + 'static, W: Write, +{ + let client = new_client::(config)?; + let mut block = from; + + let last = match to { + Some(v) if v == As::sa(0) => As::sa(1), + Some(v) => v, + None => client.info()?.chain.best_number, + }; + + if last < block { + return Err("Invalid block range specified".into()); + } + + let (exit_send, exit_recv) = std::sync::mpsc::channel(); + ::std::thread::spawn(move || { + let _ = exit.wait(); + let _ = exit_send.send(()); + }); + info!("Exporting blocks from #{} to #{}", block, last); + if !json { + output.write(&(last - block + As::sa(1)).encode())?; + } + + loop { + if exit_recv.try_recv().is_ok() { + break; + } + match client.block(&BlockId::number(block))? { + Some(block) => { + if json { + serde_json::to_writer(&mut output, &block).map_err(|e| format!("Eror writing JSON: {}", e))?; + } else { + output.write(&block.encode())?; + } + }, + None => break, + } + if block.as_() % 10000 == 0 { + info!("#{}", block); + } + if block == last { + break; + } + block += As::sa(1); + } + Ok(()) +} + +/// Import blocks from a binary stream. +pub fn import_blocks(config: FactoryFullConfiguration, exit: E, mut input: R) -> error::Result<()> + where F: ServiceFactory, E: Future + Send + 'static, R: Read, +{ + let client = new_client::(config)?; + + let (exit_send, exit_recv) = std::sync::mpsc::channel(); + ::std::thread::spawn(move || { + let _ = exit.wait(); + let _ = exit_send.send(()); + }); + + let count: u32 = Decode::decode(&mut input).ok_or("Error reading file")?; + info!("Importing {} blocks", count); + let mut block = 0; + for _ in 0 .. count { + if exit_recv.try_recv().is_ok() { + break; + } + match SignedBlock::decode(&mut input) { + Some(block) => { + let header = client.check_justification(block.block.header, block.justification.into())?; + client.import_block(BlockOrigin::File, header, Some(block.block.extrinsics))?; + }, + None => { + warn!("Error reading block data."); + break; + } + } + block += 1; + if block % 1000 == 0 { + info!("#{}", block); + } + } + info!("Imported {} blocks. Best: #{}", block, client.info()?.chain.best_number); + + Ok(()) +} + +/// Revert the chain. +pub fn revert_chain(config: FactoryFullConfiguration, blocks: FactoryBlockNumber) -> error::Result<()> + where F: ServiceFactory, +{ + let client = new_client::(config)?; + let reverted = client.revert(blocks)?; + let info = client.info()?.chain; + info!("Reverted {} blocks. Best: #{} ({})", reverted, info.best_number, info.best_hash); + Ok(()) +} + +/// Build a chain spec json +pub fn build_spec(spec: ChainSpec, raw: bool) -> error::Result + where G: RuntimeGenesis, +{ + Ok(spec.to_json(raw)?) +} diff --git a/core/service/src/chain_spec.rs b/core/service/src/chain_spec.rs new file mode 100644 index 00000000000..6ccd0545b44 --- /dev/null +++ b/core/service/src/chain_spec.rs @@ -0,0 +1,170 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate chain configurations. + +use std::collections::HashMap; +use std::fs::File; +use std::path::PathBuf; +use primitives::storage::{StorageKey, StorageData}; +use runtime_primitives::{BuildStorage, StorageMap}; +use serde_json as json; +use components::RuntimeGenesis; + +enum GenesisSource { + File(PathBuf), + Embedded(&'static [u8]), + Factory(fn() -> G), +} + +impl GenesisSource { + fn resolve(&self) -> Result, String> { + #[derive(Serialize, Deserialize)] + struct GenesisContainer { + genesis: Genesis, + } + + match *self { + GenesisSource::File(ref path) => { + let file = File::open(path).map_err(|e| format!("Error opening spec file: {}", e))?; + let genesis: GenesisContainer = json::from_reader(file).map_err(|e| format!("Error parsing spec file: {}", e))?; + Ok(genesis.genesis) + }, + GenesisSource::Embedded(buf) => { + let genesis: GenesisContainer = json::from_reader(buf).map_err(|e| format!("Error parsing embedded file: {}", e))?; + Ok(genesis.genesis) + }, + GenesisSource::Factory(f) => Ok(Genesis::Runtime(f())), + } + } +} + +impl<'a, G: RuntimeGenesis> BuildStorage for &'a ChainSpec { + fn build_storage(self) -> Result { + match self.genesis.resolve()? { + Genesis::Runtime(gc) => gc.build_storage(), + Genesis::Raw(map) => Ok(map.into_iter().map(|(k, v)| (k.0, v.0)).collect()), + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +enum Genesis { + Runtime(G), + Raw(HashMap), +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ChainSpecFile { + pub name: String, + pub id: String, + pub boot_nodes: Vec, + pub telemetry_url: Option, +} + +/// A configuration of a chain. Can be used to build a genesis block. +pub struct ChainSpec { + spec: ChainSpecFile, + genesis: GenesisSource, +} + +impl ChainSpec { + pub fn boot_nodes(&self) -> &[String] { + &self.spec.boot_nodes + } + + pub fn name(&self) -> &str { + &self.spec.name + } + + pub fn id(&self) -> &str { + &self.spec.id + } + + pub fn telemetry_url(&self) -> Option<&str> { + self.spec.telemetry_url.as_ref().map(String::as_str) + } + + /// Parse json content into a `ChainSpec` + pub fn from_embedded(json: &'static [u8]) -> Result { + let spec = json::from_slice(json).map_err(|e| format!("Error parsing spec file: {}", e))?; + Ok(ChainSpec { + spec, + genesis: GenesisSource::Embedded(json), + }) + } + + /// Parse json file into a `ChainSpec` + pub fn from_json_file(path: PathBuf) -> Result { + let file = File::open(&path).map_err(|e| format!("Error opening spec file: {}", e))?; + let spec = json::from_reader(file).map_err(|e| format!("Error parsing spec file: {}", e))?; + Ok(ChainSpec { + spec, + genesis: GenesisSource::File(path), + }) + } + + /// Create hardcoded spec. + pub fn from_genesis( + name: &str, + id: &str, + constructor: fn() -> G, + boot_nodes: Vec, + telemetry_url: Option<&str> + ) -> Self + { + let spec = ChainSpecFile { + name: name.to_owned(), + id: id.to_owned(), + boot_nodes: boot_nodes, + telemetry_url: telemetry_url.map(str::to_owned), + }; + ChainSpec { + spec, + genesis: GenesisSource::Factory(constructor), + } + } + + /// Dump to json string. + pub fn to_json(self, raw: bool) -> Result { + #[derive(Serialize, Deserialize)] + struct Container { + #[serde(flatten)] + spec: ChainSpecFile, + #[serde(flatten)] + genesis: Genesis, + + }; + let genesis = match (raw, self.genesis.resolve()?) { + (true, Genesis::Runtime(g)) => { + let storage = g.build_storage()?.into_iter() + .map(|(k, v)| (StorageKey(k), StorageData(v))) + .collect(); + + Genesis::Raw(storage) + }, + (_, genesis) => genesis, + }; + let spec = Container { + spec: self.spec, + genesis, + }; + json::to_string_pretty(&spec).map_err(|e| format!("Error generating spec json: {}", e)) + } +} diff --git a/core/service/src/components.rs b/core/service/src/components.rs new file mode 100644 index 00000000000..38088b90506 --- /dev/null +++ b/core/service/src/components.rs @@ -0,0 +1,256 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Polkadot service components. + +use std::fmt; +use std::sync::Arc; +use std::marker::PhantomData; +use serde::{Serialize, de::DeserializeOwned}; +use chain_spec::ChainSpec; +use client_db; +use client::{self, Client}; +use error; +use network::{self, OnDemand}; +use substrate_executor::{NativeExecutor, NativeExecutionDispatch}; +use extrinsic_pool::{self, Options as ExtrinsicPoolOptions, Pool as ExtrinsicPool}; +use runtime_primitives::{traits::Block as BlockT, traits::Header as HeaderT, BuildStorage}; +use config::Configuration; +use primitives::{Blake2Hasher, RlpCodec, H256}; + +// Type aliases. +// These exist mainly to avoid typing `::Foo` all over the code. +/// Network service type for a factory. +pub type NetworkService = network::Service< + ::Block, + ::NetworkProtocol, + ::ExtrinsicHash, +>; + +/// Code executor type for a factory. +pub type CodeExecutor = NativeExecutor<::RuntimeDispatch>; + +/// Full client backend type for a factory. +pub type FullBackend = client_db::Backend<::Block>; + +/// Full client executor type for a factory. +pub type FullExecutor = client::LocalCallExecutor< + client_db::Backend<::Block>, + CodeExecutor, +>; + +/// Light client backend type for a factory. +pub type LightBackend = client::light::backend::Backend< + client_db::light::LightStorage<::Block>, + network::OnDemand<::Block, NetworkService>, +>; + +/// Light client executor type for a factory. +pub type LightExecutor = client::light::call_executor::RemoteCallExecutor< + client::light::blockchain::Blockchain< + client_db::light::LightStorage<::Block>, + network::OnDemand<::Block, NetworkService> + >, + network::OnDemand<::Block, NetworkService>, + Blake2Hasher, + RlpCodec, +>; + +/// Full client type for a factory. +pub type FullClient = Client, FullExecutor, ::Block>; + +/// Light client type for a factory. +pub type LightClient = Client, LightExecutor, ::Block>; + +/// `ChainSpec` specialization for a factory. +pub type FactoryChainSpec = ChainSpec<::Genesis>; + +/// `Genesis` specialization for a factory. +pub type FactoryGenesis = ::Genesis; + +/// `Block` type for a factory. +pub type FactoryBlock = ::Block; + +/// `Number` type for a factory. +pub type FactoryBlockNumber = < as BlockT>::Header as HeaderT>::Number; + +/// Full `Configuration` type for a factory. +pub type FactoryFullConfiguration = Configuration<::Configuration, FactoryGenesis>; + +/// Client type for `Components`. +pub type ComponentClient = Client< + ::Backend, + ::Executor, + FactoryBlock<::Factory> +>; + +/// Block type for `Components` +pub type ComponentBlock = <::Factory as ServiceFactory>::Block; + +/// Extrinsic hash type for `Components` +pub type ComponentExHash = <::ExtrinsicPoolApi as extrinsic_pool::ChainApi>::Hash; + +/// Extrinsic type. +pub type ComponentExtrinsic = as BlockT>::Extrinsic; + +/// Extrinsic pool API type for `Components`. +pub type PoolApi = ::ExtrinsicPoolApi; + +/// A set of traits for the runtime genesis config. +pub trait RuntimeGenesis: Serialize + DeserializeOwned + BuildStorage {} +impl RuntimeGenesis for T {} + +/// A collection of types and methods to build a service on top of the substrate service. +pub trait ServiceFactory: 'static { + /// Block type. + type Block: BlockT; + /// Extrinsic hash type. + type ExtrinsicHash: ::std::hash::Hash + Eq + Copy + fmt::Debug + fmt::LowerHex + Serialize + DeserializeOwned + ::std::str::FromStr + Send + Sync + Default + 'static; + /// Network protocol extensions. + type NetworkProtocol: network::specialization::Specialization; + /// Chain runtime. + type RuntimeDispatch: NativeExecutionDispatch + Send + Sync + 'static; + /// Extrinsic pool backend type for the full client. + type FullExtrinsicPoolApi: extrinsic_pool::ChainApi + Send + 'static; + /// Extrinsic pool backend type for the light client. + type LightExtrinsicPoolApi: extrinsic_pool::ChainApi + 'static; + /// Genesis configuration for the runtime. + type Genesis: RuntimeGenesis; + /// Other configuration for service members. + type Configuration: Default; + + /// Network protocol id. + const NETWORK_PROTOCOL_ID: network::ProtocolId; + + //TODO: replace these with a constructor trait. that ExtrinsicPool implements. + /// Extrinsic pool constructor for the full client. + fn build_full_extrinsic_pool(config: ExtrinsicPoolOptions, client: Arc>) + -> Result, error::Error>; + /// Extrinsic pool constructor for the light client. + fn build_light_extrinsic_pool(config: ExtrinsicPoolOptions, client: Arc>) + -> Result, error::Error>; + + /// Build network protocol. + fn build_network_protocol(config: &FactoryFullConfiguration) + -> Result; +} + +/// A collection of types and function to generalise over full / light client type. +pub trait Components: 'static { + /// Associated service factory. + type Factory: ServiceFactory; + /// Client backend. + type Backend: 'static + client::backend::Backend, Blake2Hasher, RlpCodec>; + /// Client executor. + type Executor: 'static + client::CallExecutor, Blake2Hasher, RlpCodec> + Send + Sync; + /// Extrinsic pool type. + type ExtrinsicPoolApi: 'static + extrinsic_pool::ChainApi::ExtrinsicHash, Block=FactoryBlock>; + + /// Create client. + fn build_client( + config: &FactoryFullConfiguration, + executor: CodeExecutor, + ) + -> Result<( + Arc>, + Option, NetworkService>>> + ), error::Error>; + + /// Create extrinsic pool. + fn build_extrinsic_pool(config: ExtrinsicPoolOptions, client: Arc>) + -> Result, error::Error>; +} + +/// A struct that implement `Components` for the full client. +pub struct FullComponents { + _factory: PhantomData, +} + +impl Components for FullComponents { + type Factory = Factory; + type Executor = FullExecutor; + type Backend = FullBackend; + type ExtrinsicPoolApi = ::FullExtrinsicPoolApi; + + fn build_client( + config: &FactoryFullConfiguration, + executor: CodeExecutor, + ) + -> Result<( + Arc>, + Option, NetworkService>>> + ), error::Error> + { + let db_settings = client_db::DatabaseSettings { + cache_size: None, + path: config.database_path.as_str().into(), + pruning: config.pruning.clone(), + }; + Ok((Arc::new(client_db::new_client(db_settings, executor, &config.chain_spec, config.execution_strategy)?), None)) + } + + fn build_extrinsic_pool(config: ExtrinsicPoolOptions, client: Arc>) + -> Result, error::Error> + { + Factory::build_full_extrinsic_pool(config, client) + } +} + +/// A struct that implement `Components` for the light client. +pub struct LightComponents { + _factory: PhantomData, +} + +impl Components for LightComponents + where + <::Block as BlockT>::Hash: From, + H256: From<<::Block as BlockT>::Hash>, +{ + type Factory = Factory; + type Executor = LightExecutor; + type Backend = LightBackend; + type ExtrinsicPoolApi = ::LightExtrinsicPoolApi; + + fn build_client( + config: &FactoryFullConfiguration, + executor: CodeExecutor, + ) + -> Result<( + Arc>, + Option, + NetworkService>>> + ), error::Error> + { + let db_settings = client_db::DatabaseSettings { + cache_size: None, + path: config.database_path.as_str().into(), + pruning: config.pruning.clone(), + }; + let db_storage = client_db::light::LightStorage::new(db_settings)?; + let light_blockchain = client::light::new_light_blockchain(db_storage); + let fetch_checker = Arc::new(client::light::new_fetch_checker::<_, Blake2Hasher, RlpCodec>(executor)); + let fetcher = Arc::new(network::OnDemand::new(fetch_checker)); + let client_backend = client::light::new_light_backend(light_blockchain, fetcher.clone()); + let client = client::light::new_light(client_backend, fetcher.clone(), &config.chain_spec)?; + Ok((Arc::new(client), Some(fetcher))) + } + + fn build_extrinsic_pool(config: ExtrinsicPoolOptions, client: Arc>) + -> Result, error::Error> + { + Factory::build_light_extrinsic_pool(config, client) + } +} diff --git a/core/service/src/config.rs b/core/service/src/config.rs new file mode 100644 index 00000000000..fd0db61f8dd --- /dev/null +++ b/core/service/src/config.rs @@ -0,0 +1,121 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Service configuration. + +use std::net::SocketAddr; +use extrinsic_pool; +use chain_spec::ChainSpec; +pub use client::ExecutionStrategy; +pub use network::Roles; +pub use network::NetworkConfiguration; +pub use client_db::PruningMode; +use runtime_primitives::BuildStorage; +use serde::{Serialize, de::DeserializeOwned}; +use target_info::Target; + +/// Service configuration. +pub struct Configuration { + /// Implementation name + pub impl_name: &'static str, + /// Implementation version + pub impl_version: &'static str, + /// Git commit if any. + pub impl_commit: &'static str, + /// Node roles. + pub roles: Roles, + /// Extrinsic pool configuration. + pub extrinsic_pool: extrinsic_pool::Options, + /// Network configuration. + pub network: NetworkConfiguration, + /// Path to key files. + pub keystore_path: String, + /// Path to the database. + pub database_path: String, + /// Pruning settings. + pub pruning: PruningMode, + /// Additional key seeds. + pub keys: Vec, + /// Chain configuration. + pub chain_spec: ChainSpec, + /// Custom configuration. + pub custom: C, + /// Telemetry server URL, optional - only `Some` if telemetry reporting is enabled + pub telemetry: Option, + /// Node name. + pub name: String, + /// Execution strategy. + pub execution_strategy: ExecutionStrategy, + /// RPC over HTTP binding address. `None` if disabled. + pub rpc_http: Option, + /// RPC over Websockets binding address. `None` if disabled. + pub rpc_ws: Option, + /// Telemetry service URL. `None` if disabled. + pub telemetry_url: Option, +} + +impl Configuration { + /// Create default config for given chain spec. + pub fn default_with_spec(chain_spec: ChainSpec) -> Self { + let mut configuration = Configuration { + impl_name: "parity-substrate", + impl_version: "0.0.0", + impl_commit: "", + chain_spec, + name: Default::default(), + roles: Roles::FULL, + extrinsic_pool: Default::default(), + network: Default::default(), + keystore_path: Default::default(), + database_path: Default::default(), + keys: Default::default(), + custom: Default::default(), + telemetry: Default::default(), + pruning: PruningMode::default(), + execution_strategy: ExecutionStrategy::Both, + rpc_http: None, + rpc_ws: None, + telemetry_url: None, + }; + configuration.network.boot_nodes = configuration.chain_spec.boot_nodes().to_vec(); + configuration.telemetry_url = configuration.chain_spec.telemetry_url().map(str::to_owned); + configuration + } + + /// Returns full version string of this configuration. + pub fn full_version(&self) -> String { + full_version_from_strs(self.impl_version, self.impl_commit) + } + + /// Implementation id and version. + pub fn client_id(&self) -> String { + format!("{}/v{}", self.impl_name, self.full_version()) + } +} + +/// Returns platform info +pub fn platform() -> String { + let env = Target::env(); + let env_dash = if env.is_empty() { "" } else { "-" }; + format!("{}-{}{}{}", Target::arch(), Target::os(), env_dash, env) +} + +/// Returns full version string, using supplied version and commit. +pub fn full_version_from_strs(impl_version: &str, impl_commit: &str) -> String { + let commit_dash = if impl_commit.is_empty() { "" } else { "-" }; + format!("{}{}{}-{}", impl_version, commit_dash, impl_commit, platform()) +} + diff --git a/core/service/src/error.rs b/core/service/src/error.rs new file mode 100644 index 00000000000..abc09cb0c3c --- /dev/null +++ b/core/service/src/error.rs @@ -0,0 +1,35 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Errors that can occur during the service operation. + +use client; +use network; +use keystore; + +error_chain! { + foreign_links { + Io(::std::io::Error) #[doc="IO error"]; + } + links { + Client(client::error::Error, client::error::ErrorKind) #[doc="Client error"]; + Network(network::error::Error, network::error::ErrorKind) #[doc="Network error"]; + Keystore(keystore::Error, keystore::ErrorKind) #[doc="Keystore error"]; + } + + errors { + } +} diff --git a/core/service/src/lib.rs b/core/service/src/lib.rs new file mode 100644 index 00000000000..d7c1e90e599 --- /dev/null +++ b/core/service/src/lib.rs @@ -0,0 +1,424 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Substrate service. Starts a thread that spins the network, the client and the extrinsic pool. +//! Manages communication between them. +// end::description[] + +#![warn(unused_extern_crates)] + +extern crate futures; +extern crate exit_future; +extern crate serde; +extern crate serde_json; +extern crate substrate_keystore as keystore; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_network as network; +extern crate substrate_executor; +extern crate substrate_client as client; +extern crate substrate_client_db as client_db; +extern crate substrate_codec as codec; +extern crate substrate_extrinsic_pool as extrinsic_pool; +extern crate substrate_rpc; +extern crate substrate_rpc_servers as rpc; +extern crate target_info; +extern crate tokio; + +#[macro_use] +extern crate substrate_telemetry as tel; +#[macro_use] +extern crate error_chain; +#[macro_use] +extern crate slog; // needed until we can reexport `slog_info` from `substrate_telemetry` +#[macro_use] +extern crate log; +#[macro_use] +extern crate serde_derive; + +mod components; +mod error; +mod chain_spec; +pub mod config; +pub mod chain_ops; + +use std::io; +use std::net::SocketAddr; +use std::sync::Arc; +use std::collections::HashMap; +use futures::prelude::*; +use keystore::Store as Keystore; +use client::BlockchainEvents; +use runtime_primitives::traits::{Header, As}; +use runtime_primitives::generic::BlockId; +use exit_future::Signal; +use tokio::runtime::TaskExecutor; +use substrate_executor::NativeExecutor; +use codec::{Encode, Decode}; + +pub use self::error::{ErrorKind, Error}; +pub use config::{Configuration, Roles, PruningMode}; +pub use chain_spec::ChainSpec; +pub use extrinsic_pool::{Pool as ExtrinsicPool, Options as ExtrinsicPoolOptions, ChainApi, VerifiedTransaction, IntoPoolError}; +pub use client::ExecutionStrategy; + +pub use components::{ServiceFactory, FullBackend, FullExecutor, LightBackend, + LightExecutor, Components, PoolApi, ComponentClient, + ComponentBlock, FullClient, LightClient, FullComponents, LightComponents, + CodeExecutor, NetworkService, FactoryChainSpec, FactoryBlock, + FactoryFullConfiguration, RuntimeGenesis, FactoryGenesis, + ComponentExHash, ComponentExtrinsic, +}; + +/// Substrate service. +pub struct Service { + client: Arc>, + network: Option>>, + extrinsic_pool: Arc>, + keystore: Keystore, + exit: ::exit_future::Exit, + signal: Option, + _rpc_http: Option, + _rpc_ws: Option, + _telemetry: Option, +} + +/// Creates bare client without any networking. +pub fn new_client(config: FactoryFullConfiguration) + -> Result>>, error::Error> +{ + let executor = NativeExecutor::new(); + let (client, _) = components::FullComponents::::build_client( + &config, + executor, + )?; + Ok(client) +} + +impl Service + where + Components: components::Components, +{ + /// Creates a new service. + pub fn new( + config: FactoryFullConfiguration, + task_executor: TaskExecutor, + ) + -> Result + { + let (signal, exit) = ::exit_future::signal(); + + // Create client + let executor = NativeExecutor::new(); + + let mut keystore = Keystore::open(config.keystore_path.as_str().into())?; + for seed in &config.keys { + keystore.generate_from_seed(seed)?; + } + + // Keep the public key for telemetry + let public_key = match keystore.contents()?.get(0) { + Some(public_key) => public_key.clone(), + None => { + let key = keystore.generate("")?; + let public_key = key.public(); + info!("Generated a new keypair: {:?}", public_key); + + public_key + } + }; + + let (client, on_demand) = Components::build_client(&config, executor)?; + let best_header = client.best_block_header()?; + + let version = config.full_version(); + info!("Best block: #{}", best_header.number()); + telemetry!("node.start"; "height" => best_header.number().as_(), "best" => ?best_header.hash()); + + let network_protocol = ::build_network_protocol(&config)?; + let extrinsic_pool = Arc::new( + Components::build_extrinsic_pool(config.extrinsic_pool, client.clone())? + ); + let extrinsic_pool_adapter = ExtrinsicPoolAdapter:: { + imports_external_transactions: !config.roles == Roles::LIGHT, + pool: extrinsic_pool.clone(), + client: client.clone(), + }; + + let network_params = network::Params { + config: network::ProtocolConfig { + roles: config.roles, + }, + network_config: config.network, + chain: client.clone(), + on_demand: on_demand.clone() + .map(|d| d as Arc>>), + transaction_pool: Arc::new(extrinsic_pool_adapter), + specialization: network_protocol, + }; + + let network = network::Service::new(network_params, Components::Factory::NETWORK_PROTOCOL_ID)?; + on_demand.map(|on_demand| on_demand.set_service_link(Arc::downgrade(&network))); + + { + // block notifications + let network = network.clone(); + let txpool = extrinsic_pool.clone(); + + let events = client.import_notification_stream() + .for_each(move |notification| { + network.on_block_imported(notification.hash, ¬ification.header); + txpool.cull(&BlockId::hash(notification.hash)) + .map_err(|e| warn!("Error removing extrinsics: {:?}", e))?; + Ok(()) + }) + .select(exit.clone()) + .then(|_| Ok(())); + task_executor.spawn(events); + } + + { + // extrinsic notifications + let network = network.clone(); + let events = extrinsic_pool.import_notification_stream() + // TODO [ToDr] Consider throttling? + .for_each(move |_| { + network.trigger_repropagate(); + Ok(()) + }) + .select(exit.clone()) + .then(|_| Ok(())); + + task_executor.spawn(events); + } + + // RPC + let rpc_config = RpcConfig { + chain_name: config.chain_spec.name().to_string(), + impl_name: config.impl_name, + impl_version: config.impl_version, + }; + + let (rpc_http, rpc_ws) = { + let handler = || { + let client = client.clone(); + let chain = rpc::apis::chain::Chain::new(client.clone(), task_executor.clone()); + let state = rpc::apis::state::State::new(client.clone(), task_executor.clone()); + let author = rpc::apis::author::Author::new(client.clone(), extrinsic_pool.clone(), task_executor.clone()); + rpc::rpc_handler::, ComponentExHash, _, _, _, _, _>( + state, + chain, + author, + rpc_config.clone(), + ) + }; + ( + maybe_start_server(config.rpc_http, |address| rpc::start_http(address, handler()))?, + maybe_start_server(config.rpc_ws, |address| rpc::start_ws(address, handler()))?, + ) + }; + + // Telemetry + let telemetry = match config.telemetry_url { + Some(url) => { + let is_authority = config.roles == Roles::AUTHORITY; + let pubkey = format!("{}", public_key); + let name = config.name.clone(); + let impl_name = config.impl_name.to_owned(); + let version = version.clone(); + let chain_name = config.chain_spec.name().to_owned(); + Some(tel::init_telemetry(tel::TelemetryConfig { + url: url, + on_connect: Box::new(move || { + telemetry!("system.connected"; + "name" => name.clone(), + "implementation" => impl_name.clone(), + "version" => version.clone(), + "config" => "", + "chain" => chain_name.clone(), + "pubkey" => &pubkey, + "authority" => is_authority + ); + }), + })) + }, + None => None, + }; + + Ok(Service { + client: client, + network: Some(network), + extrinsic_pool: extrinsic_pool, + signal: Some(signal), + keystore: keystore, + exit, + _rpc_http: rpc_http, + _rpc_ws: rpc_ws, + _telemetry: telemetry, + }) + } + + /// Get shared client instance. + pub fn client(&self) -> Arc> { + self.client.clone() + } + + /// Get shared network instance. + pub fn network(&self) -> Arc> { + self.network.as_ref().expect("self.network always Some").clone() + } + + /// Get shared extrinsic pool instance. + pub fn extrinsic_pool(&self) -> Arc> { + self.extrinsic_pool.clone() + } + + /// Get shared keystore. + pub fn keystore(&self) -> &Keystore { + &self.keystore + } + + /// Get a handle to a future that will resolve on exit. + pub fn on_exit(&self) -> ::exit_future::Exit { + self.exit.clone() + } +} + +impl Drop for Service where Components: components::Components { + fn drop(&mut self) { + debug!(target: "service", "Substrate service shutdown"); + + drop(self.network.take()); + + if let Some(signal) = self.signal.take() { + signal.fire(); + } + } +} + +fn maybe_start_server(address: Option, start: F) -> Result, io::Error> where + F: Fn(&SocketAddr) -> Result, +{ + Ok(match address { + Some(mut address) => Some(start(&address) + .or_else(|e| match e.kind() { + io::ErrorKind::AddrInUse | + io::ErrorKind::PermissionDenied => { + warn!("Unable to bind server to {}. Trying random port.", address); + address.set_port(0); + start(&address) + }, + _ => Err(e), + })?), + None => None, + }) +} + +#[derive(Clone)] +struct RpcConfig { + chain_name: String, + impl_name: &'static str, + impl_version: &'static str, +} + +impl substrate_rpc::system::SystemApi for RpcConfig { + fn system_name(&self) -> substrate_rpc::system::error::Result { + Ok(self.impl_name.into()) + } + + fn system_version(&self) -> substrate_rpc::system::error::Result { + Ok(self.impl_version.into()) + } + + fn system_chain(&self) -> substrate_rpc::system::error::Result { + Ok(self.chain_name.clone()) + } +} + +/// Transaction pool adapter. +pub struct ExtrinsicPoolAdapter { + imports_external_transactions: bool, + pool: Arc>, + client: Arc>, +} + +impl ExtrinsicPoolAdapter { + fn best_block_id(&self) -> Option>> { + self.client.info() + .map(|info| BlockId::hash(info.chain.best_hash)) + .map_err(|e| { + debug!("Error getting best block: {:?}", e); + }) + .ok() + } +} + +impl network::TransactionPool, ComponentBlock> for ExtrinsicPoolAdapter { + fn transactions(&self) -> Vec<(ComponentExHash, ComponentExtrinsic)> { + let best_block_id = match self.best_block_id() { + Some(id) => id, + None => return vec![], + }; + self.pool.cull_and_get_pending(&best_block_id, |pending| pending + .map(|t| { + let hash = t.hash().clone(); + let ex: ComponentExtrinsic = t.original.clone(); + (hash, ex) + }) + .collect() + ).unwrap_or_else(|e| { + warn!("Error retrieving pending set: {}", e); + vec![] + }) + } + + fn import(&self, transaction: &ComponentExtrinsic) -> Option> { + if !self.imports_external_transactions { + return None; + } + + let encoded = transaction.encode(); + if let Some(uxt) = Decode::decode(&mut &encoded[..]) { + let best_block_id = self.best_block_id()?; + match self.pool.submit_one(&best_block_id, uxt) { + Ok(xt) => Some(*xt.hash()), + Err(e) => match e.into_pool_error() { + Ok(e) => match e.kind() { + extrinsic_pool::ErrorKind::AlreadyImported(hash) => + Some(::std::str::FromStr::from_str(&hash).map_err(|_| {}) + .expect("Hash string is always valid")), + _ => { + debug!("Error adding transaction to the pool: {:?}", e); + None + }, + }, + Err(e) => { + debug!("Error converting pool error: {:?}", e); + None + } + } + } + } else { + debug!("Error decoding transaction"); + None + } + } + + fn on_broadcasted(&self, propagations: HashMap, Vec>) { + self.pool.on_broadcasted(propagations) + } +} diff --git a/core/state-db/Cargo.toml b/core/state-db/Cargo.toml new file mode 100644 index 00000000000..95e47981922 --- /dev/null +++ b/core/state-db/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "substrate-state-db" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +parking_lot = "0.5" +log = "0.4" +substrate-primitives = { path = "../primitives" } +substrate-codec = { path = "../codec" } +substrate-codec-derive = { path = "../codec/derive" } + +[dev-dependencies] +env_logger = "0.4" diff --git a/core/state-db/README.adoc b/core/state-db/README.adoc new file mode 100644 index 00000000000..f9934ed8d9a --- /dev/null +++ b/core/state-db/README.adoc @@ -0,0 +1,13 @@ + += State DB + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/state-db/src/lib.rs b/core/state-db/src/lib.rs new file mode 100644 index 00000000000..7e7dd3369c9 --- /dev/null +++ b/core/state-db/src/lib.rs @@ -0,0 +1,407 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! State database maintenance. Handles finalization and pruning in the database. The input to +//! this module is a `ChangeSet` which is basically a list of key-value pairs (trie nodes) that +//! were added or deleted during block execution. +//! +//! # Finalization. +//! Finalization window tracks a tree of blocks identified by header hash. The in-memory +//! overlay allows to get any node that was was inserted in any any of the blocks within the window. +//! The tree is journaled to the backing database and rebuilt on startup. +//! Finalization function select one root from the top of the tree and discards all other roots and +//! their subtrees. +//! +//! # Pruning. +//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each finalization until pruning +//! constraints are satisfied. +//! +// end::description[] + +#[macro_use] extern crate log; +#[macro_use] extern crate substrate_codec_derive; +extern crate parking_lot; +extern crate substrate_codec as codec; +extern crate substrate_primitives as primitives; + +mod unfinalized; +mod pruning; +#[cfg(test)] mod test; + +use std::fmt; +use parking_lot::RwLock; +use codec::Codec; +use std::collections::HashSet; +use unfinalized::UnfinalizedOverlay; +use pruning::RefWindow; + +/// Database value type. +pub type DBValue = Vec; + +/// Basic set of requirements for the Block hash and node key types. +pub trait Hash: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static {} +impl Hash for T {} + +/// Backend database trait. Read-only. +pub trait MetaDb { + type Error: fmt::Debug; + + /// Get meta value, such as the journal. + fn get_meta(&self, key: &[u8]) -> Result, Self::Error>; +} + + +/// Backend database trait. Read-only. +pub trait HashDb { + type Hash: Hash; + type Error: fmt::Debug; + + /// Get state trie node. + fn get(&self, key: &Self::Hash) -> Result, Self::Error>; +} + +/// Error type. +pub enum Error { + /// Database backend error. + Db(E), + /// `Codec` decoding error. + Decoding, +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + match self { + Error::Db(e) => e.fmt(f), + Error::Decoding => write!(f, "Error decoding slicable value"), + } + } +} + +/// A set of state node changes. +#[derive(Default, Debug, Clone)] +pub struct ChangeSet { + /// Inserted nodes. + pub inserted: Vec<(H, DBValue)>, + /// Deleted nodes. + pub deleted: Vec, +} + + +/// A set of changes to the backing database. +#[derive(Default, Debug, Clone)] +pub struct CommitSet { + /// State node changes. + pub data: ChangeSet, + /// Metadata changes. + pub meta: ChangeSet>, +} + +/// Pruning constraints. If none are specified pruning is +#[derive(Default, Debug, Clone)] +pub struct Constraints { + /// Maximum blocks. Defaults to 0 when unspecified, effectively keeping only unfinalized states. + pub max_blocks: Option, + /// Maximum memory in the pruning overlay. + pub max_mem: Option, +} + +/// Pruning mode. +#[derive(Debug, Clone)] +pub enum PruningMode { + /// Maintain a pruning window. + Constrained(Constraints), + /// No pruning. Finalization is a no-op. + ArchiveAll, + /// Finalization discards unfinalized nodes. All the finalized nodes are kept in the DB. + ArchiveCanonical, +} + +impl PruningMode { + /// Create a mode that keeps given number of blocks. + pub fn keep_blocks(n: u32) -> PruningMode { + PruningMode::Constrained(Constraints { + max_blocks: Some(n), + max_mem: None, + }) + } +} + +impl Default for PruningMode { + fn default() -> Self { + PruningMode::keep_blocks(256) + } +} + +fn to_meta_key(suffix: &[u8], data: &S) -> Vec { + let mut buffer = data.encode(); + buffer.extend(suffix); + buffer +} + +struct StateDbSync { + mode: PruningMode, + unfinalized: UnfinalizedOverlay, + pruning: Option>, + pinned: HashSet, +} + +impl StateDbSync { + pub fn new(mode: PruningMode, db: &D) -> Result, Error> { + trace!("StateDb settings: {:?}", mode); + let unfinalized: UnfinalizedOverlay = UnfinalizedOverlay::new(db)?; + let pruning: Option> = match mode { + PruningMode::Constrained(Constraints { + max_mem: Some(_), + .. + }) => unimplemented!(), + PruningMode::Constrained(_) => Some(RefWindow::new(db)?), + PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => None, + }; + Ok(StateDbSync { + mode, + unfinalized, + pruning: pruning, + pinned: Default::default(), + }) + } + + pub fn insert_block(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet) -> CommitSet { + if number == 0 { + return CommitSet { + data: changeset, + meta: Default::default(), + } + } + match self.mode { + PruningMode::ArchiveAll => { + changeset.deleted.clear(); + // write changes immediately + CommitSet { + data: changeset, + meta: Default::default(), + } + }, + PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => { + self.unfinalized.insert(hash, number, parent_hash, changeset) + } + } + } + + pub fn finalize_block(&mut self, hash: &BlockHash) -> CommitSet { + // clear the temporary overlay from the previous finalization. + self.unfinalized.clear_overlay(); + let mut commit = match self.mode { + PruningMode::ArchiveAll => { + CommitSet::default() + }, + PruningMode::ArchiveCanonical => { + let mut commit = self.unfinalized.finalize(hash); + commit.data.deleted.clear(); + commit + }, + PruningMode::Constrained(_) => { + self.unfinalized.finalize(hash) + }, + }; + if let Some(ref mut pruning) = self.pruning { + pruning.note_finalized(hash, &mut commit); + } + self.prune(&mut commit); + commit + } + + pub fn best_finalized(&self) -> u64 { + return self.unfinalized.last_finalized_block_number() + } + + pub fn is_pruned(&self, number: u64) -> bool { + self.pruning.as_ref().map_or(false, |pruning| number < pruning.pending()) + } + + fn prune(&mut self, commit: &mut CommitSet) { + if let (&mut Some(ref mut pruning), &PruningMode::Constrained(ref constraints)) = (&mut self.pruning, &self.mode) { + loop { + if pruning.window_size() <= constraints.max_blocks.unwrap_or(0) as u64 { + break; + } + + if constraints.max_mem.map_or(false, |m| pruning.mem_used() > m) { + break; + } + + let pinned = &self.pinned; + if pruning.next_hash().map_or(false, |h| pinned.contains(&h)) { + break; + } + pruning.prune_one(commit); + } + } + } + + /// Revert all unfinalized blocks with the best block number. + /// Returns a database commit or `None` if not possible. + /// For archive an empty commit set is returned. + pub fn revert_one(&mut self) -> Option> { + match self.mode { + PruningMode::ArchiveAll => { + Some(CommitSet::default()) + }, + PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => { + self.unfinalized.revert_one() + }, + } + } + + pub fn pin(&mut self, hash: &BlockHash) { + self.pinned.insert(hash.clone()); + } + + pub fn unpin(&mut self, hash: &BlockHash) { + self.pinned.remove(hash); + } + + pub fn get>(&self, key: &Key, db: &D) -> Result, Error> { + if let Some(value) = self.unfinalized.get(key) { + return Ok(Some(value)); + } + db.get(key).map_err(|e| Error::Db(e)) + } +} + +/// State DB maintenance. See module description. +/// Can be shared across threads. +pub struct StateDb { + db: RwLock>, +} + +impl StateDb { + /// Creates a new instance. Does not expect any metadata in the database. + pub fn new(mode: PruningMode, db: &D) -> Result, Error> { + Ok(StateDb { + db: RwLock::new(StateDbSync::new(mode, db)?) + }) + } + + /// Add a new unfinalized block. + pub fn insert_block(&self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet) -> CommitSet { + self.db.write().insert_block(hash, number, parent_hash, changeset) + } + + /// Finalize a previously inserted block. + pub fn finalize_block(&self, hash: &BlockHash) -> CommitSet { + self.db.write().finalize_block(hash) + } + + /// Prevents pruning of specified block and its descendants. + pub fn pin(&self, hash: &BlockHash) { + self.db.write().pin(hash) + } + + /// Allows pruning of specified block. + pub fn unpin(&self, hash: &BlockHash) { + self.db.write().unpin(hash) + } + + /// Get a value from unfinalized/pruning overlay or the backing DB. + pub fn get>(&self, key: &Key, db: &D) -> Result, Error> { + self.db.read().get(key, db) + } + + /// Revert all unfinalized blocks with the best block number. + /// Returns a database commit or `None` if not possible. + /// For archive an empty commit set is returned. + pub fn revert_one(&self) -> Option> { + self.db.write().revert_one() + } + + /// Returns last finalized block number. + pub fn best_finalized(&self) -> u64 { + return self.db.read().best_finalized() + } + + /// Check if block is pruned away. + pub fn is_pruned(&self, number: u64) -> bool { + return self.db.read().is_pruned(number) + } +} + +#[cfg(test)] +mod tests { + use primitives::H256; + use {StateDb, PruningMode, Constraints}; + use test::{make_db, make_changeset, TestDb}; + + fn make_test_db(settings: PruningMode) -> (TestDb, StateDb) { + let mut db = make_db(&[91, 921, 922, 93, 94]); + let state_db = StateDb::new(settings, &db).unwrap(); + + db.commit(&state_db.insert_block(&H256::from(1), 1, &H256::from(0), make_changeset(&[1], &[91]))); + db.commit(&state_db.insert_block(&H256::from(21), 2, &H256::from(1), make_changeset(&[21], &[921, 1]))); + db.commit(&state_db.insert_block(&H256::from(22), 2, &H256::from(1), make_changeset(&[22], &[922]))); + db.commit(&state_db.insert_block(&H256::from(3), 3, &H256::from(21), make_changeset(&[3], &[93]))); + db.commit(&state_db.finalize_block(&H256::from(1))); + db.commit(&state_db.insert_block(&H256::from(4), 4, &H256::from(3), make_changeset(&[4], &[94]))); + db.commit(&state_db.finalize_block(&H256::from(21))); + db.commit(&state_db.finalize_block(&H256::from(3))); + + (db, state_db) + } + + #[test] + fn full_archive_keeps_everything() { + let (db, _) = make_test_db(PruningMode::ArchiveAll); + assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94]))); + } + + #[test] + fn canonical_archive_keeps_canonical() { + let (db, _) = make_test_db(PruningMode::ArchiveCanonical); + assert!(db.data_eq(&make_db(&[1, 21, 3, 91, 921, 922, 93, 94]))); + } + + #[test] + fn prune_window_0() { + let (db, _) = make_test_db(PruningMode::Constrained(Constraints { + max_blocks: Some(0), + max_mem: None, + })); + assert!(db.data_eq(&make_db(&[21, 3, 922, 94]))); + } + + #[test] + fn prune_window_1() { + let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints { + max_blocks: Some(1), + max_mem: None, + })); + assert!(sdb.is_pruned(0)); + assert!(sdb.is_pruned(1)); + assert!(!sdb.is_pruned(2)); + assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94]))); + } + + #[test] + fn prune_window_2() { + let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints { + max_blocks: Some(2), + max_mem: None, + })); + assert!(sdb.is_pruned(0)); + assert!(!sdb.is_pruned(1)); + assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94]))); + } +} diff --git a/core/state-db/src/pruning.rs b/core/state-db/src/pruning.rs new file mode 100644 index 00000000000..aaacbd09b66 --- /dev/null +++ b/core/state-db/src/pruning.rs @@ -0,0 +1,267 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Pruning window. +//! +//! For each block we maintain a list of nodes pending deletion. +//! There is also a global index of node key to block number. +//! If a node is re-inserted into the window it gets removed from +//! the death list. +//! The changes are journaled in the DB. + +use std::collections::{HashMap, HashSet, VecDeque}; +use codec::{Encode, Decode}; +use {CommitSet, Error, MetaDb, to_meta_key, Hash}; + +const LAST_PRUNED: &[u8] = b"last_pruned"; +const PRUNING_JOURNAL: &[u8] = b"pruning_journal"; + +/// See module documentation. +pub struct RefWindow { + death_rows: VecDeque>, + death_index: HashMap, + pending_number: u64, +} + +#[derive(Debug, PartialEq, Eq)] +struct DeathRow { + hash: BlockHash, + journal_key: Vec, + deleted: HashSet, +} + +#[derive(Encode, Decode)] +struct JournalRecord { + hash: BlockHash, + inserted: Vec, + deleted: Vec, +} + +fn to_journal_key(block: u64) -> Vec { + to_meta_key(PRUNING_JOURNAL, &block) +} + +impl RefWindow { + pub fn new(db: &D) -> Result, Error> { + let last_pruned = db.get_meta(&to_meta_key(LAST_PRUNED, &())) + .map_err(|e| Error::Db(e))?; + let pending_number: u64 = match last_pruned { + Some(buffer) => u64::decode(&mut buffer.as_slice()).ok_or(Error::Decoding)? + 1, + None => 0, + }; + let mut block = pending_number; + let mut pruning = RefWindow { + death_rows: Default::default(), + death_index: Default::default(), + pending_number: pending_number, + }; + // read the journal + trace!(target: "state-db", "Reading pruning journal. Pending #{}", pending_number); + loop { + let journal_key = to_journal_key(block); + match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { + Some(record) => { + let record: JournalRecord = Decode::decode(&mut record.as_slice()).ok_or(Error::Decoding)?; + trace!(target: "state-db", "Pruning journal entry {} ({} inserted, {} deleted)", block, record.inserted.len(), record.deleted.len()); + pruning.import(&record.hash, journal_key, record.inserted.into_iter(), record.deleted); + }, + None => break, + } + block += 1; + } + Ok(pruning) + } + + fn import>(&mut self, hash: &BlockHash, journal_key: Vec, inserted: I, deleted: Vec) { + // remove all re-inserted keys from death rows + for k in inserted { + if let Some(block) = self.death_index.remove(&k) { + self.death_rows[(block - self.pending_number) as usize].deleted.remove(&k); + } + } + + // add new keys + let imported_block = self.pending_number + self.death_rows.len() as u64; + for k in deleted.iter() { + self.death_index.insert(k.clone(), imported_block); + } + self.death_rows.push_back( + DeathRow { + hash: hash.clone(), + deleted: deleted.into_iter().collect(), + journal_key: journal_key, + } + ); + } + + pub fn window_size(&self) -> u64 { + self.death_rows.len() as u64 + } + + pub fn next_hash(&self) -> Option { + self.death_rows.front().map(|r| r.hash.clone()) + } + + pub fn mem_used(&self) -> usize { + 0 + } + + pub fn pending(&self) -> u64 { + self.pending_number + } + + /// Prune next block. Expects at least one block in the window. Adds changes to `commit`. + pub fn prune_one(&mut self, commit: &mut CommitSet) { + let pruned = self.death_rows.pop_front().expect("prune_one is only called with a non-empty window"); + trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len()); + for k in pruned.deleted.iter() { + self.death_index.remove(&k); + } + commit.data.deleted.extend(pruned.deleted.into_iter()); + commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), self.pending_number.encode())); + commit.meta.deleted.push(pruned.journal_key); + self.pending_number += 1; + } + + /// Add a change set to the window. Creates a journal record and pushes it to `commit` + pub fn note_finalized(&mut self, hash: &BlockHash, commit: &mut CommitSet) { + trace!(target: "state-db", "Adding to pruning window: {:?} ({} inserted, {} deleted)", hash, commit.data.inserted.len(), commit.data.deleted.len()); + let inserted = commit.data.inserted.iter().map(|(k, _)| k.clone()).collect(); + let deleted = ::std::mem::replace(&mut commit.data.deleted, Vec::new()); + let journal_record = JournalRecord { + hash: hash.clone(), + inserted, + deleted, + }; + let block = self.pending_number + self.window_size(); + let journal_key = to_journal_key(block); + commit.meta.inserted.push((journal_key.clone(), journal_record.encode())); + + self.import(hash, journal_key, journal_record.inserted.into_iter(), journal_record.deleted); + } +} + +#[cfg(test)] +mod tests { + use super::RefWindow; + use primitives::H256; + use {CommitSet}; + use test::{make_db, make_commit, TestDb}; + + fn check_journal(pruning: &RefWindow, db: &TestDb) { + let restored: RefWindow = RefWindow::new(db).unwrap(); + assert_eq!(pruning.pending_number, restored.pending_number); + assert_eq!(pruning.death_rows, restored.death_rows); + assert_eq!(pruning.death_index, restored.death_index); + } + + #[test] + fn created_from_empty_db() { + let db = make_db(&[]); + let pruning: RefWindow = RefWindow::new(&db).unwrap(); + assert_eq!(pruning.pending_number, 0); + assert!(pruning.death_rows.is_empty()); + assert!(pruning.death_index.is_empty()); + } + + #[test] + #[should_panic] + fn prune_empty_panics() { + let db = make_db(&[]); + let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut commit = CommitSet::default(); + pruning.prune_one(&mut commit); + } + + #[test] + fn prune_one() { + let mut db = make_db(&[1, 2, 3]); + let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut commit = make_commit(&[4, 5], &[1, 3]); + let h = H256::random(); + pruning.note_finalized(&h, &mut commit); + db.commit(&commit); + assert!(commit.data.deleted.is_empty()); + assert_eq!(pruning.death_rows.len(), 1); + assert_eq!(pruning.death_index.len(), 2); + assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); + check_journal(&pruning, &db); + + let mut commit = CommitSet::default(); + pruning.prune_one(&mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[2, 4, 5]))); + assert!(pruning.death_rows.is_empty()); + assert!(pruning.death_index.is_empty()); + assert_eq!(pruning.pending_number, 1); + } + + #[test] + fn prune_two() { + let mut db = make_db(&[1, 2, 3]); + let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut commit = make_commit(&[4], &[1]); + pruning.note_finalized(&H256::random(), &mut commit); + db.commit(&commit); + let mut commit = make_commit(&[5], &[2]); + pruning.note_finalized(&H256::random(), &mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5]))); + + check_journal(&pruning, &db); + + let mut commit = CommitSet::default(); + pruning.prune_one(&mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[2, 3, 4, 5]))); + let mut commit = CommitSet::default(); + pruning.prune_one(&mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[3, 4, 5]))); + assert_eq!(pruning.pending_number, 2); + } + + #[test] + fn reinserted_survives() { + let mut db = make_db(&[1, 2, 3]); + let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut commit = make_commit(&[], &[2]); + pruning.note_finalized(&H256::random(), &mut commit); + db.commit(&commit); + let mut commit = make_commit(&[2], &[]); + pruning.note_finalized(&H256::random(), &mut commit); + db.commit(&commit); + let mut commit = make_commit(&[], &[2]); + pruning.note_finalized(&H256::random(), &mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[1, 2, 3]))); + + check_journal(&pruning, &db); + + let mut commit = CommitSet::default(); + pruning.prune_one(&mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[1, 2, 3]))); + let mut commit = CommitSet::default(); + pruning.prune_one(&mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[1, 2, 3]))); + pruning.prune_one(&mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[1, 3]))); + assert_eq!(pruning.pending_number, 3); + } +} diff --git a/core/state-db/src/test.rs b/core/state-db/src/test.rs new file mode 100644 index 00000000000..4931ed95069 --- /dev/null +++ b/core/state-db/src/test.rs @@ -0,0 +1,83 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Test utils + +use std::collections::HashMap; +use primitives::H256; +use {DBValue, ChangeSet, CommitSet, MetaDb, HashDb}; + +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct TestDb { + pub data: HashMap, + pub meta: HashMap, DBValue>, +} + +impl MetaDb for TestDb { + type Error = (); + + fn get_meta(&self, key: &[u8]) -> Result, ()> { + Ok(self.meta.get(key).cloned()) + } +} + +impl HashDb for TestDb { + type Error = (); + type Hash = H256; + + fn get(&self, key: &H256) -> Result, ()> { + Ok(self.data.get(key).cloned()) + } +} + +impl TestDb { + pub fn commit(&mut self, commit: &CommitSet) { + self.data.extend(commit.data.inserted.iter().cloned()); + for k in commit.data.deleted.iter() { + self.data.remove(k); + } + self.meta.extend(commit.meta.inserted.iter().cloned()); + for k in commit.meta.deleted.iter() { + self.meta.remove(k); + } + } + + pub fn data_eq(&self, other: &TestDb) -> bool { + self.data == other.data + } +} + +pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet { + ChangeSet { + inserted: inserted.iter().map(|v| (H256::from(*v), H256::from(*v).to_vec())).collect(), + deleted: deleted.iter().map(|v| H256::from(*v)).collect(), + } +} + +pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet { + CommitSet { + data: make_changeset(inserted, deleted), + meta: ChangeSet::default(), + } +} + +pub fn make_db(inserted: &[u64]) -> TestDb { + TestDb { + data: inserted.iter().map(|v| (H256::from(*v), H256::from(*v).to_vec())).collect(), + meta: Default::default(), + } +} + diff --git a/core/state-db/src/unfinalized.rs b/core/state-db/src/unfinalized.rs new file mode 100644 index 00000000000..728c21ae4c1 --- /dev/null +++ b/core/state-db/src/unfinalized.rs @@ -0,0 +1,533 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Finalization window. +//! Maintains trees of block overlays and allows discarding trees/roots +//! The overlays are added in `insert` and removed in `finalize`. +//! Last finalized overlay is kept in memory until next call to `finalize` or +//! `clear_overlay` + +use std::collections::{HashMap, VecDeque}; +use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key}; +use codec::{Decode, Encode}; + +const UNFINALIZED_JOURNAL: &[u8] = b"unfinalized_journal"; +const LAST_FINALIZED: &[u8] = b"last_finalized"; + +/// See module documentation. +pub struct UnfinalizedOverlay { + last_finalized: Option<(BlockHash, u64)>, + levels: VecDeque>>, + parents: HashMap, + last_finalized_overlay: HashMap, +} + +#[derive(Encode, Decode)] +struct JournalRecord { + hash: BlockHash, + parent_hash: BlockHash, + inserted: Vec<(Key, DBValue)>, + deleted: Vec, +} + +fn to_journal_key(block: u64, index: u64) -> Vec { + to_meta_key(UNFINALIZED_JOURNAL, &(block, index)) +} + +#[cfg_attr(test, derive(PartialEq, Debug))] +struct BlockOverlay { + hash: BlockHash, + journal_key: Vec, + values: HashMap, + deleted: Vec, +} + +impl UnfinalizedOverlay { + /// Creates a new instance. Does not expect any metadata to be present in the DB. + pub fn new(db: &D) -> Result, Error> { + let last_finalized = db.get_meta(&to_meta_key(LAST_FINALIZED, &())) + .map_err(|e| Error::Db(e))?; + let last_finalized = match last_finalized { + Some(buffer) => Some(<(BlockHash, u64)>::decode(&mut buffer.as_slice()).ok_or(Error::Decoding)?), + None => None, + }; + let mut levels = VecDeque::new(); + let mut parents = HashMap::new(); + if let Some((ref hash, mut block)) = last_finalized { + // read the journal + trace!(target: "state-db", "Reading unfinalized journal. Last finalized #{} ({:?})", block, hash); + let mut total: u64 = 0; + block += 1; + loop { + let mut index: u64 = 0; + let mut level = Vec::new(); + loop { + let journal_key = to_journal_key(block, index); + match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? { + Some(record) => { + let record: JournalRecord = Decode::decode(&mut record.as_slice()).ok_or(Error::Decoding)?; + let overlay = BlockOverlay { + hash: record.hash.clone(), + journal_key, + values: record.inserted.into_iter().collect(), + deleted: record.deleted, + }; + trace!(target: "state-db", "Unfinalized journal entry {}.{} ({} inserted, {} deleted)", block, index, overlay.values.len(), overlay.deleted.len()); + level.push(overlay); + parents.insert(record.hash, record.parent_hash); + index += 1; + total += 1; + }, + None => break, + } + } + if level.is_empty() { + break; + } + levels.push_back(level); + block += 1; + } + trace!(target: "state-db", "Finished reading unfinalized journal, {} entries", total); + } + Ok(UnfinalizedOverlay { + last_finalized: last_finalized, + levels, + parents, + last_finalized_overlay: Default::default(), + }) + } + + /// Insert a new block into the overlay. If inserted on the second level or lover expects parent to be present in the window. + pub fn insert(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet) -> CommitSet { + let mut commit = CommitSet::default(); + if self.levels.is_empty() && self.last_finalized.is_none() { + // assume that parent was finalized + let last_finalized = (parent_hash.clone(), number - 1); + commit.meta.inserted.push((to_meta_key(LAST_FINALIZED, &()), last_finalized.encode())); + self.last_finalized = Some(last_finalized); + } else if self.last_finalized.is_some() { + assert!(number >= self.front_block_number() && number < (self.front_block_number() + self.levels.len() as u64 + 1)); + // check for valid parent if inserting on second level or higher + if number == self.front_block_number() { + assert!(self.last_finalized.as_ref().map_or(false, |&(ref h, n)| h == parent_hash && n == number - 1)); + } else { + assert!(self.parents.contains_key(&parent_hash)); + } + } + let level = if self.levels.is_empty() || number == self.front_block_number() + self.levels.len() as u64 { + self.levels.push_back(Vec::new()); + self.levels.back_mut().expect("can't be empty after insertion; qed") + } else { + let front_block_number = self.front_block_number(); + self.levels.get_mut((number - front_block_number) as usize) + .expect("number is [front_block_number .. front_block_number + levels.len()) is asserted in precondition; qed") + }; + + let index = level.len() as u64; + let journal_key = to_journal_key(number, index); + + let overlay = BlockOverlay { + hash: hash.clone(), + journal_key: journal_key.clone(), + values: changeset.inserted.iter().cloned().collect(), + deleted: changeset.deleted.clone(), + }; + level.push(overlay); + self.parents.insert(hash.clone(), parent_hash.clone()); + let journal_record = JournalRecord { + hash: hash.clone(), + parent_hash: parent_hash.clone(), + inserted: changeset.inserted, + deleted: changeset.deleted, + }; + trace!(target: "state-db", "Inserted unfinalized changeset {}.{} ({} inserted, {} deleted)", number, index, journal_record.inserted.len(), journal_record.deleted.len()); + let journal_record = journal_record.encode(); + commit.meta.inserted.push((journal_key, journal_record)); + commit + } + + fn discard( + levels: &mut [Vec>], + parents: &mut HashMap, + discarded_journals: &mut Vec>, + number: u64, + hash: &BlockHash, + ) { + if let Some((level, sublevels)) = levels.split_first_mut() { + level.retain(|ref overlay| { + let parent = parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone(); + if parent == *hash { + parents.remove(&overlay.hash); + discarded_journals.push(overlay.journal_key.clone()); + Self::discard(sublevels, parents, discarded_journals, number + 1, &overlay.hash); + false + } else { + true + } + }); + } + } + + fn front_block_number(&self) -> u64 { + self.last_finalized.as_ref().map(|&(_, n)| n + 1).unwrap_or(0) + } + + pub fn last_finalized_block_number(&self) -> u64 { + self.last_finalized.as_ref().map(|&(_, n)| n).unwrap_or(0) + } + + /// This may be called when the last finalization commit was applied to the database. + pub fn clear_overlay(&mut self) { + self.last_finalized_overlay.clear(); + } + + /// Select a top-level root and finalized it. Discards all sibling subtrees and the root. + /// Returns a set of changes that need to be added to the DB. + pub fn finalize(&mut self, hash: &BlockHash) -> CommitSet { + trace!(target: "state-db", "Finalizing {:?}", hash); + let level = self.levels.pop_front().expect("no blocks to finalize"); + let index = level.iter().position(|overlay| overlay.hash == *hash) + .expect("attempting to finalize unknown block"); + + let mut commit = CommitSet::default(); + let mut discarded_journals = Vec::new(); + for (i, overlay) in level.into_iter().enumerate() { + self.parents.remove(&overlay.hash); + if i == index { + self.last_finalized_overlay = overlay.values; + // that's the one we need to finalize + commit.data.inserted = self.last_finalized_overlay.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + commit.data.deleted = overlay.deleted; + } else { + // TODO: borrow checker won't allow us to split out mutable refernces + // required for recursive processing. A more efficient implementaion + // that does not require converting to vector is possible + let mut vec: Vec<_> = self.levels.drain(..).collect(); + Self::discard(&mut vec, &mut self.parents, &mut discarded_journals, 0, &overlay.hash); + self.levels.extend(vec.into_iter()); + } + // cleanup journal entry + discarded_journals.push(overlay.journal_key); + } + commit.meta.deleted.append(&mut discarded_journals); + let last_finalized = (hash.clone(), self.front_block_number()); + commit.meta.inserted.push((to_meta_key(LAST_FINALIZED, &()), last_finalized.encode())); + self.last_finalized = Some(last_finalized); + trace!(target: "state-db", "Discarded {} records", commit.meta.deleted.len()); + commit + } + + /// Get a value from the node overlay. This searches in every existing changeset. + pub fn get(&self, key: &Key) -> Option { + if let Some(value) = self.last_finalized_overlay.get(&key) { + return Some(value.clone()); + } + for level in self.levels.iter() { + for overlay in level.iter() { + if let Some(value) = overlay.values.get(&key) { + return Some(value.clone()); + } + } + } + None + } + + /// Revert a single level. Returns commit set that deletes the journal or `None` if not possible. + pub fn revert_one(&mut self) -> Option> { + self.levels.pop_back().map(|level| { + let mut commit = CommitSet::default(); + for overlay in level.into_iter() { + commit.meta.deleted.push(overlay.journal_key); + self.parents.remove(&overlay.hash); + } + commit + }) + } +} + +#[cfg(test)] +mod tests { + use super::UnfinalizedOverlay; + use {ChangeSet}; + use primitives::H256; + use test::{make_db, make_changeset}; + + fn contains(overlay: &UnfinalizedOverlay, key: u64) -> bool { + overlay.get(&H256::from(key)) == Some(H256::from(key).to_vec()) + } + + #[test] + fn created_from_empty_db() { + let db = make_db(&[]); + let overlay: UnfinalizedOverlay = UnfinalizedOverlay::new(&db).unwrap(); + assert_eq!(overlay.last_finalized, None); + assert!(overlay.levels.is_empty()); + assert!(overlay.parents.is_empty()); + } + + #[test] + #[should_panic] + fn finalize_empty_panics() { + let db = make_db(&[]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + overlay.finalize(&H256::default()); + } + + #[test] + #[should_panic] + fn insert_ahead_panics() { + let db = make_db(&[]); + let h1 = H256::random(); + let h2 = H256::random(); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + overlay.insert(&h1, 2, &H256::default(), ChangeSet::default()); + overlay.insert(&h2, 1, &h1, ChangeSet::default()); + } + + #[test] + #[should_panic] + fn insert_behind_panics() { + let h1 = H256::random(); + let h2 = H256::random(); + let db = make_db(&[]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + overlay.insert(&h1, 1, &H256::default(), ChangeSet::default()); + overlay.insert(&h2, 3, &h1, ChangeSet::default()); + } + + #[test] + #[should_panic] + fn insert_unknown_parent_panics() { + let db = make_db(&[]); + let h1 = H256::random(); + let h2 = H256::random(); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + overlay.insert(&h1, 1, &H256::default(), ChangeSet::default()); + overlay.insert(&h2, 2, &H256::default(), ChangeSet::default()); + } + + #[test] + #[should_panic] + fn finalize_unknown_panics() { + let h1 = H256::random(); + let h2 = H256::random(); + let db = make_db(&[]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + overlay.insert(&h1, 1, &H256::default(), ChangeSet::default()); + overlay.finalize(&h2); + } + + #[test] + fn insert_finalize_one() { + let h1 = H256::random(); + let mut db = make_db(&[1, 2]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + let changeset = make_changeset(&[3, 4], &[2]); + let insertion = overlay.insert(&h1, 1, &H256::default(), changeset.clone()); + assert_eq!(insertion.data.inserted.len(), 0); + assert_eq!(insertion.data.deleted.len(), 0); + assert_eq!(insertion.meta.inserted.len(), 2); + assert_eq!(insertion.meta.deleted.len(), 0); + db.commit(&insertion); + let finalization = overlay.finalize(&h1); + assert_eq!(finalization.data.inserted.len(), changeset.inserted.len()); + assert_eq!(finalization.data.deleted.len(), changeset.deleted.len()); + assert_eq!(finalization.meta.inserted.len(), 1); + assert_eq!(finalization.meta.deleted.len(), 1); + db.commit(&finalization); + assert!(db.data_eq(&make_db(&[1, 3, 4]))); + } + + #[test] + fn restore_from_journal() { + let h1 = H256::random(); + let h2 = H256::random(); + let mut db = make_db(&[1, 2]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + db.commit(&overlay.insert(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))); + db.commit(&overlay.insert(&h2, 11, &h1, make_changeset(&[5], &[3]))); + assert_eq!(db.meta.len(), 3); + + let overlay2 = UnfinalizedOverlay::::new(&db).unwrap(); + assert_eq!(overlay.levels, overlay2.levels); + assert_eq!(overlay.parents, overlay2.parents); + assert_eq!(overlay.last_finalized, overlay2.last_finalized); + } + + #[test] + fn restore_from_journal_after_finalize() { + let h1 = H256::random(); + let h2 = H256::random(); + let mut db = make_db(&[1, 2]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + db.commit(&overlay.insert(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))); + db.commit(&overlay.insert(&h2, 11, &h1, make_changeset(&[5], &[3]))); + db.commit(&overlay.finalize(&h1)); + assert_eq!(overlay.levels.len(), 1); + + let overlay2 = UnfinalizedOverlay::::new(&db).unwrap(); + assert_eq!(overlay.levels, overlay2.levels); + assert_eq!(overlay.parents, overlay2.parents); + assert_eq!(overlay.last_finalized, overlay2.last_finalized); + } + + #[test] + fn insert_finalize_two() { + let h1 = H256::random(); + let h2 = H256::random(); + let mut db = make_db(&[1, 2, 3, 4]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + let changeset1 = make_changeset(&[5, 6], &[2]); + let changeset2 = make_changeset(&[7, 8], &[5, 3]); + db.commit(&overlay.insert(&h1, 1, &H256::default(), changeset1)); + assert!(contains(&overlay, 5)); + db.commit(&overlay.insert(&h2, 2, &h1, changeset2)); + assert!(contains(&overlay, 7)); + assert!(contains(&overlay, 5)); + assert_eq!(overlay.levels.len(), 2); + assert_eq!(overlay.parents.len(), 2); + db.commit(&overlay.finalize(&h1)); + assert_eq!(overlay.levels.len(), 1); + assert_eq!(overlay.parents.len(), 1); + assert!(contains(&overlay, 5)); + overlay.clear_overlay(); + assert!(!contains(&overlay, 5)); + assert!(contains(&overlay, 7)); + db.commit(&overlay.finalize(&h2)); + overlay.clear_overlay(); + assert_eq!(overlay.levels.len(), 0); + assert_eq!(overlay.parents.len(), 0); + assert!(db.data_eq(&make_db(&[1, 4, 6, 7, 8]))); + } + + + #[test] + fn complex_tree() { + let mut db = make_db(&[]); + + // - 1 - 1_1 - 1_1_1 + // \ 1_2 - 1_2_1 + // \ 1_2_2 + // \ 1_2_3 + // + // - 2 - 2_1 - 2_1_1 + // \ 2_2 + // + // 1_2_2 is the winner + + let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[])); + let (h_2, c_2) = (H256::random(), make_changeset(&[2], &[])); + + let (h_1_1, c_1_1) = (H256::random(), make_changeset(&[11], &[])); + let (h_1_2, c_1_2) = (H256::random(), make_changeset(&[12], &[])); + let (h_2_1, c_2_1) = (H256::random(), make_changeset(&[21], &[])); + let (h_2_2, c_2_2) = (H256::random(), make_changeset(&[22], &[])); + + let (h_1_1_1, c_1_1_1) = (H256::random(), make_changeset(&[111], &[])); + let (h_1_2_1, c_1_2_1) = (H256::random(), make_changeset(&[121], &[])); + let (h_1_2_2, c_1_2_2) = (H256::random(), make_changeset(&[122], &[])); + let (h_1_2_3, c_1_2_3) = (H256::random(), make_changeset(&[123], &[])); + let (h_2_1_1, c_2_1_1) = (H256::random(), make_changeset(&[211], &[])); + + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + db.commit(&overlay.insert(&h_1, 1, &H256::default(), c_1)); + + db.commit(&overlay.insert(&h_1_1, 2, &h_1, c_1_1)); + db.commit(&overlay.insert(&h_1_2, 2, &h_1, c_1_2)); + + db.commit(&overlay.insert(&h_2, 1, &H256::default(), c_2)); + + db.commit(&overlay.insert(&h_2_1, 2, &h_2, c_2_1)); + db.commit(&overlay.insert(&h_2_2, 2, &h_2, c_2_2)); + + db.commit(&overlay.insert(&h_1_1_1, 3, &h_1_1, c_1_1_1)); + db.commit(&overlay.insert(&h_1_2_1, 3, &h_1_2, c_1_2_1)); + db.commit(&overlay.insert(&h_1_2_2, 3, &h_1_2, c_1_2_2)); + db.commit(&overlay.insert(&h_1_2_3, 3, &h_1_2, c_1_2_3)); + db.commit(&overlay.insert(&h_2_1_1, 3, &h_2_1, c_2_1_1)); + + assert!(contains(&overlay, 2)); + assert!(contains(&overlay, 11)); + assert!(contains(&overlay, 21)); + assert!(contains(&overlay, 111)); + assert!(contains(&overlay, 122)); + assert!(contains(&overlay, 211)); + assert_eq!(overlay.levels.len(), 3); + assert_eq!(overlay.parents.len(), 11); + assert_eq!(overlay.last_finalized, Some((H256::default(), 0))); + + // check if restoration from journal results in the same tree + let overlay2 = UnfinalizedOverlay::::new(&db).unwrap(); + assert_eq!(overlay.levels, overlay2.levels); + assert_eq!(overlay.parents, overlay2.parents); + assert_eq!(overlay.last_finalized, overlay2.last_finalized); + + // finalize 1. 2 and all its children should be discarded + db.commit(&overlay.finalize(&h_1)); + overlay.clear_overlay(); + assert_eq!(overlay.levels.len(), 2); + assert_eq!(overlay.parents.len(), 6); + assert!(!contains(&overlay, 1)); + assert!(!contains(&overlay, 2)); + assert!(!contains(&overlay, 21)); + assert!(!contains(&overlay, 22)); + assert!(!contains(&overlay, 211)); + assert!(contains(&overlay, 111)); + + // finalize 1_2. 1_1 and all its children should be discarded + db.commit(&overlay.finalize(&h_1_2)); + overlay.clear_overlay(); + assert_eq!(overlay.levels.len(), 1); + assert_eq!(overlay.parents.len(), 3); + assert!(!contains(&overlay, 11)); + assert!(!contains(&overlay, 111)); + assert!(contains(&overlay, 121)); + assert!(contains(&overlay, 122)); + assert!(contains(&overlay, 123)); + + // finalize 1_2_2 + db.commit(&overlay.finalize(&h_1_2_2)); + overlay.clear_overlay(); + assert_eq!(overlay.levels.len(), 0); + assert_eq!(overlay.parents.len(), 0); + assert!(db.data_eq(&make_db(&[1, 12, 122]))); + assert_eq!(overlay.last_finalized, Some((h_1_2_2, 3))); + } + + #[test] + fn insert_revert() { + let h1 = H256::random(); + let h2 = H256::random(); + let mut db = make_db(&[1, 2, 3, 4]); + let mut overlay = UnfinalizedOverlay::::new(&db).unwrap(); + assert!(overlay.revert_one().is_none()); + let changeset1 = make_changeset(&[5, 6], &[2]); + let changeset2 = make_changeset(&[7, 8], &[5, 3]); + db.commit(&overlay.insert(&h1, 1, &H256::default(), changeset1)); + db.commit(&overlay.insert(&h2, 2, &h1, changeset2)); + assert!(contains(&overlay, 7)); + db.commit(&overlay.revert_one().unwrap()); + assert_eq!(overlay.parents.len(), 1); + assert!(contains(&overlay, 5)); + assert!(!contains(&overlay, 7)); + db.commit(&overlay.revert_one().unwrap()); + assert_eq!(overlay.levels.len(), 0); + assert_eq!(overlay.parents.len(), 0); + assert!(overlay.revert_one().is_none()); + } + +} + diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml new file mode 100644 index 00000000000..3635627999f --- /dev/null +++ b/core/state-machine/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "substrate-state-machine" +version = "0.1.0" +authors = ["Parity Technologies "] +description = "Substrate State Machine" + +[dependencies] +byteorder = "1.1" +hex-literal = "0.1.0" +log = "0.3" +parking_lot = "0.4" +heapsize = "0.4" +hashdb = "0.2.1" +memorydb = "0.2.1" +patricia-trie = "0.2.1" +triehash = "0.2" +rlp = "0.2.4" + +substrate-primitives = { path = "../primitives", version = "0.1.0" } +substrate-codec = { path = "../codec", default_features = false } + diff --git a/core/state-machine/README.adoc b/core/state-machine/README.adoc new file mode 100644 index 00000000000..aad08bed989 --- /dev/null +++ b/core/state-machine/README.adoc @@ -0,0 +1,13 @@ + += State Machine + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/state-machine/src/backend.rs b/core/state-machine/src/backend.rs new file mode 100644 index 00000000000..7db1e64d3f2 --- /dev/null +++ b/core/state-machine/src/backend.rs @@ -0,0 +1,193 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! State machine backends. These manage the code and storage of contracts. + +use std::{error, fmt}; +use std::cmp::Ord; +use std::collections::HashMap; +use std::marker::PhantomData; +use std::sync::Arc; + +use hashdb::Hasher; +use rlp::Encodable; +use trie_backend::{TryIntoTrieBackend, TrieBackend}; +use patricia_trie::{TrieDBMut, TrieMut, NodeCodec}; +use heapsize::HeapSizeOf; + +/// A state backend is used to read state data and can have changes committed +/// to it. +/// +/// The clone operation (if implemented) should be cheap. +pub trait Backend>: TryIntoTrieBackend { + /// An error type when fetching data is not possible. + type Error: super::Error; + + /// Changes to be applied if committing + type Transaction; + + /// Get keyed storage associated with specific address, or None if there is nothing associated. + fn storage(&self, key: &[u8]) -> Result>, Self::Error>; + + /// true if a key exists in storage. + fn exists_storage(&self, key: &[u8]) -> Result { + Ok(self.storage(key)?.is_some()) + } + + /// Retrieve all entries keys of which start with the given prefix and + /// call `f` for each of those keys. + fn for_keys_with_prefix(&self, prefix: &[u8], f: F); + + /// Calculate the storage root, with given delta over what is already stored in + /// the backend, and produce a "transaction" that can be used to commit. + fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) + where + I: IntoIterator, Option>)>, + H::Out: Ord + Encodable; + + /// Get all key/value pairs into a Vec. + fn pairs(&self) -> Vec<(Vec, Vec)>; +} + +/// Error impossible. +// TODO: use `!` type when stabilized. +#[derive(Debug)] +pub enum Void {} + +impl fmt::Display for Void { + fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result { + match *self {} + } +} + +impl error::Error for Void { + fn description(&self) -> &str { "unreachable error" } +} + +/// In-memory backend. Fully recomputes tries on each commit but useful for +/// tests. +#[derive(Eq)] +pub struct InMemory { + inner: Arc, Vec>>, + _hasher: PhantomData, + _codec: PhantomData, +} + +impl Default for InMemory { + fn default() -> Self { + InMemory { + inner: Arc::new(Default::default()), + _hasher: PhantomData, + _codec: PhantomData, + } + } +} + +impl Clone for InMemory { + fn clone(&self) -> Self { + InMemory { + inner: self.inner.clone(), _hasher: PhantomData, _codec: PhantomData, + } + } +} + +impl PartialEq for InMemory { + fn eq(&self, other: &Self) -> bool { + self.inner.eq(&other.inner) + } +} + +impl> InMemory where H::Out: HeapSizeOf { + /// Copy the state, with applied updates + pub fn update(&self, changes: >::Transaction) -> Self { + let mut inner: HashMap<_, _> = (&*self.inner).clone(); + for (key, val) in changes { + match val { + Some(v) => { inner.insert(key, v); }, + None => { inner.remove(&key); }, + } + } + + inner.into() + } +} + +impl From, Vec>> for InMemory { + fn from(inner: HashMap, Vec>) -> Self { + InMemory { + inner: Arc::new(inner), _hasher: PhantomData, _codec: PhantomData + } + } +} + +impl super::Error for Void {} + +impl> Backend for InMemory where H::Out: HeapSizeOf { + type Error = Void; + type Transaction = Vec<(Vec, Option>)>; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + Ok(self.inner.get(key).map(Clone::clone)) + } + + fn exists_storage(&self, key: &[u8]) -> Result { + Ok(self.inner.get(key).is_some()) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + self.inner.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f); + } + + fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) + where + I: IntoIterator, Option>)>, + ::Out: Ord + Encodable, + { + let existing_pairs = self.inner.iter().map(|(k, v)| (k.clone(), Some(v.clone()))); + + let transaction: Vec<_> = delta.into_iter().collect(); + let root = ::triehash::trie_root::(existing_pairs.chain(transaction.iter().cloned()) + .collect::>() + .into_iter() + .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) + ); + + (root, transaction) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.inner.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } +} + +impl> TryIntoTrieBackend for InMemory where H::Out: HeapSizeOf { + fn try_into_trie_backend(self) -> Option> { + use memorydb::MemoryDB; + let mut root = ::Out::default(); + let mut mdb = MemoryDB::new(); + { + let mut trie = TrieDBMut::::new(&mut mdb, &mut root); + for (key, value) in self.inner.iter() { + if let Err(e) = trie.insert(&key, &value) { + warn!(target: "trie", "Failed to write to trie: {}", e); + return None; + } + } + } + + Some(TrieBackend::with_memorydb(mdb, root)) + } +} diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs new file mode 100644 index 00000000000..8504f42b593 --- /dev/null +++ b/core/state-machine/src/ext.rs @@ -0,0 +1,171 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Conrete externalities implementation. + +use std::{error, fmt, cmp::Ord}; +use backend::Backend; +use {Externalities, OverlayedChanges}; +use hashdb::Hasher; +use rlp::Encodable; +use patricia_trie::NodeCodec; + +/// Errors that can occur when interacting with the externalities. +#[derive(Debug, Copy, Clone)] +pub enum Error { + /// Failure to load state data from the backend. + #[allow(unused)] + Backend(B), + /// Failure to execute a function. + #[allow(unused)] + Executor(E), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Error::Backend(ref e) => write!(f, "Storage backend error: {}", e), + Error::Executor(ref e) => write!(f, "Sub-call execution error: {}", e), + } + } +} + +impl error::Error for Error { + fn description(&self) -> &str { + match *self { + Error::Backend(..) => "backend error", + Error::Executor(..) => "executor error", + } + } +} + +/// Wraps a read-only backend, call executor, and current overlayed changes. +pub struct Ext<'a, H, C, B> +where + H: Hasher, + C: NodeCodec, + B: 'a + Backend, +{ + // The overlayed changes to write to. + overlay: &'a mut OverlayedChanges, + // The storage backend to read from. + backend: &'a B, + // The transaction necessary to commit to the backend. + transaction: Option<(B::Transaction, H::Out)>, +} + +impl<'a, H, C, B> Ext<'a, H, C, B> +where + H: Hasher, + C: NodeCodec, + B: 'a + Backend, + H::Out: Ord + Encodable +{ + /// Create a new `Ext` from overlayed changes and read-only backend + pub fn new(overlay: &'a mut OverlayedChanges, backend: &'a B) -> Self { + Ext { + overlay, + backend, + transaction: None, + } + } + + /// Get the transaction necessary to update the backend. + pub fn transaction(mut self) -> B::Transaction { + let _ = self.storage_root(); + self.transaction.expect("transaction always set after calling storage root; qed").0 + } + + /// Invalidates the currently cached storage root and the db transaction. + /// + /// Called when there are changes that likely will invalidate the storage root. + fn mark_dirty(&mut self) { + self.transaction = None; + } +} + +#[cfg(test)] +impl<'a, H, C, B> Ext<'a, H, C, B> +where + H: Hasher, + C: NodeCodec, + B: 'a + Backend, +{ + pub fn storage_pairs(&self) -> Vec<(Vec, Vec)> { + use std::collections::HashMap; + + self.backend.pairs().iter() + .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) + .chain(self.overlay.committed.clone().into_iter()) + .chain(self.overlay.prospective.clone().into_iter()) + .collect::>() + .into_iter() + .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) + .collect() + } +} + +impl<'a, B: 'a, H, C> Externalities for Ext<'a, H, C, B> +where + H: Hasher, + C: NodeCodec, + B: 'a + Backend, + H::Out: Ord + Encodable +{ + fn storage(&self, key: &[u8]) -> Option> { + self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| + self.backend.storage(key).expect("Externalities not allowed to fail within runtime")) + } + + fn exists_storage(&self, key: &[u8]) -> bool { + match self.overlay.storage(key) { + Some(x) => x.is_some(), + _ => self.backend.exists_storage(key).expect("Externalities not allowed to fail within runtime"), + } + } + + fn place_storage(&mut self, key: Vec, value: Option>) { + self.mark_dirty(); + self.overlay.set_storage(key, value); + } + + fn clear_prefix(&mut self, prefix: &[u8]) { + self.mark_dirty(); + self.overlay.clear_prefix(prefix); + self.backend.for_keys_with_prefix(prefix, |key| { + self.overlay.set_storage(key.to_vec(), None); + }); + } + + fn chain_id(&self) -> u64 { + 42 + } + + fn storage_root(&mut self) -> H::Out { + if let Some((_, ref root)) = self.transaction { + return root.clone(); + } + + // compute and memoize + let delta = self.overlay.committed.iter() + .chain(self.overlay.prospective.iter()) + .map(|(k, v)| (k.clone(), v.clone())); + + let (root, transaction) = self.backend.storage_root(delta); + self.transaction = Some((transaction, root)); + root + } +} diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs new file mode 100644 index 00000000000..7dab6e6af63 --- /dev/null +++ b/core/state-machine/src/lib.rs @@ -0,0 +1,691 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Substrate state machine implementation. +// end::description[] + +#![warn(missing_docs)] + +#[cfg_attr(test, macro_use)] +extern crate hex_literal; + +#[macro_use] +extern crate log; + +extern crate hashdb; +extern crate memorydb; +extern crate triehash; +extern crate patricia_trie; +extern crate byteorder; +extern crate parking_lot; +extern crate rlp; +extern crate heapsize; +#[cfg(test)] +extern crate substrate_primitives as primitives; +extern crate substrate_codec as codec; + +use std::collections::HashMap; +use std::fmt; +use hashdb::Hasher; +use patricia_trie::NodeCodec; +use rlp::Encodable; +use heapsize::HeapSizeOf; +use codec::Decode; + +pub mod backend; +mod ext; +mod testing; +mod proving_backend; +mod trie_backend; + +pub use testing::TestExternalities; +pub use ext::Ext; +pub use backend::Backend; +pub use trie_backend::{TryIntoTrieBackend, TrieBackend, Storage, DBValue}; + +/// The overlayed changes to state to be queried on top of the backend. +/// +/// A transaction shares all prospective changes within an inner overlay +/// that can be cleared. +#[derive(Debug, Default, Clone)] +pub struct OverlayedChanges { + prospective: HashMap, Option>>, + committed: HashMap, Option>>, +} + +impl OverlayedChanges { + /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered + /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose + /// value has been set. + pub fn storage(&self, key: &[u8]) -> Option> { + self.prospective.get(key) + .or_else(|| self.committed.get(key)) + .map(|x| x.as_ref().map(AsRef::as_ref)) + } + + /// Inserts the given key-value pair into the prospective change set. + /// + /// `None` can be used to delete a value specified by the given key. + fn set_storage(&mut self, key: Vec, val: Option>) { + self.prospective.insert(key, val); + } + + /// Removes all key-value pairs which keys share the given prefix. + /// + /// NOTE that this doesn't take place immediately but written into the prospective + /// change set, and still can be reverted by [`discard_prospective`]. + /// + /// [`discard_prospective`]: #method.discard_prospective + fn clear_prefix(&mut self, prefix: &[u8]) { + // Iterate over all prospective and mark all keys that share + // the given prefix as removed (None). + for (key, value) in self.prospective.iter_mut() { + if key.starts_with(prefix) { + *value = None; + } + } + + // Then do the same with keys from commited changes. + // NOTE that we are making changes in the prospective change set. + for key in self.committed.keys() { + if key.starts_with(prefix) { + self.prospective.insert(key.to_owned(), None); + } + } + } + + /// Discard prospective changes to state. + pub fn discard_prospective(&mut self) { + self.prospective.clear(); + } + + /// Commit prospective changes to state. + pub fn commit_prospective(&mut self) { + if self.committed.is_empty() { + ::std::mem::swap(&mut self.prospective, &mut self.committed); + } else { + self.committed.extend(self.prospective.drain()); + } + } + + /// Drain committed changes to an iterator. + /// + /// Panics: + /// Will panic if there are any uncommitted prospective changes. + pub fn drain<'a>(&'a mut self) -> impl Iterator, Option>)> + 'a { + assert!(self.prospective.is_empty()); + self.committed.drain() + } + + /// Consume `OverlayedChanges` and take committed set. + /// + /// Panics: + /// Will panic if there are any uncommitted prospective changes. + pub fn into_committed(self) -> impl Iterator, Option>)> { + assert!(self.prospective.is_empty()); + self.committed.into_iter() + } +} + +/// State Machine Error bound. +/// +/// This should reflect WASM error type bound for future compatibility. +pub trait Error: 'static + fmt::Debug + fmt::Display + Send {} + +impl Error for ExecutionError {} + +/// Externalities Error. +/// +/// Externalities are not really allowed to have errors, since it's assumed that dependent code +/// would not be executed unless externalities were available. This is included for completeness, +/// and as a transition away from the pre-existing framework. +#[derive(Debug, Eq, PartialEq)] +pub enum ExecutionError { + /// The entry `:code` doesn't exist in storage so there's no way we can execute anything. + CodeEntryDoesNotExist, + /// Backend is incompatible with execution proof generation process. + UnableToGenerateProof, + /// Invalid execution proof. + InvalidProof, +} + +impl fmt::Display for ExecutionError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "Externalities Error") } +} + +/// Externalities: pinned to specific active address. +pub trait Externalities { + /// Read storage of current contract being called. + fn storage(&self, key: &[u8]) -> Option>; + + /// Set storage entry `key` of current contract being called (effective immediately). + fn set_storage(&mut self, key: Vec, value: Vec) { + self.place_storage(key, Some(value)); + } + + /// Clear a storage entry (`key`) of current contract being called (effective immediately). + fn clear_storage(&mut self, key: &[u8]) { + self.place_storage(key.to_vec(), None); + } + + /// Clear a storage entry (`key`) of current contract being called (effective immediately). + fn exists_storage(&self, key: &[u8]) -> bool { + self.storage(key).is_some() + } + + /// Clear storage entries which keys are start with the given prefix. + fn clear_prefix(&mut self, prefix: &[u8]); + + /// Set or clear a storage entry (`key`) of current contract being called (effective immediately). + fn place_storage(&mut self, key: Vec, value: Option>); + + /// Get the identity of the chain. + fn chain_id(&self) -> u64; + + /// Get the trie root of the current storage map. + fn storage_root(&mut self) -> H::Out where H::Out: Ord + Encodable; +} + +/// Code execution engine. +pub trait CodeExecutor: Sized + Send + Sync { + /// Externalities error type. + type Error: Error; + + /// Call a given method in the runtime. Returns a tuple of the result (either the output data + /// or an execution error) together with a `bool`, which is true if native execution was used. + fn call>( + &self, + ext: &mut E, + heap_pages: usize, + code: &[u8], + method: &str, + data: &[u8], + use_native: bool + ) -> (Result, Self::Error>, bool); +} + +/// Strategy for executing a call into the runtime. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum ExecutionStrategy { + /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. + NativeWhenPossible, + /// Use the given wasm module. + AlwaysWasm, + /// Run with both the wasm and the native variant (if compatible). Report any discrepency as an error. + Both, +} + +/// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure. +pub enum ExecutionManager { + /// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm. + NativeWhenPossible, + /// Use the given wasm module. + AlwaysWasm, + /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepency. + Both(F), +} + +impl<'a, F> From<&'a ExecutionManager> for ExecutionStrategy { + fn from(s: &'a ExecutionManager) -> Self { + match *s { + ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible, + ExecutionManager::AlwaysWasm => ExecutionStrategy::AlwaysWasm, + ExecutionManager::Both(_) => ExecutionStrategy::Both, + } + } +} + +/// Evaluate to ExecutionManager::NativeWhenPossible, without having to figure out the type. +pub fn native_when_possible() -> ExecutionManager, E>, Result, E>)->Result, E>> { + ExecutionManager::NativeWhenPossible +} + +/// Evaluate to ExecutionManager::NativeWhenPossible, without having to figure out the type. +pub fn always_wasm() -> ExecutionManager, E>, Result, E>)->Result, E>> { + ExecutionManager::AlwaysWasm +} + +/// Execute a call using the given state backend, overlayed changes, and call executor. +/// Produces a state-backend-specific "transaction" which can be used to apply the changes +/// to the backing store, such as the disk. +/// +/// On an error, no prospective changes are written to the overlay. +/// +/// Note: changes to code will be in place if this call is made again. For running partial +/// blocks (e.g. a transaction at a time), ensure a different method is used. +pub fn execute( + backend: &B, + overlay: &mut OverlayedChanges, + exec: &Exec, + method: &str, + call_data: &[u8], + strategy: ExecutionStrategy, +) -> Result<(Vec, B::Transaction), Box> +where + H: Hasher, + C: NodeCodec, + Exec: CodeExecutor, + B: Backend, + H::Out: Ord + Encodable +{ + execute_using_consensus_failure_handler( + backend, + overlay, + exec, + method, + call_data, + match strategy { + ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm, + ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible, + ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| { + warn!("Consensus error between wasm {:?} and native {:?}. Using wasm.", wasm_result, native_result); + wasm_result + }), + }, + ) +} + +/// Execute a call using the given state backend, overlayed changes, and call executor. +/// Produces a state-backend-specific "transaction" which can be used to apply the changes +/// to the backing store, such as the disk. +/// +/// On an error, no prospective changes are written to the overlay. +/// +/// Note: changes to code will be in place if this call is made again. For running partial +/// blocks (e.g. a transaction at a time), ensure a different method is used. +pub fn execute_using_consensus_failure_handler( + backend: &B, + overlay: &mut OverlayedChanges, + exec: &Exec, + method: &str, + call_data: &[u8], + manager: ExecutionManager, +) -> Result<(Vec, B::Transaction), Box> +where + H: Hasher, + C: NodeCodec, + Exec: CodeExecutor, + B: Backend, + H::Out: Ord + Encodable, + Handler: FnOnce(Result, Exec::Error>, Result, Exec::Error>) -> Result, Exec::Error> +{ + let strategy: ExecutionStrategy = (&manager).into(); + + // make a copy. + let code = ext::Ext::new(overlay, backend).storage(b":code") + .ok_or_else(|| Box::new(ExecutionError::CodeEntryDoesNotExist) as Box)? + .to_vec(); + + let heap_pages = ext::Ext::new(overlay, backend).storage(b":heappages") + .and_then(|v| u64::decode(&mut &v[..])).unwrap_or(8) as usize; + + let result = { + let mut orig_prospective = overlay.prospective.clone(); + + let (result, was_native, delta) = { + let ((result, was_native), delta) = { + let mut externalities = ext::Ext::new(overlay, backend); + ( + exec.call( + &mut externalities, + heap_pages, + &code, + method, + call_data, + // attempt to run native first, if we're not directed to run wasm only + strategy != ExecutionStrategy::AlwaysWasm, + ), + externalities.transaction() + ) + }; + (result, was_native, delta) + }; + + // run wasm separately if we did run native the first time and we're meant to run both + let (result, delta) = if let (true, ExecutionManager::Both(on_consensus_failure)) = + (was_native, manager) + { + overlay.prospective = orig_prospective.clone(); + + let (wasm_result, wasm_delta) = { + let ((result, _), delta) = { + let mut externalities = ext::Ext::new(overlay, backend); + ( + exec.call( + &mut externalities, + heap_pages, + &code, + method, + call_data, + false, + ), + externalities.transaction() + ) + }; + (result, delta) + }; + + if (result.is_ok() && wasm_result.is_ok() && result.as_ref().unwrap() == wasm_result.as_ref().unwrap()/* && delta == wasm_delta*/) + || (result.is_err() && wasm_result.is_err()) + { + (result, delta) + } else { + // Consensus error. + (on_consensus_failure(wasm_result, result), wasm_delta) + } + } else { + (result, delta) + }; + result.map(move |out| (out, delta)) + }; + + result.map_err(|e| Box::new(e) as _) +} + +/// Prove execution using the given state backend, overlayed changes, and call executor. +/// Produces a state-backend-specific "transaction" which can be used to apply the changes +/// to the backing store, such as the disk. +/// Execution proof is the set of all 'touched' storage DBValues from the backend. +/// +/// On an error, no prospective changes are written to the overlay. +/// +/// Note: changes to code will be in place if this call is made again. For running partial +/// blocks (e.g. a transaction at a time), ensure a different method is used. +pub fn prove_execution( + backend: B, + overlay: &mut OverlayedChanges, + exec: &Exec, + method: &str, + call_data: &[u8], +) -> Result<(Vec, Vec>, as Backend>::Transaction), Box> +where + H: Hasher, + Exec: CodeExecutor, + C: NodeCodec, + B: TryIntoTrieBackend, + H::Out: Ord + Encodable + HeapSizeOf, +{ + let trie_backend = backend.try_into_trie_backend() + .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; + let proving_backend = proving_backend::ProvingBackend::new(trie_backend); + let (result, transaction) = execute::(&proving_backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)?; + let proof = proving_backend.extract_proof(); + Ok((result, proof, transaction)) +} + +/// Check execution proof, generated by `prove_execution` call. +pub fn execution_proof_check( + root: H::Out, + proof: Vec>, + overlay: &mut OverlayedChanges, + exec: &Exec, + method: &str, + call_data: &[u8], +) -> Result<(Vec, memorydb::MemoryDB), Box> +where +H: Hasher, +C: NodeCodec, +Exec: CodeExecutor, +H::Out: Ord + Encodable + HeapSizeOf, +{ + let backend = proving_backend::create_proof_check_backend::(root.into(), proof)?; + execute::(&backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible) +} + +/// Generate storage read proof. +pub fn prove_read( + backend: B, + key: &[u8] +) -> Result<(Option>, Vec>), Box> +where + B: TryIntoTrieBackend, + H: Hasher, + C: NodeCodec, + H::Out: Ord + Encodable + HeapSizeOf +{ + let trie_backend = backend.try_into_trie_backend() + .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; + let proving_backend = proving_backend::ProvingBackend::::new(trie_backend); + let result = proving_backend.storage(key).map_err(|e| Box::new(e) as Box)?; + Ok((result, proving_backend.extract_proof())) +} + +/// Check storage read proof, generated by `prove_read` call. +pub fn read_proof_check( + root: H::Out, + proof: Vec>, + key: &[u8], +) -> Result>, Box> +where + H: Hasher, + C: NodeCodec, + H::Out: Ord + Encodable + HeapSizeOf +{ + let backend = proving_backend::create_proof_check_backend::(root, proof)?; + backend.storage(key).map_err(|e| Box::new(e) as Box) +} + +#[cfg(test)] +mod tests { + use super::*; + use super::backend::InMemory; + use super::ext::Ext; + use primitives::{Blake2Hasher, RlpCodec, H256}; + + struct DummyCodeExecutor { + native_available: bool, + native_succeeds: bool, + fallback_succeeds: bool, + } + + impl CodeExecutor for DummyCodeExecutor { + type Error = u8; + + fn call>( + &self, + ext: &mut E, + _heap_pages: usize, + _code: &[u8], + _method: &str, + _data: &[u8], + use_native: bool + ) -> (Result, Self::Error>, bool) { + let using_native = use_native && self.native_available; + match (using_native, self.native_succeeds, self.fallback_succeeds) { + (true, true, _) | (false, _, true) => + (Ok(vec![ext.storage(b"value1").unwrap()[0] + ext.storage(b"value2").unwrap()[0]]), using_native), + _ => (Err(0), using_native), + } + } + } + + impl Error for u8 {} + + #[test] + fn overlayed_storage_works() { + let mut overlayed = OverlayedChanges::default(); + + let key = vec![42, 69, 169, 142]; + + assert!(overlayed.storage(&key).is_none()); + + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.commit_prospective(); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.set_storage(key.clone(), Some(vec![])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..])); + + overlayed.set_storage(key.clone(), None); + assert!(overlayed.storage(&key).unwrap().is_none()); + + overlayed.discard_prospective(); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.set_storage(key.clone(), None); + overlayed.commit_prospective(); + assert!(overlayed.storage(&key).unwrap().is_none()); + } + + macro_rules! map { + ($( $name:expr => $value:expr ),*) => ( + vec![ $( ( $name, $value ) ),* ].into_iter().collect() + ) + } + + #[test] + fn overlayed_storage_root_works() { + let initial: HashMap<_, _> = map![ + b"doe".to_vec() => b"reindeer".to_vec(), + b"dog".to_vec() => b"puppyXXX".to_vec(), + b"dogglesworth".to_vec() => b"catXXX".to_vec(), + b"doug".to_vec() => b"notadog".to_vec() + ]; + let backend = InMemory::::from(initial); + let mut overlay = OverlayedChanges { + committed: map![ + b"dog".to_vec() => Some(b"puppy".to_vec()), + b"dogglesworth".to_vec() => Some(b"catYYY".to_vec()), + b"doug".to_vec() => Some(vec![]) + ], + prospective: map![ + b"dogglesworth".to_vec() => Some(b"cat".to_vec()), + b"doug".to_vec() => None + ], + }; + let mut ext = Ext::new(&mut overlay, &backend); + const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b"); + assert_eq!(ext.storage_root(), H256(ROOT)); + } + + #[test] + fn execute_works() { + assert_eq!(execute( + &trie_backend::tests::test_trie(), + &mut Default::default(), + &DummyCodeExecutor { + native_available: true, + native_succeeds: true, + fallback_succeeds: true, + }, + "test", + &[], + ExecutionStrategy::NativeWhenPossible + ).unwrap().0, vec![66]); + } + + #[test] + fn dual_execution_strategy_detects_consensus_failure() { + let mut consensus_failed = false; + assert!(execute_using_consensus_failure_handler( + &trie_backend::tests::test_trie(), + &mut Default::default(), + &DummyCodeExecutor { + native_available: true, + native_succeeds: true, + fallback_succeeds: false, + }, + "test", + &[], + ExecutionManager::Both(|we, _ne| { + consensus_failed = true; + println!("HELLO!"); + we + }), + ).is_err()); + assert!(consensus_failed); + } + + #[test] + fn prove_execution_and_proof_check_works() { + let executor = DummyCodeExecutor { + native_available: true, + native_succeeds: true, + fallback_succeeds: true, + }; + + // fetch execution proof from 'remote' full node + let remote_backend = trie_backend::tests::test_trie(); + let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let (remote_result, remote_proof, _) = prove_execution(remote_backend, + &mut Default::default(), &executor, "test", &[]).unwrap(); + + // check proof locally + let (local_result, _) = execution_proof_check::(remote_root, remote_proof, + &mut Default::default(), &executor, "test", &[]).unwrap(); + + // check that both results are correct + assert_eq!(remote_result, vec![66]); + assert_eq!(remote_result, local_result); + } + + #[test] + fn clear_prefix_in_ext_works() { + let initial: HashMap<_, _> = map![ + b"aaa".to_vec() => b"0".to_vec(), + b"abb".to_vec() => b"1".to_vec(), + b"abc".to_vec() => b"2".to_vec(), + b"bbb".to_vec() => b"3".to_vec() + ]; + let backend = InMemory::::from(initial).try_into_trie_backend().unwrap(); + let mut overlay = OverlayedChanges { + committed: map![ + b"aba".to_vec() => Some(b"1312".to_vec()), + b"bab".to_vec() => Some(b"228".to_vec()) + ], + prospective: map![ + b"abd".to_vec() => Some(b"69".to_vec()), + b"bbd".to_vec() => Some(b"42".to_vec()) + ], + }; + + { + let mut ext = Ext::new(&mut overlay, &backend); + ext.clear_prefix(b"ab"); + } + overlay.commit_prospective(); + + assert_eq!( + overlay.committed, + map![ + b"abb".to_vec() => None, + b"abc".to_vec() => None, + b"aba".to_vec() => None, + b"abd".to_vec() => None, + + b"bab".to_vec() => Some(b"228".to_vec()), + b"bbd".to_vec() => Some(b"42".to_vec()) + ], + ); + } + + #[test] + fn prove_read_and_proof_check_works() { + // fetch read proof from 'remote' full node + let remote_backend = trie_backend::tests::test_trie(); + let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let remote_proof = prove_read(remote_backend, b"value2").unwrap().1; + // check proof locally + let local_result1 = read_proof_check::(remote_root, remote_proof.clone(), b"value2").unwrap(); + let local_result2 = read_proof_check::(remote_root, remote_proof.clone(), &[0xff]).is_ok(); + // check that results are correct + assert_eq!(local_result1, Some(vec![24])); + assert_eq!(local_result2, false); + } +} diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs new file mode 100644 index 00000000000..25a2f49c75a --- /dev/null +++ b/core/state-machine/src/proving_backend.rs @@ -0,0 +1,183 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Proving state machine backend. + +use std::cell::RefCell; +use hashdb::{Hasher, HashDB}; +use memorydb::MemoryDB; +use patricia_trie::{TrieDB, Trie, Recorder, NodeCodec}; +use trie_backend::{TrieBackend, Ephemeral}; +use {Error, ExecutionError, Backend, TryIntoTrieBackend}; +use rlp::Encodable; +use heapsize::HeapSizeOf; + +/// Patricia trie-based backend which also tracks all touched storage trie values. +/// These can be sent to remote node and used as a proof of execution. +pub struct ProvingBackend> { + backend: TrieBackend, + proof_recorder: RefCell>, +} + +impl> ProvingBackend { + /// Create new proving backend. + pub fn new(backend: TrieBackend) -> Self { + ProvingBackend { + backend, + proof_recorder: RefCell::new(Recorder::new()), + } + } + + /// Consume the backend, extracting the gathered proof in lexicographical order + /// by value. + pub fn extract_proof(self) -> Vec> { + self.proof_recorder.into_inner().drain() + .into_iter() + .map(|n| n.data.to_vec()) + .collect() + } +} + +impl Backend for ProvingBackend +where + H: Hasher, + C: NodeCodec, + H::Out: Ord + Encodable + HeapSizeOf +{ + type Error = String; + type Transaction = MemoryDB; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + let mut read_overlay = MemoryDB::new(); + let eph = Ephemeral::new( + self.backend.backend_storage(), + &mut read_overlay, + ); + let map_e = |e| format!("Trie lookup error: {}", e); + + let mut proof_recorder = self.proof_recorder.try_borrow_mut() + .expect("only fails when already borrowed; storage() is non-reentrant; qed"); + TrieDB::::new(&eph, &self.backend.root()).map_err(map_e)? + .get_with(key, &mut *proof_recorder).map(|x| x.map(|val| val.to_vec())).map_err(map_e) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + self.backend.for_keys_with_prefix(prefix, f) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.backend.pairs() + } + + fn storage_root(&self, delta: I) -> (H::Out, MemoryDB) + where I: IntoIterator, Option>)> + { + self.backend.storage_root(delta) + } +} + +impl> TryIntoTrieBackend for ProvingBackend { + fn try_into_trie_backend(self) -> Option> { + None + } +} + +/// Create proof check backend. +pub fn create_proof_check_backend( + root: H::Out, + proof: Vec> +) -> Result, Box> +where + H: Hasher, + C: NodeCodec, + H::Out: HeapSizeOf, +{ + let mut db = MemoryDB::new(); + for item in proof { + db.insert(&item); + } + + if !db.contains(&root) { + return Err(Box::new(ExecutionError::InvalidProof) as Box); + } + + + Ok(TrieBackend::with_memorydb(db, root)) +} + +#[cfg(test)] +mod tests { + use backend::{InMemory}; + use trie_backend::tests::test_trie; + use super::*; + use primitives::{Blake2Hasher, RlpCodec}; + + fn test_proving() -> ProvingBackend { + ProvingBackend::new(test_trie()) + } + + #[test] + fn proof_is_empty_until_value_is_read() { + assert!(test_proving().extract_proof().is_empty()); + } + + #[test] + fn proof_is_non_empty_after_value_is_read() { + let backend = test_proving(); + assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec())); + assert!(!backend.extract_proof().is_empty()); + } + + #[test] + fn proof_is_invalid_when_does_not_contains_root() { + assert!(create_proof_check_backend::(1.into(), vec![]).is_err()); + } + + #[test] + fn passes_throgh_backend_calls() { + let trie_backend = test_trie(); + let proving_backend = test_proving(); + assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); + assert_eq!(trie_backend.pairs(), proving_backend.pairs()); + + let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty()); + let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty()); + assert_eq!(trie_root, proving_root); + assert_eq!(trie_mdb.drain(), proving_mdb.drain()); + } + + #[test] + fn proof_recorded_and_checked() { + let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); + let in_memory = InMemory::::default(); + let in_memory = in_memory.update(contents); + let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; + (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); + + let trie = in_memory.try_into_trie_backend().unwrap(); + let trie_root = trie.storage_root(::std::iter::empty()).0; + assert_eq!(in_memory_root, trie_root); + (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); + + let proving = ProvingBackend::new(trie); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); + + let proof = proving.extract_proof(); + + let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); + } +} diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs new file mode 100644 index 00000000000..bcb02f6bb06 --- /dev/null +++ b/core/state-machine/src/testing.rs @@ -0,0 +1,118 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Test implementation for Externalities. + +use std::collections::HashMap; +use std::cmp::Ord; +use super::Externalities; +use triehash::trie_root; +use hashdb::Hasher; +use rlp::Encodable; +use std::marker::PhantomData; +use std::iter::FromIterator; + +/// Simple HashMap-based Externalities impl. +#[derive(Debug)] +pub struct TestExternalities { + inner: HashMap, Vec>, + _hasher: PhantomData, +} + +impl TestExternalities { + /// Create a new instance of `TestExternalities` + pub fn new() -> Self { + TestExternalities {inner: HashMap::new(), _hasher: PhantomData} + } + /// Insert key/value + pub fn insert(&mut self, k: Vec, v: Vec) -> Option> { + self.inner.insert(k, v) + } +} + +impl PartialEq for TestExternalities { + fn eq(&self, other: &TestExternalities) -> bool { + self.inner.eq(&other.inner) + } +} + +impl FromIterator<(Vec, Vec)> for TestExternalities { + fn from_iter, Vec)>>(iter: I) -> Self { + let mut t = Self::new(); + for i in iter { + t.inner.insert(i.0, i.1); + } + t + } +} + +impl Default for TestExternalities { + fn default() -> Self { Self::new() } +} + +impl From> for HashMap, Vec> { + fn from(tex: TestExternalities) -> Self { + tex.inner.into() + } +} + +impl From< HashMap, Vec> > for TestExternalities { + fn from(hashmap: HashMap, Vec>) -> Self { + TestExternalities { inner: hashmap, _hasher: PhantomData } + } +} + + +impl Externalities for TestExternalities where H::Out: Ord + Encodable { + fn storage(&self, key: &[u8]) -> Option> { + self.inner.get(key).map(|x| x.to_vec()) + } + + fn place_storage(&mut self, key: Vec, maybe_value: Option>) { + match maybe_value { + Some(value) => { self.inner.insert(key, value); } + None => { self.inner.remove(&key); } + } + } + + fn clear_prefix(&mut self, prefix: &[u8]) { + self.inner.retain(|key, _| + !key.starts_with(prefix) + ) + } + + fn chain_id(&self) -> u64 { 42 } + + fn storage_root(&mut self) -> H::Out { + trie_root::(self.inner.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use primitives::{Blake2Hasher, H256}; + + #[test] + fn commit_should_work() { + let mut ext = TestExternalities::::new(); + ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); + ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); + ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); + const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b"); + assert_eq!(ext.storage_root(), H256(ROOT)); + } +} diff --git a/core/state-machine/src/trie_backend.rs b/core/state-machine/src/trie_backend.rs new file mode 100644 index 00000000000..565cc5387f4 --- /dev/null +++ b/core/state-machine/src/trie_backend.rs @@ -0,0 +1,362 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Trie-based state machine backend. +use Backend; +use hashdb::{Hasher, HashDB, AsHashDB}; +use memorydb::MemoryDB; +use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut, NodeCodec}; +use std::collections::HashMap; +use std::sync::Arc; +use std::marker::PhantomData; +use heapsize::HeapSizeOf; + +pub use hashdb::DBValue; + +/// Backend trie storage trait. +pub trait Storage: Send + Sync { + /// Get a trie node. + fn get(&self, key: &H::Out) -> Result, String>; +} + +/// Try convert into trie-based backend. +pub trait TryIntoTrieBackend> { + /// Try to convert self into trie backend. + fn try_into_trie_backend(self) -> Option>; +} + +/// Patricia trie-based backend. Transaction type is an overlay of changes to commit. +#[derive(Clone)] +pub struct TrieBackend> { + storage: TrieBackendStorage, + root: H::Out, + _codec: PhantomData +} + +impl> TrieBackend where H::Out: HeapSizeOf { + /// Create new trie-based backend. + pub fn with_storage(db: Arc>, root: H::Out) -> Self { + TrieBackend { + storage: TrieBackendStorage::Storage(db), + root, + _codec: PhantomData, + } + } + + /// Create new trie-based backend for genesis block. + pub fn with_storage_for_genesis(db: Arc>) -> Self { + let mut root = ::Out::default(); + let mut mdb = MemoryDB::::new(); + TrieDBMut::::new(&mut mdb, &mut root); + + Self::with_storage(db, root) + } + + /// Create new trie-based backend backed by MemoryDb storage. + pub fn with_memorydb(db: MemoryDB, root: H::Out) -> Self { + // TODO: check that root is a part of db??? + TrieBackend { + storage: TrieBackendStorage::MemoryDb(db), + root, + _codec: PhantomData, + } + } + + /// Get backend storage reference. + pub fn backend_storage(&self) -> &TrieBackendStorage { + &self.storage + } + + /// Get trie root. + pub fn root(&self) -> &H::Out { + &self.root + } +} + +impl super::Error for String {} + +impl> Backend for TrieBackend where H::Out: HeapSizeOf { + type Error = String; + type Transaction = MemoryDB; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + let mut read_overlay = MemoryDB::new(); + let eph = Ephemeral { + storage: &self.storage, + overlay: &mut read_overlay, + }; + + let map_e = |e| format!("Trie lookup error: {}", e); + TrieDB::::new(&eph, &self.root).map_err(map_e)? + .get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], mut f: F) { + let mut read_overlay = MemoryDB::new(); + let eph = Ephemeral { + storage: &self.storage, + overlay: &mut read_overlay, + }; + + let mut iter = move || -> Result<(), Box>> { + let trie = TrieDB::::new(&eph, &self.root)?; + let mut iter = trie.iter()?; + + iter.seek(prefix)?; + + for x in iter { + let (key, _) = x?; + + if !key.starts_with(prefix) { + break; + } + + f(&key); + } + + Ok(()) + }; + + if let Err(e) = iter() { + debug!(target: "trie", "Error while iterating by prefix: {}", e); + } + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + let mut read_overlay = MemoryDB::new(); + let eph = Ephemeral { + storage: &self.storage, + overlay: &mut read_overlay, + }; + + let collect_all = || -> Result<_, Box>> { + let trie = TrieDB::::new(&eph, &self.root)?; + let mut v = Vec::new(); + for x in trie.iter()? { + let (key, value) = x?; + v.push((key.to_vec(), value.to_vec())); + } + + Ok(v) + }; + + match collect_all() { + Ok(v) => v, + Err(e) => { + debug!(target: "trie", "Error extracting trie values: {}", e); + Vec::new() + } + } + } + + fn storage_root(&self, delta: I) -> (H::Out, MemoryDB) + where I: IntoIterator, Option>)> + { + let mut write_overlay = MemoryDB::new(); + let mut root = self.root; + { + let mut eph = Ephemeral { + storage: &self.storage, + overlay: &mut write_overlay, + }; + + let mut trie = TrieDBMut::::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully + for (key, change) in delta { + let result = match change { + Some(val) => trie.insert(&key, &val), + None => trie.remove(&key), // TODO: archive mode + }; + + if let Err(e) = result { + warn!(target: "trie", "Failed to write to trie: {}", e); + } + } + } + + (root, write_overlay) + } +} + +impl> TryIntoTrieBackend for TrieBackend { + fn try_into_trie_backend(self) -> Option> { + Some(self) + } +} + +pub struct Ephemeral<'a, H: 'a + Hasher> { + storage: &'a TrieBackendStorage, + overlay: &'a mut MemoryDB, +} + +impl<'a, H: Hasher> AsHashDB for Ephemeral<'a, H> where H::Out: HeapSizeOf { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +impl<'a, H: Hasher> Ephemeral<'a, H> { + pub fn new(storage: &'a TrieBackendStorage, overlay: &'a mut MemoryDB) -> Self { + Ephemeral { + storage, + overlay, + } + } +} + +impl<'a, H: Hasher> HashDB for Ephemeral<'a, H> where H::Out: HeapSizeOf { + fn keys(&self) -> HashMap { + self.overlay.keys() // TODO: iterate backing + } + + fn get(&self, key: &H::Out) -> Option { + match self.overlay.raw(key) { + Some((val, i)) => { + if i <= 0 { + None + } else { + Some(val) + } + } + None => match self.storage.get(key) { + Ok(x) => x, + Err(e) => { + warn!(target: "trie", "Failed to read from DB: {}", e); + None + }, + }, + } + } + + fn contains(&self, key: &H::Out) -> bool { + self.get(key).is_some() + } + + fn insert(&mut self, value: &[u8]) -> H::Out { + self.overlay.insert(value) + } + + fn emplace(&mut self, key: H::Out, value: DBValue) { + self.overlay.emplace(key, value) + } + + fn remove(&mut self, key: &H::Out) { + self.overlay.remove(key) + } +} + +#[derive(Clone)] +pub enum TrieBackendStorage { + /// Key value db + storage column. + Storage(Arc>), + /// Hash db. + MemoryDb(MemoryDB), +} + +impl TrieBackendStorage { + pub fn get(&self, key: &H::Out) -> Result, String> { + match *self { + TrieBackendStorage::Storage(ref db) => + db.get(key) + .map_err(|e| format!("Trie lookup error: {}", e)), + TrieBackendStorage::MemoryDb(ref db) => + Ok(db.get(key)), + } + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + use std::collections::HashSet; + use primitives::{Blake2Hasher, RlpCodec, H256}; + + fn test_db() -> (MemoryDB, H256) { + let mut root = H256::default(); + let mut mdb = MemoryDB::::new(); + { + let mut trie = TrieDBMut::<_, RlpCodec>::new(&mut mdb, &mut root); + trie.insert(b"key", b"value").expect("insert failed"); + trie.insert(b"value1", &[42]).expect("insert failed"); + trie.insert(b"value2", &[24]).expect("insert failed"); + trie.insert(b":code", b"return 42").expect("insert failed"); + for i in 128u8..255u8 { + trie.insert(&[i], &[i]).unwrap(); + } + } + (mdb, root) + } + + pub(crate) fn test_trie() -> TrieBackend { + let (mdb, root) = test_db(); + TrieBackend::with_memorydb(mdb, root) + } + + #[test] + fn read_from_storage_returns_some() { + assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec())); + } + + #[test] + fn read_from_storage_returns_none() { + assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None); + } + + #[test] + fn pairs_are_not_empty_on_non_empty_storage() { + assert!(!test_trie().pairs().is_empty()); + } + + #[test] + fn pairs_are_empty_on_empty_storage() { + let db = TrieBackend::::with_memorydb( + MemoryDB::new(), + Default::default() + ); + assert!(db.pairs().is_empty()); + } + + #[test] + fn storage_root_is_non_default() { + assert!(test_trie().storage_root(::std::iter::empty()).0 != H256([0; 32])); + } + + #[test] + fn storage_root_transaction_is_empty() { + assert!(test_trie().storage_root(::std::iter::empty()).1.drain().is_empty()); + } + + #[test] + fn storage_root_transaction_is_non_empty() { + let (new_root, mut tx) = test_trie().storage_root(vec![(b"new-key".to_vec(), Some(b"new-value".to_vec()))]); + assert!(!tx.drain().is_empty()); + assert!(new_root != test_trie().storage_root(::std::iter::empty()).0); + } + + #[test] + fn prefix_walking_works() { + let trie = test_trie(); + + let mut seen = HashSet::new(); + trie.for_keys_with_prefix(b"value", |key| { + let for_first_time = seen.insert(key.to_vec()); + assert!(for_first_time, "Seen key '{:?}' more than once", key); + }); + + let mut expected = HashSet::new(); + expected.insert(b"value1".to_vec()); + expected.insert(b"value2".to_vec()); + assert_eq!(seen, expected); + } +} diff --git a/core/telemetry/Cargo.toml b/core/telemetry/Cargo.toml new file mode 100644 index 00000000000..2e4e8db6d6b --- /dev/null +++ b/core/telemetry/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "substrate-telemetry" +version = "0.3.0" +authors = ["Parity Technologies "] +description = "Telemetry utils" + +[dependencies] +parking_lot = "0.4" +lazy_static = "1.0" +log = "0.3" +slog = "^2" +slog-json = "^2" +slog-async = "^2" +slog-scope = "^4" +websocket = "^0.20" diff --git a/core/telemetry/README.adoc b/core/telemetry/README.adoc new file mode 100644 index 00000000000..9759c5bc50b --- /dev/null +++ b/core/telemetry/README.adoc @@ -0,0 +1,13 @@ + += Telemetry + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/telemetry/src/lib.rs b/core/telemetry/src/lib.rs new file mode 100644 index 00000000000..21e8685b360 --- /dev/null +++ b/core/telemetry/src/lib.rs @@ -0,0 +1,160 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Telemetry utils. +//! +//! `telemetry` macro be used from whereever in the Substrate codebase +//! in order to send real-time logging information to the telemetry +//! server (if there is one). We use the async drain adapter of `slog` +//! so that the logging thread doesn't get held up at all. +// end::description[] + +extern crate parking_lot; +extern crate websocket as ws; +extern crate slog_async; +extern crate slog_json; +#[macro_use] +extern crate log; +#[macro_use(o, kv)] +extern crate slog; +extern crate slog_scope; + +use std::{io, time}; +use parking_lot::Mutex; +use slog::Drain; +pub use slog_scope::with_logger; + +/// Configuration for telemetry. +pub struct TelemetryConfig { + /// URL of the telemetry WebSocket server. + pub url: String, + /// What do do when we connect to the server. + pub on_connect: Box, +} + +/// Telemetry service guard. +pub type Telemetry = slog_scope::GlobalLoggerGuard; + +/// Size of the channel for passing messages to telemetry thread. +const CHANNEL_SIZE: usize = 262144; + +/// Initialise telemetry. +pub fn init_telemetry(config: TelemetryConfig) -> slog_scope::GlobalLoggerGuard { + let client = ws::ClientBuilder::new(&config.url).ok().and_then(|mut x| x.connect(None).ok()); + let log = slog::Logger::root( + slog_async::Async::new( + slog_json::Json::default( + TelemetryWriter { + buffer: vec![], + out: Mutex::new(client), + config, + last_time: None, // ensures that on_connect will be called. + } + ).fuse() + ).chan_size(CHANNEL_SIZE) + .overflow_strategy(slog_async::OverflowStrategy::DropAndReport) + .build().fuse(), o!() + ); + slog_scope::set_global_logger(log) +} + +/// Exactly equivalent to `slog_scope::info`, provided as a convenience. +#[macro_export] +macro_rules! telemetry { + ( $($t:tt)* ) => { $crate::with_logger(|l| slog_info!(l, $($t)* )) } +} + +struct TelemetryWriter { + buffer: Vec, + out: Mutex>>>, + config: TelemetryConfig, + last_time: Option, +} + +/// Every two minutes we reconnect to the telemetry server otherwise we don't get notified +/// of a flakey connection that has been dropped and needs to be reconnected. We can remove +/// this once we introduce a keepalive ping/pong. +const RECONNECT_PERIOD: u64 = 120; + +impl TelemetryWriter { + fn ensure_connected(&mut self) { + let mut client = self.out.lock(); + + let controlled_disconnect = if let Some(t) = self.last_time { + if t.elapsed().as_secs() > RECONNECT_PERIOD && client.is_some() { + trace!(target: "telemetry", "Performing controlled drop of the telemetry connection."); + let _ = client.as_mut().and_then(|socket| + socket.send_message(&ws::Message::text("{\"msg\":\"system.reconnect\"}")).ok() + ); + *client = None; + true + } else { + false + } + } else { + false + }; + + let just_connected = if client.is_none() { + if !controlled_disconnect { + info!(target: "telemetry", "Connection dropped unexpectedly. Reconnecting to telemetry server..."); + } + *client = ws::ClientBuilder::new(&self.config.url).ok().and_then(|mut x| x.connect(None).ok()); + client.is_some() + } else { + self.last_time.is_none() + }; + + drop(client); + if just_connected { + if !controlled_disconnect { + info!("Reconnected to telemetry server: {}", self.config.url); + } + self.last_time = Some(time::Instant::now()); + (self.config.on_connect)(); + } + } +} + +impl io::Write for TelemetryWriter { + fn write(&mut self, msg: &[u8]) -> io::Result { + if msg.iter().any(|x| *x == b'\n') { + let _ = self.flush(); + } else { + self.buffer.extend_from_slice(msg); + } + Ok(msg.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.ensure_connected(); + + let mut l = self.out.lock(); + let socket_closed = if let Some(ref mut socket) = *l { + if let Ok(s) = ::std::str::from_utf8(&self.buffer[..]) { + let r = socket.send_message(&ws::Message::text(s)); + trace!(target: "telemetry", "Sent to telemetry: {} -> {:?}", s, r); + r.is_err() + } else { false } + } else { false }; + if socket_closed { + *l = None; + } + self.buffer.clear(); + Ok(()) + } +} diff --git a/core/test-client/Cargo.toml b/core/test-client/Cargo.toml new file mode 100644 index 00000000000..eae01feafe9 --- /dev/null +++ b/core/test-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substrate-test-client" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +rhododendron = "0.3" +substrate-bft = { path = "../bft" } +substrate-client = { path = "../client" } +substrate-codec = { path = "../codec" } +substrate-executor = { path = "../executor" } +substrate-keyring = { path = "../../core/keyring" } +substrate-primitives = { path = "../primitives" } +substrate-runtime-support = { path = "../../runtime/support" } +substrate-test-runtime = { path = "../test-runtime" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +hashdb = "0.2.1" + diff --git a/core/test-client/README.adoc b/core/test-client/README.adoc new file mode 100644 index 00000000000..e56c4c7f66e --- /dev/null +++ b/core/test-client/README.adoc @@ -0,0 +1,13 @@ + += Test client + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/test-client/src/block_builder_ext.rs b/core/test-client/src/block_builder_ext.rs new file mode 100644 index 00000000000..4cc0854771a --- /dev/null +++ b/core/test-client/src/block_builder_ext.rs @@ -0,0 +1,42 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block Builder extensions for tests. + +use codec; +use client; +use keyring; +use runtime; + +use {Backend, Executor}; +use primitives::{Blake2Hasher, RlpCodec}; + +/// Extension trait for test block builder. +pub trait BlockBuilderExt { + /// Add transfer extrinsic to the block. + fn push_transfer(&mut self, transfer: runtime::Transfer) -> Result<(), client::error::Error>; +} + +impl BlockBuilderExt for client::block_builder::BlockBuilder { + fn push_transfer(&mut self, transfer: runtime::Transfer) -> Result<(), client::error::Error> { + self.push(sign_tx(transfer)) + } +} + +fn sign_tx(transfer: runtime::Transfer) -> runtime::Extrinsic { + let signature = keyring::Keyring::from_raw_public(transfer.from.0.clone()).unwrap().sign(&codec::Encode::encode(&transfer)).into(); + runtime::Extrinsic { transfer, signature } +} diff --git a/core/test-client/src/client_ext.rs b/core/test-client/src/client_ext.rs new file mode 100644 index 00000000000..386df691583 --- /dev/null +++ b/core/test-client/src/client_ext.rs @@ -0,0 +1,103 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Client extension for tests. + +use client::{self, Client}; +use keyring::Keyring; +use runtime_primitives::StorageMap; +use runtime::genesismap::{GenesisConfig, additional_storage_with_genesis}; +use executor::NativeExecutor; +use runtime; +use bft; +use {Backend, Executor}; + +/// Extension trait for a test client. +pub trait TestClient { + /// Crates new client instance for tests. + fn new_for_tests() -> Self; + + /// Justify and import block to the chain. + fn justify_and_import(&self, origin: client::BlockOrigin, block: runtime::Block) -> client::error::Result<()>; + + /// Returns hash of the genesis block. + fn genesis_hash(&self) -> runtime::Hash; +} + +impl TestClient for Client { + fn new_for_tests() -> Self { + client::new_in_mem(NativeExecutor::new(), genesis_storage()).unwrap() + } + + fn justify_and_import(&self, origin: client::BlockOrigin, block: runtime::Block) -> client::error::Result<()> { + let justification = fake_justify(&block.header); + let justified = self.check_justification(block.header, justification)?; + self.import_block(origin, justified, Some(block.extrinsics))?; + + Ok(()) + } + + fn genesis_hash(&self) -> runtime::Hash { + self.block_hash(0).unwrap().unwrap() + } +} + +/// Prepare fake justification for the header. +/// +/// since we are in the client module we can create falsely justified +/// headers. +/// TODO: remove this in favor of custom verification pipelines for the +/// client +fn fake_justify(header: &runtime::Header) -> bft::UncheckedJustification { + let hash = header.hash(); + let authorities = vec![ + Keyring::Alice.into(), + Keyring::Bob.into(), + Keyring::Charlie.into(), + ]; + + bft::UncheckedJustification::new( + hash, + authorities.iter().map(|key| { + let msg = bft::sign_message::( + ::rhododendron::Vote::Commit(1, hash).into(), + key, + header.parent_hash + ); + + match msg { + ::rhododendron::LocalizedMessage::Vote(vote) => vote.signature, + _ => panic!("signing vote leads to signed vote"), + } + }).collect(), + 1, + ) +} + +fn genesis_config() -> GenesisConfig { + GenesisConfig::new_simple(vec![ + Keyring::Alice.to_raw_public().into(), + Keyring::Bob.to_raw_public().into(), + Keyring::Charlie.to_raw_public().into(), + ], 1000) +} + +fn genesis_storage() -> StorageMap { + let mut storage = genesis_config().genesis_map(); + let block: runtime::Block = client::genesis::construct_genesis_block(&storage); + storage.extend(additional_storage_with_genesis(&block)); + storage +} diff --git a/core/test-client/src/lib.rs b/core/test-client/src/lib.rs new file mode 100644 index 00000000000..3aeefed0faf --- /dev/null +++ b/core/test-client/src/lib.rs @@ -0,0 +1,66 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Client testing utilities. +// end::description[] + +#![warn(missing_docs)] + +extern crate rhododendron; +extern crate substrate_bft as bft; +extern crate substrate_codec as codec; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_support as runtime_support; +extern crate substrate_runtime_primitives as runtime_primitives; +#[macro_use] extern crate substrate_executor as executor; +extern crate hashdb; + +pub extern crate substrate_client as client; +pub extern crate substrate_keyring as keyring; +pub extern crate substrate_test_runtime as runtime; + +mod client_ext; +mod block_builder_ext; + +pub use client_ext::TestClient; +pub use block_builder_ext::BlockBuilderExt; + +use primitives::{Blake2Hasher, RlpCodec}; + +mod local_executor { + #![allow(missing_docs)] + use super::runtime; + // TODO: change the macro and pass in the `BlakeHasher` that dispatch needs from here instead + native_executor_instance!(pub LocalExecutor, runtime::api::dispatch, runtime::VERSION, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm")); +} + +/// Native executor used for tests. +pub use local_executor::LocalExecutor; + +/// Test client database backend. +pub type Backend = client::in_mem::Backend; + +/// Test client executor. +pub type Executor = client::LocalCallExecutor< + Backend, + executor::NativeExecutor, +>; + +/// Creates new client instance used for tests. +pub fn new() -> client::Client { + TestClient::new_for_tests() +} diff --git a/core/test-runtime/Cargo.toml b/core/test-runtime/Cargo.toml new file mode 100644 index 00000000000..784c7252e27 --- /dev/null +++ b/core/test-runtime/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "substrate-test-runtime" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +log = { version = "0.3", optional = true } +hex-literal = { version = "0.1.0", optional = true } +serde = { version = "1.0", optional = true } +serde_derive = { version = "1.0", optional = true } +substrate-keyring = { path = "../keyring", optional = true } +substrate-codec = { path = "../codec", default-features = false } +substrate-codec-derive = { path = "../codec/derive", default-features = false } +substrate-runtime-std = { path = "../runtime-std", default-features = false } +substrate-runtime-io = { path = "../runtime-io", default-features = false } +substrate-runtime-support = { path = "../../runtime/support", default-features = false } +substrate-primitives = { path = "../primitives", default-features = false } +substrate-runtime-primitives = { path = "../../runtime/primitives", default-features = false } +substrate-runtime-version = { path = "../../runtime/version", default-features = false } + +[features] +default = ["std"] +std = [ + "log", + "hex-literal", + "serde", + "serde_derive", + "substrate-keyring", + "substrate-codec/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-primitives/std", + "substrate-runtime-primitives/std", + "substrate-runtime-version/std" +] diff --git a/core/test-runtime/README.adoc b/core/test-runtime/README.adoc new file mode 100644 index 00000000000..15b3c4c4ac3 --- /dev/null +++ b/core/test-runtime/README.adoc @@ -0,0 +1,13 @@ + += Test runtime + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- diff --git a/core/test-runtime/src/genesismap.rs b/core/test-runtime/src/genesismap.rs new file mode 100644 index 00000000000..fe5fd84acd0 --- /dev/null +++ b/core/test-runtime/src/genesismap.rs @@ -0,0 +1,67 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tool for creating the genesis block. + +use std::collections::HashMap; +use runtime_io::twox_128; +use codec::{KeyedVec, Joiner}; +use primitives::AuthorityId; +use runtime_primitives::traits::Block; + +/// Configuration of a general Substrate test genesis block. +pub struct GenesisConfig { + pub authorities: Vec, + pub balances: Vec<(AuthorityId, u64)>, +} + +impl GenesisConfig { + pub fn new_simple(authorities: Vec, balance: u64) -> Self { + GenesisConfig { + authorities: authorities.clone(), + balances: authorities.into_iter().map(|a| (a, balance)).collect(), + } + } + + pub fn genesis_map(&self) -> HashMap, Vec> { + let wasm_runtime = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm").to_vec(); + self.balances.iter() + .map(|&(account, balance)| (account.to_keyed_vec(b"balance:"), vec![].and(&balance))) + .map(|(k, v)| (twox_128(&k[..])[..].to_vec(), v.to_vec())) + .chain(vec![ + (b":code"[..].into(), wasm_runtime), + (b":heappages"[..].into(), vec![].and(&(16 as u64))), + (b":auth:len"[..].into(), vec![].and(&(self.authorities.len() as u32))), + ].into_iter()) + .chain(self.authorities.iter() + .enumerate() + .map(|(i, account)| ((i as u32).to_keyed_vec(b":auth:"), vec![].and(account))) + ) + .collect() + } +} + +macro_rules! map { + ($( $name:expr => $value:expr ),*) => ( + vec![ $( ( $name, $value ) ),* ].into_iter().collect() + ) +} + +pub fn additional_storage_with_genesis(genesis_block: &::Block) -> HashMap, Vec> { + map![ + twox_128(&b"latest"[..]).to_vec() => genesis_block.hash().0.to_vec() + ] +} diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs new file mode 100644 index 00000000000..f8fe9d01371 --- /dev/null +++ b/core/test-runtime/src/lib.rs @@ -0,0 +1,156 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! The Substrate runtime. This can be compiled with #[no_std], ready for Wasm. +// end::description[] + +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate substrate_runtime_std as rstd; +extern crate substrate_codec as codec; +extern crate substrate_runtime_primitives as runtime_primitives; + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; +#[macro_use] +extern crate substrate_codec_derive; +#[macro_use] +extern crate substrate_runtime_io as runtime_io; +#[macro_use] +extern crate substrate_runtime_version as runtime_version; + + +#[cfg(test)] +#[macro_use] +extern crate hex_literal; +#[cfg(test)] +extern crate substrate_keyring as keyring; +#[cfg_attr(test, macro_use)] +extern crate substrate_primitives as primitives; + +#[cfg(feature = "std")] pub mod genesismap; +pub mod system; + +use rstd::prelude::*; +use codec::{Encode, Decode}; + +use runtime_primitives::traits::{BlindCheckable, BlakeTwo256}; +use runtime_primitives::Ed25519Signature; +use runtime_version::RuntimeVersion; +pub use primitives::hash::H256; + +/// Test runtime version. +pub const VERSION: RuntimeVersion = RuntimeVersion { + spec_name: ver_str!("test"), + impl_name: ver_str!("parity-test"), + authoring_version: 1, + spec_version: 1, + impl_version: 1, +}; + +fn version() -> RuntimeVersion { + VERSION +} + +/// Calls in transactions. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct Transfer { + pub from: AccountId, + pub to: AccountId, + pub amount: u64, + pub nonce: u64, +} + +/// Extrinsic for test-runtime. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct Extrinsic { + pub transfer: Transfer, + pub signature: Ed25519Signature, +} + +impl BlindCheckable for Extrinsic { + type Checked = Self; + + fn check(self) -> Result { + if ::runtime_primitives::verify_encoded_lazy(&self.signature, &self.transfer, &self.transfer.from) { + Ok(self) + } else { + Err("bad signature") + } + } +} + +/// An identifier for an account on this system. +pub type AccountId = H256; +/// A simple hash type for all our hashing. +pub type Hash = H256; +/// The block number type used in this runtime. +pub type BlockNumber = u64; +/// Index of a transaction. +pub type Index = u64; +/// The item of a block digest. +pub type DigestItem = runtime_primitives::generic::DigestItem; +/// The digest of a block. +pub type Digest = runtime_primitives::generic::Digest; +/// A test block. +pub type Block = runtime_primitives::generic::Block; +/// A test block's header. +pub type Header = runtime_primitives::generic::Header; + +/// Run whatever tests we have. +pub fn run_tests(mut input: &[u8]) -> Vec { + use runtime_io::print; + + print("run_tests..."); + let block = Block::decode(&mut input).unwrap(); + print("deserialised block."); + let stxs = block.extrinsics.iter().map(Encode::encode).collect::>(); + print("reserialised transactions."); + [stxs.len() as u8].encode() +} + +fn test_event_json() -> &'static str { + "hallo" +} + +pub mod api { + use system; + impl_stubs!( + version => |()| super::version(), + json_metadata => |()| { + let mut vec = ::runtime_support::metadata::Vec::new(); + vec.push(::runtime_support::metadata::JSONMetadata::Events { + name: "Test", events: &[ ("event", super::test_event_json) ] + }); + vec + }, + authorities => |()| system::authorities(), + initialise_block => |header| system::initialise_block(header), + execute_block => |block| system::execute_block(block), + apply_extrinsic => |utx| system::execute_transaction(utx), + finalise_block => |()| system::finalise_block() + ); +} diff --git a/core/test-runtime/src/system.rs b/core/test-runtime/src/system.rs new file mode 100644 index 00000000000..b60cfa7872c --- /dev/null +++ b/core/test-runtime/src/system.rs @@ -0,0 +1,281 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! System manager: Handles all of the top-level stuff; executing block/transaction, setting code +//! and depositing logs. + +use rstd::prelude::*; +use runtime_io::{storage_root, enumerated_trie_root}; +use runtime_support::storage::{self, StorageValue, StorageMap}; +use runtime_primitives::traits::{Hash as HashT, BlakeTwo256}; +use runtime_primitives::{ApplyError, ApplyOutcome, ApplyResult}; +use codec::{KeyedVec, Encode}; +use super::{AccountId, BlockNumber, Extrinsic, H256 as Hash, Block, Header}; +use primitives::Blake2Hasher; + +const NONCE_OF: &[u8] = b"nonce:"; +const BALANCE_OF: &[u8] = b"balance:"; +const AUTHORITY_AT: &'static[u8] = b":auth:"; +const AUTHORITY_COUNT: &'static[u8] = b":auth:len"; + +storage_items! { + ExtrinsicIndex: b"sys:xti" => required u32; + ExtrinsicData: b"sys:xtd" => required map [ u32 => Vec ]; + // The current block number being processed. Set by `execute_block`. + Number: b"sys:num" => required BlockNumber; + ParentHash: b"sys:pha" => required Hash; +} + +pub fn balance_of(who: AccountId) -> u64 { + storage::get_or(&who.to_keyed_vec(BALANCE_OF), 0) +} + +pub fn nonce_of(who: AccountId) -> u64 { + storage::get_or(&who.to_keyed_vec(NONCE_OF), 0) +} + +/// Get authorities ar given block. +pub fn authorities() -> Vec<::primitives::AuthorityId> { + let len: u32 = storage::unhashed::get(AUTHORITY_COUNT).expect("There are always authorities in test-runtime"); + (0..len) + .map(|i| storage::unhashed::get(&i.to_keyed_vec(AUTHORITY_AT)).expect("Authority is properly encoded in test-runtime")) + .collect() +} + +pub fn initialise_block(header: Header) { + // populate environment. + ::put(&header.number); + ::put(&header.parent_hash); + ::put(0); +} + +/// Actually execute all transitioning for `block`. +pub fn execute_block(block: Block) { + let ref header = block.header; + + // check transaction trie root represents the transactions. + let txs = block.extrinsics.iter().map(Encode::encode).collect::>(); + let txs = txs.iter().map(Vec::as_slice).collect::>(); + let txs_root = enumerated_trie_root::(&txs).into(); + info_expect_equal_hash(&txs_root, &header.extrinsics_root); + assert!(txs_root == header.extrinsics_root, "Transaction trie root must be valid."); + + // execute transactions + block.extrinsics.iter().for_each(|e| { execute_transaction_backend(e).map_err(|_| ()).expect("Extrinsic error"); }); + + // check storage root. + let storage_root = storage_root().into(); + info_expect_equal_hash(&storage_root, &header.state_root); + assert!(storage_root == header.state_root, "Storage root must match that calculated."); +} + +/// Execute a transaction outside of the block execution function. +/// This doesn't attempt to validate anything regarding the block. +pub fn execute_transaction(utx: Extrinsic) -> ApplyResult { + let extrinsic_index = ExtrinsicIndex::get(); + ExtrinsicData::insert(extrinsic_index, utx.encode()); + ExtrinsicIndex::put(extrinsic_index + 1); + execute_transaction_backend(&utx) +} + +/// Finalise the block. +pub fn finalise_block() -> Header { + let extrinsic_index = ExtrinsicIndex::take(); + let txs: Vec<_> = (0..extrinsic_index).map(ExtrinsicData::take).collect(); + let txs = txs.iter().map(Vec::as_slice).collect::>(); + let extrinsics_root = enumerated_trie_root::(&txs).into(); + + let number = ::take(); + let parent_hash = ::take(); + let storage_root = BlakeTwo256::storage_root(); + + Header { + number, + extrinsics_root, + state_root: storage_root, + parent_hash, + digest: Default::default(), + } +} + +fn execute_transaction_backend(utx: &Extrinsic) -> ApplyResult { + use runtime_primitives::traits::BlindCheckable; + + // check signature + let utx = match utx.clone().check() { + Ok(tx) => tx, + Err(_) => return Err(ApplyError::BadSignature), + }; + + let tx: ::Transfer = utx.transfer; + + // check nonce + let nonce_key = tx.from.to_keyed_vec(NONCE_OF); + let expected_nonce: u64 = storage::get_or(&nonce_key, 0); + if !(tx.nonce == expected_nonce) { + return Err(ApplyError::Stale) + } + + // increment nonce in storage + storage::put(&nonce_key, &(expected_nonce + 1)); + + // check sender balance + let from_balance_key = tx.from.to_keyed_vec(BALANCE_OF); + let from_balance: u64 = storage::get_or(&from_balance_key, 0); + + // enact transfer + if !(tx.amount <= from_balance) { + return Err(ApplyError::CantPay) + } + let to_balance_key = tx.to.to_keyed_vec(BALANCE_OF); + let to_balance: u64 = storage::get_or(&to_balance_key, 0); + storage::put(&from_balance_key, &(from_balance - tx.amount)); + storage::put(&to_balance_key, &(to_balance + tx.amount)); + Ok(ApplyOutcome::Success) +} + +#[cfg(feature = "std")] +fn info_expect_equal_hash(given: &Hash, expected: &Hash) { + use primitives::hexdisplay::HexDisplay; + if given != expected { + println!("Hash: given={}, expected={}", HexDisplay::from(&given.0), HexDisplay::from(&expected.0)); + } +} + +#[cfg(not(feature = "std"))] +fn info_expect_equal_hash(given: &Hash, expected: &Hash) { + if given != expected { + ::runtime_io::print("Hash not equal"); + ::runtime_io::print(&given.0[..]); + ::runtime_io::print(&expected.0[..]); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use runtime_io::{with_externalities, twox_128, TestExternalities}; + use codec::{Joiner, KeyedVec}; + use keyring::Keyring; + use ::{Header, Digest, Extrinsic, Transfer}; + use primitives::Blake2Hasher; + + fn new_test_ext() -> TestExternalities { + map![ + twox_128(b"latest").to_vec() => vec![69u8; 32], + twox_128(b":auth:len").to_vec() => vec![].and(&3u32), + twox_128(&0u32.to_keyed_vec(b":auth:")).to_vec() => Keyring::Alice.to_raw_public().to_vec(), + twox_128(&1u32.to_keyed_vec(b":auth:")).to_vec() => Keyring::Bob.to_raw_public().to_vec(), + twox_128(&2u32.to_keyed_vec(b":auth:")).to_vec() => Keyring::Charlie.to_raw_public().to_vec(), + twox_128(&Keyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0] + ] + } + + fn construct_signed_tx(tx: Transfer) -> Extrinsic { + let signature = Keyring::from_raw_public(tx.from.0).unwrap().sign(&tx.encode()).into(); + Extrinsic { transfer: tx, signature } + } + + #[test] + fn block_import_works() { + let mut t = new_test_ext(); + + let h = Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: hex!("0c22599e15fb5e052c84f79a2aab179ba6bb238218fd86bdd4a74ebcc87adfcd").into(), + extrinsics_root: hex!("45b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c0").into(), + digest: Digest { logs: vec![], }, + }; + + let b = Block { + header: h, + extrinsics: vec![], + }; + + with_externalities(&mut t, || { + execute_block(b); + }); + } + + #[test] + fn block_import_with_transaction_works() { + let mut t = new_test_ext(); + + with_externalities(&mut t, || { + assert_eq!(balance_of(Keyring::Alice.to_raw_public().into()), 111); + assert_eq!(balance_of(Keyring::Bob.to_raw_public().into()), 0); + }); + + let b = Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: hex!("0425393fd07e2a806cfd7e990ee91dc92fe6bba34eab2bf45d5be7d67e24d467").into(), + extrinsics_root: hex!("83fd59e8fe7cee53d7421713a09fe0abae1aec5f4db94fe5193737b12195f013").into(), + digest: Digest { logs: vec![], }, + }, + extrinsics: vec![ + construct_signed_tx(Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Bob.to_raw_public().into(), + amount: 69, + nonce: 0, + }) + ], + }; + + with_externalities(&mut t, || { + execute_block(b.clone()); + + assert_eq!(balance_of(Keyring::Alice.to_raw_public().into()), 42); + assert_eq!(balance_of(Keyring::Bob.to_raw_public().into()), 69); + }); + + let b = Block { + header: Header { + parent_hash: b.header.hash(), + number: 2, + state_root: hex!("e32dd1d84d9133ca48078d2d83f2b0db19f9d47229ba98bf5ced0e9f86fac2c7").into(), + extrinsics_root: hex!("5d2d0a93201744f0df878c33b07da40cd38e24ac2358cc2811ea640835c31b68").into(), + digest: Digest { logs: vec![], }, + }, + extrinsics: vec![ + construct_signed_tx(Transfer { + from: Keyring::Bob.to_raw_public().into(), + to: Keyring::Alice.to_raw_public().into(), + amount: 27, + nonce: 0, + }), + construct_signed_tx(Transfer { + from: Keyring::Alice.to_raw_public().into(), + to: Keyring::Charlie.to_raw_public().into(), + amount: 69, + nonce: 1, + }), + ], + }; + + with_externalities(&mut t, || { + execute_block(b); + + assert_eq!(balance_of(Keyring::Alice.to_raw_public().into()), 0); + assert_eq!(balance_of(Keyring::Bob.to_raw_public().into()), 42); + assert_eq!(balance_of(Keyring::Charlie.to_raw_public().into()), 69); + }); + } +} diff --git a/core/test-runtime/wasm/Cargo.lock b/core/test-runtime/wasm/Cargo.lock new file mode 100644 index 00000000000..82f7da6df04 --- /dev/null +++ b/core/test-runtime/wasm/Cargo.lock @@ -0,0 +1,709 @@ +[[package]] +name = "arrayvec" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base58" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "blake2-rfc" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byteorder" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "constant_time_eq" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "crossbeam-deque" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crunchy" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ed25519" +version = "0.1.0" +dependencies = [ + "base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-primitives 0.1.0", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "elastic-array" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ethbloom" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types-serialize 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ethereum-types" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ethbloom 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types-serialize 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ethereum-types-serialize" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fixed-hash" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "gcc" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hashdb" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "heapsize" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hex-literal" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hex-literal-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hex-literal-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "integer-sqrt" +version = "0.1.0" +source = "git+https://github.com/paritytech/integer-sqrt-rs.git#886e9cb983c46498003878afe965d55caa762025" + +[[package]] +name = "lazy_static" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.41" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memoffset" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nan-preserving-float" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nodrop" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num-traits" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num_cpus" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parity-bytes" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "parity-wasm" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "patricia-trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "plain_hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack-impl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack-impl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pwasm-alloc" +version = "0.1.0" +dependencies = [ + "pwasm-libc 0.1.0", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pwasm-libc" +version = "0.1.0" + +[[package]] +name = "quote" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon-core" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ring" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rlp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-hex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-hex" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "scopeguard" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-codec" +version = "0.1.0" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-codec-derive" +version = "0.1.0" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-primitives" +version = "0.1.0" +dependencies = [ + "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "patricia-trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-runtime-std 0.1.0", + "twox-hash 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-runtime-io" +version = "0.1.0" +dependencies = [ + "hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "substrate-runtime-primitives" +version = "0.1.0" +dependencies = [ + "integer-sqrt 0.1.0 (git+https://github.com/paritytech/integer-sqrt-rs.git)", + "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", +] + +[[package]] +name = "substrate-runtime-std" +version = "0.1.0" +dependencies = [ + "pwasm-alloc 0.1.0", + "pwasm-libc 0.1.0", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-runtime-support" +version = "0.1.0" +dependencies = [ + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "substrate-runtime-version" +version = "0.1.0" +dependencies = [ + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", +] + +[[package]] +name = "substrate-test-runtime" +version = "0.1.0" +dependencies = [ + "ed25519 0.1.0", + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-version 0.1.0", +] + +[[package]] +name = "syn" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tiny-keccak" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "twox-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "uint" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "untrusted" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wasmi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nan-preserving-float 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-wasm 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" +"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" +"checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" +"checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9" +"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" +"checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" +"checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" +"checksum crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" +"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" +"checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" +"checksum elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88d4851b005ef16de812ea9acdb7bece2f0a40dd86c07b85631d7dafa54537bb" +"checksum ethbloom 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a93a43ce2e9f09071449da36bfa7a1b20b950ee344b6904ff23de493b03b386" +"checksum ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "35b3c5a18bc5e73a32a110ac743ec04b02bbbcd3b71d3118d40a6113d509378a" +"checksum ethereum-types-serialize 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4ac59a21a9ce98e188f3dace9eb67a6c4a3c67ec7fbc7218cb827852679dc002" +"checksum fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0d5ec8112f00ea8a483e04748a85522184418fd1cf02890b626d8fc28683f7de" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" +"checksum hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f1c71fc577cde89b3345d5f2880fecaf462a32e96c619f431279bdaf1ba5ddb1" +"checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" +"checksum hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4da5f0e01bd8a71a224a4eedecaacfcabda388dbb7a80faf04d3514287572d95" +"checksum hex-literal-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1d340b6514f232f6db1bd16db65302a5278a04fef9ce867cb932e7e5fa21130a" +"checksum integer-sqrt 0.1.0 (git+https://github.com/paritytech/integer-sqrt-rs.git)" = "" +"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" +"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" +"checksum libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)" = "ac8ebf8343a981e2fa97042b14768f02ed3e1d602eac06cae6166df3c8ced206" +"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +"checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" +"checksum nan-preserving-float 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34d4f00fcc2f4c9efa8cc971db0da9e28290e28e97af47585e48691ef10ff31f" +"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" +"checksum num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775393e285254d2f5004596d69bb8bc1149754570dcc08cf30cabeba67955e28" +"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" +"checksum parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa5168b4cf41f3835e4bc6ffb32f51bc9365dc50cb351904595b3931d917fd0c" +"checksum parity-wasm 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e1c91199d14bd5b78ecade323d4a891d094799749c1b9e82d9c590c2e2849a40" +"checksum patricia-trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fa27fc4a972a03d64e5170d7facd2c84c6ed425b38ce62ad98dcfee2f7845b3b" +"checksum plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95fa6386b1d34aaf0adb9b7dd2885dbe7c34190e6263785e5a7ec2b19044a90f" +"checksum proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ba8d4f9257b85eb6cdf13f055cea3190520aab1409ca2ab43493ea4820c25f0" +"checksum proc-macro-hack-impl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d5cb6f960ad471404618e9817c0e5d10b1ae74cfdf01fab89ea0641fe7fb2892" +"checksum proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1fa93823f53cfd0f5ac117b189aed6cfdfb2cfc0a9d82e956dd7927595ed7d46" +"checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035" +"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" +"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" +"checksum rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b614fe08b6665cb9a231d07ac1364b0ef3cb3698f1239ee0c4c3a88a524f54c8" +"checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8" +"checksum ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6f7d28b30a72c01b458428e0ae988d4149c20d902346902be881e3edc4bb325c" +"checksum rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "524c5ad554859785dfc8469df3ed5e0b5784d4d335877ed47c8d90fc0eb238fe" +"checksum rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0ceb8ce7a5e520de349e1fa172baeba4a9e8d5ef06c47471863530bc4972ee1e" +"checksum rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b03280c2813907a030785570c577fb27d3deec8da4c18566751ade94de0ace" +"checksum rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a54aa04a10c68c1c4eacb4337fd883b435997ede17a9385784b990777686b09a" +"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)" = "fba5be06346c5200249c8c8ca4ccba4a09e8747c71c16e420bd359a0db4d8f91" +"checksum serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)" = "79e4620ba6fbe051fc7506fab6f84205823564d55da18d55b695160fb3479cd8" +"checksum syn 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6dfd71b2be5a58ee30a6f8ea355ba8290d397131c00dfa55c3d34e6e13db5101" +"checksum tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e9175261fbdb60781fcd388a4d6cc7e14764a2b629a7ad94abb439aed223a44f" +"checksum twox-hash 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "475352206e7a290c5fccc27624a163e8d0d115f7bb60ca18a64fc9ce056d7435" +"checksum uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "754ba11732b9161b94c41798e5197e5e75388d012f760c42adb5000353e98646" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" +"checksum wasmi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "522fe3fdd44a56f25cd5ddcd8ccdb1cf2e982ceb28fcb00f41d8a018ae5245a8" +"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/core/test-runtime/wasm/Cargo.toml b/core/test-runtime/wasm/Cargo.toml new file mode 100644 index 00000000000..65cf0f29f20 --- /dev/null +++ b/core/test-runtime/wasm/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "substrate-test-runtime" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +log = { version = "0.3", optional = true } +hex-literal = { version = "0.1.0", optional = true } +substrate-codec = { path = "../../codec", default-features = false } +substrate-codec-derive = { path = "../../codec/derive", default-features = false } +substrate-runtime-std = { path = "../../runtime-std", default-features = false } +substrate-runtime-io = { path = "../../runtime-io", default-features = false } +substrate-primitives = { path = "../../primitives", default-features = false } +substrate-runtime-support = { path = "../../../runtime/support", default-features = false } +substrate-runtime-version = { path = "../../../runtime/version", default-features = false } +substrate-runtime-primitives = { path = "../../../runtime/primitives", default-features = false } + +[lib] +crate-type = ["cdylib"] + +[profile.release] +panic = "abort" +lto = true + +[workspace] +members = [] diff --git a/core/test-runtime/wasm/build.sh b/core/test-runtime/wasm/build.sh new file mode 100755 index 00000000000..63d9347bf46 --- /dev/null +++ b/core/test-runtime/wasm/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e + +cargo +nightly build --target=wasm32-unknown-unknown --release +for i in substrate_test_runtime +do + wasm-gc target/wasm32-unknown-unknown/release/$i.wasm target/wasm32-unknown-unknown/release/$i.compact.wasm +done diff --git a/core/test-runtime/wasm/src b/core/test-runtime/wasm/src new file mode 120000 index 00000000000..5cd551cf269 --- /dev/null +++ b/core/test-runtime/wasm/src @@ -0,0 +1 @@ +../src \ No newline at end of file diff --git a/framework/client/Cargo.toml b/framework/client/Cargo.toml new file mode 100644 index 00000000000..886410f74ee --- /dev/null +++ b/framework/client/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-client" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +error-chain = "0.12" +fnv = "1.0" +log = "0.3" +parking_lot = "0.4" +triehash = "0.2" +hex-literal = "0.1" +futures = "0.1.17" + +slog = "^2" +heapsize = "0.4" +substrate-bft = { path = "../bft" } +substrate-codec = { path = "../codec" } +substrate-executor = { path = "../executor" } +substrate-primitives = { path = "../primitives" } +substrate-runtime-io = { path = "../runtime-io" } +substrate-runtime-support = { path = "../../runtime/support" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-state-machine = { path = "../state-machine" } +substrate-keyring = { path = "../keyring" } +substrate-telemetry = { path = "../telemetry" } +hashdb = "0.2.1" +patricia-trie = "0.2.1" +rlp = "0.2.4" + +[dev-dependencies] +substrate-test-client = { path = "../test-client" } diff --git a/framework/executor/Cargo.toml b/framework/executor/Cargo.toml new file mode 100644 index 00000000000..4c30827dc1b --- /dev/null +++ b/framework/executor/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "substrate-executor" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +error-chain = "0.12" +substrate-codec = { path = "../codec" } +substrate-runtime-io = { path = "../runtime-io" } +substrate-primitives = { path = "../primitives" } +substrate-serializer = { path = "../serializer" } +substrate-state-machine = { path = "../state-machine" } +substrate-runtime-version = { path = "../../runtime/version" } + +serde = "1.0" +serde_derive = "1.0" +wasmi = "0.4" +byteorder = "1.1" +triehash = "0.2" +twox-hash = "1.1.0" +lazy_static = "1.0" +parking_lot = "*" +log = "0.3" +hashdb = "0.2.1" + +[dev-dependencies] +assert_matches = "1.1" +wabt = "0.4" +hex-literal = "0.1.0" + +[features] +default = [] +wasm-extern-trace = [] diff --git a/framework/network/Cargo.toml b/framework/network/Cargo.toml new file mode 100644 index 00000000000..fdd31aa7d02 --- /dev/null +++ b/framework/network/Cargo.toml @@ -0,0 +1,30 @@ +[package] +description = "Polkadot network protocol" +name = "substrate-network" +version = "0.1.0" +license = "GPL-3.0" +authors = ["Parity Technologies "] + +[lib] + +[dependencies] +log = "0.3" +parking_lot = "0.4" +error-chain = "0.12" +bitflags = "1.0" +futures = "0.1.17" +linked-hash-map = "0.5" +rustc-hex = "1.0" +ethcore-io = { git = "https://github.com/paritytech/parity.git" } + +substrate-primitives = { path = "../primitives" } +substrate-client = { path = "../client" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-codec = { path = "../codec" } +substrate-codec-derive = { path = "../codec/derive" } +substrate-network-libp2p = { path = "../network-libp2p" } + +[dev-dependencies] +env_logger = "0.4" +substrate-keyring = { path = "../keyring" } +substrate-test-client = { path = "../test-client" } diff --git a/framework/runtime-support/Cargo.toml b/framework/runtime-support/Cargo.toml new file mode 100644 index 00000000000..bf7794a0832 --- /dev/null +++ b/framework/runtime-support/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-runtime-support" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = { version = "0.1.0", optional = true } +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-runtime-std = { path = "../runtime-std", default_features = false } +substrate-runtime-io = { path = "../runtime-io", default_features = false } +substrate-primitives = { path = "../primitives", default_features = false } +substrate-codec = { path = "../codec", default_features = false } + +[dev-dependencies] +pretty_assertions = "0.5.1" +serde_json = { version = "1.0" } +substrate-codec-derive = { path = "../codec/derive" } + +[features] +default = ["std"] +std = [ + "hex-literal", + "serde/std", + "serde_derive", + "substrate-primitives/std", + "substrate-runtime-io/std", + "substrate-codec/std", + "substrate-runtime-std/std", +] +nightly = [] +strict = [] diff --git a/framework/test-runtime/Cargo.toml b/framework/test-runtime/Cargo.toml new file mode 100644 index 00000000000..784c7252e27 --- /dev/null +++ b/framework/test-runtime/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "substrate-test-runtime" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +log = { version = "0.3", optional = true } +hex-literal = { version = "0.1.0", optional = true } +serde = { version = "1.0", optional = true } +serde_derive = { version = "1.0", optional = true } +substrate-keyring = { path = "../keyring", optional = true } +substrate-codec = { path = "../codec", default-features = false } +substrate-codec-derive = { path = "../codec/derive", default-features = false } +substrate-runtime-std = { path = "../runtime-std", default-features = false } +substrate-runtime-io = { path = "../runtime-io", default-features = false } +substrate-runtime-support = { path = "../../runtime/support", default-features = false } +substrate-primitives = { path = "../primitives", default-features = false } +substrate-runtime-primitives = { path = "../../runtime/primitives", default-features = false } +substrate-runtime-version = { path = "../../runtime/version", default-features = false } + +[features] +default = ["std"] +std = [ + "log", + "hex-literal", + "serde", + "serde_derive", + "substrate-keyring", + "substrate-codec/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-primitives/std", + "substrate-runtime-primitives/std", + "substrate-runtime-version/std" +] diff --git a/node/Cargo.toml b/node/Cargo.toml new file mode 100644 index 00000000000..8746954b5a8 --- /dev/null +++ b/node/Cargo.toml @@ -0,0 +1,18 @@ +[[bin]] +name = "substrate" +path = "src/main.rs" + +[package] +name = "substrate" +version = "0.1.0" +authors = ["Parity Technologies "] +build = "build.rs" + +[dependencies] +error-chain = "0.12" +node-cli = { path = "cli" } +futures = "0.1" +ctrlc = { version = "3.0", features = ["termination"] } + +[build-dependencies] +vergen = "0.1" diff --git a/node/api/Cargo.toml b/node/api/Cargo.toml new file mode 100644 index 00000000000..5a0cee13877 --- /dev/null +++ b/node/api/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "node-api" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +node-runtime = { path = "../runtime" } +node-primitives = { path = "../primitives" } +substrate-client = { path = "../../core/client" } +substrate-primitives = { path = "../../core/primitives" } + +[dev-dependencies] +substrate-keyring = { path = "../../core/keyring" } diff --git a/node/api/src/lib.rs b/node/api/src/lib.rs new file mode 100644 index 00000000000..81cf7ffae30 --- /dev/null +++ b/node/api/src/lib.rs @@ -0,0 +1,155 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Strongly typed API for Substrate runtime. + +#![warn(missing_docs)] +#![warn(unused_extern_crates)] + +extern crate node_primitives as primitives; +extern crate node_runtime as runtime; +extern crate substrate_client as client; +extern crate substrate_primitives; + +pub use client::error::{Error, ErrorKind, Result}; +use runtime::Address; +use client::backend::Backend; +use client::block_builder::BlockBuilder as ClientBlockBuilder; +use client::{Client, CallExecutor}; +use primitives::{ + AccountId, Block, BlockId, Hash, Index, InherentData, + SessionKey, Timestamp, UncheckedExtrinsic, +}; +use substrate_primitives::{Blake2Hasher, RlpCodec}; + +/// Build new blocks. +pub trait BlockBuilder { + /// Push an extrinsic onto the block. Fails if the extrinsic is invalid. + fn push_extrinsic(&mut self, extrinsic: UncheckedExtrinsic) -> Result<()>; + + /// Bake the block with provided extrinsics. + fn bake(self) -> Result; +} + +/// Trait encapsulating the demo API. +/// +/// All calls should fail when the exact runtime is unknown. +pub trait Api { + /// The block builder for this API type. + type BlockBuilder: BlockBuilder; + + /// Get session keys at a given block. + fn session_keys(&self, at: &BlockId) -> Result>; + + /// Get validators at a given block. + fn validators(&self, at: &BlockId) -> Result>; + + /// Get the value of the randomness beacon at a given block. + fn random_seed(&self, at: &BlockId) -> Result; + + /// Get the timestamp registered at a block. + fn timestamp(&self, at: &BlockId) -> Result; + + /// Get the nonce (né index) of an account at a block. + fn index(&self, at: &BlockId, account: AccountId) -> Result; + + /// Get the account id of an address at a block. + fn lookup(&self, at: &BlockId, address: Address) -> Result>; + + /// Evaluate a block. Returns true if the block is good, false if it is known to be bad, + /// and an error if we can't evaluate for some reason. + fn evaluate_block(&self, at: &BlockId, block: Block) -> Result; + + /// Build a block on top of the given, with inherent extrinsics pre-pushed. + fn build_block(&self, at: &BlockId, inherent_data: InherentData) -> Result; + + /// Attempt to produce the (encoded) inherent extrinsics for a block being built upon the given. + /// This may vary by runtime and will fail if a runtime doesn't follow the same API. + fn inherent_extrinsics(&self, at: &BlockId, inherent_data: InherentData) -> Result>; +} + +impl BlockBuilder for ClientBlockBuilder +where + B: Backend, + E: CallExecutor+ Clone, +{ + fn push_extrinsic(&mut self, extrinsic: UncheckedExtrinsic) -> Result<()> { + self.push(extrinsic).map_err(Into::into) + } + + /// Bake the block with provided extrinsics. + fn bake(self) -> Result { + ClientBlockBuilder::bake(self).map_err(Into::into) + } +} + +impl Api for Client +where + B: Backend, + E: CallExecutor + Clone, +{ + type BlockBuilder = ClientBlockBuilder; + + fn session_keys(&self, at: &BlockId) -> Result> { + Ok(self.authorities_at(at)?) + } + + fn validators(&self, at: &BlockId) -> Result> { + self.call_api(at, "validators", &()) + } + + fn random_seed(&self, at: &BlockId) -> Result { + self.call_api(at, "random_seed", &()) + } + + fn timestamp(&self, at: &BlockId) -> Result { + self.call_api(at, "timestamp", &()) + } + + fn evaluate_block(&self, at: &BlockId, block: Block) -> Result { + let res: Result<()> = self.call_api(at, "execute_block", &block); + match res { + Ok(()) => Ok(true), + Err(err) => match err.kind() { + &client::error::ErrorKind::Execution(_) => Ok(false), + _ => Err(err) + } + } + } + + fn index(&self, at: &BlockId, account: AccountId) -> Result { + self.call_api(at, "account_nonce", &account) + } + + fn lookup(&self, at: &BlockId, address: Address) -> Result> { + self.call_api(at, "lookup_address", &address) + } + + fn build_block(&self, at: &BlockId, inherent_data: InherentData) -> Result { + let mut block_builder = self.new_block_at(at)?; + for inherent in self.inherent_extrinsics(at, inherent_data)? { + block_builder.push(inherent)?; + } + + Ok(block_builder) + } + + fn inherent_extrinsics(&self, at: &BlockId, inherent_data: InherentData) -> Result> { + let runtime_version = self.runtime_version_at(at)?; + self.call_api(at, "inherent_extrinsics", &(inherent_data, runtime_version.spec_version)) + } +} + diff --git a/node/build.rs b/node/build.rs new file mode 100644 index 00000000000..4e48353aeac --- /dev/null +++ b/node/build.rs @@ -0,0 +1,24 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +extern crate vergen; + +const ERROR_MSG: &'static str = "Failed to generate metadata files"; + +fn main() { + vergen::vergen(vergen::SHORT_SHA).expect(ERROR_MSG); + println!("cargo:rerun-if-changed=../../.git/HEAD"); +} diff --git a/node/cli/Cargo.toml b/node/cli/Cargo.toml new file mode 100644 index 00000000000..d26a47aade0 --- /dev/null +++ b/node/cli/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "node-cli" +version = "0.1.0" +authors = ["Parity Technologies "] +description = "Substrate node implementation in Rust." + +[dependencies] +log = "0.3" +tokio = "0.1.7" +exit-future = "0.1" +substrate-cli = { path = "../../core/cli" } +node-service = { path = "../service" } diff --git a/node/cli/src/cli.yml b/node/cli/src/cli.yml new file mode 100644 index 00000000000..6c1fe186e79 --- /dev/null +++ b/node/cli/src/cli.yml @@ -0,0 +1,12 @@ +name: substrate-node +author: "Parity Team " +about: Substrate Node Rust Implementation +args: + - log: + short: l + value_name: LOG_PATTERN + help: Sets a custom logging + takes_value: true +subcommands: + - validator: + about: Run validator node diff --git a/node/cli/src/error.rs b/node/cli/src/error.rs new file mode 100644 index 00000000000..a83466fbe67 --- /dev/null +++ b/node/cli/src/error.rs @@ -0,0 +1,29 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Initialization errors. + +use client; + +error_chain! { + foreign_links { + Io(::std::io::Error) #[doc="IO error"]; + Cli(::clap::Error) #[doc="CLI error"]; + } + links { + Client(client::error::Error, client::error::ErrorKind) #[doc="Client error"]; + } +} diff --git a/node/cli/src/lib.rs b/node/cli/src/lib.rs new file mode 100644 index 00000000000..75cbf5d003f --- /dev/null +++ b/node/cli/src/lib.rs @@ -0,0 +1,122 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate CLI library. + +#![warn(missing_docs)] +#![warn(unused_extern_crates)] + +extern crate tokio; + +extern crate substrate_cli as cli; +extern crate node_service as service; +extern crate exit_future; + +#[macro_use] +extern crate log; + +pub use cli::error; + +use tokio::runtime::Runtime; +pub use service::{Components as ServiceComponents, Service, CustomConfiguration}; +pub use cli::{VersionInfo, IntoExit}; + +/// The chain specification option. +#[derive(Clone, Debug)] +pub enum ChainSpec { + /// Whatever the current runtime is, with just Alice as an auth. + Development, + /// Whatever the current runtime is, with simple Alice/Bob auths. + LocalTestnet, + /// The PoC-1 & PoC-2 era testnet. + Testnet, + /// Whatever the current runtime is with the "global testnet" defaults. + StagingTestnet, +} + +/// Get a chain config from a spec setting. +impl ChainSpec { + pub(crate) fn load(self) -> Result { + Ok(match self { + ChainSpec::Testnet => service::chain_spec::testnet_config()?, + ChainSpec::Development => service::chain_spec::development_config(), + ChainSpec::LocalTestnet => service::chain_spec::local_testnet_config(), + ChainSpec::StagingTestnet => service::chain_spec::staging_testnet_config(), + }) + } + + pub(crate) fn from(s: &str) -> Option { + match s { + "dev" => Some(ChainSpec::Development), + "local" => Some(ChainSpec::LocalTestnet), + "" | "test" => Some(ChainSpec::Testnet), + "staging" => Some(ChainSpec::StagingTestnet), + _ => None, + } + } +} + +fn load_spec(id: &str) -> Result, String> { + Ok(match ChainSpec::from(id) { + Some(spec) => Some(spec.load()?), + None => None, + }) +} + +/// Parse command line arguments into service configuration. +pub fn run(args: I, exit: E, version: cli::VersionInfo) -> error::Result<()> where + I: IntoIterator, + T: Into + Clone, + E: IntoExit, +{ + match cli::prepare_execution::(args, exit, version, load_spec, "substrate-node")? { + cli::Action::ExecutedInternally => (), + cli::Action::RunService((config, exit)) => { + info!("Parity ·:· Substrate"); + info!(" version {}", config.full_version()); + info!(" by Parity Technologies, 2017, 2018"); + info!("Chain specification: {}", config.chain_spec.name()); + info!("Node name: {}", config.name); + info!("Roles: {:?}", config.roles); + let mut runtime = Runtime::new()?; + let executor = runtime.executor(); + match config.roles == service::Roles::LIGHT { + true => run_until_exit(&mut runtime, service::new_light(config, executor)?, exit)?, + false => run_until_exit(&mut runtime, service::new_full(config, executor)?, exit)?, + } + } + } + Ok(()) +} + +fn run_until_exit( + runtime: &mut Runtime, + service: service::Service, + e: E, +) -> error::Result<()> + where + C: service::Components, + E: IntoExit, +{ + let (exit_send, exit) = exit_future::signal(); + + let executor = runtime.executor(); + cli::informant::start(&service, exit.clone(), executor.clone()); + + let _ = runtime.block_on(e.into_exit()); + exit_send.fire(); + Ok(()) +} diff --git a/node/consensus/Cargo.toml b/node/consensus/Cargo.toml new file mode 100644 index 00000000000..e6c6ac74170 --- /dev/null +++ b/node/consensus/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "node-consensus" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +futures = "0.1.17" +parking_lot = "0.4" +tokio = "0.1.7" +error-chain = "0.12" +log = "0.3" +exit-future = "0.1" +rhododendron = "0.3" +node-api = { path = "../api" } +node-primitives = { path = "../primitives" } +node-runtime = { path = "../runtime" } +node-transaction-pool = { path = "../transaction-pool" } +substrate-bft = { path = "../../core/bft" } +substrate-codec = { path = "../../core/codec" } +substrate-primitives = { path = "../../core/primitives" } +substrate-runtime-support = { path = "../../runtime/support" } +substrate-client = { path = "../../core/client" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } + +[dev-dependencies] +substrate-keyring = { path = "../../core/keyring" } diff --git a/node/consensus/README.adoc b/node/consensus/README.adoc new file mode 100644 index 00000000000..a3ac5f631c3 --- /dev/null +++ b/node/consensus/README.adoc @@ -0,0 +1,5 @@ + += Polkadot Consensus + +placeholder +//TODO Write content :) diff --git a/node/consensus/src/error.rs b/node/consensus/src/error.rs new file mode 100644 index 00000000000..01823a8e503 --- /dev/null +++ b/node/consensus/src/error.rs @@ -0,0 +1,51 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Errors that can occur during the consensus process. + +use primitives::AuthorityId; + +error_chain! { + links { + Api(::node_api::Error, ::node_api::ErrorKind); + Bft(::bft::Error, ::bft::ErrorKind); + } + + errors { + NotValidator(id: AuthorityId) { + description("Local account ID not a validator at this block."), + display("Local account ID ({:?}) not a validator at this block.", id), + } + PrematureDestruction { + description("Proposer destroyed before finishing proposing or evaluating"), + display("Proposer destroyed before finishing proposing or evaluating"), + } + Timer(e: ::tokio::timer::Error) { + description("Failed to register or resolve async timer."), + display("Timer failed: {}", e), + } + Executor(e: ::futures::future::ExecuteErrorKind) { + description("Unable to dispatch agreement future"), + display("Unable to dispatch agreement future: {:?}", e), + } + } +} + +impl From<::bft::InputStreamConcluded> for Error { + fn from(err: ::bft::InputStreamConcluded) -> Self { + ::bft::Error::from(err).into() + } +} diff --git a/node/consensus/src/evaluation.rs b/node/consensus/src/evaluation.rs new file mode 100644 index 00000000000..9d4d5b09119 --- /dev/null +++ b/node/consensus/src/evaluation.rs @@ -0,0 +1,96 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block evaluation and evaluation errors. + +use super::MAX_TRANSACTIONS_SIZE; + +use codec::{Decode, Encode}; +use node_runtime::{Block as GenericBlock, CheckedBlock}; +use node_primitives::{Block, Hash, BlockNumber, Timestamp}; + +error_chain! { + links { + Api(::node_api::Error, ::node_api::ErrorKind); + } + + errors { + BadProposalFormat { + description("Proposal provided not a block."), + display("Proposal provided not a block."), + } + TimestampInFuture { + description("Proposal had timestamp too far in the future."), + display("Proposal had timestamp too far in the future."), + } + WrongParentHash(expected: Hash, got: Hash) { + description("Proposal had wrong parent hash."), + display("Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got), + } + WrongNumber(expected: BlockNumber, got: BlockNumber) { + description("Proposal had wrong number."), + display("Proposal had wrong number. Expected {:?}, got {:?}", expected, got), + } + ProposalTooLarge(size: usize) { + description("Proposal exceeded the maximum size."), + display( + "Proposal exceeded the maximum size of {} by {} bytes.", + MAX_TRANSACTIONS_SIZE, size.saturating_sub(MAX_TRANSACTIONS_SIZE) + ), + } + } +} + +/// Attempt to evaluate a substrate block as a node block, returning error +/// upon any initial validity checks failing. +pub fn evaluate_initial( + proposal: &Block, + now: Timestamp, + parent_hash: &Hash, + parent_number: BlockNumber, +) -> Result { + const MAX_TIMESTAMP_DRIFT: Timestamp = 60; + + let encoded = Encode::encode(proposal); + let proposal = GenericBlock::decode(&mut &encoded[..]) + .and_then(|b| CheckedBlock::new(b).ok()) + .ok_or_else(|| ErrorKind::BadProposalFormat)?; + + let transactions_size = proposal.extrinsics.iter().fold(0, |a, tx| { + a + Encode::encode(tx).len() + }); + + if transactions_size > MAX_TRANSACTIONS_SIZE { + bail!(ErrorKind::ProposalTooLarge(transactions_size)) + } + + if proposal.header.parent_hash != *parent_hash { + bail!(ErrorKind::WrongParentHash(*parent_hash, proposal.header.parent_hash)); + } + + if proposal.header.number != parent_number + 1 { + bail!(ErrorKind::WrongNumber(parent_number + 1, proposal.header.number)); + } + + let block_timestamp = proposal.timestamp(); + + // lenient maximum -- small drifts will just be delayed using a timer. + if block_timestamp > now + MAX_TIMESTAMP_DRIFT { + bail!(ErrorKind::TimestampInFuture) + } + + Ok(proposal) +} diff --git a/node/consensus/src/lib.rs b/node/consensus/src/lib.rs new file mode 100644 index 00000000000..2135beebd0c --- /dev/null +++ b/node/consensus/src/lib.rs @@ -0,0 +1,445 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! This service uses BFT consensus provided by the substrate. + +extern crate parking_lot; +extern crate node_api; +extern crate node_transaction_pool as transaction_pool; +extern crate node_runtime; +extern crate node_primitives; + +extern crate substrate_bft as bft; +extern crate substrate_codec as codec; +extern crate substrate_primitives as primitives; +extern crate substrate_runtime_support as runtime_support; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_client as client; + +extern crate exit_future; +extern crate tokio; +extern crate rhododendron; + +#[macro_use] +extern crate error_chain; +extern crate futures; + +#[macro_use] +extern crate log; + +#[cfg(test)] +extern crate substrate_keyring; + +use std::sync::Arc; +use std::time::{self, Duration, Instant}; + +use codec::{Decode, Encode}; +use node_api::Api; +use node_primitives::{AccountId, Hash, Block, BlockId, BlockNumber, Header, Timestamp, SessionKey}; +use primitives::{AuthorityId, ed25519}; +use transaction_pool::TransactionPool; +use tokio::runtime::TaskExecutor; +use tokio::timer::Delay; + +use futures::prelude::*; +use futures::future; +use parking_lot::RwLock; + +pub use self::error::{ErrorKind, Error}; +pub use self::offline_tracker::OfflineTracker; +pub use service::Service; + +mod evaluation; +mod error; +mod offline_tracker; +mod service; + +/// Shared offline validator tracker. +pub type SharedOfflineTracker = Arc>; + +// block size limit. +const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024; + +/// A long-lived network which can create BFT message routing processes on demand. +pub trait Network { + /// The input stream of BFT messages. Should never logically conclude. + type Input: Stream,Error=Error>; + /// The output sink of BFT messages. Messages sent here should eventually pass to all + /// current authorities. + type Output: Sink,SinkError=Error>; + + /// Instantiate input and output streams. + fn communication_for( + &self, + validators: &[SessionKey], + local_id: SessionKey, + parent_hash: Hash, + task_executor: TaskExecutor + ) -> (Self::Input, Self::Output); +} + +/// Proposer factory. +pub struct ProposerFactory + where + P: Api + Send + Sync + 'static +{ + /// The client instance. + pub client: Arc

, + /// The transaction pool. + pub transaction_pool: Arc>, + /// The backing network handle. + pub network: N, + /// handle to remote task executor + pub handle: TaskExecutor, + /// Offline-tracker. + pub offline: SharedOfflineTracker, +} + +impl bft::Environment for ProposerFactory + where + N: Network, + P: Api + Send + Sync + 'static, +{ + type Proposer = Proposer

; + type Input = N::Input; + type Output = N::Output; + type Error = Error; + + fn init( + &self, + parent_header: &Header, + authorities: &[AuthorityId], + sign_with: Arc, + ) -> Result<(Self::Proposer, Self::Input, Self::Output), Error> { + use runtime_primitives::traits::{Hash as HashT, BlakeTwo256}; + + // force delay in evaluation this long. + const FORCE_DELAY: Timestamp = 5; + + let parent_hash = parent_header.hash().into(); + + let id = BlockId::hash(parent_hash); + let random_seed = self.client.random_seed(&id)?; + let random_seed = BlakeTwo256::hash(&*random_seed); + + let validators = self.client.validators(&id)?; + self.offline.write().note_new_block(&validators[..]); + + info!("Starting consensus session on top of parent {:?}", parent_hash); + + let local_id = sign_with.public().0.into(); + let (input, output) = self.network.communication_for( + authorities, + local_id, + parent_hash.clone(), + self.handle.clone(), + ); + let now = Instant::now(); + let proposer = Proposer { + client: self.client.clone(), + start: now, + local_key: sign_with, + parent_hash, + parent_id: id, + parent_number: parent_header.number, + random_seed, + transaction_pool: self.transaction_pool.clone(), + offline: self.offline.clone(), + validators, + minimum_timestamp: current_timestamp() + FORCE_DELAY, + }; + + Ok((proposer, input, output)) + } +} + +/// The proposer logic. +pub struct Proposer { + client: Arc, + start: Instant, + local_key: Arc, + parent_hash: Hash, + parent_id: BlockId, + parent_number: BlockNumber, + random_seed: Hash, + transaction_pool: Arc>, + offline: SharedOfflineTracker, + validators: Vec, + minimum_timestamp: u64, +} + +impl Proposer { + fn primary_index(&self, round_number: usize, len: usize) -> usize { + use primitives::uint::U256; + + let big_len = U256::from(len); + let offset = U256::from_big_endian(&self.random_seed.0) % big_len; + let offset = offset.low_u64() as usize + round_number; + offset % len + } +} + +impl bft::Proposer for Proposer + where + C: Api + Send + Sync, +{ + type Create = Result; + type Error = Error; + type Evaluate = Box>; + + fn propose(&self) -> Result { + use node_api::BlockBuilder; + use runtime_primitives::traits::{Hash as HashT, BlakeTwo256}; + use node_primitives::InherentData; + + const MAX_VOTE_OFFLINE_SECONDS: Duration = Duration::from_secs(60); + + // TODO: handle case when current timestamp behind that in state. + let timestamp = ::std::cmp::max(self.minimum_timestamp, current_timestamp()); + + let elapsed_since_start = self.start.elapsed(); + let offline_indices = if elapsed_since_start > MAX_VOTE_OFFLINE_SECONDS { + Vec::new() + } else { + self.offline.read().reports(&self.validators[..]) + }; + + if !offline_indices.is_empty() { + info!( + "Submitting offline validators {:?} for slash-vote", + offline_indices.iter().map(|&i| self.validators[i as usize]).collect::>(), + ) + } + + let inherent_data = InherentData { + timestamp, + offline_indices, + }; + + let mut block_builder = self.client.build_block(&self.parent_id, inherent_data)?; + + { + let mut unqueue_invalid = Vec::new(); + let result = self.transaction_pool.cull_and_get_pending(&BlockId::hash(self.parent_hash), |pending_iterator| { + let mut pending_size = 0; + for pending in pending_iterator { + if pending_size + pending.verified.encoded_size() >= MAX_TRANSACTIONS_SIZE { break } + + match block_builder.push_extrinsic(pending.original.clone()) { + Ok(()) => { + pending_size += pending.verified.encoded_size(); + } + Err(e) => { + trace!(target: "transaction-pool", "Invalid transaction: {}", e); + unqueue_invalid.push(pending.verified.hash().clone()); + } + } + } + }); + if let Err(e) = result { + warn!("Unable to get the pending set: {:?}", e); + } + + self.transaction_pool.remove(&unqueue_invalid, false); + } + + let block = block_builder.bake()?; + + info!("Proposing block [number: {}; hash: {}; parent_hash: {}; extrinsics: [{}]]", + block.header.number, + Hash::from(block.header.hash()), + block.header.parent_hash, + block.extrinsics.iter() + .map(|xt| format!("{}", BlakeTwo256::hash_of(xt))) + .collect::>() + .join(", ") + ); + + let substrate_block = Decode::decode(&mut block.encode().as_slice()) + .expect("blocks are defined to serialize to substrate blocks correctly; qed"); + + assert!(evaluation::evaluate_initial( + &substrate_block, + timestamp, + &self.parent_hash, + self.parent_number, + ).is_ok()); + + Ok(substrate_block) + } + + fn evaluate(&self, unchecked_proposal: &Block) -> Self::Evaluate { + debug!(target: "bft", "evaluating block on top of parent ({}, {:?})", self.parent_number, self.parent_hash); + + let current_timestamp = current_timestamp(); + + // do initial serialization and structural integrity checks. + let maybe_proposal = evaluation::evaluate_initial( + unchecked_proposal, + current_timestamp, + &self.parent_hash, + self.parent_number, + ); + + let proposal = match maybe_proposal { + Ok(p) => p, + Err(e) => { + // TODO: these errors are easily re-checked in runtime. + debug!(target: "bft", "Invalid proposal: {:?}", e); + return Box::new(future::ok(false)); + } + }; + + let vote_delays = { + let now = Instant::now(); + + // the duration until the given timestamp is current + let proposed_timestamp = ::std::cmp::max(self.minimum_timestamp, proposal.timestamp()); + let timestamp_delay = if proposed_timestamp > current_timestamp { + let delay_s = proposed_timestamp - current_timestamp; + debug!(target: "bft", "Delaying evaluation of proposal for {} seconds", delay_s); + Some(now + Duration::from_secs(delay_s)) + } else { + None + }; + + match timestamp_delay { + Some(duration) => future::Either::A( + Delay::new(duration).map_err(|e| Error::from(ErrorKind::Timer(e))) + ), + None => future::Either::B(future::ok(())), + } + }; + + // refuse to vote if this block says a validator is offline that we + // think isn't. + let offline = proposal.noted_offline(); + if !self.offline.read().check_consistency(&self.validators[..], offline) { + return Box::new(futures::empty()); + } + + // evaluate whether the block is actually valid. + // TODO: is it better to delay this until the delays are finished? + let evaluated = self.client + .evaluate_block(&self.parent_id, unchecked_proposal.clone()) + .map_err(Into::into); + + let future = future::result(evaluated).and_then(move |good| { + let end_result = future::ok(good); + if good { + // delay a "good" vote. + future::Either::A(vote_delays.and_then(|_| end_result)) + } else { + // don't delay a "bad" evaluation. + future::Either::B(end_result) + } + }); + + Box::new(future) as Box<_> + } + + fn round_proposer(&self, round_number: usize, authorities: &[AuthorityId]) -> AuthorityId { + let offset = self.primary_index(round_number, authorities.len()); + let proposer = authorities[offset].clone(); + trace!(target: "bft", "proposer for round {} is {}", round_number, proposer); + + proposer + } + + fn import_misbehavior(&self, misbehavior: Vec<(AuthorityId, bft::Misbehavior)>) { + use rhododendron::Misbehavior as GenericMisbehavior; + use runtime_primitives::bft::{MisbehaviorKind, MisbehaviorReport}; + use node_primitives::UncheckedExtrinsic as GenericExtrinsic; + use node_runtime::{Call, UncheckedExtrinsic, ConsensusCall}; + + let local_id = self.local_key.public().0.into(); + let mut next_index = { + let cur_index = self.transaction_pool.cull_and_get_pending(&BlockId::hash(self.parent_hash), |pending| pending + .filter(|tx| tx.verified.sender == local_id) + .last() + .map(|tx| Ok(tx.verified.index())) + .unwrap_or_else(|| self.client.index(&self.parent_id, local_id)) + ); + + match cur_index { + Ok(Ok(cur_index)) => cur_index + 1, + Ok(Err(e)) => { + warn!(target: "consensus", "Error computing next transaction index: {}", e); + return; + } + Err(e) => { + warn!(target: "consensus", "Error computing next transaction index: {}", e); + return; + } + } + }; + + for (target, misbehavior) in misbehavior { + let report = MisbehaviorReport { + parent_hash: self.parent_hash, + parent_number: self.parent_number, + target, + misbehavior: match misbehavior { + GenericMisbehavior::ProposeOutOfTurn(_, _, _) => continue, + GenericMisbehavior::DoublePropose(_, _, _) => continue, + GenericMisbehavior::DoublePrepare(round, (h1, s1), (h2, s2)) + => MisbehaviorKind::BftDoublePrepare(round as u32, (h1, s1.signature), (h2, s2.signature)), + GenericMisbehavior::DoubleCommit(round, (h1, s1), (h2, s2)) + => MisbehaviorKind::BftDoubleCommit(round as u32, (h1, s1.signature), (h2, s2.signature)), + } + }; + let payload = (next_index, Call::Consensus(ConsensusCall::report_misbehavior(report))); + let signature = self.local_key.sign(&payload.encode()).into(); + next_index += 1; + + let local_id = self.local_key.public().0.into(); + let extrinsic = UncheckedExtrinsic { + signature: Some((node_runtime::RawAddress::Id(local_id), signature)), + index: payload.0, + function: payload.1, + }; + let uxt: GenericExtrinsic = Decode::decode(&mut extrinsic.encode().as_slice()).expect("Encoded extrinsic is valid"); + self.transaction_pool.submit_one(&BlockId::hash(self.parent_hash), uxt) + .expect("locally signed extrinsic is valid; qed"); + } + } + + fn on_round_end(&self, round_number: usize, was_proposed: bool) { + let primary_validator = self.validators[ + self.primary_index(round_number, self.validators.len()) + ]; + + + // alter the message based on whether we think the empty proposer was forced to skip the round. + // this is determined by checking if our local validator would have been forced to skip the round. + if !was_proposed { + let public = ed25519::Public::from_raw(primary_validator.0); + info!( + "Potential Offline Validator: {} failed to propose during assigned slot: {}", + public, + round_number, + ); + } + + self.offline.write().note_round_end(primary_validator, was_proposed); + } +} + +fn current_timestamp() -> Timestamp { + time::SystemTime::now().duration_since(time::UNIX_EPOCH) + .expect("now always later than unix epoch; qed") + .as_secs() +} diff --git a/node/consensus/src/offline_tracker.rs b/node/consensus/src/offline_tracker.rs new file mode 100644 index 00000000000..18845dd68b2 --- /dev/null +++ b/node/consensus/src/offline_tracker.rs @@ -0,0 +1,137 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tracks offline validators. + +use node_primitives::AccountId; + +use std::collections::HashMap; +use std::time::{Instant, Duration}; + +// time before we report a validator. +const REPORT_TIME: Duration = Duration::from_secs(60 * 5); + +struct Observed { + last_round_end: Instant, + offline_since: Instant, +} + +impl Observed { + fn new() -> Observed { + let now = Instant::now(); + Observed { + last_round_end: now, + offline_since: now, + } + } + + fn note_round_end(&mut self, was_online: bool) { + let now = Instant::now(); + + self.last_round_end = now; + if was_online { + self.offline_since = now; + } + } + + fn is_active(&self) -> bool { + // can happen if clocks are not monotonic + if self.offline_since > self.last_round_end { return true } + self.last_round_end.duration_since(self.offline_since) < REPORT_TIME + } +} + +/// Tracks offline validators and can issue a report for those offline. +pub struct OfflineTracker { + observed: HashMap, +} + +impl OfflineTracker { + /// Create a new tracker. + pub fn new() -> Self { + OfflineTracker { observed: HashMap::new() } + } + + /// Note new consensus is starting with the given set of validators. + pub fn note_new_block(&mut self, validators: &[AccountId]) { + use std::collections::HashSet; + + let set: HashSet<_> = validators.iter().cloned().collect(); + self.observed.retain(|k, _| set.contains(k)); + } + + /// Note that a round has ended. + pub fn note_round_end(&mut self, validator: AccountId, was_online: bool) { + self.observed.entry(validator) + .or_insert_with(Observed::new) + .note_round_end(was_online); + } + + /// Generate a vector of indices for offline account IDs. + pub fn reports(&self, validators: &[AccountId]) -> Vec { + validators.iter() + .enumerate() + .filter_map(|(i, v)| if self.is_online(v) { + None + } else { + Some(i as u32) + }) + .collect() + } + + /// Whether reports on a validator set are consistent with our view of things. + pub fn check_consistency(&self, validators: &[AccountId], reports: &[u32]) -> bool { + reports.iter().cloned().all(|r| { + let v = match validators.get(r as usize) { + Some(v) => v, + None => return false, + }; + + // we must think all validators reported externally are offline. + let thinks_online = self.is_online(v); + !thinks_online + }) + } + + fn is_online(&self, v: &AccountId) -> bool { + self.observed.get(v).map(Observed::is_active).unwrap_or(true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validator_offline() { + let mut tracker = OfflineTracker::new(); + let v = [0; 32].into(); + let v2 = [1; 32].into(); + let v3 = [2; 32].into(); + tracker.note_round_end(v, true); + tracker.note_round_end(v2, true); + tracker.note_round_end(v3, true); + + let slash_time = REPORT_TIME + Duration::from_secs(5); + tracker.observed.get_mut(&v).unwrap().offline_since -= slash_time; + tracker.observed.get_mut(&v2).unwrap().offline_since -= slash_time; + + assert_eq!(tracker.reports(&[v, v2, v3]), vec![0, 1]); + + tracker.note_new_block(&[v, v3]); + assert_eq!(tracker.reports(&[v, v2, v3]), vec![0]); + } +} diff --git a/node/consensus/src/service.rs b/node/consensus/src/service.rs new file mode 100644 index 00000000000..43b5980ddf1 --- /dev/null +++ b/node/consensus/src/service.rs @@ -0,0 +1,172 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Consensus service. + +/// Consensus service. A long running service that manages BFT agreement +/// the network. +use std::thread; +use std::time::{Duration, Instant}; +use std::sync::Arc; + +use bft::{self, BftService}; +use client::{BlockchainEvents, ChainHead, BlockBody}; +use primitives::ed25519; +use futures::prelude::*; +use node_api::Api; +use node_primitives::{Block, Header}; +use transaction_pool::TransactionPool; + +use tokio::executor::current_thread::TaskExecutor as LocalThreadHandle; +use tokio::runtime::TaskExecutor as ThreadPoolHandle; +use tokio::runtime::current_thread::Runtime as LocalRuntime; +use tokio::timer::Interval; + +use super::{Network, ProposerFactory}; +use error; + +const TIMER_DELAY_MS: u64 = 5000; +const TIMER_INTERVAL_MS: u64 = 500; + +// spin up an instance of BFT agreement on the current thread's executor. +// panics if there is no current thread executor. +fn start_bft( + header: Header, + bft_service: Arc>, +) where + F: bft::Environment + 'static, + C: bft::BlockImport + bft::Authorities + 'static, + F::Error: ::std::fmt::Debug, + >::Error: ::std::fmt::Display + Into, + >::Error: ::std::fmt::Display +{ + let mut handle = LocalThreadHandle::current(); + match bft_service.build_upon(&header) { + Ok(Some(bft_work)) => if let Err(e) = handle.spawn_local(Box::new(bft_work)) { + warn!(target: "bft", "Couldn't initialize BFT agreement: {:?}", e); + } + Ok(None) => trace!(target: "bft", "Could not start agreement on top of {}", header.hash()), + Err(e) => warn!(target: "bft", "BFT agreement error: {}", e), + } +} + +/// Consensus service. Starts working when created. +pub struct Service { + thread: Option>, + exit_signal: Option<::exit_future::Signal>, +} + +impl Service { + /// Create and start a new instance. + pub fn new( + client: Arc, + api: Arc, + network: N, + transaction_pool: Arc>, + thread_pool: ThreadPoolHandle, + key: ed25519::Pair, + ) -> Service + where + A: Api + Send + Sync + 'static, + C: BlockchainEvents + ChainHead + BlockBody, + C: bft::BlockImport + bft::Authorities + Send + Sync + 'static, + N: Network + Send + 'static, + { + use parking_lot::RwLock; + use super::OfflineTracker; + + let (signal, exit) = ::exit_future::signal(); + let thread = thread::spawn(move || { + let mut runtime = LocalRuntime::new().expect("Could not create local runtime"); + let key = Arc::new(key); + + let factory = ProposerFactory { + client: api.clone(), + transaction_pool: transaction_pool.clone(), + network, + handle: thread_pool.clone(), + offline: Arc::new(RwLock::new(OfflineTracker::new())), + }; + let bft_service = Arc::new(BftService::new(client.clone(), key, factory)); + + let notifications = { + let client = client.clone(); + let bft_service = bft_service.clone(); + + client.import_notification_stream().for_each(move |notification| { + if notification.is_new_best { + start_bft(notification.header, bft_service.clone()); + } + Ok(()) + }) + }; + + let interval = Interval::new( + Instant::now() + Duration::from_millis(TIMER_DELAY_MS), + Duration::from_millis(TIMER_INTERVAL_MS), + ); + + let mut prev_best = match client.best_block_header() { + Ok(header) => header.hash(), + Err(e) => { + warn!("Cant's start consensus service. Error reading best block header: {:?}", e); + return; + } + }; + + let timed = { + let c = client.clone(); + let s = bft_service.clone(); + + interval.map_err(|e| debug!(target: "bft", "Timer error: {:?}", e)).for_each(move |_| { + if let Ok(best_block) = c.best_block_header() { + let hash = best_block.hash(); + + if hash == prev_best { + debug!(target: "bft", "Starting consensus round after a timeout"); + start_bft(best_block, s.clone()); + } + prev_best = hash; + } + Ok(()) + }) + }; + + runtime.spawn(notifications); + runtime.spawn(timed); + + if let Err(e) = runtime.block_on(exit) { + debug!("BFT event loop error {:?}", e); + } + }); + Service { + thread: Some(thread), + exit_signal: Some(signal), + } + } +} + +impl Drop for Service { + fn drop(&mut self) { + if let Some(signal) = self.exit_signal.take() { + signal.fire(); + } + + if let Some(thread) = self.thread.take() { + thread.join().expect("The service thread has panicked"); + } + } +} diff --git a/node/executor/Cargo.toml b/node/executor/Cargo.toml new file mode 100644 index 00000000000..2974a60a943 --- /dev/null +++ b/node/executor/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "node-executor" +version = "0.1.0" +authors = ["Parity Technologies "] +description = "Substrate node implementation in Rust." + +[dependencies] +hex-literal = "0.1" +triehash = "0.2" +substrate-codec = { path = "../../core/codec" } +substrate-runtime-io = { path = "../../core/runtime-io" } +substrate-runtime-support = { path = "../../runtime/support" } +substrate-state-machine = { path = "../../core/state-machine" } +substrate-executor = { path = "../../core/executor" } +substrate-primitives = { path = "../../core/primitives" } +node-primitives = { path = "../primitives" } +node-runtime = { path = "../runtime" } + +[dev-dependencies] +substrate-keyring = { path = "../../core/keyring" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-runtime-balances = { path = "../../runtime/balances" } +substrate-runtime-session = { path = "../../runtime/session" } +substrate-runtime-staking = { path = "../../runtime/staking" } +substrate-runtime-system = { path = "../../runtime/system" } +substrate-runtime-consensus = { path = "../../runtime/consensus" } +substrate-runtime-timestamp = { path = "../../runtime/timestamp" } +substrate-runtime-treasury = { path = "../../runtime/treasury" } diff --git a/node/executor/src/lib.rs b/node/executor/src/lib.rs new file mode 100644 index 00000000000..9c92e42d0d8 --- /dev/null +++ b/node/executor/src/lib.rs @@ -0,0 +1,523 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! A `CodeExecutor` specialisation which uses natively compiled runtime when the wasm to be +//! executed is equivalent to the natively compiled code. + +extern crate node_runtime; +#[macro_use] extern crate substrate_executor; +extern crate substrate_codec as codec; +extern crate substrate_state_machine as state_machine; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_primitives as primitives; +extern crate node_primitives; +extern crate triehash; + +#[cfg(test)] extern crate substrate_keyring as keyring; +#[cfg(test)] extern crate substrate_runtime_primitives as runtime_primitives; +#[cfg(test)] extern crate substrate_runtime_support as runtime_support; +#[cfg(test)] extern crate substrate_runtime_balances as balances; +#[cfg(test)] extern crate substrate_runtime_session as session; +#[cfg(test)] extern crate substrate_runtime_staking as staking; +#[cfg(test)] extern crate substrate_runtime_system as system; +#[cfg(test)] extern crate substrate_runtime_consensus as consensus; +#[cfg(test)] extern crate substrate_runtime_timestamp as timestamp; +#[cfg(test)] extern crate substrate_runtime_treasury as treasury; +#[cfg(test)] #[macro_use] extern crate hex_literal; + +pub use substrate_executor::NativeExecutor; +native_executor_instance!(pub Executor, node_runtime::api::dispatch, node_runtime::VERSION, include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.compact.wasm")); + +#[cfg(test)] +mod tests { + use runtime_io; + use super::Executor; + use substrate_executor::{WasmExecutor, NativeExecutionDispatch}; + use codec::{Encode, Decode, Joiner}; + use keyring::Keyring; + use runtime_support::{Hashable, StorageValue, StorageMap}; + use state_machine::{CodeExecutor, TestExternalities}; + use primitives::{twox_128, Blake2Hasher, ed25519::{Public, Pair}}; + use node_primitives::{Hash, BlockNumber, AccountId}; + use runtime_primitives::traits::Header as HeaderT; + use runtime_primitives::{ApplyOutcome, ApplyError, ApplyResult}; + use {balances, staking, session, system, consensus, timestamp, treasury}; + use system::{EventRecord, Phase}; + use node_runtime::{Header, Block, UncheckedExtrinsic, CheckedExtrinsic, Call, Runtime, Balances, + BuildStorage, GenesisConfig, BalancesConfig, SessionConfig, StakingConfig, System, Event}; + + const BLOATY_CODE: &[u8] = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.wasm"); + const COMPACT_CODE: &[u8] = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.compact.wasm"); + + // TODO: move into own crate. + macro_rules! map { + ($( $name:expr => $value:expr ),*) => ( + vec![ $( ( $name, $value ) ),* ].into_iter().collect() + ) + } + + fn alice() -> AccountId { + AccountId::from(Keyring::Alice.to_raw_public()) + } + + fn bob() -> AccountId { + AccountId::from(Keyring::Bob.to_raw_public()) + } + + fn sign(xt: CheckedExtrinsic) -> UncheckedExtrinsic { + match xt.signed { + Some(signed) => { + let payload = (xt.index, xt.function); + let pair = Pair::from(Keyring::from_public(Public::from_raw(signed.clone().into())).unwrap()); + let signature = pair.sign(&payload.encode()).into(); + UncheckedExtrinsic { + signature: Some((balances::address::Address::Id(signed), signature)), + index: payload.0, + function: payload.1, + } + } + None => UncheckedExtrinsic { + signature: None, + index: xt.index, + function: xt.function, + }, + } + } + + fn xt() -> UncheckedExtrinsic { + sign(CheckedExtrinsic { + signed: Some(alice()), + index: 0, + function: Call::Balances(balances::Call::transfer::(bob().into(), 69)), + }) + } + + fn from_block_number(n: u64) -> Header { + Header::new(n, Default::default(), Default::default(), [69; 32].into(), Default::default()) + } + + fn executor() -> ::substrate_executor::NativeExecutor { + ::substrate_executor::NativeExecutor::new() + } + + #[test] + fn panic_execution_with_foreign_code_gives_error() { + let mut t: TestExternalities = map![ + twox_128(&>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![70u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(&>::key_for(0)).to_vec() => vec![0u8; 32] + ]; + + let r = executor().call(&mut t, 8, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0; + assert!(r.is_ok()); + let v = executor().call(&mut t, 8, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap(); + let r = ApplyResult::decode(&mut &v[..]).unwrap(); + assert_eq!(r, Err(ApplyError::CantPay)); + } + + #[test] + fn bad_extrinsic_with_native_equivalent_code_gives_error() { + let mut t: TestExternalities = map![ + twox_128(&>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![70u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(&>::key_for(0)).to_vec() => vec![0u8; 32] + ]; + + let r = executor().call(&mut t, 8, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0; + assert!(r.is_ok()); + let v = executor().call(&mut t, 8, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap(); + let r = ApplyResult::decode(&mut &v[..]).unwrap(); + assert_eq!(r, Err(ApplyError::CantPay)); + } + + #[test] + fn successful_execution_with_native_equivalent_code_gives_ok() { + let mut t: TestExternalities = map![ + twox_128(&>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(&>::key_for(0)).to_vec() => vec![0u8; 32] + ]; + + let r = executor().call(&mut t, 8, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0; + assert!(r.is_ok()); + let r = executor().call(&mut t, 8, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0; + assert!(r.is_ok()); + + runtime_io::with_externalities(&mut t, || { + assert_eq!(Balances::total_balance(&alice()), 42); + assert_eq!(Balances::total_balance(&bob()), 69); + }); + } + + #[test] + fn successful_execution_with_foreign_code_gives_ok() { + let mut t: TestExternalities = map![ + twox_128(&>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(&>::key_for(0)).to_vec() => vec![0u8; 32] + ]; + + let r = executor().call(&mut t, 8, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0; + assert!(r.is_ok()); + let r = executor().call(&mut t, 8, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0; + assert!(r.is_ok()); + + runtime_io::with_externalities(&mut t, || { + assert_eq!(Balances::total_balance(&alice()), 42); + assert_eq!(Balances::total_balance(&bob()), 69); + }); + } + + fn new_test_ext() -> TestExternalities { + use keyring::Keyring::*; + let three = [3u8; 32].into(); + GenesisConfig { + consensus: Some(Default::default()), + system: Some(Default::default()), + balances: Some(BalancesConfig { + balances: vec![(alice(), 111)], + transaction_base_fee: 1, + transaction_byte_fee: 0, + existential_deposit: 0, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + }), + session: Some(SessionConfig { + session_length: 2, + validators: vec![One.to_raw_public().into(), Two.to_raw_public().into(), three], + }), + staking: Some(StakingConfig { + sessions_per_era: 2, + current_era: 0, + intentions: vec![alice(), bob(), Charlie.to_raw_public().into()], + validator_count: 3, + minimum_validator_count: 0, + bonding_duration: 0, + offline_slash: 0, + session_reward: 0, + offline_slash_grace: 0, + }), + democracy: Some(Default::default()), + council: Some(Default::default()), + timestamp: Some(Default::default()), + treasury: Some(Default::default()), + }.build_storage().unwrap().into() + } + + fn construct_block(number: BlockNumber, parent_hash: Hash, state_root: Hash, extrinsics: Vec) -> (Vec, Hash) { + use triehash::ordered_trie_root; + + let extrinsics = extrinsics.into_iter().map(sign).collect::>(); + let extrinsics_root = ordered_trie_root::(extrinsics.iter().map(Encode::encode)).0.into(); + + let header = Header { + parent_hash, + number, + state_root, + extrinsics_root, + digest: Default::default(), + }; + let hash = header.blake2_256(); + + (Block { header, extrinsics }.encode(), hash.into()) + } + + fn block1() -> (Vec, Hash) { + construct_block( + 1, + [69u8; 32].into(), + hex!("c2fcc528c92b3c958b0e0914f26e05f151903ed43c87f29b20f9c8f0450d7484").into(), + vec![ + CheckedExtrinsic { + signed: None, + index: 0, + function: Call::Timestamp(timestamp::Call::set(42)), + }, + CheckedExtrinsic { + signed: Some(alice()), + index: 0, + function: Call::Balances(balances::Call::transfer(bob().into(), 69)), + }, + ] + ) + } + + fn block2() -> (Vec, Hash) { + construct_block( + 2, + block1().1, + hex!("62e5879f10338fa6136161c60ae0ffc35936f7b8c3fdb38095ddd0e044309762").into(), + vec![ + CheckedExtrinsic { + signed: None, + index: 0, + function: Call::Timestamp(timestamp::Call::set(52)), + }, + CheckedExtrinsic { + signed: Some(bob()), + index: 0, + function: Call::Balances(balances::Call::transfer(alice().into(), 5)), + }, + CheckedExtrinsic { + signed: Some(alice()), + index: 1, + function: Call::Balances(balances::Call::transfer(bob().into(), 15)), + } + ] + ) + } + + fn block1big() -> (Vec, Hash) { + construct_block( + 1, + [69u8; 32].into(), + hex!("789b19bc7beaa83ae70412f65ad0ac02435fd79e0226ba3394865a052e56fbd8").into(), + vec![ + CheckedExtrinsic { + signed: None, + index: 0, + function: Call::Timestamp(timestamp::Call::set(42)), + }, + CheckedExtrinsic { + signed: Some(alice()), + index: 0, + function: Call::Consensus(consensus::Call::remark(vec![0; 120000])), + } + ] + ) + } + + #[test] + fn full_native_block_import_works() { + let mut t = new_test_ext(); + + executor().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap(); + + runtime_io::with_externalities(&mut t, || { + assert_eq!(Balances::total_balance(&alice()), 41); + assert_eq!(Balances::total_balance(&bob()), 69); + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: Event::system(system::Event::ExtrinsicSuccess) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::balances(balances::RawEvent::NewAccount(bob(), 1, balances::NewAccountOutcome::NoHint)) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::balances(balances::RawEvent::Transfer( + hex!["d172a74cda4c865912c32ba0a80a57ae69abae410e5ccb59dee84e2f4432db4f"].into(), + hex!["d7568e5f0a7eda67a82691ff379ac4bba4f9c9b859fe779b5d46363b61ad2db9"].into(), + 69, + 0 + )) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::system(system::Event::ExtrinsicSuccess) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::treasury(treasury::RawEvent::Spending(0)) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::treasury(treasury::RawEvent::Burnt(0)) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::treasury(treasury::RawEvent::Rollover(0)) + } + ]); + }); + + executor().call(&mut t, 8, COMPACT_CODE, "execute_block", &block2().0, true).0.unwrap(); + + runtime_io::with_externalities(&mut t, || { + assert_eq!(Balances::total_balance(&alice()), 30); + assert_eq!(Balances::total_balance(&bob()), 78); + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: Event::system(system::Event::ExtrinsicSuccess) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::balances( + balances::RawEvent::Transfer( + hex!["d7568e5f0a7eda67a82691ff379ac4bba4f9c9b859fe779b5d46363b61ad2db9"].into(), + hex!["d172a74cda4c865912c32ba0a80a57ae69abae410e5ccb59dee84e2f4432db4f"].into(), + 5, + 0 + ) + ) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::system(system::Event::ExtrinsicSuccess) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(2), + event: Event::balances( + balances::RawEvent::Transfer( + hex!["d172a74cda4c865912c32ba0a80a57ae69abae410e5ccb59dee84e2f4432db4f"].into(), + hex!["d7568e5f0a7eda67a82691ff379ac4bba4f9c9b859fe779b5d46363b61ad2db9"].into(), + 15, + 0 + ) + ) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(2), + event: Event::system(system::Event::ExtrinsicSuccess) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::session(session::RawEvent::NewSession(1)) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::staking(staking::RawEvent::Reward(0)) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::treasury(treasury::RawEvent::Spending(0)) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::treasury(treasury::RawEvent::Burnt(0)) + }, + EventRecord { + phase: Phase::Finalization, + event: Event::treasury(treasury::RawEvent::Rollover(0)) + } + ]); + }); + } + + #[test] + fn full_wasm_block_import_works() { + let mut t = new_test_ext(); + + WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1().0).unwrap(); + + runtime_io::with_externalities(&mut t, || { + assert_eq!(Balances::total_balance(&alice()), 41); + assert_eq!(Balances::total_balance(&bob()), 69); + }); + + WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block2().0).unwrap(); + + runtime_io::with_externalities(&mut t, || { + assert_eq!(Balances::total_balance(&alice()), 30); + assert_eq!(Balances::total_balance(&bob()), 78); + }); + } + + #[test] + fn wasm_big_block_import_fails() { + let mut t = new_test_ext(); + + let r = WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0); + assert!(!r.is_ok()); + } + + #[test] + fn native_big_block_import_succeeds() { + let mut t = new_test_ext(); + + let r = Executor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0, true).0; + assert!(r.is_ok()); + } + + #[test] + fn native_big_block_import_fails_on_fallback() { + let mut t = new_test_ext(); + + let r = Executor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0, false).0; + assert!(!r.is_ok()); + } + + #[test] + fn panic_execution_gives_error() { + let mut t: TestExternalities = map![ + twox_128(&>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![70u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(&>::key_for(0)).to_vec() => vec![0u8; 32] + ]; + + let foreign_code = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.wasm"); + let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64))); + assert!(r.is_ok()); + let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap(); + let r = ApplyResult::decode(&mut &r[..]).unwrap(); + assert_eq!(r, Err(ApplyError::CantPay)); + } + + #[test] + fn successful_execution_gives_ok() { + let mut t: TestExternalities = map![ + twox_128(&>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(>::key()).to_vec() => vec![0u8; 8], + twox_128(&>::key_for(0)).to_vec() => vec![0u8; 32] + ]; + + let foreign_code = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.compact.wasm"); + let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64))); + assert!(r.is_ok()); + let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap(); + let r = ApplyResult::decode(&mut &r[..]).unwrap(); + assert_eq!(r, Ok(ApplyOutcome::Success)); + + runtime_io::with_externalities(&mut t, || { + assert_eq!(Balances::total_balance(&alice()), 42); + assert_eq!(Balances::total_balance(&bob()), 69); + }); + } +} diff --git a/node/network/Cargo.toml b/node/network/Cargo.toml new file mode 100644 index 00000000000..b44864ef9a9 --- /dev/null +++ b/node/network/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "node-network" +version = "0.1.0" +authors = ["Parity Technologies "] +description = "Substrate node networking protocol" + +[dependencies] +node-api = { path = "../api" } +node-consensus = { path = "../consensus" } +node-primitives = { path = "../primitives" } +substrate-primitives = { path = "../../core/primitives" } +substrate-bft = { path = "../../core/bft" } +substrate-network = { path = "../../core/network" } +futures = "0.1" +tokio = "0.1.7" +log = "0.4" +rhododendron = "0.3" diff --git a/node/network/src/consensus.rs b/node/network/src/consensus.rs new file mode 100644 index 00000000000..6a6ab77ca67 --- /dev/null +++ b/node/network/src/consensus.rs @@ -0,0 +1,297 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! The "consensus" networking code built on top of the base network service. +//! This fulfills the `node_consensus::Network` trait, providing a hook to be called +//! each time consensus begins on a new chain head. + +use bft; +use primitives::ed25519; +use substrate_network::{self as net, generic_message as msg}; +use substrate_network::consensus_gossip::ConsensusMessage; +use node_api::Api; +use node_consensus::Network; +use node_primitives::{Block, Hash, SessionKey}; +use rhododendron; + +use futures::prelude::*; +use futures::sync::mpsc; + +use std::sync::Arc; + +use tokio::runtime::TaskExecutor; + +use super::NetworkService; + +/// Sink for output BFT messages. +pub struct BftSink { + network: Arc, + parent_hash: Hash, + _marker: ::std::marker::PhantomData, +} + +impl Sink for BftSink { + type SinkItem = bft::Communication; + // TODO: replace this with the ! type when that's stabilized + type SinkError = E; + + fn start_send(&mut self, message: bft::Communication) + -> ::futures::StartSend, E> + { + let network_message = net::LocalizedBftMessage { + message: match message { + rhododendron::Communication::Consensus(c) => msg::BftMessage::Consensus(match c { + rhododendron::LocalizedMessage::Propose(proposal) => msg::SignedConsensusMessage::Propose(msg::SignedConsensusProposal { + round_number: proposal.round_number as u32, + proposal: proposal.proposal, + digest: proposal.digest, + sender: proposal.sender, + digest_signature: proposal.digest_signature.signature, + full_signature: proposal.full_signature.signature, + }), + rhododendron::LocalizedMessage::Vote(vote) => msg::SignedConsensusMessage::Vote(msg::SignedConsensusVote { + sender: vote.sender, + signature: vote.signature.signature, + vote: match vote.vote { + rhododendron::Vote::Prepare(r, h) => msg::ConsensusVote::Prepare(r as u32, h), + rhododendron::Vote::Commit(r, h) => msg::ConsensusVote::Commit(r as u32, h), + rhododendron::Vote::AdvanceRound(r) => msg::ConsensusVote::AdvanceRound(r as u32), + } + }), + }), + rhododendron::Communication::Auxiliary(justification) => { + let unchecked: bft::UncheckedJustification<_> = justification.uncheck().into(); + msg::BftMessage::Auxiliary(unchecked.into()) + } + }, + parent_hash: self.parent_hash, + }; + self.network.with_spec( + move |spec, ctx| spec.consensus_gossip.multicast_bft_message(ctx, network_message) + ); + Ok(::futures::AsyncSink::Ready) + } + + fn poll_complete(&mut self) -> ::futures::Poll<(), E> { + Ok(Async::Ready(())) + } +} + +// check signature and authority validity of message. +fn process_bft_message( + msg: msg::LocalizedBftMessage, + local_id: &SessionKey, + authorities: &[SessionKey] + ) -> Result>, bft::Error> +{ + Ok(Some(match msg.message { + msg::BftMessage::Consensus(c) => rhododendron::Communication::Consensus(match c { + msg::SignedConsensusMessage::Propose(proposal) => rhododendron::LocalizedMessage::Propose({ + if &proposal.sender == local_id { return Ok(None) } + let proposal = rhododendron::LocalizedProposal { + round_number: proposal.round_number as usize, + proposal: proposal.proposal, + digest: proposal.digest, + sender: proposal.sender, + digest_signature: ed25519::LocalizedSignature { + signature: proposal.digest_signature, + signer: ed25519::Public(proposal.sender.into()), + }, + full_signature: ed25519::LocalizedSignature { + signature: proposal.full_signature, + signer: ed25519::Public(proposal.sender.into()), + } + }; + bft::check_proposal(authorities, &msg.parent_hash, &proposal)?; + + trace!(target: "bft", "importing proposal message for round {} from {}", proposal.round_number, Hash::from(proposal.sender.0)); + proposal + }), + msg::SignedConsensusMessage::Vote(vote) => rhododendron::LocalizedMessage::Vote({ + if &vote.sender == local_id { return Ok(None) } + let vote = rhododendron::LocalizedVote { + sender: vote.sender, + signature: ed25519::LocalizedSignature { + signature: vote.signature, + signer: ed25519::Public(vote.sender.0), + }, + vote: match vote.vote { + msg::ConsensusVote::Prepare(r, h) => rhododendron::Vote::Prepare(r as usize, h), + msg::ConsensusVote::Commit(r, h) => rhododendron::Vote::Commit(r as usize, h), + msg::ConsensusVote::AdvanceRound(r) => rhododendron::Vote::AdvanceRound(r as usize), + } + }; + bft::check_vote::(authorities, &msg.parent_hash, &vote)?; + + trace!(target: "bft", "importing vote {:?} from {}", vote.vote, Hash::from(vote.sender.0)); + vote + }), + }), + msg::BftMessage::Auxiliary(a) => { + let justification = bft::UncheckedJustification::from(a); + // TODO: get proper error + let justification: Result<_, bft::Error> = bft::check_prepare_justification::(authorities, msg.parent_hash, justification) + .map_err(|_| bft::ErrorKind::InvalidJustification.into()); + rhododendron::Communication::Auxiliary(justification?) + }, + })) +} + +// task that processes all gossipped consensus messages, +// checking signatures +struct MessageProcessTask { + inner_stream: mpsc::UnboundedReceiver>, + bft_messages: mpsc::UnboundedSender>, + validators: Vec, + local_id: SessionKey, +} + +impl MessageProcessTask { + fn process_message(&self, msg: ConsensusMessage) -> Option> { + match msg { + ConsensusMessage::Bft(msg) => { + match process_bft_message(msg, &self.local_id, &self.validators[..]) { + Ok(Some(msg)) => { + if let Err(_) = self.bft_messages.unbounded_send(msg) { + // if the BFT receiving stream has ended then + // we should just bail. + trace!(target: "bft", "BFT message stream appears to have closed"); + return Some(Async::Ready(())); + } + } + Ok(None) => {} // ignored local message + Err(e) => { + debug!("Message validation failed: {:?}", e); + } + } + } + ConsensusMessage::ChainSpecific(_, _) => { + panic!("ChainSpecific messages are not allowed by the top level message handler"); + } + } + + None + } +} + +impl Future for MessageProcessTask { + type Item = (); + type Error = (); + + fn poll(&mut self) -> Poll<(), ()> { + loop { + match self.inner_stream.poll() { + Ok(Async::Ready(Some(val))) => if let Some(async) = self.process_message(val) { + return Ok(async); + }, + Ok(Async::Ready(None)) => return Ok(Async::Ready(())), + Ok(Async::NotReady) => return Ok(Async::NotReady), + Err(e) => { + debug!(target: "node-network", "Error getting consensus message: {:?}", e); + return Err(e); + }, + } + } + } +} + +/// Input stream from the consensus network. +pub struct InputAdapter { + input: mpsc::UnboundedReceiver>, +} + +impl Stream for InputAdapter { + type Item = bft::Communication; + type Error = ::node_consensus::Error; + + fn poll(&mut self) -> Poll, Self::Error> { + match self.input.poll() { + Err(_) | Ok(Async::Ready(None)) => Err(bft::InputStreamConcluded.into()), + Ok(x) => Ok(x) + } + } +} + +/// Wrapper around the network service +pub struct ConsensusNetwork

{ + network: Arc, + api: Arc

, +} + +impl

ConsensusNetwork

{ + /// Create a new consensus networking object. + pub fn new(network: Arc, api: Arc

) -> Self { + ConsensusNetwork { network, api } + } +} + +impl

Clone for ConsensusNetwork

{ + fn clone(&self) -> Self { + ConsensusNetwork { + network: self.network.clone(), + api: self.api.clone(), + } + } +} + +/// A long-lived network which can create parachain statement and BFT message routing processes on demand. +impl Network for ConsensusNetwork

{ + /// The input stream of BFT messages. Should never logically conclude. + type Input = InputAdapter; + /// The output sink of BFT messages. Messages sent here should eventually pass to all + /// current validators. + type Output = BftSink<::node_consensus::Error>; + + /// Get input and output streams of BFT messages. + fn communication_for( + &self, validators: &[SessionKey], + local_id: SessionKey, + parent_hash: Hash, + task_executor: TaskExecutor + ) -> (Self::Input, Self::Output) + { + let sink = BftSink { + network: self.network.clone(), + parent_hash, + _marker: Default::default(), + }; + + let (bft_send, bft_recv) = mpsc::unbounded(); + + // spin up a task in the background that processes all incoming statements + // TODO: propagate statements on a timer? + let process_task = self.network.with_spec(|spec, _ctx| { + spec.new_consensus(parent_hash); + MessageProcessTask { + inner_stream: spec.consensus_gossip.messages_for(parent_hash), + bft_messages: bft_send, + validators: validators.to_vec(), + local_id, + } + }); + + match process_task { + Some(task) => task_executor.spawn(task), + None => warn!(target: "node-network", "Cannot process incoming messages: network appears to be down"), + } + + (InputAdapter { input: bft_recv }, sink) + } +} + +/// Error when the network appears to be down. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NetworkDown; diff --git a/node/network/src/lib.rs b/node/network/src/lib.rs new file mode 100644 index 00000000000..5b02f4d8ff5 --- /dev/null +++ b/node/network/src/lib.rs @@ -0,0 +1,117 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate-specific network implementation. +//! +//! This manages gossip of consensus messages for BFT. + +#![warn(unused_extern_crates)] + +extern crate substrate_primitives as primitives; +extern crate substrate_bft as bft; +extern crate substrate_network; + +extern crate node_api; +extern crate node_consensus; +extern crate node_primitives; + +extern crate futures; +extern crate tokio; +extern crate rhododendron; + +#[macro_use] +extern crate log; + +pub mod consensus; + +use node_primitives::{Block, Hash, Header}; +use substrate_network::{NodeIndex, Context, Severity}; +use substrate_network::consensus_gossip::ConsensusGossip; +use substrate_network::{message, generic_message}; +use substrate_network::specialization::Specialization; +use substrate_network::StatusMessage as GenericFullStatus; + +/// Demo protocol id. +pub const PROTOCOL_ID: ::substrate_network::ProtocolId = *b"dot"; + +type FullStatus = GenericFullStatus; + +/// Specialization of the network service for the node protocol. +pub type NetworkService = ::substrate_network::Service; + + +/// Demo protocol attachment for substrate. +pub struct Protocol { + consensus_gossip: ConsensusGossip, + live_consensus: Option, +} + +impl Protocol { + /// Instantiate a node protocol handler. + pub fn new() -> Self { + Protocol { + consensus_gossip: ConsensusGossip::new(), + live_consensus: None, + } + } + + /// Note new consensus session. + fn new_consensus(&mut self, parent_hash: Hash) { + let old_consensus = self.live_consensus.take(); + self.live_consensus = Some(parent_hash); + self.consensus_gossip.collect_garbage(old_consensus.as_ref()); + } +} + +impl Specialization for Protocol { + fn status(&self) -> Vec { + Vec::new() + } + + fn on_connect(&mut self, ctx: &mut Context, who: NodeIndex, status: FullStatus) { + self.consensus_gossip.new_peer(ctx, who, status.roles); + } + + fn on_disconnect(&mut self, ctx: &mut Context, who: NodeIndex) { + self.consensus_gossip.peer_disconnected(ctx, who); + } + + fn on_message(&mut self, ctx: &mut Context, who: NodeIndex, message: message::Message) { + match message { + generic_message::Message::BftMessage(msg) => { + trace!(target: "node-network", "BFT message from {}: {:?}", who, msg); + // TODO: check signature here? what if relevant block is unknown? + self.consensus_gossip.on_bft_message(ctx, who, msg) + } + generic_message::Message::ChainSpecific(_) => { + trace!(target: "node-network", "Bad message from {}", who); + ctx.report_peer(who, Severity::Bad("Invalid node protocol message format")); + } + _ => {} + } + } + + fn on_abort(&mut self) { + self.consensus_gossip.abort(); + } + + fn maintain_peers(&mut self, _ctx: &mut Context) { + } + + fn on_block_imported(&mut self, _ctx: &mut Context, _hash: Hash, _header: &Header) { + } +} + diff --git a/node/primitives/Cargo.toml b/node/primitives/Cargo.toml new file mode 100644 index 00000000000..f6a74b058fa --- /dev/null +++ b/node/primitives/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "node-primitives" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-primitives = { path = "../../runtime/primitives", default_features = false } + +[dev-dependencies] +substrate-serializer = { path = "../../core/serializer" } +pretty_assertions = "0.4" + +[features] +default = ["std"] +std = [ + "substrate-codec-derive/std", + "substrate-codec/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-primitives/std", + "serde_derive", + "serde/std", +] diff --git a/node/primitives/src/lib.rs b/node/primitives/src/lib.rs new file mode 100644 index 00000000000..433412eaf3f --- /dev/null +++ b/node/primitives/src/lib.rs @@ -0,0 +1,94 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Low-level types used throughout the Substrate code. + +#![warn(missing_docs)] + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_runtime_std as rstd; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_primitives as primitives; +extern crate substrate_codec as codec; + +use rstd::prelude::*; +use runtime_primitives::generic; +#[cfg(feature = "std")] +use primitives::bytes; +use runtime_primitives::traits::BlakeTwo256; + +/// An index to a block. +pub type BlockNumber = u64; + +/// Alias to Ed25519 pubkey that identifies an account on the chain. This will almost +/// certainly continue to be the same as the substrate's `AuthorityId`. +pub type AccountId = ::primitives::H256; + +/// The type for looking up accounts. We don't expect more than 4 billion of them, but you +/// never know... +pub type AccountIndex = u32; + +/// Balance of an account. +pub type Balance = u64; + +/// The Ed25519 pub key of an session that belongs to an authority of the chain. This is +/// exactly equivalent to what the substrate calls an "authority". +pub type SessionKey = primitives::AuthorityId; + +/// Index of a transaction in the chain. +pub type Index = u64; + +/// A hash of some data used by the chain. +pub type Hash = primitives::H256; + +/// Alias to 512-bit hash when used in the context of a signature on the chain. +pub type Signature = runtime_primitives::Ed25519Signature; + +/// A timestamp: seconds since the unix epoch. +pub type Timestamp = u64; + +/// Header type. +pub type Header = generic::Header>; +/// Block type. +pub type Block = generic::Block; +/// Block ID. +pub type BlockId = generic::BlockId; + +/// Opaque, encoded, unchecked extrinsic. +#[derive(PartialEq, Eq, Clone, Default, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +pub struct UncheckedExtrinsic(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec); + +/// Inherent data to include in a block. +#[derive(Encode, Decode)] +pub struct InherentData { + /// Current timestamp. + pub timestamp: Timestamp, + /// Indices of offline validators. + pub offline_indices: Vec, +} diff --git a/node/runtime/Cargo.toml b/node/runtime/Cargo.toml new file mode 100644 index 00000000000..31db56e360f --- /dev/null +++ b/node/runtime/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "node-runtime" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +rustc-hex = "1.0" +hex-literal = "0.1.0" +log = { version = "0.3", optional = true } +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +safe-mix = { version = "1.0", default_features = false} +substrate-codec = { path = "../../core/codec" } +substrate-codec-derive = { path = "../../core/codec/derive" } +substrate-runtime-std = { path = "../../core/runtime-std" } +substrate-runtime-io = { path = "../../core/runtime-io" } +substrate-primitives = { path = "../../core/primitives" } +substrate-keyring = { path = "../../core/keyring" } +substrate-runtime-support = { path = "../../runtime/support" } +substrate-runtime-balances = { path = "../../runtime/balances" } +substrate-runtime-consensus = { path = "../../runtime/consensus" } +substrate-runtime-contract = { path = "../../runtime/contract" } +substrate-runtime-council = { path = "../../runtime/council" } +substrate-runtime-democracy = { path = "../../runtime/democracy" } +substrate-runtime-executive = { path = "../../runtime/executive" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } +substrate-runtime-session = { path = "../../runtime/session" } +substrate-runtime-staking = { path = "../../runtime/staking" } +substrate-runtime-system = { path = "../../runtime/system" } +substrate-runtime-timestamp = { path = "../../runtime/timestamp" } +substrate-runtime-treasury = { path = "../../runtime/treasury" } +substrate-runtime-version = { path = "../../runtime/version" } +node-primitives = { path = "../primitives" } + +[features] +default = ["std"] +std = [ + "substrate-codec/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-balances/std", + "substrate-runtime-consensus/std", + "substrate-runtime-contract/std", + "substrate-runtime-council/std", + "substrate-runtime-democracy/std", + "substrate-runtime-executive/std", + "substrate-runtime-primitives/std", + "substrate-runtime-session/std", + "substrate-runtime-staking/std", + "substrate-runtime-system/std", + "substrate-runtime-timestamp/std", + "substrate-runtime-treasury/std", + "substrate-runtime-version/std", + "node-primitives/std", + "serde_derive", + "serde/std", + "log", + "safe-mix/std" +] diff --git a/node/runtime/src/checked_block.rs b/node/runtime/src/checked_block.rs new file mode 100644 index 00000000000..bba748dd15f --- /dev/null +++ b/node/runtime/src/checked_block.rs @@ -0,0 +1,94 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Typesafe block interaction. + +use super::{Call, Block, TIMESTAMP_SET_POSITION, NOTE_OFFLINE_POSITION}; +use timestamp::Call as TimestampCall; +use consensus::Call as ConsensusCall; + +/// Provides a type-safe wrapper around a structurally valid block. +pub struct CheckedBlock { + inner: Block, + file_line: Option<(&'static str, u32)>, +} + +impl CheckedBlock { + /// Create a new checked block. Fails if the block is not structurally valid. + pub fn new(block: Block) -> Result { + let has_timestamp = block.extrinsics.get(TIMESTAMP_SET_POSITION as usize).map_or(false, |xt| { + !xt.is_signed() && match xt.function { + Call::Timestamp(TimestampCall::set(_)) => true, + _ => false, + } + }); + + if !has_timestamp { return Err(block) } + + Ok(CheckedBlock { + inner: block, + file_line: None, + }) + } + + // Creates a new checked block, asserting that it is valid. + #[doc(hidden)] + pub fn new_unchecked(block: Block, file: &'static str, line: u32) -> Self { + CheckedBlock { + inner: block, + file_line: Some((file, line)), + } + } + + /// Extract the timestamp from the block. + pub fn timestamp(&self) -> ::node_primitives::Timestamp { + let x = self.inner.extrinsics.get(TIMESTAMP_SET_POSITION as usize).and_then(|xt| match xt.function { + Call::Timestamp(TimestampCall::set(x)) => Some(x), + _ => None + }); + + match x { + Some(x) => x, + None => panic!("Invalid block asserted at {:?}", self.file_line), + } + } + + /// Extract the noted missed proposal validator indices (if any) from the block. + pub fn noted_offline(&self) -> &[u32] { + self.inner.extrinsics.get(NOTE_OFFLINE_POSITION as usize).and_then(|xt| match xt.function { + Call::Consensus(ConsensusCall::note_offline(ref x)) => Some(&x[..]), + _ => None, + }).unwrap_or(&[]) + } + + /// Convert into inner block. + pub fn into_inner(self) -> Block { self.inner } +} + +impl ::std::ops::Deref for CheckedBlock { + type Target = Block; + + fn deref(&self) -> &Block { &self.inner } +} + +/// Assert that a block is structurally valid. May lead to panic in the future +/// in case it isn't. +#[macro_export] +macro_rules! assert_node_block { + ($block: expr) => { + $crate::CheckedBlock::new_unchecked($block, file!(), line!()) + } +} diff --git a/node/runtime/src/lib.rs b/node/runtime/src/lib.rs new file mode 100644 index 00000000000..948507d5b2e --- /dev/null +++ b/node/runtime/src/lib.rs @@ -0,0 +1,386 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! The Substrate runtime. This can be compiled with ``#[no_std]`, ready for Wasm. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[macro_use] +extern crate substrate_runtime_io as runtime_io; + +#[macro_use] +extern crate substrate_runtime_support; + +#[macro_use] +extern crate substrate_runtime_primitives as runtime_primitives; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[cfg(feature = "std")] +extern crate serde; + +extern crate substrate_codec as codec; +extern crate substrate_primitives; + +#[macro_use] +extern crate substrate_codec_derive; + +#[cfg_attr(not(feature = "std"), macro_use)] +extern crate substrate_runtime_std as rstd; +extern crate substrate_runtime_balances as balances; +extern crate substrate_runtime_consensus as consensus; +extern crate substrate_runtime_contract as contract; +extern crate substrate_runtime_council as council; +extern crate substrate_runtime_democracy as democracy; +extern crate substrate_runtime_executive as executive; +extern crate substrate_runtime_session as session; +extern crate substrate_runtime_staking as staking; +extern crate substrate_runtime_system as system; +extern crate substrate_runtime_timestamp as timestamp; +extern crate substrate_runtime_treasury as treasury; +#[macro_use] +extern crate substrate_runtime_version as version; +extern crate node_primitives; + +#[cfg(feature = "std")] +mod checked_block; + +use rstd::prelude::*; +use substrate_primitives::u32_trait::{_2, _4}; +use codec::{Encode, Decode, Input}; +use node_primitives::{AccountId, AccountIndex, Balance, BlockNumber, Hash, Index, SessionKey, Signature, InherentData}; +use runtime_primitives::generic; +use runtime_primitives::traits::{Convert, BlakeTwo256, DigestItem}; +use version::RuntimeVersion; +use council::{motions as council_motions, voting as council_voting}; + +#[cfg(any(feature = "std", test))] +pub use runtime_primitives::BuildStorage; +pub use consensus::Call as ConsensusCall; +pub use timestamp::Call as TimestampCall; +pub use runtime_primitives::Permill; +#[cfg(any(feature = "std", test))] +pub use checked_block::CheckedBlock; + +const TIMESTAMP_SET_POSITION: u32 = 0; +const NOTE_OFFLINE_POSITION: u32 = 1; + +// Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +/// Runtime type used to collate and parameterize the various modules. +pub struct Runtime; + +/// Runtime version. +pub const VERSION: RuntimeVersion = RuntimeVersion { + spec_name: ver_str!("node"), + impl_name: ver_str!("substrate-node"), + authoring_version: 1, + spec_version: 1, + impl_version: 0, +}; + +impl system::Trait for Runtime { + type Origin = Origin; + type Index = Index; + type BlockNumber = BlockNumber; + type Hash = Hash; + type Hashing = BlakeTwo256; + type Digest = generic::Digest; + type AccountId = AccountId; + type Header = generic::Header; + type Event = Event; +} + +/// System module for this concrete runtime. +pub type System = system::Module; + +impl balances::Trait for Runtime { + type Balance = Balance; + type AccountIndex = AccountIndex; + type OnFreeBalanceZero = (Staking, Contract); + type EnsureAccountLiquid = Staking; + type Event = Event; +} + +/// Staking module for this concrete runtime. +pub type Balances = balances::Module; + +impl consensus::Trait for Runtime { + const NOTE_OFFLINE_POSITION: u32 = NOTE_OFFLINE_POSITION; + type Log = Log; + type SessionKey = SessionKey; + type OnOfflineValidator = Staking; +} + +/// Consensus module for this concrete runtime. +pub type Consensus = consensus::Module; + +impl timestamp::Trait for Runtime { + const TIMESTAMP_SET_POSITION: u32 = TIMESTAMP_SET_POSITION; + type Moment = u64; +} + +/// Timestamp module for this concrete runtime. +pub type Timestamp = timestamp::Module; + +/// Session key conversion. +pub struct SessionKeyConversion; +impl Convert for SessionKeyConversion { + fn convert(a: AccountId) -> SessionKey { + a.0.into() + } +} + +impl session::Trait for Runtime { + type ConvertAccountIdToSessionKey = SessionKeyConversion; + type OnSessionChange = Staking; + type Event = Event; +} + +/// Session module for this concrete runtime. +pub type Session = session::Module; + +impl staking::Trait for Runtime { + type OnRewardMinted = Treasury; + type Event = Event; +} + +/// Staking module for this concrete runtime. +pub type Staking = staking::Module; + +impl democracy::Trait for Runtime { + type Proposal = Call; + type Event = Event; +} + +/// Democracy module for this concrete runtime. +pub type Democracy = democracy::Module; + +impl council::Trait for Runtime { + type Event = Event; +} + +/// Council module for this concrete runtime. +pub type Council = council::Module; + +impl council::voting::Trait for Runtime { + type Event = Event; +} + +/// Council voting module for this concrete runtime. +pub type CouncilVoting = council::voting::Module; + +impl council::motions::Trait for Runtime { + type Origin = Origin; + type Proposal = Call; + type Event = Event; +} + +/// Council motions module for this concrete runtime. +pub type CouncilMotions = council_motions::Module; + +impl treasury::Trait for Runtime { + type ApproveOrigin = council_motions::EnsureMembers<_4>; + type RejectOrigin = council_motions::EnsureMembers<_2>; + type Event = Event; +} + +/// Treasury module for this concrete runtime. +pub type Treasury = treasury::Module; + +/// Address calculated from the code (of the constructor), input data to the constructor +/// and account id which requested the account creation. +/// +/// Formula: `blake2_256(blake2_256(code) + blake2_256(data) + origin)` +pub struct DetermineContractAddress; +impl contract::ContractAddressFor for DetermineContractAddress { + fn contract_address_for(code: &[u8], data: &[u8], origin: &AccountId) -> AccountId { + use runtime_primitives::traits::Hash; + + let code_hash = BlakeTwo256::hash(code); + let data_hash = BlakeTwo256::hash(data); + let mut buf = [0u8, 32 + 32 + 32]; + &mut buf[0..32].copy_from_slice(&code_hash); + &mut buf[32..64].copy_from_slice(&data_hash); + &mut buf[64..96].copy_from_slice(origin); + AccountId::from(BlakeTwo256::hash(&buf[..])) + } +} + +impl contract::Trait for Runtime { + type Gas = u64; + type DetermineContractAddress = DetermineContractAddress; +} + +/// Contract module for this concrete runtime. +pub type Contract = contract::Module; + +impl_outer_event! { + pub enum Event for Runtime { + //consensus, + balances, + //timetstamp, + session, + staking, + democracy, + council, + council_voting, + council_motions, + treasury + } +} + +impl_outer_log! { + pub enum Log(InternalLog: DigestItem) for Runtime { + consensus(AuthoritiesChange) + } +} + +impl_outer_origin! { + pub enum Origin for Runtime { + council_motions + } +} + +impl_outer_dispatch! { + pub enum Call where origin: Origin { + Consensus, + Balances, + Timestamp, + Session, + Staking, + Democracy, + Council, + CouncilVoting, + CouncilMotions, + Treasury, + Contract, + } +} + +impl_outer_config! { + pub struct GenesisConfig for Runtime { + SystemConfig => system, + ConsensusConfig => consensus, + BalancesConfig => balances, + TimestampConfig => timestamp, + SessionConfig => session, + StakingConfig => staking, + DemocracyConfig => democracy, + CouncilConfig => council, + TreasuryConfig => treasury, + } +} + +type AllModules = ( + Consensus, + Balances, + Timestamp, + Session, + Staking, + Democracy, + Council, + CouncilVoting, + CouncilMotions, + Treasury, + Contract, +); + +impl_json_metadata!( + for Runtime with modules + system::Module with Storage, + consensus::Module with Storage, + balances::Module with Storage, + timestamp::Module with Storage, + session::Module with Storage, + staking::Module with Storage, + democracy::Module with Storage, + council::Module with Storage, + council_voting::Module with Storage, + council_motions::Module with Storage, + treasury::Module with Storage, + contract::Module with Storage, +); + +impl DigestItem for Log { + type AuthorityId = SessionKey; + + fn as_authorities_change(&self) -> Option<&[Self::AuthorityId]> { + match self.0 { + InternalLog::consensus(ref item) => item.as_authorities_change(), + } + } +} + +/// The address format for describing accounts. +pub use balances::address::Address as RawAddress; +/// The address format for describing accounts. +pub type Address = balances::Address; +/// Block header type as expected by this runtime. +pub type Header = generic::Header; +/// Block type as expected by this runtime. +pub type Block = generic::Block; +/// BlockId type as expected by this runtime. +pub type BlockId = generic::BlockId; +/// Unchecked extrinsic type as expected by this runtime. +pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; +/// Extrinsic type that has already been checked. +pub type CheckedExtrinsic = generic::CheckedExtrinsic; +/// Executive: handles dispatch to the various modules. +pub type Executive = executive::Executive; + +pub mod api { + impl_stubs!( + version => |()| super::VERSION, + json_metadata => |()| super::Runtime::json_metadata(), + authorities => |()| super::Consensus::authorities(), + initialise_block => |header| super::Executive::initialise_block(&header), + apply_extrinsic => |extrinsic| super::Executive::apply_extrinsic(extrinsic), + execute_block => |block| super::Executive::execute_block(block), + finalise_block => |()| super::Executive::finalise_block(), + inherent_extrinsics => |(inherent, spec_version)| super::inherent_extrinsics(inherent, spec_version), + validator_count => |()| super::Session::validator_count(), + validators => |()| super::Session::validators(), + timestamp => |()| super::Timestamp::get(), + random_seed => |()| super::System::random_seed(), + account_nonce => |account| super::System::account_nonce(&account), + lookup_address => |address| super::Balances::lookup_address(address) + ); +} + +/// Produces the list of inherent extrinsics. +fn inherent_extrinsics(data: InherentData, _spec_version: u32) -> Vec { + let make_inherent = |function| UncheckedExtrinsic { + signature: Default::default(), + function, + index: 0, + }; + + let mut inherent = vec![ + make_inherent(Call::Timestamp(TimestampCall::set(data.timestamp))), + ]; + + if !data.offline_indices.is_empty() { + inherent.push(make_inherent( + Call::Consensus(ConsensusCall::note_offline(data.offline_indices)) + )); + } + + inherent +} diff --git a/node/runtime/wasm/Cargo.lock b/node/runtime/wasm/Cargo.lock new file mode 100644 index 00000000000..78ff2c0cedd --- /dev/null +++ b/node/runtime/wasm/Cargo.lock @@ -0,0 +1,568 @@ +[[package]] +name = "arrayvec" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byteorder" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "crunchy" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fixed-hash" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hashdb" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hex-literal" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hex-literal-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hex-literal-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "integer-sqrt" +version = "0.1.0" +source = "git+https://github.com/paritytech/integer-sqrt-rs.git#886e9cb983c46498003878afe965d55caa762025" + +[[package]] +name = "log" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "node-primitives" +version = "0.1.0" +dependencies = [ + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "node-runtime" +version = "0.1.0" +dependencies = [ + "integer-sqrt 0.1.0 (git+https://github.com/paritytech/integer-sqrt-rs.git)", + "node-primitives 0.1.0", + "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-balances 0.1.0", + "substrate-runtime-consensus 0.1.0", + "substrate-runtime-contract 0.1.0", + "substrate-runtime-council 0.1.0", + "substrate-runtime-democracy 0.1.0", + "substrate-runtime-executive 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-session 0.1.0", + "substrate-runtime-staking 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", + "substrate-runtime-timestamp 0.1.0", + "substrate-runtime-treasury 0.1.0", + "substrate-runtime-version 0.1.0", +] + +[[package]] +name = "nodrop" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num-traits" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "parity-wasm" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "plain_hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack-impl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack-impl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pwasm-alloc" +version = "0.1.0" +dependencies = [ + "pwasm-libc 0.1.0", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pwasm-libc" +version = "0.1.0" + +[[package]] +name = "pwasm-utils" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-wasm 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-hex" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "safe-mix" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "substrate-codec" +version = "0.1.0" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-codec-derive" +version = "0.1.0" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-primitives" +version = "0.1.0" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-runtime-std 0.1.0", + "uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-runtime-balances" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-consensus" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-contract" +version = "0.1.0" +dependencies = [ + "parity-wasm 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pwasm-utils 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-balances 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-sandbox 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-council" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "integer-sqrt 0.1.0 (git+https://github.com/paritytech/integer-sqrt-rs.git)", + "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-balances 0.1.0", + "substrate-runtime-consensus 0.1.0", + "substrate-runtime-democracy 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-democracy" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-balances 0.1.0", + "substrate-runtime-consensus 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-executive" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-io" +version = "0.1.0" +dependencies = [ + "hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "substrate-runtime-primitives" +version = "0.1.0" +dependencies = [ + "integer-sqrt 0.1.0 (git+https://github.com/paritytech/integer-sqrt-rs.git)", + "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", +] + +[[package]] +name = "substrate-runtime-sandbox" +version = "0.1.0" +dependencies = [ + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "substrate-runtime-session" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-consensus 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", + "substrate-runtime-timestamp 0.1.0", +] + +[[package]] +name = "substrate-runtime-staking" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-balances 0.1.0", + "substrate-runtime-consensus 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-sandbox 0.1.0", + "substrate-runtime-session 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", + "substrate-runtime-timestamp 0.1.0", +] + +[[package]] +name = "substrate-runtime-std" +version = "0.1.0" +dependencies = [ + "pwasm-alloc 0.1.0", + "pwasm-libc 0.1.0", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "substrate-runtime-support" +version = "0.1.0" +dependencies = [ + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-std 0.1.0", +] + +[[package]] +name = "substrate-runtime-system" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", +] + +[[package]] +name = "substrate-runtime-timestamp" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-consensus 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-treasury" +version = "0.1.0" +dependencies = [ + "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-primitives 0.1.0", + "substrate-runtime-balances 0.1.0", + "substrate-runtime-io 0.1.0", + "substrate-runtime-primitives 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", + "substrate-runtime-system 0.1.0", +] + +[[package]] +name = "substrate-runtime-version" +version = "0.1.0" +dependencies = [ + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-codec 0.1.0", + "substrate-codec-derive 0.1.0", + "substrate-runtime-std 0.1.0", + "substrate-runtime-support 0.1.0", +] + +[[package]] +name = "syn" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "uint" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9" +"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" +"checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" +"checksum fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0d5ec8112f00ea8a483e04748a85522184418fd1cf02890b626d8fc28683f7de" +"checksum hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f1c71fc577cde89b3345d5f2880fecaf462a32e96c619f431279bdaf1ba5ddb1" +"checksum hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4da5f0e01bd8a71a224a4eedecaacfcabda388dbb7a80faf04d3514287572d95" +"checksum hex-literal-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1d340b6514f232f6db1bd16db65302a5278a04fef9ce867cb932e7e5fa21130a" +"checksum integer-sqrt 0.1.0 (git+https://github.com/paritytech/integer-sqrt-rs.git)" = "" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" +"checksum num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775393e285254d2f5004596d69bb8bc1149754570dcc08cf30cabeba67955e28" +"checksum parity-wasm 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e1c91199d14bd5b78ecade323d4a891d094799749c1b9e82d9c590c2e2849a40" +"checksum plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95fa6386b1d34aaf0adb9b7dd2885dbe7c34190e6263785e5a7ec2b19044a90f" +"checksum proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ba8d4f9257b85eb6cdf13f055cea3190520aab1409ca2ab43493ea4820c25f0" +"checksum proc-macro-hack-impl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d5cb6f960ad471404618e9817c0e5d10b1ae74cfdf01fab89ea0641fe7fb2892" +"checksum proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1fa93823f53cfd0f5ac117b189aed6cfdfb2cfc0a9d82e956dd7927595ed7d46" +"checksum pwasm-utils 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "efd695333cfae6e9dbe2703a6d040e252b57a6fc3b9a65c712615ac042b2e0c5" +"checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035" +"checksum rustc-hex 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b03280c2813907a030785570c577fb27d3deec8da4c18566751ade94de0ace" +"checksum rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a54aa04a10c68c1c4eacb4337fd883b435997ede17a9385784b990777686b09a" +"checksum safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f7bf422d23a88c16d5090d455f182bc99c60af4df6a345c63428acf5129e347" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)" = "fba5be06346c5200249c8c8ca4ccba4a09e8747c71c16e420bd359a0db4d8f91" +"checksum syn 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6dfd71b2be5a58ee30a6f8ea355ba8290d397131c00dfa55c3d34e6e13db5101" +"checksum uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "754ba11732b9161b94c41798e5197e5e75388d012f760c42adb5000353e98646" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" diff --git a/node/runtime/wasm/Cargo.toml b/node/runtime/wasm/Cargo.toml new file mode 100644 index 00000000000..db62ba1d175 --- /dev/null +++ b/node/runtime/wasm/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "node-runtime" +version = "0.1.0" +authors = ["Parity Technologies "] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +integer-sqrt = { git = "https://github.com/paritytech/integer-sqrt-rs.git", branch = "master" } +safe-mix = { version = "1.0", default_features = false} +node-primitives = { path = "../../primitives", default-features = false } +substrate-codec-derive = { path = "../../../core/codec/derive" } +substrate-codec = { path = "../../../core/codec", default-features = false } +substrate-primitives = { path = "../../../core/primitives", default-features = false } +substrate-runtime-std = { path = "../../../core/runtime-std", default-features = false } +substrate-runtime-io = { path = "../../../core/runtime-io", default-features = false } +substrate-runtime-support = { path = "../../../runtime/support", default-features = false } +substrate-runtime-balances = { path = "../../../runtime/balances", default-features = false } +substrate-runtime-consensus = { path = "../../../runtime/consensus", default-features = false } +substrate-runtime-contract = { path = "../../../runtime/contract", default-features = false } +substrate-runtime-council = { path = "../../../runtime/council", default-features = false } +substrate-runtime-democracy = { path = "../../../runtime/democracy", default-features = false } +substrate-runtime-executive = { path = "../../../runtime/executive", default-features = false } +substrate-runtime-primitives = { path = "../../../runtime/primitives", default-features = false } +substrate-runtime-session = { path = "../../../runtime/session", default-features = false } +substrate-runtime-staking = { path = "../../../runtime/staking", default-features = false } +substrate-runtime-system = { path = "../../../runtime/system", default-features = false } +substrate-runtime-timestamp = { path = "../../../runtime/timestamp", default-features = false } +substrate-runtime-treasury = { path = "../../../runtime/treasury", default-features = false } +substrate-runtime-version = { path = "../../../runtime/version", default-features = false } + +[profile.release] +panic = "abort" +lto = true + +[workspace] +members = [] diff --git a/node/runtime/wasm/build.sh b/node/runtime/wasm/build.sh new file mode 100755 index 00000000000..9fe3f0ca10d --- /dev/null +++ b/node/runtime/wasm/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e + +cargo +nightly build --target=wasm32-unknown-unknown --release +for i in node_runtime +do + wasm-gc target/wasm32-unknown-unknown/release/$i.wasm target/wasm32-unknown-unknown/release/$i.compact.wasm +done diff --git a/node/runtime/wasm/src b/node/runtime/wasm/src new file mode 120000 index 00000000000..5cd551cf269 --- /dev/null +++ b/node/runtime/wasm/src @@ -0,0 +1 @@ +../src \ No newline at end of file diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml new file mode 100644 index 00000000000..5890c2ab745 --- /dev/null +++ b/node/service/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "node-service" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +parking_lot = "0.4" +error-chain = "0.12" +lazy_static = "1.0" +log = "0.3" +slog = "^2" +tokio = "0.1.7" +hex-literal = "0.1" +node-api = { path = "../api" } +node-primitives = { path = "../primitives" } +node-runtime = { path = "../runtime" } +node-executor = { path = "../executor" } +node-consensus = { path = "../consensus" } +node-network = { path = "../network" } +node-transaction-pool = { path = "../transaction-pool" } +substrate-runtime-io = { path = "../../core/runtime-io" } +substrate-primitives = { path = "../../core/primitives" } +substrate-network = { path = "../../core/network" } +substrate-client = { path = "../../core/client" } +substrate-service = { path = "../../core/service" } +substrate-telemetry = { path = "../../core/telemetry" } diff --git a/node/service/src/chain_spec.rs b/node/service/src/chain_spec.rs new file mode 100644 index 00000000000..b26de88f759 --- /dev/null +++ b/node/service/src/chain_spec.rs @@ -0,0 +1,209 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate chain configurations. + +use primitives::{AuthorityId, ed25519}; +use node_runtime::{GenesisConfig, ConsensusConfig, CouncilConfig, DemocracyConfig, + SessionConfig, StakingConfig, TimestampConfig, BalancesConfig, TreasuryConfig, Permill}; +use service::ChainSpec; + +const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; + +pub fn testnet_config() -> Result, String> { + //ChainSpec::from_embedded(include_bytes!("../res/node.json")) + Ok(staging_testnet_config()) +} + +fn staging_testnet_config_genesis() -> GenesisConfig { + let initial_authorities = vec![ + hex!["82c39b31a2b79a90f8e66e7a77fdb85a4ed5517f2ae39f6a80565e8ecae85cf5"].into(), + hex!["4de37a07567ebcbf8c64568428a835269a566723687058e017b6d69db00a77e7"].into(), + hex!["063d7787ebca768b7445dfebe7d62cbb1625ff4dba288ea34488da266dd6dca5"].into(), + hex!["8101764f45778d4980dadaceee6e8af2517d3ab91ac9bec9cd1714fa5994081c"].into(), + ]; + let endowed_accounts = vec![ + hex!["f295940fa750df68a686fcf4abd4111c8a9c5a5a5a83c4c8639c451a94a7adfd"].into(), + ]; + GenesisConfig { + consensus: Some(ConsensusConfig { + code: include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.compact.wasm").to_vec(), // TODO change + authorities: initial_authorities.clone(), + }), + system: None, + balances: Some(BalancesConfig { + transaction_base_fee: 100, + transaction_byte_fee: 1, + existential_deposit: 500, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + balances: endowed_accounts.iter().map(|&k|(k, 1u64 << 60)).collect(), + }), + session: Some(SessionConfig { + validators: initial_authorities.iter().cloned().map(Into::into).collect(), + session_length: 60, // that's 5 minutes per session. + }), + staking: Some(StakingConfig { + current_era: 0, + intentions: initial_authorities.iter().cloned().map(Into::into).collect(), + offline_slash: 10000, + session_reward: 100, + validator_count: 12, + sessions_per_era: 12, // 1 hour per era + bonding_duration: 24 * 60 * 12, // 1 day per bond. + offline_slash_grace: 4, + minimum_validator_count: 4, + }), + democracy: Some(DemocracyConfig { + launch_period: 12 * 60 * 24, // 1 day per public referendum + voting_period: 12 * 60 * 24 * 3, // 3 days to discuss & vote on an active referendum + minimum_deposit: 5000, // 12000 as the minimum deposit for a referendum + }), + council: Some(CouncilConfig { + active_council: vec![], + candidacy_bond: 5000, // 5000 to become a council candidate + voter_bond: 1000, // 1000 down to vote for a candidate + present_slash_per_voter: 1, // slash by 1 per voter for an invalid presentation. + carry_count: 6, // carry over the 6 runners-up to the next council election + presentation_duration: 12 * 60 * 24, // one day for presenting winners. + approval_voting_period: 12 * 60 * 24 * 2, // two days period between possible council elections. + term_duration: 12 * 60 * 24 * 24, // 24 day term duration for the council. + desired_seats: 0, // start with no council: we'll raise this once the stake has been dispersed a bit. + inactive_grace_period: 1, // one addition vote should go by before an inactive voter can be reaped. + + cooloff_period: 12 * 60 * 24 * 4, // 4 day cooling off period if council member vetoes a proposal. + voting_period: 12 * 60 * 24, // 1 day voting period for council members. + }), + timestamp: Some(TimestampConfig { + period: 5, // 5 second block time. + }), + treasury: Some(TreasuryConfig { + proposal_bond: Permill::from_percent(5), + proposal_bond_minimum: 1_000_000, + spend_period: 12 * 60 * 24, + burn: Permill::from_percent(50), + }), + } +} + +/// Staging testnet config. +pub fn staging_testnet_config() -> ChainSpec { + let boot_nodes = vec![]; + ChainSpec::from_genesis( + "Staging Testnet", + "staging_testnet", + staging_testnet_config_genesis, + boot_nodes, + Some(STAGING_TELEMETRY_URL.into()), + ) +} + +fn testnet_genesis(initial_authorities: Vec) -> GenesisConfig { + let endowed_accounts = vec![ + ed25519::Pair::from_seed(b"Alice ").public().0.into(), + ed25519::Pair::from_seed(b"Bob ").public().0.into(), + ed25519::Pair::from_seed(b"Charlie ").public().0.into(), + ed25519::Pair::from_seed(b"Dave ").public().0.into(), + ed25519::Pair::from_seed(b"Eve ").public().0.into(), + ed25519::Pair::from_seed(b"Ferdie ").public().0.into(), + ]; + GenesisConfig { + consensus: Some(ConsensusConfig { + code: include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.compact.wasm").to_vec(), + authorities: initial_authorities.clone(), + }), + system: None, + balances: Some(BalancesConfig { + transaction_base_fee: 1, + transaction_byte_fee: 0, + existential_deposit: 500, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + balances: endowed_accounts.iter().map(|&k|(k, (1u64 << 60))).collect(), + }), + session: Some(SessionConfig { + validators: initial_authorities.iter().cloned().map(Into::into).collect(), + session_length: 10, + }), + staking: Some(StakingConfig { + current_era: 0, + intentions: initial_authorities.iter().cloned().map(Into::into).collect(), + minimum_validator_count: 1, + validator_count: 2, + sessions_per_era: 5, + bonding_duration: 2 * 60 * 12, + offline_slash: 0, + session_reward: 0, + offline_slash_grace: 0, + }), + democracy: Some(DemocracyConfig { + launch_period: 9, + voting_period: 18, + minimum_deposit: 10, + }), + council: Some(CouncilConfig { + active_council: endowed_accounts.iter() + .filter(|a| initial_authorities.iter().find(|&b| a.0 == b.0).is_none()) + .map(|a| (a.clone(), 1000000)).collect(), + candidacy_bond: 10, + voter_bond: 2, + present_slash_per_voter: 1, + carry_count: 4, + presentation_duration: 10, + approval_voting_period: 20, + term_duration: 1000000, + desired_seats: (endowed_accounts.len() - initial_authorities.len()) as u32, + inactive_grace_period: 1, + + cooloff_period: 75, + voting_period: 20, + }), + timestamp: Some(TimestampConfig { + period: 5, // 5 second block time. + }), + treasury: Some(TreasuryConfig { + proposal_bond: Permill::from_percent(5), + proposal_bond_minimum: 1_000_000, + spend_period: 12 * 60 * 24, + burn: Permill::from_percent(50), + }), + } +} + +fn development_config_genesis() -> GenesisConfig { + testnet_genesis(vec![ + ed25519::Pair::from_seed(b"Alice ").public().into(), + ]) +} + +/// Development config (single validator Alice) +pub fn development_config() -> ChainSpec { + ChainSpec::from_genesis("Development", "development", development_config_genesis, vec![], None) +} + +fn local_testnet_genesis() -> GenesisConfig { + testnet_genesis(vec![ + ed25519::Pair::from_seed(b"Alice ").public().into(), + ed25519::Pair::from_seed(b"Bob ").public().into(), + ]) +} + +/// Local testnet config (multivalidator Alice + Bob) +pub fn local_testnet_config() -> ChainSpec { + ChainSpec::from_genesis("Local Testnet", "local_testnet", local_testnet_genesis, vec![], None) +} diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs new file mode 100644 index 00000000000..469349d1c4a --- /dev/null +++ b/node/service/src/lib.rs @@ -0,0 +1,213 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![warn(unused_extern_crates)] + +//! Substrate service. Specialized wrapper over substrate service. + +extern crate node_api; +extern crate node_primitives; +extern crate node_runtime; +extern crate node_executor; +extern crate node_network; +extern crate node_transaction_pool as transaction_pool; +extern crate node_consensus as consensus; +extern crate substrate_primitives as primitives; +extern crate substrate_network as network; +extern crate substrate_client as client; +extern crate substrate_service as service; +extern crate tokio; + +#[macro_use] +extern crate log; +#[macro_use] +extern crate hex_literal; + +pub mod chain_spec; + +use std::sync::Arc; + +use transaction_pool::TransactionPool; +use node_api::Api; +use node_primitives::{Block, Hash}; +use node_runtime::GenesisConfig; +use client::Client; +use node_network::{Protocol as DemoProtocol, consensus::ConsensusNetwork}; +use tokio::runtime::TaskExecutor; +use service::FactoryFullConfiguration; +use primitives::{Blake2Hasher, RlpCodec}; + +pub use service::{Roles, PruningMode, ExtrinsicPoolOptions, + ErrorKind, Error, ComponentBlock, LightComponents, FullComponents}; +pub use client::ExecutionStrategy; + +/// Specialised `ChainSpec`. +pub type ChainSpec = service::ChainSpec; +/// Client type for specialised `Components`. +pub type ComponentClient = Client<::Backend, ::Executor, Block>; +pub type NetworkService = network::Service::NetworkProtocol, Hash>; + +/// A collection of type to generalise specific components over full / light client. +pub trait Components: service::Components { + /// Demo API. + type Api: 'static + Api + Send + Sync; + /// Client backend. + type Backend: 'static + client::backend::Backend; + /// Client executor. + type Executor: 'static + client::CallExecutor + Send + Sync; +} + +impl Components for service::LightComponents { + type Api = service::LightClient; + type Executor = service::LightExecutor; + type Backend = service::LightBackend; +} + +impl Components for service::FullComponents { + type Api = service::FullClient; + type Executor = service::FullExecutor; + type Backend = service::FullBackend; +} + +/// All configuration for the node. +pub type Configuration = FactoryFullConfiguration; + +/// Demo-specific configuration. +#[derive(Default)] +pub struct CustomConfiguration; + +/// Config for the substrate service. +pub struct Factory; + +impl service::ServiceFactory for Factory { + type Block = Block; + type ExtrinsicHash = Hash; + type NetworkProtocol = DemoProtocol; + type RuntimeDispatch = node_executor::Executor; + type FullExtrinsicPoolApi = transaction_pool::ChainApi>; + type LightExtrinsicPoolApi = transaction_pool::ChainApi>; + type Genesis = GenesisConfig; + type Configuration = CustomConfiguration; + + const NETWORK_PROTOCOL_ID: network::ProtocolId = ::node_network::PROTOCOL_ID; + + fn build_full_extrinsic_pool(config: ExtrinsicPoolOptions, client: Arc>) + -> Result>, Error> + { + Ok(TransactionPool::new(config, transaction_pool::ChainApi::new(client))) + } + + fn build_light_extrinsic_pool(config: ExtrinsicPoolOptions, client: Arc>) + -> Result>, Error> + { + Ok(TransactionPool::new(config, transaction_pool::ChainApi::new(client))) + } + + fn build_network_protocol(_config: &Configuration) + -> Result + { + Ok(DemoProtocol::new()) + } +} + +/// Demo service. +pub struct Service { + inner: service::Service, + client: Arc>, + network: Arc, + api: Arc<::Api>, + _consensus: Option, +} + +impl Service { + pub fn client(&self) -> Arc> { + self.client.clone() + } + + pub fn network(&self) -> Arc { + self.network.clone() + } + + pub fn api(&self) -> Arc<::Api> { + self.api.clone() + } +} + +/// Creates light client and register protocol with the network service +pub fn new_light(config: Configuration, executor: TaskExecutor) + -> Result>, Error> +{ + let service = service::Service::>::new(config, executor.clone())?; + let api = service.client(); + Ok(Service { + client: service.client(), + network: service.network(), + api: api, + inner: service, + _consensus: None, + }) +} + +/// Creates full client and register protocol with the network service +pub fn new_full(config: Configuration, executor: TaskExecutor) + -> Result>, Error> +{ + let is_validator = (config.roles & Roles::AUTHORITY) == Roles::AUTHORITY; + let service = service::Service::>::new(config, executor.clone())?; + // Spin consensus service if configured + let consensus = if is_validator { + // Load the first available key + let key = service.keystore().load(&service.keystore().contents()?[0], "")?; + info!("Using authority key {}", key.public()); + + let client = service.client(); + + let consensus_net = ConsensusNetwork::new(service.network(), client.clone()); + Some(consensus::Service::new( + client.clone(), + client.clone(), + consensus_net, + service.extrinsic_pool(), + executor, + key, + )) + } else { + None + }; + + Ok(Service { + client: service.client(), + network: service.network(), + api: service.client(), + inner: service, + _consensus: consensus, + }) +} + +/// Creates bare client without any networking. +pub fn new_client(config: Configuration) + -> Result>>, Error> +{ + service::new_client::(config) +} + +impl ::std::ops::Deref for Service { + type Target = service::Service; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} diff --git a/node/src/main.rs b/node/src/main.rs new file mode 100644 index 00000000000..dca5e5ed909 --- /dev/null +++ b/node/src/main.rs @@ -0,0 +1,69 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate CLI + +#![warn(missing_docs)] + +extern crate node_cli as cli; +extern crate ctrlc; +extern crate futures; + +#[macro_use] +extern crate error_chain; + +use cli::VersionInfo; +use futures::sync::oneshot; +use futures::{future, Future}; + +use std::cell::RefCell; + +mod vergen { + #![allow(unused)] + include!(concat!(env!("OUT_DIR"), "/version.rs")); +} + +// handles ctrl-c +struct Exit; +impl cli::IntoExit for Exit { + type Exit = future::MapErr, fn(oneshot::Canceled) -> ()>; + fn into_exit(self) -> Self::Exit { + // can't use signal directly here because CtrlC takes only `Fn`. + let (exit_send, exit) = oneshot::channel(); + + let exit_send_cell = RefCell::new(Some(exit_send)); + ctrlc::set_handler(move || { + if let Some(exit_send) = exit_send_cell.try_borrow_mut().expect("signal handler not reentrant; qed").take() { + exit_send.send(()).expect("Error sending exit notification"); + } + }).expect("Error setting Ctrl-C handler"); + + exit.map_err(drop) + } +} + +quick_main!(run); + +fn run() -> cli::error::Result<()> { + let version = VersionInfo { + commit: vergen::short_sha(), + version: env!("CARGO_PKG_VERSION"), + executable_name: "substrate", + author: "Parity Team ", + description: "Generic substrate node", + }; + cli::run(::std::env::args(), Exit, version) +} diff --git a/node/transaction-pool/Cargo.toml b/node/transaction-pool/Cargo.toml new file mode 100644 index 00000000000..0db2796437c --- /dev/null +++ b/node/transaction-pool/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "node-transaction-pool" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +log = "0.3.0" +error-chain = "0.12" +parking_lot = "0.4" +node-api = { path = "../api" } +node-primitives = { path = "../primitives" } +node-runtime = { path = "../runtime" } +substrate-client = { path = "../../core/client" } +substrate-codec = { path = "../../core/codec" } +substrate-keyring = { path = "../../core/keyring" } +substrate-extrinsic-pool = { path = "../../core/extrinsic-pool" } +substrate-primitives = { path = "../../core/primitives" } +substrate-runtime-primitives = { path = "../../runtime/primitives" } diff --git a/node/transaction-pool/src/error.rs b/node/transaction-pool/src/error.rs new file mode 100644 index 00000000000..7d1712a55ac --- /dev/null +++ b/node/transaction-pool/src/error.rs @@ -0,0 +1,73 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use extrinsic_pool; +use node_api; +use primitives::Hash; +use runtime::{Address, UncheckedExtrinsic}; + +error_chain! { + links { + Pool(extrinsic_pool::Error, extrinsic_pool::ErrorKind); + Api(node_api::Error, node_api::ErrorKind); + } + errors { + /// Unexpected extrinsic format submitted + InvalidExtrinsicFormat { + description("Invalid extrinsic format."), + display("Invalid extrinsic format."), + } + /// Attempted to queue an inherent transaction. + IsInherent(xt: UncheckedExtrinsic) { + description("Inherent transactions cannot be queued."), + display("Inherent transactions cannot be queued."), + } + /// Attempted to queue a transaction with bad signature. + BadSignature(e: &'static str) { + description("Transaction had bad signature."), + display("Transaction had bad signature: {}", e), + } + /// Attempted to queue a transaction that is already in the pool. + AlreadyImported(hash: Hash) { + description("Transaction is already in the pool."), + display("Transaction {:?} is already in the pool.", hash), + } + /// Import error. + Import(err: Box<::std::error::Error + Send>) { + description("Error importing transaction"), + display("Error importing transaction: {}", err.description()), + } + /// Runtime failure. + UnrecognisedAddress(who: Address) { + description("Unrecognised address in extrinsic"), + display("Unrecognised address in extrinsic: {}", who), + } + /// Extrinsic too large + TooLarge(got: usize, max: usize) { + description("Extrinsic too large"), + display("Extrinsic is too large ({} > {})", got, max), + } + } +} + +impl extrinsic_pool::IntoPoolError for Error { + fn into_pool_error(self) -> ::std::result::Result { + match self { + Error(ErrorKind::Pool(e), c) => Ok(extrinsic_pool::Error(e, c)), + e => Err(e), + } + } +} diff --git a/node/transaction-pool/src/lib.rs b/node/transaction-pool/src/lib.rs new file mode 100644 index 00000000000..97ef5b59f2f --- /dev/null +++ b/node/transaction-pool/src/lib.rs @@ -0,0 +1,236 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +extern crate substrate_client as client; +extern crate substrate_codec as codec; +extern crate substrate_extrinsic_pool as extrinsic_pool; +extern crate substrate_primitives; +extern crate substrate_runtime_primitives; +extern crate node_runtime as runtime; +extern crate node_primitives as primitives; +extern crate node_api; +extern crate parking_lot; + +#[cfg(test)] +extern crate substrate_keyring; + +#[macro_use] +extern crate error_chain; + +#[macro_use] +extern crate log; + +mod error; + +use std::{ + cmp::Ordering, + collections::HashMap, + sync::Arc, +}; + +use codec::{Decode, Encode}; +use extrinsic_pool::{Readiness, scoring::{Change, Choice}, VerifiedFor, ExtrinsicFor}; +use node_api::Api; +use primitives::{AccountId, BlockId, Block, Hash, Index}; +use runtime::{Address, UncheckedExtrinsic, RawAddress}; +use substrate_runtime_primitives::traits::{Bounded, Checkable, Hash as HashT, BlakeTwo256}; + +pub use extrinsic_pool::{Options, Status, LightStatus, VerifiedTransaction as VerifiedTransactionOps}; +pub use error::{Error, ErrorKind, Result}; + +/// Maximal size of a single encoded extrinsic. +const MAX_TRANSACTION_SIZE: usize = 4 * 1024 * 1024; + +/// Type alias for convenience. +pub type CheckedExtrinsic = std::result::Result>>::Checked; + +/// Type alias for the transaction pool. +pub type TransactionPool = extrinsic_pool::Pool>; + +/// A verified transaction which should be includable and non-inherent. +#[derive(Clone, Debug)] +pub struct VerifiedTransaction { + /// Transaction hash. + pub hash: Hash, + /// Transaction sender. + pub sender: AccountId, + /// Transaction index. + pub index: Index, + encoded_size: usize, +} + +impl VerifiedTransaction { + /// Get the 256-bit hash of this transaction. + pub fn hash(&self) -> &Hash { + &self.hash + } + + /// Get the account ID of the sender of this transaction. + pub fn index(&self) -> Index { + self.index + } + + /// Get encoded size of the transaction. + pub fn encoded_size(&self) -> usize { + self.encoded_size + } +} + +impl extrinsic_pool::VerifiedTransaction for VerifiedTransaction { + type Hash = Hash; + type Sender = AccountId; + + fn hash(&self) -> &Self::Hash { + &self.hash + } + + fn sender(&self) -> &Self::Sender { + &self.sender + } + + fn mem_usage(&self) -> usize { + self.encoded_size // TODO + } +} + +/// The transaction pool logic. +pub struct ChainApi { + api: Arc, +} + +impl ChainApi where + A: Api, +{ + /// Create a new instance. + pub fn new(api: Arc) -> Self { + ChainApi { + api, + } + } +} + + +impl extrinsic_pool::ChainApi for ChainApi where + A: Api + Send + Sync, +{ + type Block = Block; + type Hash = Hash; + type Sender = AccountId; + type VEx = VerifiedTransaction; + type Ready = HashMap; + type Error = Error; + type Score = u64; + type Event = (); + + fn verify_transaction(&self, _at: &BlockId, xt: &ExtrinsicFor) -> Result { + let encoded = xt.encode(); + let uxt = UncheckedExtrinsic::decode(&mut encoded.as_slice()).ok_or_else(|| ErrorKind::InvalidExtrinsicFormat)?; + if !uxt.is_signed() { + bail!(ErrorKind::IsInherent(uxt)) + } + + let (encoded_size, hash) = (encoded.len(), BlakeTwo256::hash(&encoded)); + if encoded_size > MAX_TRANSACTION_SIZE { + bail!(ErrorKind::TooLarge(encoded_size, MAX_TRANSACTION_SIZE)); + } + + debug!(target: "transaction-pool", "Transaction submitted: {}", ::substrate_primitives::hexdisplay::HexDisplay::from(&encoded)); + let checked = uxt.clone().check_with(|a| { + match a { + RawAddress::Id(id) => Ok(id), + RawAddress::Index(_) => Err("Index based addresses are not supported".into()),// TODO: Make index addressing optional in substrate + } + })?; + let sender = checked.signed.expect("Only signed extrinsics are allowed at this point"); + + + if encoded_size < 1024 { + debug!(target: "transaction-pool", "Transaction verified: {} => {:?}", hash, uxt); + } else { + debug!(target: "transaction-pool", "Transaction verified: {} ({} bytes is too large to display)", hash, encoded_size); + } + + Ok(VerifiedTransaction { + index: checked.index, + sender, + hash, + encoded_size, + }) + } + + fn ready(&self) -> Self::Ready { + HashMap::default() + } + + fn is_ready(&self, at: &BlockId, known_nonces: &mut Self::Ready, xt: &VerifiedFor) -> Readiness { + let sender = xt.verified.sender().clone(); + trace!(target: "transaction-pool", "Checking readiness of {} (from {})", xt.verified.hash, sender); + + // TODO: find a way to handle index error properly -- will need changes to + // transaction-pool trait. + let api = &self.api; + let next_index = known_nonces.entry(sender) + .or_insert_with(|| api.index(at, sender).ok().unwrap_or_else(Bounded::max_value)); + + trace!(target: "transaction-pool", "Next index for sender is {}; xt index is {}", next_index, xt.verified.index); + + let result = match xt.verified.index.cmp(&next_index) { + // TODO: this won't work perfectly since accounts can now be killed, returning the nonce + // to zero. + // We should detect if the index was reset and mark all transactions as `Stale` for cull to work correctly. + // Otherwise those transactions will keep occupying the queue. + // Perhaps we could mark as stale if `index - state_index` > X? + Ordering::Greater => Readiness::Future, + Ordering::Equal => Readiness::Ready, + // TODO [ToDr] Should mark transactions referencing too old blockhash as `Stale` as well. + Ordering::Less => Readiness::Stale, + }; + + // remember to increment `next_index` + *next_index = next_index.saturating_add(1); + + result + } + + fn compare(old: &VerifiedFor, other: &VerifiedFor) -> Ordering { + old.verified.index().cmp(&other.verified.index()) + } + + fn choose(old: &VerifiedFor, new: &VerifiedFor) -> Choice { + if old.verified.index() == new.verified.index() { + return Choice::ReplaceOld; + } + Choice::InsertNew + } + + fn update_scores( + xts: &[extrinsic_pool::Transaction>], + scores: &mut [Self::Score], + _change: Change<()> + ) { + for i in 0..xts.len() { + // all the same score since there are no fees. + // TODO: prioritize things like misbehavior or fishermen reports + scores[i] = 1; + } + } + + fn should_replace(_old: &VerifiedFor, _new: &VerifiedFor) -> Choice { + // Don't allow new transactions if we are reaching the limit. + Choice::RejectNew + } +} + diff --git a/runtime/README.adoc b/runtime/README.adoc new file mode 100644 index 00000000000..616c12568a0 --- /dev/null +++ b/runtime/README.adoc @@ -0,0 +1,6 @@ + += Runtime + +Set of libs for the substrate runtime. + +TODO: Add READMEs to packages. diff --git a/runtime/balances/Cargo.toml b/runtime/balances/Cargo.toml new file mode 100644 index 00000000000..8729db0b5d8 --- /dev/null +++ b/runtime/balances/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "substrate-runtime-balances" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +safe-mix = { version = "1.0", default_features = false} +substrate-keyring = { path = "../../core/keyring", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "safe-mix/std", + "substrate-keyring", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-system/std", +] diff --git a/runtime/balances/src/address.rs b/runtime/balances/src/address.rs new file mode 100644 index 00000000000..01fb2543950 --- /dev/null +++ b/runtime/balances/src/address.rs @@ -0,0 +1,111 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Address type that is union of index and id for an account. + +#[cfg(feature = "std")] +use std::fmt; +use super::{Member, Decode, Encode, As, Input, Output}; + +/// A vetted and verified extrinsic from the external world. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Hash))] +pub enum Address where + AccountId: Member, + AccountIndex: Member, +{ + /// It's an account ID (pubkey). + #[cfg_attr(feature = "std", serde(deserialize_with="AccountId::deserialize"))] + Id(AccountId), + /// It's an account index. + #[cfg_attr(feature = "std", serde(deserialize_with="AccountIndex::deserialize"))] + Index(AccountIndex), +} + +#[cfg(feature = "std")] +impl fmt::Display for Address where + AccountId: Member, + AccountIndex: Member, +{ + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{:?}", self) + } +} + +impl From for Address where + AccountId: Member, + AccountIndex: Member, +{ + fn from(a: AccountId) -> Self { + Address::Id(a) + } +} + +fn need_more_than(a: T, b: T) -> Option { + if a < b { Some(a) } else { None } +} + +impl Decode for Address where + AccountId: Member + Decode, + AccountIndex: Member + Decode + PartialOrd + Ord + As + As + As + Copy, +{ + fn decode(input: &mut I) -> Option { + Some(match input.read_byte()? { + x @ 0x00...0xef => Address::Index(As::sa(x)), + 0xfc => Address::Index(As::sa(need_more_than(0xef, u16::decode(input)?)?)), + 0xfd => Address::Index(As::sa(need_more_than(0xffff, u32::decode(input)?)?)), + 0xfe => Address::Index(need_more_than(As::sa(0xffffffffu32), Decode::decode(input)?)?), + 0xff => Address::Id(Decode::decode(input)?), + _ => return None, + }) + } +} + +impl Encode for Address where + AccountId: Member + Encode, + AccountIndex: Member + Encode + PartialOrd + Ord + As + As + As + Copy, +{ + fn encode_to(&self, dest: &mut T) { + match *self { + Address::Id(ref i) => { + dest.push_byte(255); + dest.push(i); + } + Address::Index(i) if i > As::sa(0xffffffffu32) => { + dest.push_byte(254); + dest.push(&i); + } + Address::Index(i) if i > As::sa(0xffffu32) => { + dest.push_byte(253); + dest.push(&As::::as_(i)); + } + Address::Index(i) if i >= As::sa(0xf0u32) => { + dest.push_byte(252); + dest.push(&As::::as_(i)); + } + Address::Index(i) => dest.push_byte(As::::as_(i)), + } + } +} + +impl Default for Address where + AccountId: Member + Default, + AccountIndex: Member, +{ + fn default() -> Self { + Address::Id(Default::default()) + } +} diff --git a/runtime/balances/src/genesis_config.rs b/runtime/balances/src/genesis_config.rs new file mode 100644 index 00000000000..1b88353c632 --- /dev/null +++ b/runtime/balances/src/genesis_config.rs @@ -0,0 +1,84 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Build a balances genesis block. + +#![cfg(feature = "std")] + +use std::collections::HashMap; +use rstd::prelude::*; +use codec::Encode; +use runtime_support::{StorageValue, StorageMap}; +use primitives::traits::{Zero, As}; +use substrate_primitives::Blake2Hasher; +use {runtime_io, primitives}; +use super::{Trait, ENUM_SET_SIZE, EnumSet, NextEnumSet, CreationFee, TransferFee, + ReclaimRebate, ExistentialDeposit, TransactionByteFee, TransactionBaseFee, TotalIssuance, + FreeBalance}; + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + pub balances: Vec<(T::AccountId, T::Balance)>, + pub transaction_base_fee: T::Balance, + pub transaction_byte_fee: T::Balance, + pub transfer_fee: T::Balance, + pub creation_fee: T::Balance, + pub reclaim_rebate: T::Balance, + pub existential_deposit: T::Balance, +} + +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + balances: vec![], + transaction_base_fee: T::Balance::sa(0), + transaction_byte_fee: T::Balance::sa(0), + transfer_fee: T::Balance::sa(0), + creation_fee: T::Balance::sa(0), + existential_deposit: T::Balance::sa(0), + reclaim_rebate: T::Balance::sa(0), + } + } +} + +impl primitives::BuildStorage for GenesisConfig { + fn build_storage(self) -> ::std::result::Result, Vec>, String> { + let total_issuance: T::Balance = self.balances.iter().fold(Zero::zero(), |acc, &(_, n)| acc + n); + + let mut r: runtime_io::TestExternalities = map![ + Self::hash(>::key()).to_vec() => T::AccountIndex::sa(self.balances.len() / ENUM_SET_SIZE).encode(), + Self::hash(>::key()).to_vec() => self.transaction_base_fee.encode(), + Self::hash(>::key()).to_vec() => self.transaction_byte_fee.encode(), + Self::hash(>::key()).to_vec() => self.transfer_fee.encode(), + Self::hash(>::key()).to_vec() => self.creation_fee.encode(), + Self::hash(>::key()).to_vec() => self.existential_deposit.encode(), + Self::hash(>::key()).to_vec() => self.reclaim_rebate.encode(), + Self::hash(>::key()).to_vec() => total_issuance.encode() + ]; + + let ids: Vec<_> = self.balances.iter().map(|x| x.0.clone()).collect(); + for i in 0..(ids.len() + ENUM_SET_SIZE - 1) / ENUM_SET_SIZE { + r.insert(Self::hash(&>::key_for(T::AccountIndex::sa(i))).to_vec(), + ids[i * ENUM_SET_SIZE..ids.len().min((i + 1) * ENUM_SET_SIZE)].to_owned().encode()); + } + for (who, value) in self.balances.into_iter() { + r.insert(Self::hash(&>::key_for(who)).to_vec(), value.encode()); + } + Ok(r.into()) + } +} diff --git a/runtime/balances/src/lib.rs b/runtime/balances/src/lib.rs new file mode 100644 index 00000000000..3451b55f6aa --- /dev/null +++ b/runtime/balances/src/lib.rs @@ -0,0 +1,660 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Balances: Handles balances. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +#[cfg_attr(feature = "std", macro_use)] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_codec as codec; +extern crate substrate_primitives; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_primitives as primitives; +extern crate substrate_runtime_system as system; + +use rstd::prelude::*; +use rstd::{cmp, result}; +use codec::{Encode, Decode, Codec, Input, Output}; +use runtime_support::{StorageValue, StorageMap, Parameter}; +use runtime_support::dispatch::Result; +use primitives::traits::{Zero, One, SimpleArithmetic, OnFinalise, MakePayment, + As, Lookup, Member, CheckedAdd, CheckedSub}; +use address::Address as RawAddress; +use system::{ensure_signed, ensure_root}; + +mod mock; + +pub mod address; +mod tests; +mod genesis_config; + +#[cfg(feature = "std")] +pub use genesis_config::GenesisConfig; + +/// Number of account IDs stored per enum set. +const ENUM_SET_SIZE: usize = 64; + +/// The byte to identify intention to reclaim an existing account index. +const RECLAIM_INDEX_MAGIC: usize = 0x69; + +pub type Address = RawAddress<::AccountId, ::AccountIndex>; + +/// The account with the given id was killed. +pub trait OnFreeBalanceZero { + /// The account was the given id was killed. + fn on_free_balance_zero(who: &AccountId); +} + +impl OnFreeBalanceZero for () { + fn on_free_balance_zero(_who: &AccountId) {} +} +impl< + AccountId, + X: OnFreeBalanceZero, + Y: OnFreeBalanceZero, +> OnFreeBalanceZero for (X, Y) { + fn on_free_balance_zero(who: &AccountId) { + X::on_free_balance_zero(who); + Y::on_free_balance_zero(who); + } +} + +/// Trait for a hook to get called when some balance has been minted, causing dilution. +pub trait OnDilution { + /// Some `portion` of the total balance just "grew" by `minted`. `portion` is the pre-growth + /// amount (it doesn't take account of the recent growth). + fn on_dilution(minted: Balance, portion: Balance); +} + +impl OnDilution for () { + fn on_dilution(_minted: Balance, _portion: Balance) {} +} + +/// Determinator for whether a given account is able to transfer balance. +pub trait EnsureAccountLiquid { + /// Returns `Ok` iff the account is able to transfer funds normally. `Err(...)` + /// with the reason why not otherwise. + fn ensure_account_liquid(who: &AccountId) -> Result; +} + +impl EnsureAccountLiquid for () { + fn ensure_account_liquid(_who: &AccountId) -> Result { Ok(()) } +} + +pub trait Trait: system::Trait { + /// The balance of an account. + type Balance: Parameter + SimpleArithmetic + Codec + Default + Copy + As + As + As; + /// Type used for storing an account's index; implies the maximum number of accounts the system + /// can hold. + type AccountIndex: Parameter + Member + Codec + SimpleArithmetic + As + As + As + As + As + Copy; + /// A function which is invoked when the free-balance has fallen below the existential deposit and + /// has been reduced to zero. + /// + /// Gives a chance to clean up resources associated with the given account. + type OnFreeBalanceZero: OnFreeBalanceZero; + + /// A function that returns true iff a given account can transfer its funds to another account. + type EnsureAccountLiquid: EnsureAccountLiquid; + + /// The overarching event type. + type Event: From> + Into<::Event>; +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn transfer(origin, dest: RawAddress, value: T::Balance) -> Result; + fn set_balance(origin, who: RawAddress, free: T::Balance, reserved: T::Balance) -> Result; + } +} + +decl_event!( + pub enum Event with RawEvent + where ::AccountId, ::AccountIndex, ::Balance { + /// A new account was created. + NewAccount(AccountId, AccountIndex, NewAccountOutcome), + /// An account was reaped. + ReapedAccount(AccountId), + /// Transfer succeeded (from, to, value, fees). + Transfer(AccountId, AccountId, Balance, Balance), + } +); + +decl_storage! { + trait Store for Module as Balances { + /// The total amount of stake on the system. + pub TotalIssuance get(total_issuance): required T::Balance; + /// The minimum amount allowed to keep an account open. + pub ExistentialDeposit get(existential_deposit): required T::Balance; + /// The amount credited to a destination's account whose index was reclaimed. + pub ReclaimRebate get(reclaim_rebate): required T::Balance; + /// The fee required to make a transfer. + pub TransferFee get(transfer_fee): required T::Balance; + /// The fee required to create an account. At least as big as ReclaimRebate. + pub CreationFee get(creation_fee): required T::Balance; + + /// The next free enumeration set. + pub NextEnumSet get(next_enum_set): required T::AccountIndex; + /// The enumeration sets. + pub EnumSet get(enum_set): default map [ T::AccountIndex => Vec ]; + + /// The 'free' balance of a given account. + /// + /// This is the only balance that matters in terms of most operations on tokens. It is + /// alone used to determine the balance when in the contract execution environment. When this + /// balance falls below the value of `ExistentialDeposit`, then the 'current account' is + /// deleted: specifically `FreeBalance`. Furthermore, `OnFreeBalanceZero` callback + /// is invoked, giving a chance to external modules to cleanup data associated with + /// the deleted account. + /// + /// `system::AccountNonce` is also deleted if `ReservedBalance` is also zero (it also gets + /// collapsed to zero if it ever becomes less than `ExistentialDeposit`. + pub FreeBalance get(free_balance): default map [ T::AccountId => T::Balance ]; + + /// The amount of the balance of a given account that is exterally reserved; this can still get + /// slashed, but gets slashed last of all. + /// + /// This balance is a 'reserve' balance that other subsystems use in order to set aside tokens + /// that are still 'owned' by the account holder, but which are unspendable. (This is different + /// and wholly unrelated to the `Bondage` system used in the staking module.) + /// + /// When this balance falls below the value of `ExistentialDeposit`, then this 'reserve account' + /// is deleted: specifically, `ReservedBalance`. + /// + /// `system::AccountNonce` is also deleted if `FreeBalance` is also zero (it also gets + /// collapsed to zero if it ever becomes less than `ExistentialDeposit`. + pub ReservedBalance get(reserved_balance): default map [ T::AccountId => T::Balance ]; + + + // Payment stuff. + + /// The fee to be paid for making a transaction; the base. + pub TransactionBaseFee get(transaction_base_fee): required T::Balance; + /// The fee to be paid for making a transaction; the per-byte portion. + pub TransactionByteFee get(transaction_byte_fee): required T::Balance; + } +} + +/// Whatever happened about the hint given when creating the new account. +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +#[derive(Encode, Decode, PartialEq, Eq, Clone, Copy)] +pub enum NewAccountOutcome { + NoHint, + GoodHint, + BadHint, +} + +/// Outcome of a balance update. +pub enum UpdateBalanceOutcome { + /// Account balance was simply updated. + Updated, + /// The update has led to killing of the account. + AccountKilled, +} + +impl Module { + + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + // PUBLIC IMMUTABLES + + /// The combined balance of `who`. + pub fn total_balance(who: &T::AccountId) -> T::Balance { + Self::free_balance(who) + Self::reserved_balance(who) + } + + /// Some result as `slash(who, value)` (but without the side-effects) assuming there are no + /// balance changes in the meantime and only the reserved balance is not taken into account. + pub fn can_slash(who: &T::AccountId, value: T::Balance) -> bool { + Self::free_balance(who) >= value + } + + /// Same result as `reserve(who, value)` (but without the side-effects) assuming there + /// are no balance changes in the meantime. + pub fn can_reserve(who: &T::AccountId, value: T::Balance) -> bool { + if T::EnsureAccountLiquid::ensure_account_liquid(who).is_ok() { + Self::free_balance(who) >= value + } else { + false + } + } + + /// Lookup an T::AccountIndex to get an Id, if there's one there. + pub fn lookup_index(index: T::AccountIndex) -> Option { + let enum_set_size = Self::enum_set_size(); + let set = Self::enum_set(index / enum_set_size); + let i: usize = (index % enum_set_size).as_(); + set.get(i).map(|x| x.clone()) + } + + /// `true` if the account `index` is ready for reclaim. + pub fn can_reclaim(try_index: T::AccountIndex) -> bool { + let enum_set_size = Self::enum_set_size(); + let try_set = Self::enum_set(try_index / enum_set_size); + let i = (try_index % enum_set_size).as_(); + i < try_set.len() && Self::total_balance(&try_set[i]).is_zero() + } + + /// Lookup an address to get an Id, if there's one there. + pub fn lookup_address(a: address::Address) -> Option { + match a { + address::Address::Id(i) => Some(i), + address::Address::Index(i) => Self::lookup_index(i), + } + } + + // PUBLIC DISPATCH + + /// Transfer some liquid free balance to another staker. + pub fn transfer(origin: T::Origin, dest: Address, value: T::Balance) -> Result { + let transactor = ensure_signed(origin)?; + + let dest = Self::lookup(dest)?; + let from_balance = Self::free_balance(&transactor); + let would_create = from_balance.is_zero(); + let fee = if would_create { Self::creation_fee() } else { Self::transfer_fee() }; + let liability = value + fee; + + let new_from_balance = match from_balance.checked_sub(&liability) { + Some(b) => b, + None => return Err("balance too low to send value"), + }; + if would_create && value < Self::existential_deposit() { + return Err("value too low to create account"); + } + T::EnsureAccountLiquid::ensure_account_liquid(&transactor)?; + + let to_balance = Self::free_balance(&dest); + // NOTE: total stake being stored in the same type means that this could never overflow + // but better to be safe than sorry. + let new_to_balance = match to_balance.checked_add(&value) { + Some(b) => b, + None => return Err("destination balance too high to receive value"), + }; + + if transactor != dest { + Self::set_free_balance(&transactor, new_from_balance); + Self::decrease_total_stake_by(fee); + Self::set_free_balance_creating(&dest, new_to_balance); + Self::deposit_event(RawEvent::Transfer(transactor, dest, value, fee)); + } + + Ok(()) + } + + /// Set the balances of a given account. + fn set_balance(origin: T::Origin, who: Address, free: T::Balance, reserved: T::Balance) -> Result { + ensure_root(origin)?; + let who = Self::lookup(who)?; + Self::set_free_balance(&who, free); + Self::set_reserved_balance(&who, reserved); + Ok(()) + } + + // PUBLIC MUTABLES (DANGEROUS) + + /// Set the free balance of an account to some new value. + /// + /// Will enforce ExistentialDeposit law, anulling the account as needed. + /// In that case it will return `AccountKilled`. + pub fn set_reserved_balance(who: &T::AccountId, balance: T::Balance) -> UpdateBalanceOutcome { + if balance < Self::existential_deposit() { + >::insert(who, balance); + Self::on_reserved_too_low(who); + UpdateBalanceOutcome::AccountKilled + } else { + >::insert(who, balance); + UpdateBalanceOutcome::Updated + } + } + + /// Set the free balance of an account to some new value. Will enforce ExistentialDeposit + /// law anulling the account as needed. + /// + /// Doesn't do any preparatory work for creating a new account, so should only be used when it + /// is known that the account already exists. + /// + /// Returns if the account was successfully updated or update has led to killing of the account. + pub fn set_free_balance(who: &T::AccountId, balance: T::Balance) -> UpdateBalanceOutcome { + // Commented out for no - but consider it instructive. + // assert!(!Self::total_balance(who).is_zero()); + if balance < Self::existential_deposit() { + >::insert(who, balance); + Self::on_free_too_low(who); + UpdateBalanceOutcome::AccountKilled + } else { + >::insert(who, balance); + UpdateBalanceOutcome::Updated + } + } + + /// Set the free balance on an account to some new value. + /// + /// Same as [`set_free_balance`], but will create a new account. + /// + /// Returns if the account was successfully updated or update has led to killing of the account. + /// + /// [`set_free_balance`]: #method.set_free_balance + pub fn set_free_balance_creating(who: &T::AccountId, balance: T::Balance) -> UpdateBalanceOutcome { + let ed = >::existential_deposit(); + // If the balance is too low, then the account is reaped. + // NOTE: There are two balances for every account: `reserved_balance` and + // `free_balance`. This contract subsystem only cares about the latter: whenever + // the term "balance" is used *here* it should be assumed to mean "free balance" + // in the rest of the module. + // Free balance can never be less than ED. If that happens, it gets reduced to zero + // and the account information relevant to this subsystem is deleted (i.e. the + // account is reaped). + // NOTE: This is orthogonal to the `Bondage` value that an account has, a high + // value of which makes even the `free_balance` unspendable. + // TODO: enforce this for the other balance-altering functions. + if balance < ed { + Self::set_free_balance(who, balance); + UpdateBalanceOutcome::AccountKilled + } else { + if !>::exists(who) { + let outcome = Self::new_account(&who, balance); + let credit = match outcome { + NewAccountOutcome::GoodHint => balance + >::reclaim_rebate(), + _ => balance, + }; + Self::set_free_balance(who, credit); + Self::increase_total_stake_by(credit - balance); + } else { + Self::set_free_balance(who, balance); + } + + UpdateBalanceOutcome::Updated + } + } + + /// Adds up to `value` to the free balance of `who`. If `who` doesn't exist, it is created. + /// + /// This is a sensitive function since it circumvents any fees associated with account + /// setup. Ensure it is only called by trusted code. + /// + /// NOTE: This assumes that the total stake remains unchanged after this operation. If + /// you mean to actually mint value into existence, then use `reward` instead. + pub fn increase_free_balance_creating(who: &T::AccountId, value: T::Balance) -> UpdateBalanceOutcome { + Self::set_free_balance_creating(who, Self::free_balance(who) + value) + } + + /// Deducts up to `value` from the combined balance of `who`, preferring to deduct from the + /// free balance. This function cannot fail. + /// + /// As much funds up to `value` will be deducted as possible. If this is less than `value`, + /// then `Some(remaining)` will be returned. Full completion is given by `None`. + pub fn slash(who: &T::AccountId, value: T::Balance) -> Option { + let free_balance = Self::free_balance(who); + let free_slash = cmp::min(free_balance, value); + Self::set_free_balance(who, free_balance - free_slash); + Self::decrease_total_stake_by(free_slash); + if free_slash < value { + Self::slash_reserved(who, value - free_slash) + } else { + None + } + } + + /// Adds up to `value` to the free balance of `who`. + /// + /// If `who` doesn't exist, nothing is done and an Err returned. + pub fn reward(who: &T::AccountId, value: T::Balance) -> Result { + if Self::total_balance(who).is_zero() { + return Err("beneficiary account must pre-exist"); + } + Self::set_free_balance(who, Self::free_balance(who) + value); + Self::increase_total_stake_by(value); + Ok(()) + } + + /// Moves `value` from balance to reserved balance. + /// + /// If the free balance is lower than `value`, then no funds will be moved and an `Err` will + /// be returned to notify of this. This is different behaviour to `unreserve`. + pub fn reserve(who: &T::AccountId, value: T::Balance) -> Result { + let b = Self::free_balance(who); + if b < value { + return Err("not enough free funds") + } + T::EnsureAccountLiquid::ensure_account_liquid(who)?; + Self::set_reserved_balance(who, Self::reserved_balance(who) + value); + Self::set_free_balance(who, b - value); + Ok(()) + } + + /// Moves up to `value` from reserved balance to balance. This function cannot fail. + /// + /// As much funds up to `value` will be deducted as possible. If this is less than `value`, + /// then `Some(remaining)` will be returned. Full completion is given by `None`. + /// NOTE: This is different to `reserve`. + pub fn unreserve(who: &T::AccountId, value: T::Balance) -> Option { + let b = Self::reserved_balance(who); + let actual = cmp::min(b, value); + Self::set_free_balance(who, Self::free_balance(who) + actual); + Self::set_reserved_balance(who, b - actual); + if actual == value { + None + } else { + Some(value - actual) + } + } + + /// Deducts up to `value` from reserved balance of `who`. This function cannot fail. + /// + /// As much funds up to `value` will be deducted as possible. If this is less than `value`, + /// then `Some(remaining)` will be returned. Full completion is given by `None`. + pub fn slash_reserved(who: &T::AccountId, value: T::Balance) -> Option { + let b = Self::reserved_balance(who); + let slash = cmp::min(b, value); + Self::set_reserved_balance(who, b - slash); + Self::decrease_total_stake_by(slash); + if value == slash { + None + } else { + Some(value - slash) + } + } + + /// Moves up to `value` from reserved balance of account `slashed` to free balance of account + /// `beneficiary`. `beneficiary` must exist for this to succeed. If it does not, `Err` will be + /// returned. + /// + /// As much funds up to `value` will be moved as possible. If this is less than `value`, then + /// `Ok(Some(remaining))` will be returned. Full completion is given by `Ok(None)`. + pub fn repatriate_reserved( + slashed: &T::AccountId, + beneficiary: &T::AccountId, + value: T::Balance + ) -> result::Result, &'static str> { + if Self::total_balance(beneficiary).is_zero() { + return Err("beneficiary account must pre-exist"); + } + let b = Self::reserved_balance(slashed); + let slash = cmp::min(b, value); + Self::set_free_balance(beneficiary, Self::free_balance(beneficiary) + slash); + Self::set_reserved_balance(slashed, b - slash); + if value == slash { + Ok(None) + } else { + Ok(Some(value - slash)) + } + } + + fn enum_set_size() -> T::AccountIndex { + T::AccountIndex::sa(ENUM_SET_SIZE) + } + + /// Register a new account (with existential balance). + fn new_account(who: &T::AccountId, balance: T::Balance) -> NewAccountOutcome { + let enum_set_size = Self::enum_set_size(); + let next_set_index = Self::next_enum_set(); + let reclaim_index_magic = T::AccountIndex::sa(RECLAIM_INDEX_MAGIC); + let reclaim_index_modulus = T::AccountIndex::sa(256usize); + let quantization = T::AccountIndex::sa(256usize); + + // A little easter-egg for reclaiming dead indexes.. + let ret = { + // we quantise the number of accounts so it stays constant over a reasonable + // period of time. + let quantized_account_count: T::AccountIndex = (next_set_index * enum_set_size / quantization + One::one()) * quantization; + // then modify the starting balance to be modulo this to allow it to potentially + // identify an account index for reuse. + let maybe_try_index = balance % >::sa(quantized_account_count * reclaim_index_modulus); + let maybe_try_index = As::::as_(maybe_try_index); + + // this identifier must end with magic byte 0x69 to trigger this check (a minor + // optimisation to ensure we don't check most unintended account creations). + if maybe_try_index % reclaim_index_modulus == reclaim_index_magic { + // reuse is probably intended. first, remove magic byte. + let try_index = maybe_try_index / reclaim_index_modulus; + + // then check to see if this balance identifies a dead account index. + let set_index = try_index / enum_set_size; + let mut try_set = Self::enum_set(set_index); + let item_index = (try_index % enum_set_size).as_(); + if item_index < try_set.len() { + if Self::total_balance(&try_set[item_index]).is_zero() { + // yup - this index refers to a dead account. can be reused. + try_set[item_index] = who.clone(); + >::insert(set_index, try_set); + + Self::deposit_event(RawEvent::NewAccount(who.clone(), try_index, NewAccountOutcome::GoodHint)); + + return NewAccountOutcome::GoodHint + } + } + NewAccountOutcome::BadHint + } else { + NewAccountOutcome::NoHint + } + }; + + // insert normally as a back up + let mut set_index = next_set_index; + // defensive only: this loop should never iterate since we keep NextEnumSet up to date later. + let mut set = loop { + let set = Self::enum_set(set_index); + if set.len() < ENUM_SET_SIZE { + break set; + } + set_index += One::one(); + }; + + let index = T::AccountIndex::sa(set_index.as_() * ENUM_SET_SIZE + set.len()); + + // update set. + set.push(who.clone()); + + // keep NextEnumSet up to date + if set.len() == ENUM_SET_SIZE { + >::put(set_index + One::one()); + } + + // write set. + >::insert(set_index, set); + + Self::deposit_event(RawEvent::NewAccount(who.clone(), index, ret)); + + ret + } + + fn reap_account(who: &T::AccountId) { + >::remove(who); + Self::deposit_event(RawEvent::ReapedAccount(who.clone())); + } + + /// Kill an account's free portion. + fn on_free_too_low(who: &T::AccountId) { + Self::decrease_total_stake_by(Self::free_balance(who)); + >::remove(who); + + T::OnFreeBalanceZero::on_free_balance_zero(who); + + if Self::reserved_balance(who).is_zero() { + Self::reap_account(who); + } + } + + /// Kill an account's reserved portion. + fn on_reserved_too_low(who: &T::AccountId) { + Self::decrease_total_stake_by(Self::reserved_balance(who)); + >::remove(who); + + if Self::free_balance(who).is_zero() { + Self::reap_account(who); + } + } + + /// Increase TotalIssuance by Value. + pub fn increase_total_stake_by(value: T::Balance) { + if let Some(v) = >::total_issuance().checked_add(&value) { + >::put(v); + } + } + /// Decrease TotalIssuance by Value. + pub fn decrease_total_stake_by(value: T::Balance) { + if let Some(v) = >::total_issuance().checked_sub(&value) { + >::put(v); + } + } +} + +impl OnFinalise for Module { + fn on_finalise(_n: T::BlockNumber) { + } +} + +impl Lookup for Module { + type Source = address::Address; + type Target = T::AccountId; + fn lookup(a: Self::Source) -> result::Result { + match a { + address::Address::Id(i) => Ok(i), + address::Address::Index(i) => >::lookup_index(i).ok_or("invalid account index"), + } + } +} + +impl MakePayment for Module { + fn make_payment(transactor: &T::AccountId, encoded_len: usize) -> Result { + let b = Self::free_balance(transactor); + let transaction_fee = Self::transaction_base_fee() + Self::transaction_byte_fee() * >::sa(encoded_len as u64); + if b < transaction_fee + Self::existential_deposit() { + return Err("not enough funds for transaction fee"); + } + Self::set_free_balance(transactor, b - transaction_fee); + Self::decrease_total_stake_by(transaction_fee); + Ok(()) + } +} diff --git a/runtime/balances/src/mock.rs b/runtime/balances/src/mock.rs new file mode 100644 index 00000000000..d85a2771581 --- /dev/null +++ b/runtime/balances/src/mock.rs @@ -0,0 +1,77 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Test utilities + +#![cfg(test)] + +use primitives::BuildStorage; +use primitives::testing::{Digest, Header}; +use substrate_primitives::{H256, Blake2Hasher}; +use runtime_io; +use {GenesisConfig, Module, Trait, system}; + +impl_outer_origin!{ + pub enum Origin for Runtime {} +} + +// Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub struct Runtime; +impl system::Trait for Runtime { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = ::primitives::traits::BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); +} +impl Trait for Runtime { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = (); + type EnsureAccountLiquid = (); + type Event = (); +} + +pub fn new_test_ext(ext_deposit: u64, monied: bool) -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + let balance_factor = if ext_deposit > 0 { + 256 + } else { + 1 + }; + t.extend(GenesisConfig::{ + balances: if monied { + vec![(1, 10 * balance_factor), (2, 20 * balance_factor), (3, 30 * balance_factor), (4, 40 * balance_factor)] + } else { + vec![(10, balance_factor), (20, balance_factor)] + }, + transaction_base_fee: 0, + transaction_byte_fee: 0, + existential_deposit: ext_deposit, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + }.build_storage().unwrap()); + t.into() +} + +pub type System = system::Module; +pub type Balances = Module; diff --git a/runtime/balances/src/tests.rs b/runtime/balances/src/tests.rs new file mode 100644 index 00000000000..7279474a1f9 --- /dev/null +++ b/runtime/balances/src/tests.rs @@ -0,0 +1,324 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tests for the module. + +#![cfg(test)] + +use super::*; +use runtime_io::with_externalities; +use mock::{Balances, System, Runtime, new_test_ext}; + +#[test] +fn reward_should_work() { + with_externalities(&mut new_test_ext(0, true), || { + assert_eq!(Balances::total_balance(&1), 10); + assert_ok!(Balances::reward(&1, 10)); + assert_eq!(Balances::total_balance(&1), 20); + assert_eq!(>::get(), 110); + }); +} + +#[test] +fn indexing_lookup_should_work() { + with_externalities(&mut new_test_ext(10, true), || { + assert_eq!(Balances::lookup_index(0), Some(1)); + assert_eq!(Balances::lookup_index(1), Some(2)); + assert_eq!(Balances::lookup_index(2), Some(3)); + assert_eq!(Balances::lookup_index(3), Some(4)); + assert_eq!(Balances::lookup_index(4), None); + }); +} + +#[test] +fn default_indexing_on_new_accounts_should_work() { + with_externalities(&mut new_test_ext(10, true), || { + assert_eq!(Balances::lookup_index(4), None); + assert_ok!(Balances::transfer(Some(1).into(), 5.into(), 10)); + assert_eq!(Balances::lookup_index(4), Some(5)); + }); +} + +#[test] +fn dust_account_removal_should_work() { + with_externalities(&mut new_test_ext(256 * 10, true), || { + System::inc_account_nonce(&2); + assert_eq!(System::account_nonce(&2), 1); + assert_eq!(Balances::total_balance(&2), 256 * 20); + + assert_ok!(Balances::transfer(Some(2).into(), 5.into(), 256 * 10 + 1)); // index 1 (account 2) becomes zombie + assert_eq!(Balances::total_balance(&2), 0); + assert_eq!(Balances::total_balance(&5), 256 * 10 + 1); + assert_eq!(System::account_nonce(&2), 0); + }); +} + +#[test] +fn reclaim_indexing_on_new_accounts_should_work() { + with_externalities(&mut new_test_ext(256 * 1, true), || { + assert_eq!(Balances::lookup_index(1), Some(2)); + assert_eq!(Balances::lookup_index(4), None); + assert_eq!(Balances::total_balance(&2), 256 * 20); + + assert_ok!(Balances::transfer(Some(2).into(), 5.into(), 256 * 20)); // account 2 becomes zombie freeing index 1 for reclaim) + assert_eq!(Balances::total_balance(&2), 0); + + assert_ok!(Balances::transfer(Some(5).into(), 6.into(), 256 * 1 + 0x69)); // account 6 takes index 1. + assert_eq!(Balances::total_balance(&6), 256 * 1 + 0x69); + assert_eq!(Balances::lookup_index(1), Some(6)); + }); +} + +#[test] +fn reserved_balance_should_prevent_reclaim_count() { + with_externalities(&mut new_test_ext(256 * 1, true), || { + System::inc_account_nonce(&2); + assert_eq!(Balances::lookup_index(1), Some(2)); + assert_eq!(Balances::lookup_index(4), None); + assert_eq!(Balances::total_balance(&2), 256 * 20); + + assert_ok!(Balances::reserve(&2, 256 * 19 + 1)); // account 2 becomes mostly reserved + assert_eq!(Balances::free_balance(&2), 0); // "free" account deleted." + assert_eq!(Balances::total_balance(&2), 256 * 19 + 1); // reserve still exists. + assert_eq!(System::account_nonce(&2), 1); + + assert_ok!(Balances::transfer(Some(4).into(), 5.into(), 256 * 1 + 0x69)); // account 4 tries to take index 1 for account 5. + assert_eq!(Balances::total_balance(&5), 256 * 1 + 0x69); + assert_eq!(Balances::lookup_index(1), Some(2)); // but fails. + assert_eq!(System::account_nonce(&2), 1); + + assert_eq!(Balances::slash(&2, 256 * 18 + 2), None); // account 2 gets slashed + assert_eq!(Balances::total_balance(&2), 0); // "free" account deleted." + assert_eq!(System::account_nonce(&2), 0); + + assert_ok!(Balances::transfer(Some(4).into(), 6.into(), 256 * 1 + 0x69)); // account 4 tries to take index 1 again for account 6. + assert_eq!(Balances::total_balance(&6), 256 * 1 + 0x69); + assert_eq!(Balances::lookup_index(1), Some(6)); // and succeeds. + }); +} + +#[test] +fn balance_works() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 42); + assert_eq!(Balances::free_balance(&1), 42); + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(Balances::total_balance(&1), 42); + assert_eq!(Balances::free_balance(&2), 0); + assert_eq!(Balances::reserved_balance(&2), 0); + assert_eq!(Balances::total_balance(&2), 0); + }); +} + +#[test] +fn balance_transfer_works() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + Balances::increase_total_stake_by(111); + assert_ok!(Balances::transfer(Some(1).into(), 2.into(), 69)); + assert_eq!(Balances::total_balance(&1), 42); + assert_eq!(Balances::total_balance(&2), 69); + }); +} + +#[test] +fn reserving_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + + assert_eq!(Balances::total_balance(&1), 111); + assert_eq!(Balances::free_balance(&1), 111); + assert_eq!(Balances::reserved_balance(&1), 0); + + assert_ok!(Balances::reserve(&1, 69)); + + assert_eq!(Balances::total_balance(&1), 111); + assert_eq!(Balances::free_balance(&1), 42); + assert_eq!(Balances::reserved_balance(&1), 69); + }); +} + +#[test] +fn balance_transfer_when_reserved_should_not_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + assert_ok!(Balances::reserve(&1, 69)); + assert_noop!(Balances::transfer(Some(1).into(), 2.into(), 69), "balance too low to send value"); + }); +} + +#[test] +fn deducting_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + assert_ok!(Balances::reserve(&1, 69)); + assert_eq!(Balances::free_balance(&1), 42); + }); +} + +#[test] +fn refunding_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 42); + Balances::set_reserved_balance(&1, 69); + Balances::unreserve(&1, 69); + assert_eq!(Balances::free_balance(&1), 111); + assert_eq!(Balances::reserved_balance(&1), 0); + }); +} + +#[test] +fn slashing_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + Balances::increase_total_stake_by(111); + assert_ok!(Balances::reserve(&1, 69)); + assert!(Balances::slash(&1, 69).is_none()); + assert_eq!(Balances::free_balance(&1), 0); + assert_eq!(Balances::reserved_balance(&1), 42); + assert_eq!(>::get(), 44); + }); +} + +#[test] +fn slashing_incomplete_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 42); + Balances::increase_total_stake_by(42); + assert_ok!(Balances::reserve(&1, 21)); + assert!(Balances::slash(&1, 69).is_some()); + assert_eq!(Balances::free_balance(&1), 0); + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(>::get(), 2); + }); +} + +#[test] +fn unreserving_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + assert_ok!(Balances::reserve(&1, 111)); + Balances::unreserve(&1, 42); + assert_eq!(Balances::reserved_balance(&1), 69); + assert_eq!(Balances::free_balance(&1), 42); + }); +} + +#[test] +fn slashing_reserved_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + Balances::increase_total_stake_by(111); + assert_ok!(Balances::reserve(&1, 111)); + assert!(Balances::slash_reserved(&1, 42).is_none()); + assert_eq!(Balances::reserved_balance(&1), 69); + assert_eq!(Balances::free_balance(&1), 0); + assert_eq!(>::get(), 71); + }); +} + +#[test] +fn slashing_incomplete_reserved_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + Balances::increase_total_stake_by(111); + assert_ok!(Balances::reserve(&1, 42)); + assert!(Balances::slash_reserved(&1, 69).is_some()); + assert_eq!(Balances::free_balance(&1), 69); + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(>::get(), 71); + }); +} + +#[test] +fn transferring_reserved_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 110); + Balances::set_free_balance(&2, 1); + assert_ok!(Balances::reserve(&1, 110)); + assert_ok!(Balances::repatriate_reserved(&1, &2, 41), None); + assert_eq!(Balances::reserved_balance(&1), 69); + assert_eq!(Balances::free_balance(&1), 0); + assert_eq!(Balances::reserved_balance(&2), 0); + assert_eq!(Balances::free_balance(&2), 42); + }); +} + +#[test] +fn transferring_reserved_balance_to_nonexistent_should_fail() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 111); + assert_ok!(Balances::reserve(&1, 111)); + assert_noop!(Balances::repatriate_reserved(&1, &2, 42), "beneficiary account must pre-exist"); + }); +} + +#[test] +fn transferring_incomplete_reserved_balance_should_work() { + with_externalities(&mut new_test_ext(0, false), || { + Balances::set_free_balance(&1, 110); + Balances::set_free_balance(&2, 1); + assert_ok!(Balances::reserve(&1, 41)); + assert!(Balances::repatriate_reserved(&1, &2, 69).unwrap().is_some()); + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(Balances::free_balance(&1), 69); + assert_eq!(Balances::reserved_balance(&2), 0); + assert_eq!(Balances::free_balance(&2), 42); + }); +} + +#[test] +fn transferring_too_high_value_should_not_panic() { + with_externalities(&mut new_test_ext(0, false), || { + >::insert(1, u64::max_value()); + >::insert(2, 1); + + assert_err!( + Balances::transfer(Some(1).into(), 2.into(), u64::max_value()), + "destination balance too high to receive value" + ); + + assert_eq!(Balances::free_balance(&1), u64::max_value()); + assert_eq!(Balances::free_balance(&2), 1); + }); +} + +#[test] +fn account_removal_on_free_too_low() { + with_externalities(&mut new_test_ext(100, false), || { + // Setup two accounts with free balance above the exsistential threshold. + { + Balances::set_free_balance(&1, 110); + Balances::increase_total_stake_by(110); + + Balances::set_free_balance(&2, 110); + Balances::increase_total_stake_by(110); + + assert_eq!(>::get(), 732); + } + + // Transfer funds from account 1 of such amount that after this transfer + // the balance of account 1 will be below the exsistential threshold. + // This should lead to the removal of all balance of this account. + assert_ok!(Balances::transfer(Some(1).into(), 2.into(), 20)); + + // Verify free balance removal of account 1. + assert_eq!(Balances::free_balance(&1), 0); + + // Verify that TotalIssuance tracks balance removal when free balance is too low. + assert_eq!(>::get(), 642); + }); +} diff --git a/runtime/consensus/Cargo.toml b/runtime/consensus/Cargo.toml new file mode 100644 index 00000000000..6b2e11c6c8e --- /dev/null +++ b/runtime/consensus/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-runtime-consensus" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-system/std", +] diff --git a/runtime/consensus/src/lib.rs b/runtime/consensus/src/lib.rs new file mode 100644 index 00000000000..07ff10d0714 --- /dev/null +++ b/runtime/consensus/src/lib.rs @@ -0,0 +1,267 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Conensus module for runtime; manages the authority set ready for the native code. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[allow(unused_imports)] +#[macro_use] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_primitives as primitives; +extern crate substrate_codec as codec; +extern crate substrate_runtime_system as system; +extern crate substrate_primitives; + +use rstd::prelude::*; +use runtime_support::{storage, Parameter}; +use runtime_support::dispatch::Result; +use runtime_support::storage::StorageValue; +use runtime_support::storage::unhashed::StorageVec; +use primitives::traits::{MaybeSerializeDebug, OnFinalise, Member, DigestItem}; +use primitives::bft::MisbehaviorReport; +use system::{ensure_signed, ensure_inherent, ensure_root}; + +#[cfg(any(feature = "std", test))] +use substrate_primitives::Blake2Hasher; +#[cfg(any(feature = "std", test))] +use std::collections::HashMap; + +pub const AUTHORITY_AT: &'static [u8] = b":auth:"; +pub const AUTHORITY_COUNT: &'static [u8] = b":auth:len"; + +struct AuthorityStorageVec(rstd::marker::PhantomData); +impl StorageVec for AuthorityStorageVec { + type Item = S; + const PREFIX: &'static [u8] = AUTHORITY_AT; +} + +pub const CODE: &'static [u8] = b":code"; + +pub type KeyValue = (Vec, Vec); + +pub trait OnOfflineValidator { + fn on_offline_validator(validator_index: usize); +} + +impl OnOfflineValidator for () { + fn on_offline_validator(_validator_index: usize) {} +} + +pub type Log = RawLog< + ::SessionKey, +>; + +/// A logs in this module. +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +#[derive(Encode, Decode, PartialEq, Eq, Clone)] +pub enum RawLog { + /// Authorities set has been changed. Contains the new set of authorities. + AuthoritiesChange(Vec), +} + +impl DigestItem for RawLog { + type AuthorityId = SessionKey; + + /// Try to cast the log entry as AuthoritiesChange log entry. + fn as_authorities_change(&self) -> Option<&[SessionKey]> { + match *self { + RawLog::AuthoritiesChange(ref item) => Some(&item), + } + } +} + +// Implementation for tests outside of this crate. +impl From> for u64 { + fn from(log: RawLog) -> u64 { + match log { + RawLog::AuthoritiesChange(_) => 1, + } + } +} + +pub trait Trait: system::Trait { + /// The allowed extrinsic position for `note_offline` inherent. + const NOTE_OFFLINE_POSITION: u32; + + /// Type for all log entries of this module. + type Log: From> + Into>; + + type SessionKey: Parameter + Default + MaybeSerializeDebug; + type OnOfflineValidator: OnOfflineValidator; +} + +decl_storage! { + trait Store for Module as Consensus { + // Authorities set actual at the block execution start. IsSome only if + // the set has been changed. + OriginalAuthorities: Vec; + } +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn report_misbehavior(origin, report: MisbehaviorReport) -> Result; + fn note_offline(origin, offline_val_indices: Vec) -> Result; + fn remark(origin, remark: Vec) -> Result; + fn set_code(origin, new: Vec) -> Result; + fn set_storage(origin, items: Vec) -> Result; + } +} + +impl Module { + /// Get the current set of authorities. These are the session keys. + pub fn authorities() -> Vec { + AuthorityStorageVec::::items() + } + + /// Set the new code. + fn set_code(origin: T::Origin, new: Vec) -> Result { + ensure_root(origin)?; + storage::unhashed::put_raw(CODE, &new); + Ok(()) + } + + /// Set some items of storage. + fn set_storage(origin: T::Origin, items: Vec) -> Result { + ensure_root(origin)?; + for i in &items { + storage::unhashed::put_raw(&i.0, &i.1); + } + Ok(()) + } + + /// Report some misbehaviour. + fn report_misbehavior(origin: T::Origin, _report: MisbehaviorReport) -> Result { + ensure_signed(origin)?; + // TODO. + Ok(()) + } + + /// Note the previous block's validator missed their opportunity to propose a block. This only comes in + /// if 2/3+1 of the validators agree that no proposal was submitted. It's only relevant + /// for the previous block. + fn note_offline(origin: T::Origin, offline_val_indices: Vec) -> Result { + ensure_inherent(origin)?; + assert!( + >::extrinsic_index() == Some(T::NOTE_OFFLINE_POSITION), + "note_offline extrinsic must be at position {} in the block", + T::NOTE_OFFLINE_POSITION + ); + + for validator_index in offline_val_indices.into_iter() { + T::OnOfflineValidator::on_offline_validator(validator_index as usize); + } + + Ok(()) + } + + /// Make some on-chain remark. + fn remark(origin: T::Origin, _remark: Vec) -> Result { + ensure_signed(origin)?; + Ok(()) + } + + /// Set the current set of authorities' session keys. + /// + /// Called by `next_session` only. + pub fn set_authorities(authorities: &[T::SessionKey]) { + let current_authorities = AuthorityStorageVec::::items(); + if current_authorities != authorities { + Self::save_original_authorities(Some(current_authorities)); + AuthorityStorageVec::::set_items(authorities); + } + } + + /// Set a single authority by index. + pub fn set_authority(index: u32, key: &T::SessionKey) { + let current_authority = AuthorityStorageVec::::item(index); + if current_authority != *key { + Self::save_original_authorities(None); + AuthorityStorageVec::::set_item(index, key); + } + } + + /// Save original authorities set. + fn save_original_authorities(current_authorities: Option>) { + if OriginalAuthorities::::get().is_some() { + // if we have already saved original set before, do not overwrite + return; + } + + >::put(current_authorities.unwrap_or_else(|| + AuthorityStorageVec::::items())); + } +} + +/// Finalization hook for the consensus module. +impl OnFinalise for Module { + fn on_finalise(_n: T::BlockNumber) { + if let Some(_) = >::take() { + // TODO: call Self::deposit_log + } + } +} + +#[cfg(any(feature = "std", test))] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + pub authorities: Vec, + #[serde(with = "substrate_primitives::bytes")] + pub code: Vec, +} + +#[cfg(any(feature = "std", test))] +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + authorities: vec![], + code: vec![], + } + } +} + +#[cfg(any(feature = "std", test))] +impl primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> ::std::result::Result, Vec>, String> { + use codec::{Encode, KeyedVec}; + let auth_count = self.authorities.len() as u32; + let mut r: runtime_io::TestExternalities = self.authorities.into_iter().enumerate().map(|(i, v)| + ((i as u32).to_keyed_vec(AUTHORITY_AT), v.encode()) + ).collect(); + r.insert(AUTHORITY_COUNT.to_vec(), auth_count.encode()); + r.insert(CODE.to_vec(), self.code); + Ok(r.into()) + } +} diff --git a/runtime/contract/Cargo.toml b/runtime/contract/Cargo.toml new file mode 100644 index 00000000000..0eef76fabb6 --- /dev/null +++ b/runtime/contract/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "substrate-runtime-contract" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +parity-wasm = { version = "0.31", default_features = false } +pwasm-utils = { version = "0.3", default_features = false } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-sandbox = { path = "../../core/runtime-sandbox", default_features = false } +substrate-runtime-primitives = { path = "../../runtime/primitives", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-system = { path = "../../runtime/system", default_features = false } +substrate-runtime-balances = { path = "../balances", default_features = false } + +[dev-dependencies] +wabt = "0.4" +assert_matches = "1.1" + +[features] +default = ["std"] +std = [ + "serde_derive", + "serde/std", + "substrate-codec/std", + "substrate-primitives/std", + "substrate-runtime-primitives/std", + "substrate-runtime-io/std", + "substrate-runtime-std/std", + "substrate-runtime-balances/std", + "substrate-runtime-sandbox/std", + "substrate-runtime-support/std", + "substrate-runtime-system/std", + "parity-wasm/std", + "pwasm-utils/std", +] diff --git a/runtime/contract/src/account_db.rs b/runtime/contract/src/account_db.rs new file mode 100644 index 00000000000..d7ff094003c --- /dev/null +++ b/runtime/contract/src/account_db.rs @@ -0,0 +1,180 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Auxilliaries to help with managing partial changes to accounts state. + +use super::{CodeOf, StorageOf, Trait}; +use double_map::StorageDoubleMap; +use rstd::cell::RefCell; +use rstd::collections::btree_map::{BTreeMap, Entry}; +use rstd::prelude::*; +use runtime_support::StorageMap; +use {balances, system}; + +pub struct ChangeEntry { + balance: Option, + code: Option>, + storage: BTreeMap, Option>>, +} + +// Cannot derive(Default) since it erroneously bounds T by Default. +impl Default for ChangeEntry { + fn default() -> Self { + ChangeEntry { + balance: Default::default(), + code: Default::default(), + storage: Default::default(), + } + } +} + +pub type ChangeSet = BTreeMap<::AccountId, ChangeEntry>; + +pub trait AccountDb { + fn get_storage(&self, account: &T::AccountId, location: &[u8]) -> Option>; + fn get_code(&self, account: &T::AccountId) -> Vec; + fn get_balance(&self, account: &T::AccountId) -> T::Balance; + + fn commit(&mut self, change_set: ChangeSet); +} + +pub struct DirectAccountDb; +impl AccountDb for DirectAccountDb { + fn get_storage(&self, account: &T::AccountId, location: &[u8]) -> Option> { + >::get(account.clone(), location.to_vec()) + } + fn get_code(&self, account: &T::AccountId) -> Vec { + >::get(account) + } + fn get_balance(&self, account: &T::AccountId) -> T::Balance { + balances::Module::::free_balance(account) + } + fn commit(&mut self, s: ChangeSet) { + for (address, changed) in s.into_iter() { + if let Some(balance) = changed.balance { + if let balances::UpdateBalanceOutcome::AccountKilled = + balances::Module::::set_free_balance_creating(&address, balance) + { + // Account killed. This will ultimately lead to calling `OnFreeBalanceZero` callback + // which will make removal of CodeOf and StorageOf for this account. + // In order to avoid writing over the deleted properties we `continue` here. + continue; + } + } + if let Some(code) = changed.code { + >::insert(&address, &code); + } + for (k, v) in changed.storage.into_iter() { + if let Some(value) = v { + >::insert(address.clone(), k, value); + } else { + >::remove(address.clone(), k); + } + } + } + } +} + +pub struct OverlayAccountDb<'a, T: Trait + 'a> { + local: RefCell>, + underlying: &'a AccountDb, +} +impl<'a, T: Trait> OverlayAccountDb<'a, T> { + pub fn new(underlying: &'a AccountDb) -> OverlayAccountDb<'a, T> { + OverlayAccountDb { + local: RefCell::new(ChangeSet::new()), + underlying, + } + } + + pub fn into_change_set(self) -> ChangeSet { + self.local.into_inner() + } + + pub fn set_storage( + &mut self, + account: &T::AccountId, + location: Vec, + value: Option>, + ) { + self.local + .borrow_mut() + .entry(account.clone()) + .or_insert(Default::default()) + .storage + .insert(location, value); + } + pub fn set_code(&mut self, account: &T::AccountId, code: Vec) { + self.local + .borrow_mut() + .entry(account.clone()) + .or_insert(Default::default()) + .code = Some(code); + } + pub fn set_balance(&mut self, account: &T::AccountId, balance: T::Balance) { + self.local + .borrow_mut() + .entry(account.clone()) + .or_insert(Default::default()) + .balance = Some(balance); + } +} + +impl<'a, T: Trait> AccountDb for OverlayAccountDb<'a, T> { + fn get_storage(&self, account: &T::AccountId, location: &[u8]) -> Option> { + self.local + .borrow() + .get(account) + .and_then(|a| a.storage.get(location)) + .cloned() + .unwrap_or_else(|| self.underlying.get_storage(account, location)) + } + fn get_code(&self, account: &T::AccountId) -> Vec { + self.local + .borrow() + .get(account) + .and_then(|a| a.code.clone()) + .unwrap_or_else(|| self.underlying.get_code(account)) + } + fn get_balance(&self, account: &T::AccountId) -> T::Balance { + self.local + .borrow() + .get(account) + .and_then(|a| a.balance) + .unwrap_or_else(|| self.underlying.get_balance(account)) + } + fn commit(&mut self, s: ChangeSet) { + let mut local = self.local.borrow_mut(); + + for (address, changed) in s.into_iter() { + match local.entry(address) { + Entry::Occupied(e) => { + let mut value = e.into_mut(); + if changed.balance.is_some() { + value.balance = changed.balance; + } + if changed.code.is_some() { + value.code = changed.code; + } + value.storage.extend(changed.storage.into_iter()); + } + Entry::Vacant(e) => { + e.insert(changed); + } + } + } + } +} diff --git a/runtime/contract/src/double_map.rs b/runtime/contract/src/double_map.rs new file mode 100644 index 00000000000..6867d2a5c69 --- /dev/null +++ b/runtime/contract/src/double_map.rs @@ -0,0 +1,90 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! An implementation of double map backed by storage. +//! +//! This implementation is somewhat specialized to the tracking of the storage of accounts. + +use rstd::prelude::*; +use codec::{Codec, Encode}; +use runtime_support::storage::unhashed; +use runtime_io::{blake2_256, twox_128}; + +/// Returns only a first part of the storage key. +/// +/// Hashed by XX. +fn first_part_of_key(k1: M::Key1) -> [u8; 16] { + let mut raw_prefix = Vec::new(); + raw_prefix.extend(M::PREFIX); + raw_prefix.extend(Encode::encode(&k1)); + twox_128(&raw_prefix) +} + +/// Returns a compound key that consist of the two parts: (prefix, `k1`) and `k2`. +/// +/// The first part is hased by XX and then concatenated with a blake2 hash of `k2`. +fn full_key(k1: M::Key1, k2: M::Key2) -> Vec { + let first_part = first_part_of_key::(k1); + let second_part = blake2_256(&Encode::encode(&k2)); + + let mut k = Vec::new(); + k.extend(&first_part); + k.extend(&second_part); + k +} + +/// An implementation of a map with a two keys. +/// +/// It provides an important ability to efficiently remove all entries +/// that have a common first key. +/// +/// # Mapping of keys to a storage path +/// +/// The storage key (i.e. the key under which the `Value` will be stored) is created from two parts. +/// The first part is a XX hash of a concatenation of the `PREFIX` and `Key1`. And the second part +/// is a blake2 hash of a `Key2`. +/// +/// Blake2 is used for `Key2` is because it will be used as a key for contract's storage and +/// thus will be susceptible for a untrusted input. +pub trait StorageDoubleMap { + type Key1: Codec; + type Key2: Codec; + type Value: Codec + Default; + + const PREFIX: &'static [u8]; + + /// Insert an entry into this map. + fn insert(k1: Self::Key1, k2: Self::Key2, val: Self::Value) { + unhashed::put(&full_key::(k1, k2)[..], &val); + } + + /// Remove an entry from this map. + fn remove(k1: Self::Key1, k2: Self::Key2) { + unhashed::kill(&full_key::(k1, k2)[..]); + } + + /// Get an entry from this map. + /// + /// If there is entry stored under the given keys, returns `None`. + fn get(k1: Self::Key1, k2: Self::Key2) -> Option { + unhashed::get(&full_key::(k1, k2)[..]) + } + + /// Removes all entries that shares the `k1` as the first key. + fn remove_prefix(k1: Self::Key1) { + unhashed::kill_prefix(&first_part_of_key::(k1)) + } +} diff --git a/runtime/contract/src/exec.rs b/runtime/contract/src/exec.rs new file mode 100644 index 00000000000..ba3e0ff9676 --- /dev/null +++ b/runtime/contract/src/exec.rs @@ -0,0 +1,266 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use super::{CodeOf, MaxDepth, ContractAddressFor, Module, Trait}; +use account_db::{AccountDb, OverlayAccountDb}; +use gas::GasMeter; +use vm; + +use rstd::prelude::*; +use runtime_primitives::traits::{Zero, CheckedAdd, CheckedSub}; +use runtime_support::{StorageMap, StorageValue}; +use balances::{self, EnsureAccountLiquid}; + +pub struct CreateReceipt { + pub address: T::AccountId, +} + +pub struct CallReceipt { + pub return_data: Vec, +} + +pub struct ExecutionContext<'a, T: Trait + 'a> { + // typically should be dest + pub self_account: T::AccountId, + pub overlay: OverlayAccountDb<'a, T>, + pub depth: usize, +} + +impl<'a, T: Trait> ExecutionContext<'a, T> { + /// Make a call to the specified address. + pub fn call( + &mut self, + caller: T::AccountId, + dest: T::AccountId, + value: T::Balance, + gas_meter: &mut GasMeter, + data: &[u8], + ) -> Result { + if self.depth == >::get() as usize { + return Err("reached maximum depth, cannot make a call"); + } + + let call_base_fee = >::call_base_fee(); + if gas_meter.charge(call_base_fee).is_out_of_gas() { + return Err("not enough gas to pay base call fee"); + } + + let dest_code = >::get(&dest); + + let (exec_result, change_set) = { + let mut overlay = OverlayAccountDb::new(&self.overlay); + + if value > T::Balance::zero() { + transfer( + gas_meter, + false, + &self.self_account, + &dest, + value, + &mut overlay, + )?; + } + + let mut nested = ExecutionContext { + overlay: overlay, + self_account: dest.clone(), + depth: self.depth + 1, + }; + let exec_result = if !dest_code.is_empty() { + vm::execute( + &dest_code, + data, + &mut CallContext { + ctx: &mut nested, + _caller: caller, + }, + gas_meter, + ).map_err(|_| "vm execute returned error while call")? + } else { + // that was a plain transfer + vm::ExecutionResult { + return_data: Vec::new(), + } + }; + + (exec_result, nested.overlay.into_change_set()) + }; + + self.overlay.commit(change_set); + + Ok(CallReceipt { + return_data: exec_result.return_data, + }) + } + + pub fn create( + &mut self, + caller: T::AccountId, + endowment: T::Balance, + gas_meter: &mut GasMeter, + ctor: &[u8], + data: &[u8], + ) -> Result, &'static str> { + if self.depth == >::get() as usize { + return Err("reached maximum depth, cannot create"); + } + + let create_base_fee = >::create_base_fee(); + if gas_meter.charge(create_base_fee).is_out_of_gas() { + return Err("not enough gas to pay base create fee"); + } + + let dest = T::DetermineContractAddress::contract_address_for(ctor, data, &self.self_account); + if >::exists(&dest) { + // TODO: Is it enough? + return Err("contract already exists"); + } + + let change_set = { + let mut overlay = OverlayAccountDb::new(&self.overlay); + + if endowment > T::Balance::zero() { + transfer( + gas_meter, + true, + &self.self_account, + &dest, + endowment, + &mut overlay, + )?; + } + + let mut nested = ExecutionContext { + overlay: overlay, + self_account: dest.clone(), + depth: self.depth + 1, + }; + let exec_result = { + vm::execute( + ctor, + data, + &mut CallContext { + ctx: &mut nested, + _caller: caller, + }, + gas_meter, + ).map_err(|_| "vm execute returned error while create")? + }; + + nested.overlay.set_code(&dest, exec_result.return_data); + nested.overlay.into_change_set() + }; + + self.overlay.commit(change_set); + + Ok(CreateReceipt { + address: dest, + }) + } +} + +fn transfer( + gas_meter: &mut GasMeter, + contract_create: bool, + transactor: &T::AccountId, + dest: &T::AccountId, + value: T::Balance, + overlay: &mut OverlayAccountDb, +) -> Result<(), &'static str> { + let would_create = overlay.get_balance(transactor).is_zero(); + + let fee: T::Balance = if contract_create { + >::contract_fee() + } else { + if would_create { + >::creation_fee() + } else { + >::transfer_fee() + } + }; + + if gas_meter.charge_by_balance(fee).is_out_of_gas() { + return Err("not enough gas to pay transfer fee"); + } + + let from_balance = overlay.get_balance(transactor); + let new_from_balance = match from_balance.checked_sub(&value) { + Some(b) => b, + None => return Err("balance too low to send value"), + }; + if would_create && value < >::existential_deposit() { + return Err("value too low to create account"); + } + ::EnsureAccountLiquid::ensure_account_liquid(transactor)?; + + let to_balance = overlay.get_balance(dest); + let new_to_balance = match to_balance.checked_add(&value) { + Some(b) => b, + None => return Err("destination balance too high to receive value"), + }; + + if transactor != dest { + overlay.set_balance(transactor, new_from_balance); + overlay.set_balance(dest, new_to_balance); + } + + Ok(()) +} + +struct CallContext<'a, 'b: 'a, T: Trait + 'b> { + ctx: &'a mut ExecutionContext<'b, T>, + _caller: T::AccountId, +} + +impl<'a, 'b: 'a, T: Trait + 'b> vm::Ext for CallContext<'a, 'b, T> { + type T = T; + + fn get_storage(&self, key: &[u8]) -> Option> { + self.ctx.overlay.get_storage(&self.ctx.self_account, key) + } + + fn set_storage(&mut self, key: &[u8], value: Option>) { + self.ctx + .overlay + .set_storage(&self.ctx.self_account, key.to_vec(), value) + } + + fn create( + &mut self, + code: &[u8], + endowment: T::Balance, + gas_meter: &mut GasMeter, + data: &[u8], + ) -> Result, ()> { + let caller = self.ctx.self_account.clone(); + self.ctx + .create(caller, endowment, gas_meter, code, &data) + .map_err(|_| ()) + } + + fn call( + &mut self, + to: &T::AccountId, + value: T::Balance, + gas_meter: &mut GasMeter, + data: &[u8], + ) -> Result { + let caller = self.ctx.self_account.clone(); + self.ctx + .call(caller, to.clone(), value, gas_meter, data) + .map_err(|_| ()) + } +} diff --git a/runtime/contract/src/gas.rs b/runtime/contract/src/gas.rs new file mode 100644 index 00000000000..9d1978f7a52 --- /dev/null +++ b/runtime/contract/src/gas.rs @@ -0,0 +1,178 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use {Trait, Module, GasSpent}; +use runtime_primitives::traits::{As, CheckedMul, CheckedSub, Zero}; +use runtime_support::StorageValue; +use balances; + +#[must_use] +#[derive(Debug, PartialEq, Eq)] +pub enum GasMeterResult { + Proceed, + OutOfGas, +} + +impl GasMeterResult { + pub fn is_out_of_gas(&self) -> bool { + match *self { + GasMeterResult::OutOfGas => true, + GasMeterResult::Proceed => false, + } + } +} + +pub struct GasMeter { + limit: T::Gas, + /// Amount of gas left from initial gas limit. Can reach zero. + gas_left: T::Gas, + gas_price: T::Balance, +} +impl GasMeter { + #[cfg(test)] + pub fn with_limit(gas_limit: T::Gas, gas_price: T::Balance) -> GasMeter { + GasMeter { + limit: gas_limit, + gas_left: gas_limit, + gas_price, + } + } + + /// Account for used gas. + /// + /// Returns `OutOfGas` if there is not enough gas or addition of the specified + /// amount of gas has lead to overflow. On success returns `Proceed`. + /// + /// NOTE that `amount` is always consumed, i.e. if there is not enough gas + /// then the counter will be set to zero. + pub fn charge(&mut self, amount: T::Gas) -> GasMeterResult { + let new_value = match self.gas_left.checked_sub(&amount) { + None => None, + Some(val) if val.is_zero() => None, + Some(val) => Some(val), + }; + + // We always consume the gas even if there is not enough gas. + self.gas_left = new_value.unwrap_or_else(Zero::zero); + + match new_value { + Some(_) => GasMeterResult::Proceed, + None => GasMeterResult::OutOfGas, + } + } + + /// Account for used gas expressed in balance units. + /// + /// Same as [`charge`], but amount to be charged is converted from units of balance to + /// units of gas. + /// + /// [`charge`]: #method.charge + pub fn charge_by_balance(&mut self, amount: T::Balance) -> GasMeterResult { + let amount_in_gas: T::Balance = amount / self.gas_price; + let amount_in_gas: T::Gas = >::sa(amount_in_gas); + self.charge(amount_in_gas) + } + + /// Allocate some amount of gas and perform some work with + /// a newly created nested gas meter. + /// + /// Invokes `f` with either the gas meter that has `amount` gas left or + /// with `None`, if this gas meter has not enough gas to allocate given `amount`. + /// + /// All unused gas in the nested gas meter is returned to this gas meter. + pub fn with_nested>) -> R>( + &mut self, + amount: T::Gas, + f: F, + ) -> R { + // NOTE that it is ok to allocate all available gas since it still ensured + // by `charge` that it doesn't reach zero. + if self.gas_left < amount { + f(None) + } else { + self.gas_left = self.gas_left - amount; + let mut nested = GasMeter { + limit: amount, + gas_left: amount, + gas_price: self.gas_price, + }; + + let r = f(Some(&mut nested)); + + self.gas_left = self.gas_left + nested.gas_left; + + r + } + } + + /// Returns how much gas left from the initial budget. + pub fn gas_left(&self) -> T::Gas { + self.gas_left + } + + /// Returns how much gas was spent. + fn spent(&self) -> T::Gas { + self.limit - self.gas_left + } +} + +/// Buy the given amount of gas. +/// +/// Cost is calculated by multiplying the gas cost (taken from the storage) by the `gas_limit`. +/// The funds are deducted from `transactor`. +pub fn buy_gas( + transactor: &T::AccountId, + gas_limit: T::Gas, +) -> Result, &'static str> { + // Check if the specified amount of gas is available in the current block. + // This cannot underflow since `gas_spent` is never greater than `block_gas_limit`. + let gas_available = >::block_gas_limit() - >::gas_spent(); + if gas_limit > gas_available { + return Err("block gas limit is reached"); + } + + // Buy the specified amount of gas. + let gas_price = >::gas_price(); + let b = >::free_balance(transactor); + let cost = >::as_(gas_limit.clone()) + .checked_mul(&gas_price) + .ok_or("overflow multiplying gas limit by price")?; + if b < cost + >::existential_deposit() { + return Err("not enough funds for transaction fee"); + } + >::set_free_balance(transactor, b - cost); + >::decrease_total_stake_by(cost); + Ok(GasMeter { + limit: gas_limit, + gas_left: gas_limit, + gas_price, + }) +} + +/// Refund the unused gas. +pub fn refund_unused_gas(transactor: &T::AccountId, gas_meter: GasMeter) { + // Increase total spent gas. + // This cannot overflow, since `gas_spent` is never greater than `block_gas_limit`, which + // also has T::Gas type. + let gas_spent = >::gas_spent() + gas_meter.spent(); + >::put(gas_spent); + + // Refund gas left by the price it was bought. + let b = >::free_balance(transactor); + let refund = >::as_(gas_meter.gas_left) * gas_meter.gas_price; + >::set_free_balance(transactor, b + refund); + >::increase_total_stake_by(refund); +} diff --git a/runtime/contract/src/genesis_config.rs b/runtime/contract/src/genesis_config.rs new file mode 100644 index 00000000000..2d9a50573a6 --- /dev/null +++ b/runtime/contract/src/genesis_config.rs @@ -0,0 +1,54 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Build the contract module part of the genesis block storage. + +#![cfg(feature = "std")] + +use {Trait, ContractFee, CallBaseFee, CreateBaseFee, GasPrice, MaxDepth, BlockGasLimit}; + +use runtime_primitives; +use runtime_io::{self, twox_128}; +use runtime_support::StorageValue; +use codec::Encode; +use std::collections::HashMap; +use substrate_primitives::Blake2Hasher; + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + pub contract_fee: T::Balance, + pub call_base_fee: T::Gas, + pub create_base_fee: T::Gas, + pub gas_price: T::Balance, + pub max_depth: u32, + pub block_gas_limit: T::Gas, +} + +impl runtime_primitives::BuildStorage for GenesisConfig { + fn build_storage(self) -> ::std::result::Result, Vec>, String> { + let r: runtime_io::TestExternalities = map![ + twox_128(>::key()).to_vec() => self.contract_fee.encode(), + twox_128(>::key()).to_vec() => self.call_base_fee.encode(), + twox_128(>::key()).to_vec() => self.create_base_fee.encode(), + twox_128(>::key()).to_vec() => self.gas_price.encode(), + twox_128(>::key()).to_vec() => self.max_depth.encode(), + twox_128(>::key()).to_vec() => self.block_gas_limit.encode() + ]; + Ok(r.into()) + } +} diff --git a/runtime/contract/src/lib.rs b/runtime/contract/src/lib.rs new file mode 100644 index 00000000000..8114d265160 --- /dev/null +++ b/runtime/contract/src/lib.rs @@ -0,0 +1,278 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Smart-contract module for runtime; Allows deployment and execution of smart-contracts +//! expressed in WebAssembly. +//! +//! This module provides an ability to create smart-contract accounts and send them messages. +//! A smart-contract is an account with associated code and storage. When such an account receives a message, +//! the code associated with that account gets executed. +//! +//! The code is allowed to alter the storage entries of the associated account, +//! create smart-contracts or send messages to existing smart-contracts. +//! +//! For any actions invoked by the smart-contracts fee must be paid. The fee is paid in gas. +//! Gas is bought upfront up to the, specified in transaction, limit. Any unused gas is refunded +//! after the transaction (regardless of the execution outcome). If all gas is used, +//! then changes made for the specific call or create are reverted (including balance transfers). +//! +//! Failures are typically not cascading. That, for example, means that if contract A calls B and B errors +//! somehow, then A can decide if it should proceed or error. +//! +//! # Interaction with the system +//! +//! ## Finalization +//! +//! This module requires performing some finalization steps at the end of the block. If not performed +//! the module will have incorrect behavior. +//! +//! Call [`Module::execute`] at the end of the block. The order in relation to +//! the other module doesn't matter. +//! +//! ## Account killing +//! +//! When `staking` module determines that account is dead (e.g. account's balance fell below +//! exsistential deposit) then it reaps the account. That will lead to deletion of the associated +//! code and storage of the account. +//! +//! [`Module::execute`]: struct.Module.html#impl-OnFinalise + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[cfg(feature = "std")] +extern crate serde; + +extern crate parity_wasm; +extern crate pwasm_utils; + +extern crate substrate_codec as codec; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_sandbox as sandbox; + +#[macro_use] +extern crate substrate_runtime_std as rstd; + +extern crate substrate_runtime_balances as balances; +extern crate substrate_runtime_system as system; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_primitives; + +#[cfg(test)] +#[macro_use] +extern crate assert_matches; + +#[cfg(test)] +extern crate wabt; + +mod account_db; +mod double_map; +mod exec; +mod vm; +mod gas; + +mod genesis_config; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "std")] +pub use genesis_config::GenesisConfig; +use exec::ExecutionContext; +use account_db::{AccountDb, OverlayAccountDb}; +use double_map::StorageDoubleMap; + +use rstd::prelude::*; +use codec::Codec; +use runtime_primitives::traits::{As, SimpleArithmetic, OnFinalise}; +use runtime_support::dispatch::Result; +use runtime_support::{Parameter, StorageMap, StorageValue}; +use system::ensure_signed; + +pub trait Trait: balances::Trait { + /// Function type to get the contract address given the creator. + type DetermineContractAddress: ContractAddressFor; + + // As is needed for wasm-utils + type Gas: Parameter + Default + Codec + SimpleArithmetic + Copy + As + As + As; +} + +pub trait ContractAddressFor { + fn contract_address_for(code: &[u8], data: &[u8], origin: &AccountId) -> AccountId; +} + +decl_module! { + /// Contracts module. + pub struct Module for enum Call where origin: T::Origin { + // TODO: Change AccountId to staking::Address + fn call( + origin, + dest: T::AccountId, + value: T::Balance, + gas_limit: T::Gas, + data: Vec + ) -> Result; + + fn create( + origin, + value: T::Balance, + gas_limit: T::Gas, + ctor: Vec, + data: Vec + ) -> Result; + } +} + +decl_storage! { + trait Store for Module as Contract { + /// The fee required to create a contract. At least as big as staking's ReclaimRebate. + ContractFee get(contract_fee): required T::Balance; + /// The fee charged for a call into a contract. + CallBaseFee get(call_base_fee): required T::Gas; + /// The fee charged for a create of a contract. + CreateBaseFee get(create_base_fee): required T::Gas; + /// The price of one unit of gas. + GasPrice get(gas_price): required T::Balance; + /// The maximum nesting level of a call/create stack. + MaxDepth get(max_depth): required u32; + /// The maximum amount of gas that could be expended per block. + BlockGasLimit get(block_gas_limit): required T::Gas; + /// Gas spent so far in this block. + GasSpent get(gas_spent): default T::Gas; + + /// The code associated with an account. + pub CodeOf: default map [ T::AccountId => Vec ]; // TODO Vec values should be optimised to not do a length prefix. + } +} + +// TODO: consider storing upper-bound for contract's gas limit in fixed-length runtime +// code in contract itself and use that. + +/// The storage items associated with an account/key. +/// +/// TODO: keys should also be able to take AsRef to ensure Vecs can be passed as &[u8] +pub(crate) struct StorageOf(::rstd::marker::PhantomData); +impl double_map::StorageDoubleMap for StorageOf { + const PREFIX: &'static [u8] = b"con:sto:"; + type Key1 = T::AccountId; + type Key2 = Vec; + type Value = Vec; +} + +impl Module { + /// Make a call to a specified account, optionally transferring some balance. + fn call( + origin: ::Origin, + dest: T::AccountId, + value: T::Balance, + gas_limit: T::Gas, + data: Vec, + ) -> Result { + let origin = ensure_signed(origin)?; + + // Pay for the gas upfront. + // + // NOTE: it is very important to avoid any state changes before + // paying for the gas. + let mut gas_meter = gas::buy_gas::(&origin, gas_limit)?; + + let mut ctx = ExecutionContext { + self_account: origin.clone(), + depth: 0, + overlay: OverlayAccountDb::::new(&account_db::DirectAccountDb), + }; + let result = ctx.call(origin.clone(), dest, value, &mut gas_meter, &data); + + if let Ok(_) = result { + // Commit all changes that made it thus far into the persistant storage. + account_db::DirectAccountDb.commit(ctx.overlay.into_change_set()); + } + + // Refund cost of the unused gas. + // + // NOTE: this should go after the commit to the storage, since the storage changes + // can alter the balance of the caller. + gas::refund_unused_gas::(&origin, gas_meter); + + result.map(|_| ()) + } + + /// Create a new contract, optionally transfering some balance to the created account. + /// + /// Creation is executed as follows:ExecutionContext + /// + /// - the destination address is computed based on the sender and hash of the code. + /// - account is created at the computed address. + /// - the `ctor_code` is executed in the context of the newly created account. Buffer returned + /// after the execution is saved as the `code` of the account. That code will be invoked + /// upon any message received by this account. + fn create( + origin: ::Origin, + endowment: T::Balance, + gas_limit: T::Gas, + ctor_code: Vec, + data: Vec, + ) -> Result { + let origin = ensure_signed(origin)?; + + // Pay for the gas upfront. + // + // NOTE: it is very important to avoid any state changes before + // paying for the gas. + let mut gas_meter = gas::buy_gas::(&origin, gas_limit)?; + + let mut ctx = ExecutionContext { + self_account: origin.clone(), + depth: 0, + overlay: OverlayAccountDb::::new(&account_db::DirectAccountDb), + }; + let result = ctx.create(origin.clone(), endowment, &mut gas_meter, &ctor_code, &data); + + if let Ok(_) = result { + // Commit all changes that made it thus far into the persistant storage. + account_db::DirectAccountDb.commit(ctx.overlay.into_change_set()); + } + + // Refund cost of the unused gas. + // + // NOTE: this should go after the commit to the storage, since the storage changes + // can alter the balance of the caller. + gas::refund_unused_gas::(&origin, gas_meter); + + result.map(|_| ()) + } +} + +impl balances::OnFreeBalanceZero for Module { + fn on_free_balance_zero(who: &T::AccountId) { + >::remove(who); + >::remove_prefix(who.clone()); + } +} + +/// Finalization hook for the smart-contract module. +impl OnFinalise for Module { + fn on_finalise(_n: T::BlockNumber) { + >::kill(); + } +} diff --git a/runtime/contract/src/tests.rs b/runtime/contract/src/tests.rs new file mode 100644 index 00000000000..17f5a4d88aa --- /dev/null +++ b/runtime/contract/src/tests.rs @@ -0,0 +1,670 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use double_map::StorageDoubleMap; +use runtime_io::with_externalities; +use runtime_primitives::testing::{Digest, H256, Header}; +use runtime_primitives::traits::{BlakeTwo256}; +use runtime_primitives::BuildStorage; +use runtime_support::StorageMap; +use substrate_primitives::Blake2Hasher; +use wabt; +use { + runtime_io, balances, system, CodeOf, ContractAddressFor, + GenesisConfig, Module, StorageOf, Trait, +}; + +impl_outer_origin! { + pub enum Origin for Test {} +} + +#[derive(Clone, Eq, PartialEq)] +pub struct Test; +impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); +} +impl balances::Trait for Test { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = Contract; + type EnsureAccountLiquid = (); + type Event = (); +} +impl Trait for Test { + type Gas = u64; + type DetermineContractAddress = DummyContractAddressFor; +} + +type Balances = balances::Module; +type Contract = Module; + +pub struct DummyContractAddressFor; +impl ContractAddressFor for DummyContractAddressFor { + fn contract_address_for(_code: &[u8], _data: &[u8], origin: &u64) -> u64 { + origin + 1 + } +} + +struct ExtBuilder { + existential_deposit: u64, + gas_price: u64, + block_gas_limit: u64, +} +impl Default for ExtBuilder { + fn default() -> Self { + Self { + existential_deposit: 0, + gas_price: 2, + block_gas_limit: 100_000_000, + } + } +} +impl ExtBuilder { + fn existential_deposit(mut self, existential_deposit: u64) -> Self { + self.existential_deposit = existential_deposit; + self + } + fn gas_price(mut self, gas_price: u64) -> Self { + self.gas_price = gas_price; + self + } + fn block_gas_limit(mut self, block_gas_limit: u64) -> Self { + self.block_gas_limit = block_gas_limit; + self + } + fn build(self) -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default() + .build_storage() + .unwrap(); + t.extend( + balances::GenesisConfig:: { + balances: vec![], + transaction_base_fee: 0, + transaction_byte_fee: 0, + existential_deposit: self.existential_deposit, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + }.build_storage() + .unwrap(), + ); + t.extend( + GenesisConfig:: { + contract_fee: 21, + call_base_fee: 135, + create_base_fee: 175, + gas_price: self.gas_price, + max_depth: 100, + block_gas_limit: self.block_gas_limit, + }.build_storage() + .unwrap(), + ); + t.into() + } +} + +const CODE_TRANSFER: &str = r#" +(module + ;; ext_call( + ;; callee_ptr: u32, + ;; callee_len: u32, + ;; gas: u64, + ;; value_ptr: u32, + ;; value_len: u32, + ;; input_data_ptr: u32, + ;; input_data_len: u32 + ;; ) -> u32 + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + (func (export "call") + (drop + (call $ext_call + (i32.const 4) ;; Pointer to "callee" address. + (i32.const 8) ;; Length of "callee" address. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 12) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 0) ;; Length of input data buffer + ) + ) + ) + ;; Destination AccountId to transfer the funds. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 4) "\09\00\00\00\00\00\00\00") + ;; Amount of value to transfer. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 12) "\06\00\00\00\00\00\00\00") +) +"#; + +#[test] +fn contract_transfer() { + const CONTRACT_SHOULD_TRANSFER_VALUE: u64 = 6; + const CONTRACT_SHOULD_TRANSFER_TO: u64 = 9; + + let code_transfer = wabt::wat2wasm(CODE_TRANSFER).unwrap(); + + with_externalities(&mut ExtBuilder::default().build(), || { + >::insert(1, code_transfer.to_vec()); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + Balances::set_free_balance(&1, 11); + Balances::increase_total_stake_by(11); + + assert_ok!(Contract::call(Origin::signed(0), 1, 3, 100_000, Vec::new())); + + assert_eq!( + Balances::free_balance(&0), + // 3 - value sent with the transaction + // 2 * 10 - gas used by the contract (10) multiplied by gas price (2) + // 2 * 135 - base gas fee for call (by transaction) + // 2 * 135 - base gas fee for call (by the contract) + 100_000_000 - 3 - (2 * 10) - (2 * 135) - (2 * 135), + ); + assert_eq!( + Balances::free_balance(&1), + 11 + 3 - CONTRACT_SHOULD_TRANSFER_VALUE, + ); + assert_eq!( + Balances::free_balance(&CONTRACT_SHOULD_TRANSFER_TO), + CONTRACT_SHOULD_TRANSFER_VALUE, + ); + }); +} + +#[test] +fn contract_transfer_oog() { + const CONTRACT_SHOULD_TRANSFER_TO: u64 = 9; + + let code_transfer = wabt::wat2wasm(CODE_TRANSFER).unwrap(); + + with_externalities(&mut ExtBuilder::default().build(), || { + >::insert(1, code_transfer.to_vec()); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + Balances::set_free_balance(&1, 11); + Balances::increase_total_stake_by(11); + + assert_ok!(Contract::call(Origin::signed(0), 1, 3, 135 + 135 + 7, Vec::new())); + + assert_eq!( + Balances::free_balance(&0), + // 3 - value sent with the transaction + // 2 * 7 - gas used by the contract (7) multiplied by gas price (2) + // 2 * 135 - base gas fee for call (by transaction) + // 2 * 135 - base gas fee for call (by contract) + 100_000_000 - 3 - (2 * 7) - (2 * 135) - (2 * 135), + ); + + // Transaction level transfer should succeed. + assert_eq!(Balances::free_balance(&1), 14); + // But `ext_call` should not. + assert_eq!(Balances::free_balance(&CONTRACT_SHOULD_TRANSFER_TO), 0); + }); +} + +#[test] +fn contract_transfer_max_depth() { + const CONTRACT_SHOULD_TRANSFER_TO: u64 = 9; + + let code_transfer = wabt::wat2wasm(CODE_TRANSFER).unwrap(); + + with_externalities(&mut ExtBuilder::default().build(), || { + >::insert(CONTRACT_SHOULD_TRANSFER_TO, code_transfer.to_vec()); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + Balances::set_free_balance(&CONTRACT_SHOULD_TRANSFER_TO, 11); + Balances::increase_total_stake_by(11); + + assert_ok!(Contract::call(Origin::signed(0), CONTRACT_SHOULD_TRANSFER_TO, 3, 100_000, Vec::new())); + + assert_eq!( + Balances::free_balance(&0), + // 3 - value sent with the transaction + // 2 * 10 * 100 - gas used by the contract (10) multiplied by gas price (2) + // multiplied by max depth (100). + // 2 * 135 * 100 - base gas fee for call (by transaction) multiplied by max depth (100). + 100_000_000 - 3 - (2 * 10 * 100) - (2 * 135 * 100), + ); + assert_eq!(Balances::free_balance(&CONTRACT_SHOULD_TRANSFER_TO), 14); + }); +} + +/// Convert a byte slice to a string with hex values. +/// +/// Each value is preceeded with a `\` character. +fn escaped_bytestring(bytes: &[u8]) -> String { + use std::fmt::Write; + let mut result = String::new(); + for b in bytes { + write!(result, "\\{:02x}", b).unwrap(); + } + result +} + +/// Create a constructor for the specified code. +/// +/// When constructor is executed, it will call `ext_return` with code that +/// specified in `child_bytecode`. +fn code_ctor(child_bytecode: &[u8]) -> String { + format!( + r#" +(module + ;; ext_return(data_ptr: u32, data_len: u32) -> ! + (import "env" "ext_return" (func $ext_return (param i32 i32))) + (import "env" "memory" (memory 1 1)) + (func (export "call") + (call $ext_return + (i32.const 4) + (i32.const {code_len}) + ) + ;; ext_return is diverging, i.e. doesn't return. + unreachable + ) + (data (i32.const 4) "{escaped_bytecode}") +) +"#, + escaped_bytecode = escaped_bytestring(child_bytecode), + code_len = child_bytecode.len(), + ) +} + +/// Returns code that uses `ext_create` runtime call. +/// +/// Takes bytecode of the contract that needs to be deployed. +fn code_create(constructor: &[u8]) -> String { + format!( + r#" +(module + ;; ext_create( + ;; code_ptr: u32, + ;; code_len: u32, + ;; gas: u64, + ;; value_ptr: u32, + ;; value_len: u32, + ;; input_data_ptr: u32, + ;; input_data_len: u32, + ;; ) -> u32 + (import "env" "ext_create" (func $ext_create (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + (func (export "call") + (drop + (call $ext_create + (i32.const 12) ;; Pointer to `code` + (i32.const {code_len}) ;; Length of `code` + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 4) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 0) ;; Length of input data buffer + ) + ) + ) + ;; Amount of value to transfer. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 4) "\03\00\00\00\00\00\00\00") + ;; Embedded wasm code. + (data (i32.const 12) "{escaped_constructor}") +) +"#, + escaped_constructor = escaped_bytestring(constructor), + code_len = constructor.len(), + ) +} + +#[test] +fn contract_create() { + let code_transfer = wabt::wat2wasm(CODE_TRANSFER).unwrap(); + let code_ctor_transfer = wabt::wat2wasm(&code_ctor(&code_transfer)).unwrap(); + let code_create = wabt::wat2wasm(&code_create(&code_ctor_transfer)).unwrap(); + + with_externalities(&mut ExtBuilder::default().build(), || { + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + Balances::set_free_balance(&1, 0); + Balances::set_free_balance(&9, 30); + Balances::increase_total_stake_by(30); + + >::insert(1, code_create.to_vec()); + + // When invoked, the contract at address `1` must create a contract with 'transfer' code. + assert_ok!(Contract::call(Origin::signed(0), 1, 11, 100_000, Vec::new())); + + let derived_address = ::DetermineContractAddress::contract_address_for( + &code_ctor_transfer, + &[], + &1, + ); + + // 11 - value sent with the transaction + // 2 * 139 - gas spent by the deployer contract (139) multiplied by gas price (2) + // 2 * 135 - base gas fee for call (top level) + // 2 * 175 - base gas fee for create (by contract) + // ((21 / 2) * 2) - price per account creation + let expected_gas_after_create = + 100_000_000 - 11 - (2 * 139) - (2 * 135) - (2 * 175) - ((21 / 2) * 2); + assert_eq!(Balances::free_balance(&0), expected_gas_after_create); + assert_eq!(Balances::free_balance(&1), 8); + assert_eq!(Balances::free_balance(&derived_address), 3); + + // Initiate transfer to the newly created contract. + assert_ok!(Contract::call(Origin::signed(0), derived_address, 22, 100_000, Vec::new())); + + assert_eq!( + Balances::free_balance(&0), + // 22 - value sent with the transaction + // (2 * 10) - gas used by the contract + // (2 * 135) - base gas fee for call (top level) + // (2 * 135) - base gas fee for call (by transfer contract) + expected_gas_after_create - 22 - (2 * 10) - (2 * 135) - (2 * 135), + ); + assert_eq!(Balances::free_balance(&derived_address), 22 - 3); + assert_eq!(Balances::free_balance(&9), 36); + }); +} + +#[test] +fn top_level_create() { + let code_transfer = wabt::wat2wasm(CODE_TRANSFER).unwrap(); + let code_ctor_transfer = wabt::wat2wasm(&code_ctor(&code_transfer)).unwrap(); + + with_externalities(&mut ExtBuilder::default().gas_price(3).build(), || { + let derived_address = ::DetermineContractAddress::contract_address_for( + &code_ctor_transfer, + &[], + &0, + ); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + Balances::set_free_balance(&derived_address, 30); + Balances::increase_total_stake_by(30); + + assert_ok!(Contract::create( + Origin::signed(0), + 11, + 100_000, + code_ctor_transfer.clone(), + Vec::new(), + )); + + // 11 - value sent with the transaction + // (3 * 129) - gas spent by the ctor + // (3 * 175) - base gas fee for create (175) (top level) multipled by gas price (3) + // ((21 / 3) * 3) - price for contract creation + assert_eq!( + Balances::free_balance(&0), + 100_000_000 - 11 - (3 * 129) - (3 * 175) - ((21 / 3) * 3) + ); + assert_eq!(Balances::free_balance(&derived_address), 30 + 11); + + assert_eq!(>::get(&derived_address), code_transfer); + }); +} + +const CODE_NOP: &'static str = r#" +(module + (func (export "call") + nop + ) +) +"#; + +#[test] +fn refunds_unused_gas() { + let code_nop = wabt::wat2wasm(CODE_NOP).unwrap(); + + with_externalities(&mut ExtBuilder::default().build(), || { + >::insert(1, code_nop.to_vec()); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + + assert_ok!(Contract::call(Origin::signed(0), 1, 0, 100_000, Vec::new())); + + assert_eq!(Balances::free_balance(&0), 100_000_000 - 4 - (2 * 135)); + }); +} + +#[test] +fn call_with_zero_value() { + with_externalities(&mut ExtBuilder::default().build(), || { + >::insert(1, vec![]); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + + assert_ok!(Contract::call(Origin::signed(0), 1, 0, 100_000, Vec::new())); + + assert_eq!(Balances::free_balance(&0), 100_000_000 - (2 * 135)); + }); +} + +#[test] +fn create_with_zero_endowment() { + let code_nop = wabt::wat2wasm(CODE_NOP).unwrap(); + + with_externalities(&mut ExtBuilder::default().build(), || { + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + + assert_ok!(Contract::create(Origin::signed(0), 0, 100_000, code_nop, Vec::new())); + + assert_eq!( + Balances::free_balance(&0), + // 4 - for the gas spent by the constructor + // 2 * 175 - base gas fee for create (175) multiplied by gas price (2) (top level) + 100_000_000 - 4 - (2 * 175), + ); + }); +} + +#[test] +fn account_removal_removes_storage() { + with_externalities( + &mut ExtBuilder::default().existential_deposit(100).build(), + || { + // Setup two accounts with free balance above than exsistential threshold. + { + Balances::set_free_balance(&1, 110); + Balances::increase_total_stake_by(110); + >::insert(1, b"foo".to_vec(), b"1".to_vec()); + >::insert(1, b"bar".to_vec(), b"2".to_vec()); + + Balances::set_free_balance(&2, 110); + Balances::increase_total_stake_by(110); + >::insert(2, b"hello".to_vec(), b"3".to_vec()); + >::insert(2, b"world".to_vec(), b"4".to_vec()); + } + + // Transfer funds from account 1 of such amount that after this transfer + // the balance of account 1 is will be below than exsistential threshold. + // + // This should lead to the removal of all storage associated with this account. + assert_ok!(Balances::transfer(Origin::signed(1), 2.into(), 20)); + + // Verify that all entries from account 1 is removed, while + // entries from account 2 is in place. + { + assert_eq!(>::get(1, b"foo".to_vec()), None); + assert_eq!(>::get(1, b"bar".to_vec()), None); + + assert_eq!( + >::get(2, b"hello".to_vec()), + Some(b"3".to_vec()) + ); + assert_eq!( + >::get(2, b"world".to_vec()), + Some(b"4".to_vec()) + ); + } + }, + ); +} + +const CODE_UNREACHABLE: &'static str = r#" +(module + (func (export "call") + nop + unreachable + ) +) +"#; + +#[test] +fn top_level_call_refunds_even_if_fails() { + let code_unreachable = wabt::wat2wasm(CODE_UNREACHABLE).unwrap(); + with_externalities(&mut ExtBuilder::default().gas_price(4).build(), || { + >::insert(1, code_unreachable.to_vec()); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + + assert_err!( + Contract::call(Origin::signed(0), 1, 0, 100_000, Vec::new()), + "vm execute returned error while call" + ); + + assert_eq!(Balances::free_balance(&0), 100_000_000 - (4 * 3) - (4 * 135)); + }); +} + +const CODE_LOOP: &'static str = r#" +(module + (func (export "call") + (loop + (br 0) + ) + ) +) +"#; + +#[test] +fn block_gas_limit() { + let code_loop = wabt::wat2wasm(CODE_LOOP).unwrap(); + with_externalities( + &mut ExtBuilder::default().block_gas_limit(100_000).build(), + || { + >::insert(1, code_loop.to_vec()); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + + // Spend 50_000 units of gas (OOG). + assert_err!( + Contract::call(Origin::signed(0), 1, 0, 50_000, Vec::new()), + "vm execute returned error while call" + ); + + // Ensure we can't spend more gas than available in block gas limit. + assert_err!( + Contract::call(Origin::signed(0), 1, 0, 50_001, Vec::new()), + "block gas limit is reached" + ); + + // However, we can spend another 50_000 + assert_err!( + Contract::call(Origin::signed(0), 1, 0, 50_000, Vec::new()), + "vm execute returned error while call" + ); + }, + ); +} + +const CODE_INPUT_DATA: &'static str = r#" +(module + (import "env" "ext_input_size" (func $ext_input_size (result i32))) + (import "env" "ext_input_copy" (func $ext_input_copy (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "call") + (block $fail + ;; fail if ext_input_size != 4 + (br_if $fail + (i32.ne + (i32.const 4) + (call $ext_input_size) + ) + ) + + (call $ext_input_copy + (i32.const 0) + (i32.const 0) + (i32.const 4) + ) + + + (br_if $fail + (i32.ne + (i32.load8_u (i32.const 0)) + (i32.const 0) + ) + ) + (br_if $fail + (i32.ne + (i32.load8_u (i32.const 1)) + (i32.const 1) + ) + ) + (br_if $fail + (i32.ne + (i32.load8_u (i32.const 2)) + (i32.const 2) + ) + ) + (br_if $fail + (i32.ne + (i32.load8_u (i32.const 3)) + (i32.const 3) + ) + ) + + (return) + ) + unreachable + ) +) +"#; + +#[test] +fn input_data() { + let code_input_data = wabt::wat2wasm(CODE_INPUT_DATA).unwrap(); + with_externalities( + &mut ExtBuilder::default().build(), + || { + >::insert(1, code_input_data.to_vec()); + + Balances::set_free_balance(&0, 100_000_000); + Balances::increase_total_stake_by(100_000_000); + + assert_ok!(Contract::call(Origin::signed(0), 1, 0, 50_000, vec![0, 1, 2, 3])); + + // all asserts are made within contract code itself. + }, + ); +} diff --git a/runtime/contract/src/vm/env_def/macros.rs b/runtime/contract/src/vm/env_def/macros.rs new file mode 100644 index 00000000000..40651749eb5 --- /dev/null +++ b/runtime/contract/src/vm/env_def/macros.rs @@ -0,0 +1,285 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Definition of macros that hides boilerplate of defining external environment +//! for a wasm module. +//! +//! Typically you should use `define_env` macro. + +#[macro_export] +macro_rules! convert_args { + () => (vec![]); + ( $( $t:ty ),* ) => ( vec![ $( { use $crate::vm::env_def::ConvertibleToWasm; <$t>::VALUE_TYPE }, )* ] ); +} + +#[macro_export] +macro_rules! gen_signature { + ( ( $( $params: ty ),* ) ) => ( + { + FunctionType::new(convert_args!($($params),*), None) + } + ); + + ( ( $( $params: ty ),* ) -> $returns: ty ) => ( + { + FunctionType::new(convert_args!($($params),*), Some({ + use $crate::vm::env_def::ConvertibleToWasm; <$returns>::VALUE_TYPE + })) + } + ); +} + +/// Unmarshall arguments and then execute `body` expression and return its result. +macro_rules! unmarshall_then_body { + ( $body:tt, $ctx:ident, $args_iter:ident, $( $names:ident : $params:ty ),* ) => ({ + $( + let $names : <$params as $crate::vm::env_def::ConvertibleToWasm>::NativeType = + $args_iter.next() + .and_then(|v| <$params as $crate::vm::env_def::ConvertibleToWasm> + ::from_typed_value(v.clone())) + .expect( + "precondition: all imports should be checked against the signatures of corresponding + functions defined by `define_env!` macro by the user of the macro; + signatures of these functions defined by `$params`; + calls always made with arguments types of which are defined by the corresponding imports; + thus types of arguments should be equal to type list in `$params` and + length of argument list and $params should be equal; + thus this can never be `None`; + qed; + " + ); + )* + $body + }) +} + +/// Since we can't specify the type of closure directly at binding site: +/// +/// ```rust,ignore +/// let f: FnOnce() -> Result<::NativeType, _> = || { /* ... */ }; +/// ``` +/// +/// we use this function to constrain the type of the closure. +#[inline(always)] +pub fn constrain_closure(f: F) -> F +where + F: FnOnce() -> Result, +{ + f +} + +#[macro_export] +macro_rules! unmarshall_then_body_then_marshall { + ( $args_iter:ident, $ctx:ident, ( $( $names:ident : $params:ty ),* ) -> $returns:ty => $body:tt ) => ({ + let body = $crate::vm::env_def::macros::constrain_closure::< + <$returns as $crate::vm::env_def::ConvertibleToWasm>::NativeType, _ + >(|| { + unmarshall_then_body!($body, $ctx, $args_iter, $( $names : $params ),*) + }); + let r = body()?; + return Ok($crate::sandbox::ReturnValue::Value({ use $crate::vm::env_def::ConvertibleToWasm; r.to_typed_value() })) + }); + ( $args_iter:ident, $ctx:ident, ( $( $names:ident : $params:ty ),* ) => $body:tt ) => ({ + let body = $crate::vm::env_def::macros::constrain_closure::<(), _>(|| { + unmarshall_then_body!($body, $ctx, $args_iter, $( $names : $params ),*) + }); + body()?; + return Ok($crate::sandbox::ReturnValue::Unit) + }) +} + +#[macro_export] +macro_rules! define_func { + ( < E: $ext_ty:tt > $name:ident ( $ctx: ident $(, $names:ident : $params:ty)*) $(-> $returns:ty)* => $body:tt ) => { + fn $name< E: $ext_ty >( + $ctx: &mut $crate::vm::Runtime, + args: &[$crate::sandbox::TypedValue], + ) -> Result { + #[allow(unused)] + let mut args = args.iter(); + + unmarshall_then_body_then_marshall!( + args, + $ctx, + ( $( $names : $params ),* ) $( -> $returns )* => $body + ) + } + }; +} + +/// Define a function set that can be imported by executing wasm code. +/// +/// **NB**: Be advised that all functions defined by this macro +/// will panic if called with unexpected arguments. +/// +/// It's up to the user of this macro to check signatures of wasm code to be executed +/// and reject the code if any imported function has a mismached signature. +macro_rules! define_env { + ( $init_name:ident , < E: $ext_ty:tt > , + $( $name:ident ( $ctx:ident $( , $names:ident : $params:ty )* ) + $( -> $returns:ty )* => $body:tt , )* + ) => { + pub(crate) fn $init_name() -> HostFunctionSet { + let mut env = HostFunctionSet::new(); + + $( + env.funcs.insert( + stringify!( $name ).into(), + HostFunction::new( + gen_signature!( ( $( $params ),* ) $( -> $returns )* ), + { + define_func!( + < E: $ext_ty > $name ( $ctx $(, $names : $params )* ) $( -> $returns )* => $body + ); + $name:: + }, + ), + ); + )* + + env + } + }; +} + +#[cfg(test)] +mod tests { + use parity_wasm::elements::FunctionType; + use parity_wasm::elements::ValueType; + use runtime_primitives::traits::{As, Zero}; + use sandbox::{self, ReturnValue, TypedValue}; + use vm::env_def::{HostFunction, HostFunctionSet}; + use vm::tests::MockExt; + use vm::{Ext, Runtime}; + use Trait; + + #[test] + fn macro_unmarshall_then_body_then_marshall_value_or_trap() { + fn test_value( + _ctx: &mut u32, + args: &[sandbox::TypedValue], + ) -> Result { + let mut args = args.iter(); + unmarshall_then_body_then_marshall!( + args, + _ctx, + (a: u32, b: u32) -> u32 => { + if b == 0 { + Err(sandbox::HostError) + } else { + Ok(a / b) + } + } + ) + } + + let ctx = &mut 0; + assert_eq!( + test_value(ctx, &[TypedValue::I32(15), TypedValue::I32(3)]).unwrap(), + ReturnValue::Value(TypedValue::I32(5)), + ); + assert!(test_value(ctx, &[TypedValue::I32(15), TypedValue::I32(0)]).is_err()); + } + + #[test] + fn macro_unmarshall_then_body_then_marshall_unit() { + fn test_unit( + ctx: &mut u32, + args: &[sandbox::TypedValue], + ) -> Result { + let mut args = args.iter(); + unmarshall_then_body_then_marshall!( + args, + ctx, + (a: u32, b: u32) => { + *ctx = a + b; + Ok(()) + } + ) + } + + let ctx = &mut 0; + let result = test_unit(ctx, &[TypedValue::I32(2), TypedValue::I32(3)]).unwrap(); + assert_eq!(result, ReturnValue::Unit); + assert_eq!(*ctx, 5); + } + + #[test] + fn macro_define_func() { + define_func!( ext_gas (_ctx, amount: u32) => { + let amount = <<::T as Trait>::Gas as As>::sa(amount); + if !amount.is_zero() { + Ok(()) + } else { + Err(sandbox::HostError) + } + }); + let _f: fn(&mut Runtime, &[sandbox::TypedValue]) + -> Result = ext_gas::; + } + + #[test] + fn macro_gen_signature() { + assert_eq!( + gen_signature!((i32)), + FunctionType::new(vec![ValueType::I32], None), + ); + + assert_eq!( + gen_signature!( (i32, u32) -> u32 ), + FunctionType::new(vec![ValueType::I32, ValueType::I32], Some(ValueType::I32)), + ); + } + + #[test] + fn macro_unmarshall_then_body() { + let args = vec![TypedValue::I32(5), TypedValue::I32(3)]; + let mut args = args.iter(); + + let ctx: &mut u32 = &mut 0; + + let r = unmarshall_then_body!( + { + *ctx = a + b; + a * b + }, + ctx, + args, + a: u32, + b: u32 + ); + + assert_eq!(*ctx, 8); + assert_eq!(r, 15); + } + + #[test] + fn macro_define_env() { + define_env!(init_env, , + ext_gas( _ctx, amount: u32 ) => { + let amount = <<::T as Trait>::Gas as As>::sa(amount); + if !amount.is_zero() { + Ok(()) + } else { + Err(sandbox::HostError) + } + }, + ); + + let env = init_env::(); + assert!(env.funcs.get("ext_gas").is_some()); + } +} diff --git a/runtime/contract/src/vm/env_def/mod.rs b/runtime/contract/src/vm/env_def/mod.rs new file mode 100644 index 00000000000..dbdda705d33 --- /dev/null +++ b/runtime/contract/src/vm/env_def/mod.rs @@ -0,0 +1,315 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use super::{BalanceOf, CallReceipt, CreateReceipt, Ext, GasMeterResult, Runtime}; +use codec::Decode; +use parity_wasm::elements::{FunctionType, ValueType}; +use rstd::prelude::*; +use rstd::string::String; +use rstd::collections::btree_map::BTreeMap; +use runtime_primitives::traits::As; +use sandbox::{self, TypedValue}; +use system; +use Trait; + +#[macro_use] +mod macros; + +pub trait ConvertibleToWasm: Sized { + const VALUE_TYPE: ValueType; + type NativeType; + fn to_typed_value(self) -> TypedValue; + fn from_typed_value(TypedValue) -> Option; +} +impl ConvertibleToWasm for i32 { + type NativeType = i32; + const VALUE_TYPE: ValueType = ValueType::I32; + fn to_typed_value(self) -> TypedValue { + TypedValue::I32(self) + } + fn from_typed_value(v: TypedValue) -> Option { + v.as_i32() + } +} +impl ConvertibleToWasm for u32 { + type NativeType = u32; + const VALUE_TYPE: ValueType = ValueType::I32; + fn to_typed_value(self) -> TypedValue { + TypedValue::I32(self as i32) + } + fn from_typed_value(v: TypedValue) -> Option { + match v { + TypedValue::I32(v) => Some(v as u32), + _ => None, + } + } +} +impl ConvertibleToWasm for u64 { + type NativeType = u64; + const VALUE_TYPE: ValueType = ValueType::I64; + fn to_typed_value(self) -> TypedValue { + TypedValue::I64(self as i64) + } + fn from_typed_value(v: TypedValue) -> Option { + match v { + TypedValue::I64(v) => Some(v as u64), + _ => None, + } + } +} + +/// Represents a set of function that defined in this particular environment and +/// which can be imported and called by the module. +pub(crate) struct HostFunctionSet { + /// Functions which defined in the environment. + pub funcs: BTreeMap>, +} +impl HostFunctionSet { + pub fn new() -> Self { + HostFunctionSet { + funcs: BTreeMap::new(), + } + } +} + +pub(crate) struct HostFunction { + pub(crate) f: fn(&mut Runtime, &[sandbox::TypedValue]) + -> Result, + func_type: FunctionType, +} +impl HostFunction { + /// Create a new instance of a host function. + pub fn new( + func_type: FunctionType, + f: fn(&mut Runtime, &[sandbox::TypedValue]) + -> Result, + ) -> Self { + HostFunction { func_type, f } + } + + /// Returns a function pointer of this host function. + pub fn raw_fn_ptr( + &self, + ) -> fn(&mut Runtime, &[sandbox::TypedValue]) + -> Result { + self.f + } + + /// Check if the this function could be invoked with the given function signature. + pub fn func_type_matches(&self, func_type: &FunctionType) -> bool { + &self.func_type == func_type + } +} + +// TODO: ext_balance, ext_address, ext_callvalue, etc. + +// Define a function `fn init_env() -> HostFunctionSet` that returns +// a function set which can be imported by an executed contract. +define_env!(init_env, , + + // gas(amount: u32) + // + // Account for used gas. Traps if gas used is greater than gas limit. + // + // - amount: How much gas is used. + gas(ctx, amount: u32) => { + let amount = <<::T as Trait>::Gas as As>::sa(amount); + + match ctx.gas_meter.charge(amount) { + GasMeterResult::Proceed => Ok(()), + GasMeterResult::OutOfGas => Err(sandbox::HostError), + } + }, + + // ext_put_storage(location_ptr: u32, value_non_null: u32, value_ptr: u32); + // + // Change the value at the given location in storage or remove it. + // + // - location_ptr: pointer into the linear + // memory where the location of the requested value is placed. + // - value_non_null: if set to 0, then the entry + // at the given location will be removed. + // - value_ptr: pointer into the linear memory + // where the value to set is placed. If `value_non_null` is set to 0, then this parameter is ignored. + ext_set_storage(ctx, location_ptr: u32, value_non_null: u32, value_ptr: u32) => { + let mut location = [0; 32]; + + ctx.memory().get(location_ptr, &mut location)?; + + let value = if value_non_null != 0 { + let mut value = [0; 32]; + ctx.memory().get(value_ptr, &mut value)?; + Some(value.to_vec()) + } else { + None + }; + ctx.ext.set_storage(&location, value); + + Ok(()) + }, + + // ext_get_storage(location_ptr: u32, dest_ptr: u32); + // + // Retrieve the value at the given location from the strorage. + // If there is no entry at the given location then all-zero-value + // will be returned. + // + // - location_ptr: pointer into the linear + // memory where the location of the requested value is placed. + // - dest_ptr: pointer where contents of the specified storage location + // should be placed. + ext_get_storage(ctx, location_ptr: u32, dest_ptr: u32) => { + let mut location = [0; 32]; + ctx.memory().get(location_ptr, &mut location)?; + + if let Some(value) = ctx.ext.get_storage(&location) { + ctx.memory().set(dest_ptr, &value)?; + } else { + ctx.memory().set(dest_ptr, &[0u8; 32])?; + } + + Ok(()) + }, + + // ext_call(transfer_to_ptr: u32, transfer_to_len: u32, gas: u64, value_ptr: u32, value_len: u32, input_data_ptr: u32, input_data_len: u32) + ext_call(ctx, callee_ptr: u32, callee_len: u32, gas: u64, value_ptr: u32, value_len: u32, input_data_ptr: u32, input_data_len: u32) -> u32 => { + let mut callee = Vec::new(); + callee.resize(callee_len as usize, 0); + ctx.memory().get(callee_ptr, &mut callee)?; + let callee = + <::T as system::Trait>::AccountId::decode(&mut &callee[..]) + .ok_or_else(|| sandbox::HostError)?; + + let mut value_buf = Vec::new(); + value_buf.resize(value_len as usize, 0); + ctx.memory().get(value_ptr, &mut value_buf)?; + let value = BalanceOf::<::T>::decode(&mut &value_buf[..]) + .ok_or_else(|| sandbox::HostError)?; + + let mut input_data = Vec::new(); + input_data.resize(input_data_len as usize, 0u8); + ctx.memory().get(input_data_ptr, &mut input_data)?; + + let nested_gas_limit = if gas == 0 { + ctx.gas_meter.gas_left() + } else { + <<::T as Trait>::Gas as As>::sa(gas) + }; + let ext = &mut ctx.ext; + let call_outcome = ctx.gas_meter.with_nested(nested_gas_limit, |nested_meter| { + match nested_meter { + Some(nested_meter) => ext.call(&callee, value, nested_meter, &input_data), + // there is not enough gas to allocate for the nested call. + None => Err(()), + } + }); + + match call_outcome { + // TODO: Find a way how to pass return_data back to the this sandbox. + Ok(CallReceipt { .. }) => Ok(0), + Err(_) => Ok(1), + } + }, + + // ext_create(code_ptr: u32, code_len: u32, gas: u64, value_ptr: u32, value_len: u32, input_data_ptr: u32, input_data_len: u32) -> u32 + ext_create( + ctx, code_ptr: u32, + code_len: u32, + gas: u64, + value_ptr: u32, + value_len: u32, + input_data_ptr: u32, + input_data_len: u32 + ) -> u32 => { + let mut value_buf = Vec::new(); + value_buf.resize(value_len as usize, 0); + ctx.memory().get(value_ptr, &mut value_buf)?; + let value = BalanceOf::<::T>::decode(&mut &value_buf[..]) + .ok_or_else(|| sandbox::HostError)?; + + let mut code = Vec::new(); + code.resize(code_len as usize, 0u8); + ctx.memory().get(code_ptr, &mut code)?; + + let mut input_data = Vec::new(); + input_data.resize(input_data_len as usize, 0u8); + ctx.memory().get(input_data_ptr, &mut input_data)?; + + let nested_gas_limit = if gas == 0 { + ctx.gas_meter.gas_left() + } else { + <<::T as Trait>::Gas as As>::sa(gas) + }; + let ext = &mut ctx.ext; + let create_outcome = ctx.gas_meter.with_nested(nested_gas_limit, |nested_meter| { + match nested_meter { + Some(nested_meter) => ext.create(&code, value, nested_meter, &input_data), + // there is not enough gas to allocate for the nested call. + None => Err(()), + } + }); + + match create_outcome { + // TODO: Copy an address of the created contract in the sandbox. + Ok(CreateReceipt { .. }) => Ok(0), + Err(_) => Ok(1), + } + }, + + // ext_return(data_ptr: u32, data_len: u32) -> ! + ext_return(ctx, data_ptr: u32, data_len: u32) => { + let mut data_buf = Vec::new(); + data_buf.resize(data_len as usize, 0); + ctx.memory().get(data_ptr, &mut data_buf)?; + + ctx.store_return_data(data_buf) + .map_err(|_| sandbox::HostError)?; + + // The trap mechanism is used to immediately terminate the execution. + // This trap should be handled appropriately before returning the result + // to the user of this crate. + Err(sandbox::HostError) + }, + + // ext_input_size() -> u32 + // + // Returns size of an input buffer. + ext_input_size(ctx) -> u32 => { + Ok(ctx.input_data.len() as u32) + }, + + // ext_input_copy(dest_ptr: u32, offset: u32, len: u32) + // + // Copy data from an input buffer starting from `offset` with length `len` into the contract memory. + // The region at which the data should be put is specified by `dest_ptr`. + ext_input_copy(ctx, dest_ptr: u32, offset: u32, len: u32) => { + let offset = offset as usize; + if offset > ctx.input_data.len() { + // Offset can't be larger than input buffer length. + return Err(sandbox::HostError); + } + + // This can't panic since `offset <= ctx.input_data.len()`. + let src = &ctx.input_data[offset..]; + if src.len() != len as usize { + return Err(sandbox::HostError); + } + + ctx.memory().set(dest_ptr, src)?; + + Ok(()) + }, +); diff --git a/runtime/contract/src/vm/mod.rs b/runtime/contract/src/vm/mod.rs new file mode 100644 index 00000000000..0ca9f9e694b --- /dev/null +++ b/runtime/contract/src/vm/mod.rs @@ -0,0 +1,553 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! This module provides a means for executing contracts +//! represented in wasm. + +use exec::{CallReceipt, CreateReceipt}; +use gas::{GasMeter, GasMeterResult}; +use rstd::prelude::*; +use runtime_primitives::traits::{As, CheckedMul}; +use {sandbox, balances, system}; +use Trait; + +type BalanceOf = ::Balance; +type AccountIdOf = ::AccountId; + +mod prepare; + +#[macro_use] +mod env_def; + +use self::prepare::{prepare_contract, PreparedContract}; + +/// An interface that provides an access to the external environment in which the +/// smart-contract is executed. +/// +/// This interface is specialised to an account of the executing code, so all +/// operations are implicitly performed on that account. +pub trait Ext { + type T: Trait; + + /// Returns the storage entry of the executing account by the given key. + fn get_storage(&self, key: &[u8]) -> Option>; + + /// Sets the storage entry by the given key to the specified value. + fn set_storage(&mut self, key: &[u8], value: Option>); + + // TODO: Return the address of the created contract. + /// Create a new account for a contract. + /// + /// The newly created account will be associated with the `code`. `value` specifies the amount of value + /// transfered from this to the newly created account. + fn create( + &mut self, + code: &[u8], + value: BalanceOf, + gas_meter: &mut GasMeter, + data: &[u8], + ) -> Result, ()>; + + /// Call (possibly transfering some amount of funds) into the specified account. + fn call( + &mut self, + to: &AccountIdOf, + value: BalanceOf, + gas_meter: &mut GasMeter, + data: &[u8], + ) -> Result; +} + +/// Error that can occur while preparing or executing wasm smart-contract. +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + /// Error happened while serializing the module. + Serialization, + + /// Error happened while deserializing the module. + Deserialization, + + /// Internal memory declaration has been found in the module. + InternalMemoryDeclared, + + /// Gas instrumentation failed. + /// + /// This most likely indicates the module isn't valid. + GasInstrumentation, + + /// Stack instrumentation failed. + /// + /// This most likely indicates the module isn't valid. + StackHeightInstrumentation, + + /// Error happened during invocation of the contract's entrypoint. + /// + /// Most likely because of trap. + Invoke, + + /// Error happened during instantiation. + /// + /// This might indicate that `start` function trapped, or module isn't + /// instantiable and/or unlinkable. + Instantiate, + + /// Memory creation error. + /// + /// This might happen when the memory import has invalid descriptor or + /// requested too much resources. + Memory, +} + +/// Enumerates all possible *special* trap conditions. +/// +/// In this runtime traps used not only for signaling about errors but also +/// to just terminate quickly in some cases. +enum SpecialTrap { + // TODO: Can we pass wrapped memory instance instead of copying? + /// Signals that trap was generated in response to call `ext_return` host function. + Return(Vec), +} + +pub(crate) struct Runtime<'a, 'data, E: Ext + 'a> { + ext: &'a mut E, + input_data: &'data [u8], + config: &'a Config, + memory: sandbox::Memory, + gas_meter: &'a mut GasMeter, + special_trap: Option, +} +impl<'a, 'data, E: Ext + 'a> Runtime<'a, 'data, E> { + fn memory(&self) -> &sandbox::Memory { + &self.memory + } + /// Save a data buffer as a result of the execution. + /// + /// This function also charges gas for the returning. + /// + /// Returns `Err` if there is not enough gas. + fn store_return_data(&mut self, data: Vec) -> Result<(), ()> { + let data_len = <<::T as Trait>::Gas as As>::sa(data.len() as u64); + let price = (self.config.return_data_per_byte_cost) + .checked_mul(&data_len) + .ok_or_else(|| ())?; + + match self.gas_meter.charge(price) { + GasMeterResult::Proceed => { + self.special_trap = Some(SpecialTrap::Return(data)); + Ok(()) + } + GasMeterResult::OutOfGas => Err(()), + } + } +} + +fn to_execution_result( + runtime: Runtime, + run_err: Option, +) -> Result { + // Check the exact type of the error. It could be plain trap or + // special runtime trap the we must recognize. + let return_data = match (run_err, runtime.special_trap) { + // No traps were generated. Proceed normally. + (None, None) => Vec::new(), + // Special case. The trap was the result of the execution `return` host function. + (Some(sandbox::Error::Execution), Some(SpecialTrap::Return(rd))) => rd, + // Any other kind of a trap should result in a failure. + (Some(_), _) => return Err(Error::Invoke), + // Any other case (such as special trap flag without actual trap) signifies + // a logic error. + _ => unreachable!(), + }; + + Ok(ExecutionResult { return_data }) +} + +/// The result of execution of a smart-contract. +#[derive(PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct ExecutionResult { + /// The result produced by the execution of the contract. + /// + /// The contract can designate some buffer at the execution time via a special function. + /// If contract called this function with non-empty buffer it will be copied here. + /// + /// Note that gas is already charged for returning the data. + pub return_data: Vec, +} + +/// Execute the given code as a contract. +pub fn execute<'a, E: Ext>( + code: &[u8], + input_data: &[u8], + ext: &'a mut E, + gas_meter: &mut GasMeter, +) -> Result { + let config = Config::default(); + let env = env_def::init_env(); + + let PreparedContract { + instrumented_code, + memory, + } = prepare_contract(code, &config, &env)?; + + let mut imports = sandbox::EnvironmentDefinitionBuilder::new(); + for (func_name, ext_func) in &env.funcs { + imports.add_host_func("env", &func_name[..], ext_func.raw_fn_ptr()); + } + imports.add_memory("env", "memory", memory.clone()); + + let mut runtime = Runtime { + ext, + input_data, + config: &config, + memory, + gas_meter, + special_trap: None, + }; + + let mut instance = sandbox::Instance::new(&instrumented_code, &imports, &mut runtime) + .map_err(|_| Error::Instantiate)?; + + let run_result = instance.invoke(b"call", &[], &mut runtime); + + to_execution_result(runtime, run_result.err()) +} + +// TODO: Extract it to the root of the crate +#[derive(Clone)] +struct Config { + /// Gas cost of a growing memory by single page. + grow_mem_cost: T::Gas, + + /// Gas cost of a regular operation. + regular_op_cost: T::Gas, + + /// Gas cost per one byte returned. + return_data_per_byte_cost: T::Gas, + + /// How tall the stack is allowed to grow? + /// + /// See https://wiki.parity.io/WebAssembly-StackHeight to find out + /// how the stack frame cost is calculated. + max_stack_height: u32, + + //// What is the maximal memory pages amount is allowed to have for + /// a contract. + max_memory_pages: u32, +} + +impl Default for Config { + fn default() -> Config { + Config { + grow_mem_cost: T::Gas::sa(1), + regular_op_cost: T::Gas::sa(1), + return_data_per_byte_cost: T::Gas::sa(1), + max_stack_height: 64 * 1024, + max_memory_pages: 16, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use gas::GasMeter; + use std::collections::HashMap; + use tests::Test; + use wabt; + + #[derive(Debug, PartialEq, Eq)] + struct CreateEntry { + code: Vec, + endowment: u64, + data: Vec, + gas_left: u64, + } + #[derive(Debug, PartialEq, Eq)] + struct TransferEntry { + to: u64, + value: u64, + data: Vec, + gas_left: u64, + } + #[derive(Default)] + pub struct MockExt { + storage: HashMap, Vec>, + creates: Vec, + transfers: Vec, + next_account_id: u64, + } + impl Ext for MockExt { + type T = Test; + + fn get_storage(&self, key: &[u8]) -> Option> { + self.storage.get(key).cloned() + } + fn set_storage(&mut self, key: &[u8], value: Option>) { + *self.storage.entry(key.to_vec()).or_insert(Vec::new()) = value.unwrap_or(Vec::new()); + } + fn create( + &mut self, + code: &[u8], + endowment: u64, + gas_meter: &mut GasMeter, + data: &[u8], + ) -> Result, ()> { + self.creates.push(CreateEntry { + code: code.to_vec(), + endowment, + data: data.to_vec(), + gas_left: gas_meter.gas_left(), + }); + let address = self.next_account_id; + self.next_account_id += 1; + + Ok(CreateReceipt { address }) + } + fn call( + &mut self, + to: &u64, + value: u64, + gas_meter: &mut GasMeter, + data: &[u8], + ) -> Result { + self.transfers.push(TransferEntry { + to: *to, + value, + data: data.to_vec(), + gas_left: gas_meter.gas_left(), + }); + // Assume for now that it was just a plain transfer. + // TODO: Add tests for different call outcomes. + Ok(CallReceipt { + return_data: Vec::new(), + }) + } + } + + const CODE_TRANSFER: &str = r#" +(module + ;; ext_call( + ;; callee_ptr: u32, + ;; callee_len: u32, + ;; gas: u64, + ;; value_ptr: u32, + ;; value_len: u32, + ;; input_data_ptr: u32, + ;; input_data_len: u32 + ;;) -> u32 + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + (func (export "call") + (drop + (call $ext_call + (i32.const 4) ;; Pointer to "callee" address. + (i32.const 8) ;; Length of "callee" address. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 12) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 20) ;; Pointer to input data buffer address + (i32.const 4) ;; Length of input data buffer + ) + ) + ) + ;; Destination AccountId to transfer the funds. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 4) "\09\00\00\00\00\00\00\00") + ;; Amount of value to transfer. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 12) "\06\00\00\00\00\00\00\00") + + (data (i32.const 20) "\01\02\03\04") +) +"#; + + #[test] + fn contract_transfer() { + let code_transfer = wabt::wat2wasm(CODE_TRANSFER).unwrap(); + + let mut mock_ext = MockExt::default(); + execute( + &code_transfer, + &[], + &mut mock_ext, + &mut GasMeter::with_limit(50_000, 1), + ).unwrap(); + + assert_eq!( + &mock_ext.transfers, + &[TransferEntry { + to: 9, + value: 6, + data: vec![ + 1, 2, 3, 4, + ], + gas_left: 49990, + }] + ); + } + + const CODE_CREATE: &str = r#" +(module + ;; ext_create( + ;; code_ptr: u32, + ;; code_len: u32, + ;; gas: u64, + ;; value_ptr: u32, + ;; value_len: u32, + ;; input_data_ptr: u32, + ;; input_data_len: u32, + ;; ) -> u32 + (import "env" "ext_create" (func $ext_create (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + (func (export "call") + (drop + (call $ext_create + (i32.const 12) ;; Pointer to `code` + (i32.const 8) ;; Length of `code` + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 4) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 20) ;; Pointer to input data buffer address + (i32.const 4) ;; Length of input data buffer + ) + ) + ) + ;; Amount of value to transfer. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 4) "\03\00\00\00\00\00\00\00") + ;; Embedded wasm code. + (data (i32.const 12) "\00\61\73\6d\01\00\00\00") + ;; Input data to pass to the contract being created. + (data (i32.const 20) "\01\02\03\04") +) +"#; + + #[test] + fn contract_create() { + let code_create = wabt::wat2wasm(CODE_CREATE).unwrap(); + + let mut mock_ext = MockExt::default(); + execute( + &code_create, + &[], + &mut mock_ext, + &mut GasMeter::with_limit(50_000, 1), + ).unwrap(); + + assert_eq!( + &mock_ext.creates, + &[CreateEntry { + code: vec![0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00], + endowment: 3, + data: vec![ + 1, 2, 3, 4, + ], + gas_left: 49990, + }] + ); + } + + const CODE_MEM: &str = r#" +(module + ;; Internal memory is not allowed. + (memory 1 1) + + (func (export "call") + nop + ) +) +"#; + + #[test] + fn contract_internal_mem() { + let code_mem = wabt::wat2wasm(CODE_MEM).unwrap(); + + let mut mock_ext = MockExt::default(); + + assert_matches!( + execute( + &code_mem, + &[], + &mut mock_ext, + &mut GasMeter::with_limit(100_000, 1) + ), + Err(_) + ); + } + + const CODE_TRANSFER_LIMITED_GAS: &str = r#" +(module + ;; ext_call( + ;; callee_ptr: u32, + ;; callee_len: u32, + ;; gas: u64, + ;; value_ptr: u32, + ;; value_len: u32, + ;; input_data_ptr: u32, + ;; input_data_len: u32 + ;;) -> u32 + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + (func (export "call") + (drop + (call $ext_call + (i32.const 4) ;; Pointer to "callee" address. + (i32.const 8) ;; Length of "callee" address. + (i64.const 228) ;; How much gas to devote for the execution. + (i32.const 12) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 20) ;; Pointer to input data buffer address + (i32.const 4) ;; Length of input data buffer + ) + ) + ) + ;; Destination AccountId to transfer the funds. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 4) "\09\00\00\00\00\00\00\00") + ;; Amount of value to transfer. + ;; Represented by u64 (8 bytes long) in little endian. + (data (i32.const 12) "\06\00\00\00\00\00\00\00") + + (data (i32.const 20) "\01\02\03\04") +) +"#; + + #[test] + fn contract_call_limited_gas() { + let code_transfer = wabt::wat2wasm(CODE_TRANSFER_LIMITED_GAS).unwrap(); + + let mut mock_ext = MockExt::default(); + execute( + &code_transfer, + &[], + &mut mock_ext, + &mut GasMeter::with_limit(50_000, 1), + ).unwrap(); + + assert_eq!( + &mock_ext.transfers, + &[TransferEntry { + to: 9, + value: 6, + data: vec![ + 1, 2, 3, 4, + ], + gas_left: 228, + }] + ); + } +} diff --git a/runtime/contract/src/vm/prepare.rs b/runtime/contract/src/vm/prepare.rs new file mode 100644 index 00000000000..b15f9cbc3a9 --- /dev/null +++ b/runtime/contract/src/vm/prepare.rs @@ -0,0 +1,286 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Module that takes care of loading, checking and preprocessing of a +//! wasm module before execution. + +use super::env_def::HostFunctionSet; +use super::{Config, Error, Ext}; +use rstd::prelude::*; +use parity_wasm::elements::{self, External, MemoryType, Type}; +use pwasm_utils; +use pwasm_utils::rules; +use runtime_primitives::traits::As; +use sandbox; +use Trait; + +struct ContractModule<'a, T: Trait + 'a> { + // An `Option` is used here for loaning (`take()`-ing) the module. + // Invariant: Can't be `None` (i.e. on enter and on exit from the function + // the value *must* be `Some`). + module: Option, + config: &'a Config, +} + +impl<'a, T: Trait> ContractModule<'a, T> { + fn new(original_code: &[u8], config: &'a Config) -> Result, Error> { + let module = + elements::deserialize_buffer(original_code).map_err(|_| Error::Deserialization)?; + Ok(ContractModule { + module: Some(module), + config, + }) + } + + /// Ensures that module doesn't declare internal memories. + /// + /// In this runtime we only allow wasm module to import memory from the environment. + /// Memory section contains declarations of internal linear memories, so if we find one + /// we reject such a module. + fn ensure_no_internal_memory(&self) -> Result<(), Error> { + let module = self + .module + .as_ref() + .expect("On entry to the function `module` can't be None; qed"); + if module + .memory_section() + .map_or(false, |ms| ms.entries().len() > 0) + { + return Err(Error::InternalMemoryDeclared); + } + Ok(()) + } + + fn inject_gas_metering(&mut self) -> Result<(), Error> { + let gas_rules = rules::Set::new(self.config.regular_op_cost.as_(), Default::default()) + .with_grow_cost(self.config.grow_mem_cost.as_()) + .with_forbidden_floats(); + + let module = self + .module + .take() + .expect("On entry to the function `module` can't be `None`; qed"); + + let contract_module = pwasm_utils::inject_gas_counter(module, &gas_rules) + .map_err(|_| Error::GasInstrumentation)?; + + self.module = Some(contract_module); + Ok(()) + } + + fn inject_stack_height_metering(&mut self) -> Result<(), Error> { + let module = self + .module + .take() + .expect("On entry to the function `module` can't be `None`; qed"); + + let contract_module = + pwasm_utils::stack_height::inject_limiter(module, self.config.max_stack_height) + .map_err(|_| Error::StackHeightInstrumentation)?; + + self.module = Some(contract_module); + Ok(()) + } + + /// Scan an import section if any. + /// + /// This accomplishes two tasks: + /// + /// - checks any imported function against defined host functions set, incl. + /// their signatures. + /// - if there is a memory import, returns it's descriptor + fn scan_imports(&self, env: &HostFunctionSet) -> Result, Error> { + let module = self + .module + .as_ref() + .expect("On entry to the function `module` can't be `None`; qed"); + + let types = module.type_section().map(|ts| ts.types()).unwrap_or(&[]); + let import_entries = module + .import_section() + .map(|is| is.entries()) + .unwrap_or(&[]); + + let mut imported_mem_type = None; + + for import in import_entries { + if import.module() != "env" { + // This import tries to import something from non-"env" module, + // but all imports are located in "env" at the moment. + return Err(Error::Instantiate); + } + + let type_idx = match import.external() { + &External::Function(ref type_idx) => type_idx, + &External::Memory(ref memory_type) => { + imported_mem_type = Some(memory_type); + continue; + } + _ => continue, + }; + + let Type::Function(ref func_ty) = types + .get(*type_idx as usize) + .ok_or_else(|| Error::Instantiate)?; + + let ext_func = env + .funcs + .get(import.field()) + .ok_or_else(|| Error::Instantiate)?; + if !ext_func.func_type_matches(func_ty) { + return Err(Error::Instantiate); + } + } + Ok(imported_mem_type) + } + + fn into_wasm_code(mut self) -> Result, Error> { + elements::serialize( + self.module + .take() + .expect("On entry to the function `module` can't be `None`; qed"), + ).map_err(|_| Error::Serialization) + } +} + +pub(super) struct PreparedContract { + pub instrumented_code: Vec, + pub memory: sandbox::Memory, +} + +/// Loads the given module given in `original_code`, performs some checks on it and +/// does some preprocessing. +/// +/// The checks are: +/// +/// - module doesn't define an internal memory instance, +/// - imported memory (if any) doesn't reserve more memory than permitted by the `config`, +/// - all imported functions from the external environment matches defined by `env` module, +/// +/// The preprocessing includes injecting code for gas metering and metering the height of stack. +pub(super) fn prepare_contract( + original_code: &[u8], + config: &Config, + env: &HostFunctionSet, +) -> Result { + let mut contract_module = ContractModule::new(original_code, config)?; + contract_module.ensure_no_internal_memory()?; + contract_module.inject_gas_metering()?; + contract_module.inject_stack_height_metering()?; + + let memory = if let Some(memory_type) = contract_module.scan_imports(env)? { + // Inspect the module to extract the initial and maximum page count. + let limits = memory_type.limits(); + match (limits.initial(), limits.maximum()) { + (initial, Some(maximum)) if initial > maximum => { + // Requested initial number of pages should not exceed the requested maximum. + return Err(Error::Memory); + } + (_, Some(maximum)) if maximum > config.max_memory_pages => { + // Maximum number of pages should not exceed the configured maximum. + return Err(Error::Memory); + } + (_, None) => { + // Maximum number of pages should be always declared. + // This isn't a hard requirement and can be treated as a maxiumum set + // to configured maximum. + return Err(Error::Memory); + } + (initial, maximum) => sandbox::Memory::new(initial, maximum), + } + } else { + // If none memory imported then just crate an empty placeholder. + // Any access to it will lead to out of bounds trap. + sandbox::Memory::new(0, Some(0)) + }; + let memory = memory.map_err(|_| Error::Memory)?; + + Ok(PreparedContract { + instrumented_code: contract_module.into_wasm_code()?, + memory, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fmt; + use tests::Test; + use vm::tests::MockExt; + use wabt; + + impl fmt::Debug for PreparedContract { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "PreparedContract {{ .. }}") + } + } + + fn parse_and_prepare_wat(wat: &str) -> Result { + let wasm = wabt::Wat2Wasm::new().validate(false).convert(wat).unwrap(); + let config = Config::::default(); + let env = ::vm::env_def::init_env(); + prepare_contract::(wasm.as_ref(), &config, &env) + } + + #[test] + fn internal_memory_declaration() { + let r = parse_and_prepare_wat(r#"(module (memory 1 1))"#); + assert_matches!(r, Err(Error::InternalMemoryDeclared)); + } + + #[test] + fn memory() { + // This test assumes that maximum page number is configured to a certain number. + assert_eq!(Config::::default().max_memory_pages, 16); + + let r = parse_and_prepare_wat(r#"(module (import "env" "memory" (memory 1 1)))"#); + assert_matches!(r, Ok(_)); + + // No memory import + let r = parse_and_prepare_wat(r#"(module)"#); + assert_matches!(r, Ok(_)); + + // initial exceed maximum + let r = parse_and_prepare_wat(r#"(module (import "env" "memory" (memory 16 1)))"#); + assert_matches!(r, Err(Error::Memory)); + + // no maximum + let r = parse_and_prepare_wat(r#"(module (import "env" "memory" (memory 1)))"#); + assert_matches!(r, Err(Error::Memory)); + + // requested maximum exceed configured maximum + let r = parse_and_prepare_wat(r#"(module (import "env" "memory" (memory 1 17)))"#); + assert_matches!(r, Err(Error::Memory)); + } + + #[test] + fn imports() { + // nothing can be imported from non-"env" module for now. + let r = parse_and_prepare_wat(r#"(module (import "another_module" "memory" (memory 1 1)))"#); + assert_matches!(r, Err(Error::Instantiate)); + + let r = parse_and_prepare_wat(r#"(module (import "env" "gas" (func (param i32))))"#); + assert_matches!(r, Ok(_)); + + // wrong signature + let r = parse_and_prepare_wat(r#"(module (import "env" "gas" (func (param i64))))"#); + assert_matches!(r, Err(Error::Instantiate)); + + // unknown function name + let r = parse_and_prepare_wat(r#"(module (import "env" "unknown_func" (func)))"#); + assert_matches!(r, Err(Error::Instantiate)); + } +} diff --git a/runtime/council/Cargo.toml b/runtime/council/Cargo.toml new file mode 100644 index 00000000000..7d3b6c24d7d --- /dev/null +++ b/runtime/council/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "substrate-runtime-council" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +integer-sqrt = { git = "https://github.com/paritytech/integer-sqrt-rs.git", branch = "master" } +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +safe-mix = { version = "1.0", default_features = false} +substrate-keyring = { path = "../../core/keyring", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-runtime-consensus = { path = "../consensus", default_features = false } +substrate-runtime-balances = { path = "../balances", default_features = false } +substrate-runtime-democracy = { path = "../democracy", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "safe-mix/std", + "substrate-keyring", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-consensus/std", + "substrate-runtime-balances/std", + "substrate-runtime-democracy/std", + "substrate-runtime-system/std", +] diff --git a/runtime/council/src/lib.rs b/runtime/council/src/lib.rs new file mode 100644 index 00000000000..1c0883877fc --- /dev/null +++ b/runtime/council/src/lib.rs @@ -0,0 +1,238 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Council system: Handles the voting in and maintenance of council members. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[cfg(test)] +#[macro_use] +extern crate hex_literal; + +extern crate integer_sqrt; +extern crate substrate_codec as codec; +#[macro_use] extern crate substrate_codec_derive; +extern crate substrate_primitives; +#[cfg(feature = "std")] extern crate substrate_keyring as keyring; +#[macro_use] extern crate substrate_runtime_std as rstd; +extern crate substrate_runtime_io as runtime_io; +#[macro_use] extern crate substrate_runtime_support; +extern crate substrate_runtime_primitives as primitives; +extern crate substrate_runtime_balances as balances; +extern crate substrate_runtime_democracy as democracy; +extern crate substrate_runtime_system as system; + +#[cfg(feature = "std")] +use rstd::prelude::*; +#[cfg(feature = "std")] +use std::collections::HashMap; +#[cfg(feature = "std")] +use primitives::traits::As; +#[cfg(feature = "std")] +use substrate_runtime_support::StorageValue; + +pub mod voting; +pub mod motions; +pub mod seats; + +pub use seats::{Trait, Module, RawEvent, Event, VoteIndex}; + +#[cfg(feature = "std")] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + // for the voting onto the council + pub candidacy_bond: T::Balance, + pub voter_bond: T::Balance, + pub present_slash_per_voter: T::Balance, + pub carry_count: u32, + pub active_council: Vec<(T::AccountId, T::BlockNumber)>, + pub approval_voting_period: T::BlockNumber, + pub presentation_duration: T::BlockNumber, + pub desired_seats: u32, + pub term_duration: T::BlockNumber, + pub inactive_grace_period: T::BlockNumber, + + // for the council's votes. + pub cooloff_period: T::BlockNumber, + pub voting_period: T::BlockNumber, +} + +#[cfg(feature = "std")] +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + candidacy_bond: T::Balance::sa(9), + voter_bond: T::Balance::sa(0), + present_slash_per_voter: T::Balance::sa(1), + carry_count: 2, + inactive_grace_period: T::BlockNumber::sa(1), + active_council: vec![], + approval_voting_period: T::BlockNumber::sa(1000), + presentation_duration: T::BlockNumber::sa(1000), + desired_seats: 0, + term_duration: T::BlockNumber::sa(5), + cooloff_period: T::BlockNumber::sa(1000), + voting_period: T::BlockNumber::sa(3), + } + } +} + +#[cfg(feature = "std")] +impl primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> ::std::result::Result, Vec>, String> { + use codec::Encode; + + Ok(map![ + Self::hash(>::key()).to_vec() => self.candidacy_bond.encode(), + Self::hash(>::key()).to_vec() => self.voter_bond.encode(), + Self::hash(>::key()).to_vec() => self.present_slash_per_voter.encode(), + Self::hash(>::key()).to_vec() => self.carry_count.encode(), + Self::hash(>::key()).to_vec() => self.presentation_duration.encode(), + Self::hash(>::key()).to_vec() => self.approval_voting_period.encode(), + Self::hash(>::key()).to_vec() => self.term_duration.encode(), + Self::hash(>::key()).to_vec() => self.desired_seats.encode(), + Self::hash(>::key()).to_vec() => self.inactive_grace_period.encode(), + Self::hash(>::key()).to_vec() => self.active_council.encode(), + + Self::hash(>::key()).to_vec() => self.cooloff_period.encode(), + Self::hash(>::key()).to_vec() => self.voting_period.encode(), + Self::hash(>::key()).to_vec() => vec![0u8; 0].encode() + ]) + } +} + +#[cfg(test)] +mod tests { + // These re-exports are here for a reason, edit with care + pub use super::*; + pub use runtime_io::with_externalities; + pub use substrate_primitives::H256; + pub use primitives::BuildStorage; + pub use primitives::traits::{BlakeTwo256}; + pub use primitives::testing::{Digest, Header}; + pub use substrate_primitives::Blake2Hasher; + pub use {seats, motions, voting}; + + impl_outer_origin! { + pub enum Origin for Test { + motions + } + } + + impl_outer_event! { + pub enum Event for Test { + balances, democracy, seats, voting, motions + } + } + + impl_outer_dispatch! { + pub enum Call where origin: Origin { + Balances, + Democracy, + } + } + + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] + pub struct Test; + impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = Event; + } + impl balances::Trait for Test { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = (); + type EnsureAccountLiquid = (); + type Event = Event; + } + impl democracy::Trait for Test { + type Proposal = Call; + type Event = Event; + } + impl seats::Trait for Test { + type Event = Event; + } + impl motions::Trait for Test { + type Origin = Origin; + type Proposal = Call; + type Event = Event; + } + impl voting::Trait for Test { + type Event = Event; + } + + pub fn new_test_ext(with_council: bool) -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(balances::GenesisConfig::{ + balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], + transaction_base_fee: 0, + transaction_byte_fee: 0, + existential_deposit: 0, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + }.build_storage().unwrap()); + t.extend(democracy::GenesisConfig::{ + launch_period: 1, + voting_period: 3, + minimum_deposit: 1, + }.build_storage().unwrap()); + t.extend(GenesisConfig::{ + candidacy_bond: 9, + voter_bond: 3, + present_slash_per_voter: 1, + carry_count: 2, + inactive_grace_period: 1, + active_council: if with_council { vec![ + (1, 10), + (2, 10), + (3, 10) + ] } else { vec![] }, + approval_voting_period: 4, + presentation_duration: 2, + desired_seats: 2, + term_duration: 5, + cooloff_period: 2, + voting_period: 1, + }.build_storage().unwrap()); + t.into() + } + + pub type System = system::Module; + pub type Balances = balances::Module; + pub type Democracy = democracy::Module; + pub type Council = seats::Module; + pub type CouncilVoting = voting::Module; + pub type CouncilMotions = motions::Module; +} \ No newline at end of file diff --git a/runtime/council/src/motions.rs b/runtime/council/src/motions.rs new file mode 100644 index 00000000000..b9b26df00f0 --- /dev/null +++ b/runtime/council/src/motions.rs @@ -0,0 +1,372 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Council voting system. + +use rstd::prelude::*; +use rstd::result; +use substrate_primitives::u32_trait::Value as U32; +use primitives::traits::{Hash, EnsureOrigin, MaybeSerializeDebug, OnFinalise}; +use substrate_runtime_support::dispatch::{Result, Dispatchable, Parameter}; +use substrate_runtime_support::{StorageValue, StorageMap}; +use super::{Trait as CouncilTrait, Module as Council}; +use system::{self, ensure_signed}; + +/// Simple index type for proposal counting. +pub type ProposalIndex = u32; + +pub trait Trait: CouncilTrait + MaybeSerializeDebug { + /// The outer origin type. + type Origin: From; + + /// The outer call dispatch type. + type Proposal: Parameter + Dispatchable::Origin> + MaybeSerializeDebug; + + /// The outer event type. + type Event: From> + Into<::Event>; +} + +/// Origin for the council module. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum Origin { + /// It has been condoned by a given number of council members. + Members(u32), +} + +/// Event for this module. +decl_event!( + pub enum Event with RawEvent + where ::Hash, ::AccountId + { + /// A motion (given hash) has been proposed (by given account) with a threshold (given u32). + Proposed(AccountId, ProposalIndex, Hash, u32), + /// A motion (given hash) has been voted on by given account, leaving + /// a tally (yes votes and no votes given as u32s respectively). + Voted(AccountId, Hash, bool, u32, u32), + /// A motion was approved by the required threshold. + Approved(Hash), + /// A motion was not approved by the required threshold. + Disapproved(Hash), + /// A motion was executed; `bool` is true if returned without error. + Executed(Hash, bool), + } +); + +decl_module! { + #[cfg_attr(feature = "std", serde(bound(deserialize = "::Proposal: ::serde::de::DeserializeOwned")))] + pub struct Module for enum Call where origin: ::Origin { + fn propose(origin, threshold: u32, proposal: Box<::Proposal>) -> Result; + fn vote(origin, proposal: T::Hash, index: ProposalIndex, approve: bool) -> Result; + } +} + +decl_storage! { + trait Store for Module as CouncilMotions { + /// The (hashes of) the active proposals. + pub Proposals get(proposals): default Vec; + /// Actual proposal for a given hash, if it's current. + pub ProposalOf get(proposal_of): map [ T::Hash => ::Proposal ]; + /// Votes for a given proposal: (required_yes_votes, yes_voters, no_voters). + pub Voting get(voting): map [ T::Hash => (ProposalIndex, u32, Vec, Vec) ]; + /// Proposals so far. + pub ProposalCount get(proposal_count): default u32; + } +} + +impl Module { + + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + pub fn is_councillor(who: &T::AccountId) -> bool { + >::active_council().iter() + .any(|&(ref a, _)| a == who) + } + + // Dispatch + fn propose(origin: ::Origin, threshold: u32, proposal: Box<::Proposal>) -> Result { + let who = ensure_signed(origin)?; + + ensure!(Self::is_councillor(&who), "proposer not on council"); + + let proposal_hash = T::Hashing::hash_of(&proposal); + + ensure!(!>::exists(proposal_hash), "duplicate proposals not allowed"); + + if threshold < 2 { + let ok = proposal.dispatch(Origin::Members(1).into()).is_ok(); + Self::deposit_event(RawEvent::Executed(proposal_hash, ok)); + } else { + let index = Self::proposal_count(); + >::mutate(|i| *i += 1); + >::mutate(|proposals| proposals.push(proposal_hash)); + >::insert(proposal_hash, *proposal); + >::insert(proposal_hash, (index, threshold, vec![who.clone()], vec![])); + + Self::deposit_event(RawEvent::Proposed(who, index, proposal_hash, threshold)); + } + Ok(()) + } + + fn vote(origin: ::Origin, proposal: T::Hash, index: ProposalIndex, approve: bool) -> Result { + let who = ensure_signed(origin)?; + + ensure!(Self::is_councillor(&who), "voter not on council"); + + let mut voting = Self::voting(&proposal).ok_or("proposal must exist")?; + ensure!(voting.0 == index, "mismatched index"); + + let position_yes = voting.2.iter().position(|a| a == &who); + let position_no = voting.3.iter().position(|a| a == &who); + + if approve { + if position_yes.is_none() { + voting.2.push(who.clone()); + } else { + return Err("duplicate vote ignored") + } + if let Some(pos) = position_no { + voting.3.swap_remove(pos); + } + } else { + if position_no.is_none() { + voting.3.push(who.clone()); + } else { + return Err("duplicate vote ignored") + } + if let Some(pos) = position_yes { + voting.2.swap_remove(pos); + } + } + + let yes_votes = voting.2.len() as u32; + let no_votes = voting.3.len() as u32; + Self::deposit_event(RawEvent::Voted(who, proposal, approve, yes_votes, no_votes)); + + let threshold = voting.1; + let potential_votes = >::active_council().len() as u32; + let approved = yes_votes >= threshold; + let disapproved = potential_votes.saturating_sub(no_votes) < threshold; + if approved || disapproved { + if approved { + Self::deposit_event(RawEvent::Approved(proposal)); + + // execute motion, assuming it exists. + if let Some(p) = >::take(&proposal) { + let ok = p.dispatch(Origin::Members(threshold).into()).is_ok(); + Self::deposit_event(RawEvent::Executed(proposal, ok)); + } + } else { + // disapproved + Self::deposit_event(RawEvent::Disapproved(proposal)); + } + + // remove vote + >::remove(&proposal); + >::mutate(|proposals| proposals.retain(|h| h != &proposal)); + } else { + // update voting + >::insert(&proposal, voting); + } + + Ok(()) + } +} + +impl OnFinalise for Module { + fn on_finalise(_n: T::BlockNumber) { + } +} + +/// Ensure that the origin `o` represents at least `n` council members. Returns +/// `Ok` or an `Err` otherwise. +pub fn ensure_council_members(o: OuterOrigin, n: u32) -> result::Result + where OuterOrigin: Into> +{ + match o.into() { + Some(Origin::Members(x)) if x >= n => Ok(n), + _ => Err("bad origin: expected to be a threshold number of council members"), + } +} + +pub struct EnsureMembers(::rstd::marker::PhantomData); +impl EnsureOrigin for EnsureMembers + where O: Into> +{ + type Success = u32; + fn ensure_origin(o: O) -> result::Result { + ensure_council_members(o, N::VALUE) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::RawEvent; + use ::tests::*; + use ::tests::{Call, Origin, Event as OuterEvent}; + use substrate_runtime_support::Hashable; + use system::{EventRecord, Phase}; + + #[test] + fn motions_basic_environment_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + assert_eq!(Balances::free_balance(&42), 0); + assert_eq!(CouncilMotions::proposals(), Vec::::new()); + }); + } + + fn set_balance_proposal(value: u64) -> Call { + Call::Balances(balances::Call::set_balance(balances::address::Address::Id(42), value, 0)) + } + + #[test] + fn motions_propose_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash = proposal.blake2_256().into(); + assert_ok!(CouncilMotions::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); + assert_eq!(CouncilMotions::proposals(), vec![hash]); + assert_eq!(CouncilMotions::proposal_of(&hash), Some(proposal)); + assert_eq!(CouncilMotions::voting(&hash), Some((0, 3, vec![1], Vec::::new()))); + + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Proposed(1, 0, hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), 3)) + } + ]); + }); + } + + #[test] + fn motions_ignoring_non_council_proposals_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_noop!(CouncilMotions::propose(Origin::signed(42), 3, Box::new(proposal.clone())), "proposer not on council"); + }); + } + + #[test] + fn motions_ignoring_non_council_votes_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash: H256 = proposal.blake2_256().into(); + assert_ok!(CouncilMotions::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); + assert_noop!(CouncilMotions::vote(Origin::signed(42), hash.clone(), 0, true), "voter not on council"); + }); + } + + #[test] + fn motions_ignoring_bad_index_council_vote_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(3); + let proposal = set_balance_proposal(42); + let hash: H256 = proposal.blake2_256().into(); + assert_ok!(CouncilMotions::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); + assert_noop!(CouncilMotions::vote(Origin::signed(2), hash.clone(), 1, true), "mismatched index"); + }); + } + + #[test] + fn motions_revoting_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash: H256 = proposal.blake2_256().into(); + assert_ok!(CouncilMotions::propose(Origin::signed(1), 2, Box::new(proposal.clone()))); + assert_eq!(CouncilMotions::voting(&hash), Some((0, 2, vec![1], Vec::::new()))); + assert_noop!(CouncilMotions::vote(Origin::signed(1), hash.clone(), 0, true), "duplicate vote ignored"); + assert_ok!(CouncilMotions::vote(Origin::signed(1), hash.clone(), 0, false)); + assert_eq!(CouncilMotions::voting(&hash), Some((0, 2, Vec::::new(), vec![1]))); + assert_noop!(CouncilMotions::vote(Origin::signed(1), hash.clone(), 0, false), "duplicate vote ignored"); + + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Proposed(1, 0, hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), 2)) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Voted(1, hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), false, 0, 1)) + } + ]); + }); + } + + #[test] + fn motions_disapproval_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash: H256 = proposal.blake2_256().into(); + assert_ok!(CouncilMotions::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); + assert_ok!(CouncilMotions::vote(Origin::signed(2), hash.clone(), 0, false)); + + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Proposed(1, 0, hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), 3)) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Voted(2, hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), false, 1, 1)) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Disapproved(hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into())) + } + ]); + }); + } + + #[test] + fn motions_approval_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash: H256 = proposal.blake2_256().into(); + assert_ok!(CouncilMotions::propose(Origin::signed(1), 2, Box::new(proposal.clone()))); + assert_ok!(CouncilMotions::vote(Origin::signed(2), hash.clone(), 0, true)); + + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Proposed(1, 0, hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), 2)) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Voted(2, hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), true, 2, 0)) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Approved(hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into())) + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: OuterEvent::motions(RawEvent::Executed(hex!["a900ca23832b1f42a5d4af5d0ece88da63fbb4049cc00bac3f741eabb5a79c45"].into(), false)) + } + ]); + }); + } +} diff --git a/runtime/council/src/seats.rs b/runtime/council/src/seats.rs new file mode 100644 index 00000000000..294035a25a9 --- /dev/null +++ b/runtime/council/src/seats.rs @@ -0,0 +1,1355 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Council system: Handles the voting in and maintenance of council members. + +use rstd::prelude::*; +use primitives::traits::{Zero, One, As, Lookup, OnFinalise}; +use runtime_io::print; +use substrate_runtime_support::{StorageValue, StorageMap, dispatch::Result}; +use democracy; +use balances::{self, address::Address}; +use system::{self, ensure_signed, ensure_root}; + +// no polynomial attacks: +// +// all unbonded public operations should be constant time. +// all other public operations must be linear time in terms of prior public operations and: +// - those "valid" ones that cost nothing be limited to a constant number per single protected operation +// - the rest costing the same order as the computational complexity +// all protected operations must complete in at most O(public operations) +// +// we assume "beneficial" transactions will have the same access as attack transactions. +// +// any storage requirements should be bonded by the same order as the volume. + +// public operations: +// - express approvals (you pay in a "voter" bond the first time you do this; O(1); one extra DB entry, one DB change) +// - remove active voter (you get your "voter" bond back; O(1); one fewer DB entry, one DB change) +// - remove inactive voter (either you or the target is removed; if the target, you get their "voter" bond back; O(1); one fewer DB entry, one DB change) +// - submit candidacy (you pay a "candidate" bond; O(1); one extra DB entry, two DB changes) +// - present winner/runner-up (you may pay a "presentation" bond of O(voters) if the presentation is invalid; O(voters) compute; ) +// protected operations: +// - remove candidacy (remove all votes for a candidate) (one fewer DB entry, two DB changes) + +// to avoid a potentially problematic case of not-enough approvals prior to voting causing a +// back-to-back votes that have no way of ending, then there's a forced grace period between votes. +// to keep the system as stateless as possible (making it a bit easier to reason about), we just +// restrict when votes can begin to blocks that lie on boundaries (`voting_period`). + +// for an approval vote of C councilers: + +// top K runners-up are maintained between votes. all others are discarded. +// - candidate removed & bond returned when elected. +// - candidate removed & bond burned when discarded. + +// at the point that the vote ends (), all voters' balances are snapshotted. + +// for B blocks following, there's a counting period whereby each of the candidates that believe +// they fall in the top K+C voted can present themselves. they get the total stake +// recorded (based on the snapshot); an ordered list is maintained (the leaderboard). Noone may +// present themselves that, if elected, would result in being included twice on the council +// (important since existing councilers will will have their approval votes as it may be that they +// don't get removed), nor if existing presenters would mean they're not in the top K+C. + +// following B blocks, the top C candidates are elected and have their bond returned. the top C +// candidates and all other candidates beyond the top C+K are cleared. + +// vote-clearing happens lazily; for an approval to count, the most recent vote at the time of the +// voter's most recent vote must be no later than the most recent vote at the time that the +// candidate in the approval position was registered there. as candidates are removed from the +// register and others join in their place, this prevent an approval meant for an earlier candidate +// being used to elect a new candidate. + +// the candidate list increases as needed, but the contents (though not really the capacity) reduce +// after each vote as all but K entries are cleared. newly registering candidates must use cleared +// entries before they increase the capacity. + +pub type VoteIndex = u32; + +pub trait Trait: democracy::Trait { + type Event: From> + Into<::Event>; +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn set_approvals(origin, votes: Vec, index: VoteIndex) -> Result; + fn reap_inactive_voter(origin, reporter_index: u32, who: Address, who_index: u32, assumed_vote_index: VoteIndex) -> Result; + fn retract_voter(origin, index: u32) -> Result; + fn submit_candidacy(origin, slot: u32) -> Result; + fn present_winner(origin, candidate: Address, total: T::Balance, index: VoteIndex) -> Result; + + fn set_desired_seats(origin, count: u32) -> Result; + fn remove_member(origin, who: Address) -> Result; + fn set_presentation_duration(origin, count: T::BlockNumber) -> Result; + fn set_term_duration(origin, count: T::BlockNumber) -> Result; + } +} + +decl_storage! { + trait Store for Module as Council { + + // parameters + /// How much should be locked up in order to submit one's candidacy. + pub CandidacyBond get(candidacy_bond): required T::Balance; + /// How much should be locked up in order to be able to submit votes. + pub VotingBond get(voting_bond): required T::Balance; + /// The punishment, per voter, if you provide an invalid presentation. + pub PresentSlashPerVoter get(present_slash_per_voter): required T::Balance; + /// How many runners-up should have their approvals persist until the next vote. + pub CarryCount get(carry_count): required u32; + /// How long to give each top candidate to present themselves after the vote ends. + pub PresentationDuration get(presentation_duration): required T::BlockNumber; + /// How many votes need to go by after a voter's last vote before they can be reaped if their + /// approvals are moot. + pub InactiveGracePeriod get(inactivity_grace_period): required VoteIndex; + /// How often (in blocks) to check for new votes. + pub VotingPeriod get(voting_period): required T::BlockNumber; + /// How long each position is active for. + pub TermDuration get(term_duration): required T::BlockNumber; + /// Number of accounts that should be sitting on the council. + pub DesiredSeats get(desired_seats): required u32; + + // permanent state (always relevant, changes only at the finalisation of voting) + /// The current council. When there's a vote going on, this should still be used for executive + /// matters. + pub ActiveCouncil get(active_council): default Vec<(T::AccountId, T::BlockNumber)>; + /// The total number of votes that have happened or are in progress. + pub VoteCount get(vote_index): default VoteIndex; + + // persistent state (always relevant, changes constantly) + /// The last cleared vote index that this voter was last active at. + pub ApprovalsOf get(approvals_of): default map [ T::AccountId => Vec ]; + /// The vote index and list slot that the candidate `who` was registered or `None` if they are not + /// currently registered. + pub RegisterInfoOf get(candidate_reg_info): map [ T::AccountId => (VoteIndex, u32) ]; + /// The last cleared vote index that this voter was last active at. + pub LastActiveOf get(voter_last_active): map [ T::AccountId => VoteIndex ]; + /// The present voter list. + pub Voters get(voters): default Vec; + /// The present candidate list. + pub Candidates get(candidates): default Vec; // has holes + pub CandidateCount get(candidate_count): default u32; + + // temporary state (only relevant during finalisation/presentation) + /// The accounts holding the seats that will become free on the next tally. + pub NextFinalise get(next_finalise): (T::BlockNumber, u32, Vec); + /// The stakes as they were at the point that the vote ended. + pub SnapshotedStakes get(snapshoted_stakes): required Vec; + /// Get the leaderboard if we;re in the presentation phase. + pub Leaderboard get(leaderboard): Vec<(T::Balance, T::AccountId)>; // ORDERED low -> high + } +} + +decl_event!( + /// An event in this module. + pub enum Event with RawEvent + where ::AccountId + { + /// reaped voter, reaper + VoterReaped(AccountId, AccountId), + /// slashed reaper + BadReaperSlashed(AccountId), + /// A tally (for approval votes of council seat(s)) has started. + TallyStarted(u32), + /// A tally (for approval votes of council seat(s)) has ended (with one or more new members). + TallyFinalised(Vec, Vec), + } +); + +impl Module { + + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + // exposed immutables. + + /// True if we're currently in a presentation period. + pub fn presentation_active() -> bool { + >::exists() + } + + /// If `who` a candidate at the moment? + pub fn is_a_candidate(who: &T::AccountId) -> bool { + >::exists(who) + } + + /// Determine the block that a vote can happen on which is no less than `n`. + pub fn next_vote_from(n: T::BlockNumber) -> T::BlockNumber { + let voting_period = Self::voting_period(); + (n + voting_period - One::one()) / voting_period * voting_period + } + + /// The block number on which the tally for the next election will happen. `None` only if the + /// desired seats of the council is zero. + pub fn next_tally() -> Option { + let desired_seats = Self::desired_seats(); + if desired_seats == 0 { + None + } else { + let c = Self::active_council(); + let (next_possible, count, coming) = + if let Some((tally_end, comers, leavers)) = Self::next_finalise() { + // if there's a tally in progress, then next tally can begin immediately afterwards + (tally_end, c.len() - leavers.len() + comers as usize, comers) + } else { + (>::block_number(), c.len(), 0) + }; + if count < desired_seats as usize { + Some(next_possible) + } else { + // next tally begins once enough council members expire to bring members below desired. + if desired_seats <= coming { + // the entire amount of desired seats is less than those new members - we'll have + // to wait until they expire. + Some(next_possible + Self::term_duration()) + } else { + Some(c[c.len() - (desired_seats - coming) as usize].1) + } + }.map(Self::next_vote_from) + } + } + + // dispatch + + /// Set candidate approvals. Approval slots stay valid as long as candidates in those slots + /// are registered. + fn set_approvals(origin: T::Origin, votes: Vec, index: VoteIndex) -> Result { + let who = ensure_signed(origin)?; + + ensure!(!Self::presentation_active(), "no approval changes during presentation period"); + ensure!(index == Self::vote_index(), "incorrect vote index"); + if !>::exists(&who) { + // not yet a voter - deduct bond. + // NOTE: this must be the last potential bailer, since it changes state. + >::reserve(&who, Self::voting_bond())?; + + >::put({ + let mut v = Self::voters(); + v.push(who.clone()); + v + }); + } + >::insert(&who, index); + >::insert(&who, votes); + Ok(()) + } + + /// Remove a voter. For it not to be a bond-consuming no-op, all approved candidate indices + /// must now be either unregistered or registered to a candidate that registered the slot after + /// the voter gave their last approval set. + /// + /// May be called by anyone. Returns the voter deposit to `signed`. + fn reap_inactive_voter( + origin: T::Origin, + reporter_index: u32, + who: Address, + who_index: u32, + assumed_vote_index: VoteIndex + ) -> Result { + let reporter = ensure_signed(origin)?; + + let who = >::lookup(who)?; + ensure!(!Self::presentation_active(), "cannot reap during presentation period"); + ensure!(Self::voter_last_active(&reporter).is_some(), "reporter must be a voter"); + let last_active = Self::voter_last_active(&who).ok_or("target for inactivity cleanup must be active")?; + ensure!(assumed_vote_index == Self::vote_index(), "vote index not current"); + ensure!(last_active < assumed_vote_index - Self::inactivity_grace_period(), "cannot reap during grace perid"); + let voters = Self::voters(); + let reporter_index = reporter_index as usize; + let who_index = who_index as usize; + ensure!(reporter_index < voters.len() && voters[reporter_index] == reporter, "bad reporter index"); + ensure!(who_index < voters.len() && voters[who_index] == who, "bad target index"); + + // will definitely kill one of signed or who now. + + let valid = !Self::approvals_of(&who).iter() + .zip(Self::candidates().iter()) + .any(|(&appr, addr)| + appr && + *addr != T::AccountId::default() && + Self::candidate_reg_info(addr).map_or(false, |x| x.0 <= last_active)/*defensive only: all items in candidates list are registered*/ + ); + + Self::remove_voter( + if valid { &who } else { &reporter }, + if valid { who_index } else { reporter_index }, + voters + ); + if valid { + // This only fails if `who` doesn't exist, which it clearly must do since its the origin. + // Still, it's no more harmful to propagate any error at this point. + >::repatriate_reserved(&who, &reporter, Self::voting_bond())?; + Self::deposit_event(RawEvent::VoterReaped(who, reporter)); + } else { + >::slash_reserved(&reporter, Self::voting_bond()); + Self::deposit_event(RawEvent::BadReaperSlashed(reporter)); + } + Ok(()) + } + + /// Remove a voter. All votes are cancelled and the voter deposit is returned. + fn retract_voter(origin: T::Origin, index: u32) -> Result { + let who = ensure_signed(origin)?; + + ensure!(!Self::presentation_active(), "cannot retract when presenting"); + ensure!(>::exists(&who), "cannot retract non-voter"); + let voters = Self::voters(); + let index = index as usize; + ensure!(index < voters.len(), "retraction index invalid"); + ensure!(voters[index] == who, "retraction index mismatch"); + + Self::remove_voter(&who, index, voters); + >::unreserve(&who, Self::voting_bond()); + Ok(()) + } + + /// Submit oneself for candidacy. + /// + /// Account must have enough transferrable funds in it to pay the bond. + fn submit_candidacy(origin: T::Origin, slot: u32) -> Result { + let who = ensure_signed(origin)?; + + ensure!(!Self::is_a_candidate(&who), "duplicate candidate submission"); + let slot = slot as usize; + let count = Self::candidate_count() as usize; + let candidates = Self::candidates(); + ensure!( + (slot == count && count == candidates.len()) || + (slot < candidates.len() && candidates[slot] == T::AccountId::default()), + "invalid candidate slot" + ); + // NOTE: This must be last as it has side-effects. + >::reserve(&who, Self::candidacy_bond()) + .map_err(|_| "candidate has not enough funds")?; + + >::insert(&who, (Self::vote_index(), slot as u32)); + let mut candidates = candidates; + if slot == candidates.len() { + candidates.push(who); + } else { + candidates[slot] = who; + } + >::put(candidates); + >::put(count as u32 + 1); + Ok(()) + } + + /// Claim that `signed` is one of the top Self::carry_count() + current_vote().1 candidates. + /// Only works if the `block_number >= current_vote().0` and `< current_vote().0 + presentation_duration()`` + /// `signed` should have at least + fn present_winner( + origin: T::Origin, + candidate: Address, + total: T::Balance, + index: VoteIndex + ) -> Result { + let who = ensure_signed(origin)?; + + let candidate = >::lookup(candidate)?; + ensure!(index == Self::vote_index(), "index not current"); + let (_, _, expiring) = Self::next_finalise().ok_or("cannot present outside of presentation period")?; + let stakes = Self::snapshoted_stakes(); + let voters = Self::voters(); + let bad_presentation_punishment = Self::present_slash_per_voter() * T::Balance::sa(voters.len() as u64); + ensure!(>::can_slash(&who, bad_presentation_punishment), "presenter must have sufficient slashable funds"); + + let mut leaderboard = Self::leaderboard().ok_or("leaderboard must exist while present phase active")?; + ensure!(total > leaderboard[0].0, "candidate not worthy of leaderboard"); + + if let Some(p) = Self::active_council().iter().position(|&(ref c, _)| c == &candidate) { + ensure!(p < expiring.len(), "candidate must not form a duplicated member if elected"); + } + + let (registered_since, candidate_index): (VoteIndex, u32) = + Self::candidate_reg_info(&candidate).ok_or("presented candidate must be current")?; + let actual_total = voters.iter() + .zip(stakes.iter()) + .filter_map(|(voter, stake)| + match Self::voter_last_active(voter) { + Some(b) if b >= registered_since => + Self::approvals_of(voter).get(candidate_index as usize) + .and_then(|approved| if *approved { Some(*stake) } else { None }), + _ => None, + }) + .fold(Zero::zero(), |acc, n| acc + n); + let dupe = leaderboard.iter().find(|&&(_, ref c)| c == &candidate).is_some(); + if total == actual_total && !dupe { + // insert into leaderboard + leaderboard[0] = (total, candidate); + leaderboard.sort_by_key(|&(t, _)| t); + >::put(leaderboard); + Ok(()) + } else { + // we can rest assured it will be Ok since we checked `can_slash` earlier; still + // better safe than sorry. + let _ = >::slash(&who, bad_presentation_punishment); + Err(if dupe { "duplicate presentation" } else { "incorrect total" }) + } + } + + /// Set the desired member count; if lower than the current count, then seats will not be up + /// election when they expire. If more, then a new vote will be started if one is not already + /// in progress. + fn set_desired_seats(origin: T::Origin, count: u32) -> Result { + ensure_root(origin)?; + >::put(count); + Ok(()) + } + + /// Remove a particular member. A tally will happen instantly (if not already in a presentation + /// period) to fill the seat if removal means that the desired members are not met. + /// This is effective immediately. + fn remove_member(origin: T::Origin, who: Address) -> Result { + ensure_root(origin)?; + let who = >::lookup(who)?; + let new_council: Vec<(T::AccountId, T::BlockNumber)> = Self::active_council() + .into_iter() + .filter(|i| i.0 != who) + .collect(); + >::put(new_council); + Ok(()) + } + + /// Set the presentation duration. If there is currently a vote being presented for, will + /// invoke `finalise_vote`. + fn set_presentation_duration(origin: T::Origin, count: T::BlockNumber) -> Result { + ensure_root(origin)?; + >::put(count); + Ok(()) + } + + /// Set the presentation duration. If there is current a vote being presented for, will + /// invoke `finalise_vote`. + fn set_term_duration(origin: T::Origin, count: T::BlockNumber) -> Result { + ensure_root(origin)?; + >::put(count); + Ok(()) + } + + // private + + /// Check there's nothing to do this block + fn end_block(block_number: T::BlockNumber) -> Result { + if (block_number % Self::voting_period()).is_zero() { + if let Some(number) = Self::next_tally() { + if block_number == number { + Self::start_tally(); + } + } + } + if let Some((number, _, _)) = Self::next_finalise() { + if block_number == number { + Self::finalise_tally()? + } + } + Ok(()) + } + + /// Remove a voter from the system. Trusts that Self::voters()[index] != voter. + fn remove_voter(voter: &T::AccountId, index: usize, mut voters: Vec) { + >::put({ voters.swap_remove(index); voters }); + >::remove(voter); + >::remove(voter); + } + + /// Close the voting, snapshot the staking and the number of seats that are actually up for grabs. + fn start_tally() { + let active_council = Self::active_council(); + let desired_seats = Self::desired_seats() as usize; + let number = >::block_number(); + let expiring = active_council.iter().take_while(|i| i.1 == number).map(|i| i.0.clone()).collect::>(); + if active_council.len() - expiring.len() < desired_seats { + let empty_seats = desired_seats - (active_council.len() - expiring.len()); + >::put((number + Self::presentation_duration(), empty_seats as u32, expiring)); + + let voters = Self::voters(); + let votes = voters.iter().map(>::total_balance).collect::>(); + >::put(votes); + + // initialise leaderboard. + let leaderboard_size = empty_seats + Self::carry_count() as usize; + >::put(vec![(T::Balance::zero(), T::AccountId::default()); leaderboard_size]); + + Self::deposit_event(RawEvent::TallyStarted(empty_seats as u32)); + } + } + + /// Finalise the vote, removing each of the `removals` and inserting `seats` of the most approved + /// candidates in their place. If the total council members is less than the desired membership + /// a new vote is started. + /// Clears all presented candidates, returning the bond of the elected ones. + fn finalise_tally() -> Result { + >::kill(); + let (_, coming, expiring): (T::BlockNumber, u32, Vec) = + >::take().ok_or("finalise can only be called after a tally is started.")?; + let leaderboard: Vec<(T::Balance, T::AccountId)> = >::take().unwrap_or_default(); + let new_expiry = >::block_number() + Self::term_duration(); + + // return bond to winners. + let candidacy_bond = Self::candidacy_bond(); + let incoming: Vec = leaderboard.iter() + .rev() + .take_while(|&&(b, _)| !b.is_zero()) + .take(coming as usize) + .map(|(_, a)| a) + .cloned() + .inspect(|a| {>::unreserve(a, candidacy_bond);}) + .collect(); + let active_council = Self::active_council(); + let outgoing = active_council.iter().take(expiring.len()).map(|a| a.0.clone()).collect(); + + // set the new council. + let mut new_council: Vec<_> = active_council + .into_iter() + .skip(expiring.len()) + .chain(incoming.iter().cloned().map(|a| (a, new_expiry))) + .collect(); + new_council.sort_by_key(|&(_, expiry)| expiry); + >::put(new_council); + + // clear all except runners-up from candidate list. + let candidates = Self::candidates(); + let mut new_candidates = vec![T::AccountId::default(); candidates.len()]; // shrink later. + let runners_up = leaderboard.into_iter() + .rev() + .take_while(|&(b, _)| !b.is_zero()) + .skip(coming as usize) + .filter_map(|(_, a)| Self::candidate_reg_info(&a).map(|i| (a, i.1))); + let mut count = 0u32; + for (address, slot) in runners_up { + new_candidates[slot as usize] = address; + count += 1; + } + for (old, new) in candidates.iter().zip(new_candidates.iter()) { + if old != new { + // removed - kill it + >::remove(old); + } + } + // discard any superfluous slots. + if let Some(last_index) = new_candidates.iter().rposition(|c| *c != T::AccountId::default()) { + new_candidates.truncate(last_index + 1); + } + + Self::deposit_event(RawEvent::TallyFinalised(incoming, outgoing)); + + >::put(new_candidates); + >::put(count); + >::put(Self::vote_index() + 1); + Ok(()) + } +} + +impl OnFinalise for Module { + fn on_finalise(n: T::BlockNumber) { + if let Err(e) = Self::end_block(n) { + print("Guru meditation"); + print(e); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ::tests::*; + + #[test] + fn params_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_eq!(Council::next_vote_from(1), 4); + assert_eq!(Council::next_vote_from(4), 4); + assert_eq!(Council::next_vote_from(5), 8); + assert_eq!(Council::vote_index(), 0); + assert_eq!(Council::candidacy_bond(), 9); + assert_eq!(Council::voting_bond(), 3); + assert_eq!(Council::present_slash_per_voter(), 1); + assert_eq!(Council::presentation_duration(), 2); + assert_eq!(Council::voting_period(), 4); + assert_eq!(Council::term_duration(), 5); + assert_eq!(Council::desired_seats(), 2); + assert_eq!(Council::carry_count(), 2); + + assert_eq!(Council::active_council(), vec![]); + assert_eq!(Council::next_tally(), Some(4)); + assert_eq!(Council::presentation_active(), false); + assert_eq!(Council::next_finalise(), None); + + assert_eq!(Council::candidates(), Vec::::new()); + assert_eq!(Council::is_a_candidate(&1), false); + assert_eq!(Council::candidate_reg_info(1), None); + + assert_eq!(Council::voters(), Vec::::new()); + assert_eq!(Council::voter_last_active(1), None); + assert_eq!(Council::approvals_of(1), vec![]); + }); + } + + #[test] + fn simple_candidate_submission_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_eq!(Council::candidates(), Vec::::new()); + assert_eq!(Council::candidate_reg_info(1), None); + assert_eq!(Council::candidate_reg_info(2), None); + assert_eq!(Council::is_a_candidate(&1), false); + assert_eq!(Council::is_a_candidate(&2), false); + + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_eq!(Council::candidates(), vec![1]); + assert_eq!(Council::candidate_reg_info(1), Some((0, 0))); + assert_eq!(Council::candidate_reg_info(2), None); + assert_eq!(Council::is_a_candidate(&1), true); + assert_eq!(Council::is_a_candidate(&2), false); + + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_eq!(Council::candidates(), vec![1, 2]); + assert_eq!(Council::candidate_reg_info(1), Some((0, 0))); + assert_eq!(Council::candidate_reg_info(2), Some((0, 1))); + assert_eq!(Council::is_a_candidate(&1), true); + assert_eq!(Council::is_a_candidate(&2), true); + }); + } + + fn new_test_ext_with_candidate_holes() -> runtime_io::TestExternalities { + let mut t = new_test_ext(false); + with_externalities(&mut t, || { + >::put(vec![0, 0, 1]); + >::put(1); + >::insert(1, (0, 2)); + }); + t + } + + #[test] + fn candidate_submission_using_free_slot_should_work() { + let mut t = new_test_ext_with_candidate_holes(); + + with_externalities(&mut t, || { + System::set_block_number(1); + assert_eq!(Council::candidates(), vec![0, 0, 1]); + + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_eq!(Council::candidates(), vec![0, 2, 1]); + + assert_ok!(Council::submit_candidacy(Origin::signed(3), 0)); + assert_eq!(Council::candidates(), vec![3, 2, 1]); + }); + } + + #[test] + fn candidate_submission_using_alternative_free_slot_should_work() { + let mut t = new_test_ext_with_candidate_holes(); + + with_externalities(&mut t, || { + System::set_block_number(1); + assert_eq!(Council::candidates(), vec![0, 0, 1]); + + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_eq!(Council::candidates(), vec![2, 0, 1]); + + assert_ok!(Council::submit_candidacy(Origin::signed(3), 1)); + assert_eq!(Council::candidates(), vec![2, 3, 1]); + }); + } + + #[test] + fn candidate_submission_not_using_free_slot_should_not_work() { + with_externalities(&mut new_test_ext_with_candidate_holes(), || { + System::set_block_number(1); + assert_noop!(Council::submit_candidacy(Origin::signed(4), 3), "invalid candidate slot"); + }); + } + + #[test] + fn bad_candidate_slot_submission_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_eq!(Council::candidates(), Vec::::new()); + assert_noop!(Council::submit_candidacy(Origin::signed(1), 1), "invalid candidate slot"); + }); + } + + #[test] + fn non_free_candidate_slot_submission_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_eq!(Council::candidates(), Vec::::new()); + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_eq!(Council::candidates(), vec![1]); + assert_noop!(Council::submit_candidacy(Origin::signed(2), 0), "invalid candidate slot"); + }); + } + + #[test] + fn dupe_candidate_submission_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_eq!(Council::candidates(), Vec::::new()); + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_eq!(Council::candidates(), vec![1]); + assert_noop!(Council::submit_candidacy(Origin::signed(1), 1), "duplicate candidate submission"); + }); + } + + #[test] + fn poor_candidate_submission_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_eq!(Council::candidates(), Vec::::new()); + assert_noop!(Council::submit_candidacy(Origin::signed(7), 0), "candidate has not enough funds"); + }); + } + + #[test] + fn voting_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + + assert_ok!(Council::set_approvals(Origin::signed(1), vec![true], 0)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![true], 0)); + + assert_eq!(Council::approvals_of(1), vec![true]); + assert_eq!(Council::approvals_of(4), vec![true]); + assert_eq!(Council::voters(), vec![1, 4]); + + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 2)); + + assert_ok!(Council::set_approvals(Origin::signed(2), vec![false, true, true], 0)); + assert_ok!(Council::set_approvals(Origin::signed(3), vec![false, true, true], 0)); + + assert_eq!(Council::approvals_of(1), vec![true]); + assert_eq!(Council::approvals_of(4), vec![true]); + assert_eq!(Council::approvals_of(2), vec![false, true, true]); + assert_eq!(Council::approvals_of(3), vec![false, true, true]); + + assert_eq!(Council::voters(), vec![1, 4, 2, 3]); + }); + } + + #[test] + fn resubmitting_voting_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![true], 0)); + + assert_eq!(Council::approvals_of(4), vec![true]); + + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 2)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![true, false, true], 0)); + + assert_eq!(Council::approvals_of(4), vec![true, false, true]); + }); + } + + #[test] + fn retracting_voter_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 2)); + + assert_ok!(Council::set_approvals(Origin::signed(1), vec![true], 0)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![false, true, true], 0)); + assert_ok!(Council::set_approvals(Origin::signed(3), vec![false, true, true], 0)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![true, false, true], 0)); + + assert_eq!(Council::voters(), vec![1, 2, 3, 4]); + assert_eq!(Council::approvals_of(1), vec![true]); + assert_eq!(Council::approvals_of(2), vec![false, true, true]); + assert_eq!(Council::approvals_of(3), vec![false, true, true]); + assert_eq!(Council::approvals_of(4), vec![true, false, true]); + + assert_ok!(Council::retract_voter(Origin::signed(1), 0)); + + assert_eq!(Council::voters(), vec![4, 2, 3]); + assert_eq!(Council::approvals_of(1), Vec::::new()); + assert_eq!(Council::approvals_of(2), vec![false, true, true]); + assert_eq!(Council::approvals_of(3), vec![false, true, true]); + assert_eq!(Council::approvals_of(4), vec![true, false, true]); + + assert_ok!(Council::retract_voter(Origin::signed(2), 1)); + + assert_eq!(Council::voters(), vec![4, 3]); + assert_eq!(Council::approvals_of(1), Vec::::new()); + assert_eq!(Council::approvals_of(2), Vec::::new()); + assert_eq!(Council::approvals_of(3), vec![false, true, true]); + assert_eq!(Council::approvals_of(4), vec![true, false, true]); + + assert_ok!(Council::retract_voter(Origin::signed(3), 1)); + + assert_eq!(Council::voters(), vec![4]); + assert_eq!(Council::approvals_of(1), Vec::::new()); + assert_eq!(Council::approvals_of(2), Vec::::new()); + assert_eq!(Council::approvals_of(3), Vec::::new()); + assert_eq!(Council::approvals_of(4), vec![true, false, true]); + }); + } + + #[test] + fn invalid_retraction_index_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 0)); + assert_ok!(Council::set_approvals(Origin::signed(1), vec![true], 0)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 0)); + assert_eq!(Council::voters(), vec![1, 2]); + assert_noop!(Council::retract_voter(Origin::signed(1), 1), "retraction index mismatch"); + }); + } + + #[test] + fn overflow_retraction_index_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 0)); + assert_ok!(Council::set_approvals(Origin::signed(1), vec![true], 0)); + assert_noop!(Council::retract_voter(Origin::signed(1), 1), "retraction index invalid"); + }); + } + + #[test] + fn non_voter_retraction_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(1); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 0)); + assert_ok!(Council::set_approvals(Origin::signed(1), vec![true], 0)); + assert_noop!(Council::retract_voter(Origin::signed(2), 0), "cannot retract non-voter"); + }); + } + + #[test] + fn simple_tally_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert!(!Council::presentation_active()); + + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, true], 0)); + assert_eq!(Council::voters(), vec![2, 5]); + assert_eq!(Council::approvals_of(2), vec![true, false]); + assert_eq!(Council::approvals_of(5), vec![false, true]); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert!(Council::presentation_active()); + assert_eq!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0), Ok(())); + assert_eq!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0), Ok(())); + assert_eq!(Council::leaderboard(), Some(vec![(0, 0), (0, 0), (20, 2), (50, 5)])); + + assert_ok!(Council::end_block(System::block_number())); + + assert!(!Council::presentation_active()); + assert_eq!(Council::active_council(), vec![(5, 11), (2, 11)]); + + assert!(!Council::is_a_candidate(&2)); + assert!(!Council::is_a_candidate(&5)); + assert_eq!(Council::vote_index(), 1); + assert_eq!(Council::voter_last_active(2), Some(0)); + assert_eq!(Council::voter_last_active(5), Some(0)); + }); + } + + #[test] + fn double_presentations_should_be_punished() { + with_externalities(&mut new_test_ext(false), || { + assert!(Balances::can_slash(&4, 10)); + + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0)); + assert_eq!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0), Err("duplicate presentation")); + assert_ok!(Council::end_block(System::block_number())); + + assert_eq!(Council::active_council(), vec![(5, 11), (2, 11)]); + assert_eq!(Balances::total_balance(&4), 38); + }); + } + + #[test] + fn retracting_inactive_voter_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![true], 1)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 1)); + assert_ok!(Council::end_block(System::block_number())); + + assert_ok!(Council::reap_inactive_voter(Origin::signed(5), + Council::voters().iter().position(|&i| i == 5).unwrap() as u32, + 2.into(), Council::voters().iter().position(|&i| i == 2).unwrap() as u32, + 2 + )); + + assert_eq!(Council::voters(), vec![5]); + assert_eq!(Council::approvals_of(2).len(), 0); + assert_eq!(Balances::total_balance(&2), 17); + assert_eq!(Balances::total_balance(&5), 53); + }); + } + + #[test] + fn presenting_for_double_election_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_eq!(Council::submit_candidacy(Origin::signed(2), 0), Ok(())); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_eq!(Council::submit_candidacy(Origin::signed(2), 0), Ok(())); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 1)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_noop!(Council::present_winner(Origin::signed(4), 2.into(), 20, 1), "candidate must not form a duplicated member if elected"); + }); + } + + #[test] + fn retracting_inactive_voter_with_other_candidates_in_slots_should_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![true], 1)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 1)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(11); + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + + assert_ok!(Council::reap_inactive_voter(Origin::signed(5), + Council::voters().iter().position(|&i| i == 5).unwrap() as u32, + 2.into(), Council::voters().iter().position(|&i| i == 2).unwrap() as u32, + 2 + )); + + assert_eq!(Council::voters(), vec![5]); + assert_eq!(Council::approvals_of(2).len(), 0); + assert_eq!(Balances::total_balance(&2), 17); + assert_eq!(Balances::total_balance(&5), 53); + }); + } + + #[test] + fn retracting_inactive_voter_with_bad_reporter_index_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![true], 1)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 1)); + assert_ok!(Council::end_block(System::block_number())); + + assert_noop!(Council::reap_inactive_voter(Origin::signed(2), + 42, + 2.into(), Council::voters().iter().position(|&i| i == 2).unwrap() as u32, + 2 + ), "bad reporter index"); + }); + } + + #[test] + fn retracting_inactive_voter_with_bad_target_index_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![true], 1)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 1)); + assert_ok!(Council::end_block(System::block_number())); + + assert_noop!(Council::reap_inactive_voter(Origin::signed(2), + Council::voters().iter().position(|&i| i == 2).unwrap() as u32, + 2.into(), 42, + 2 + ), "bad target index"); + }); + } + + #[test] + fn attempting_to_retract_active_voter_should_slash_reporter() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 1)); + assert_ok!(Council::submit_candidacy(Origin::signed(4), 2)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 3)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true, false, false, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(3), vec![false, true, false, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![false, false, true, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, false, false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 3.into(), 30, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 4.into(), 40, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_ok!(Council::set_desired_seats(Origin::ROOT, 3)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 1)); + assert_ok!(Council::present_winner(Origin::signed(4), 3.into(), 30, 1)); + assert_ok!(Council::end_block(System::block_number())); + + assert_ok!(Council::reap_inactive_voter(Origin::signed(4), + Council::voters().iter().position(|&i| i == 4).unwrap() as u32, + 2.into(), Council::voters().iter().position(|&i| i == 2).unwrap() as u32, + 2 + )); + + assert_eq!(Council::voters(), vec![2, 3, 5]); + assert_eq!(Council::approvals_of(4).len(), 0); + assert_eq!(Balances::total_balance(&4), 37); + }); + } + + #[test] + fn attempting_to_retract_inactive_voter_by_nonvoter_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![true], 1)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 1)); + assert_ok!(Council::end_block(System::block_number())); + + assert_noop!(Council::reap_inactive_voter(Origin::signed(4), + 0, + 2.into(), Council::voters().iter().position(|&i| i == 2).unwrap() as u32, + 2 + ), "reporter must be a voter"); + }); + } + + #[test] + fn presenting_loser_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_ok!(Council::set_approvals(Origin::signed(6), vec![true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 2)); + assert_ok!(Council::set_approvals(Origin::signed(3), vec![false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(4), 3)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![false, false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 4)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, false, false, false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 1.into(), 60, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 3.into(), 30, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 4.into(), 40, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0)); + assert_noop!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0), "candidate not worthy of leaderboard"); + }); + } + + #[test] + fn presenting_loser_first_should_not_matter() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_ok!(Council::set_approvals(Origin::signed(6), vec![true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 2)); + assert_ok!(Council::set_approvals(Origin::signed(3), vec![false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(4), 3)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![false, false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 4)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, false, false, false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 2.into(), 20, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 1.into(), 60, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 3.into(), 30, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 4.into(), 40, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0)); + + assert_eq!(Council::leaderboard(), Some(vec![ + (30, 3), + (40, 4), + (50, 5), + (60, 1) + ])); + }); + } + + #[test] + fn present_outside_of_presentation_period_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert!(!Council::presentation_active()); + assert_noop!(Council::present_winner(Origin::signed(5), 5.into(), 1, 0), "cannot present outside of presentation period"); + }); + } + + #[test] + fn present_with_invalid_vote_index_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_noop!(Council::present_winner(Origin::signed(4), 2.into(), 20, 1), "index not current"); + }); + } + + #[test] + fn present_when_presenter_is_poor_should_not_work() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert!(!Council::presentation_active()); + + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_eq!(Balances::free_balance(&1), 1); + assert_eq!(Balances::reserved_balance(&1), 9); + assert_noop!(Council::present_winner(Origin::signed(1), 1.into(), 20, 0), "presenter must have sufficient slashable funds"); + }); + } + + #[test] + fn invalid_present_tally_should_slash() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert!(!Council::presentation_active()); + assert_eq!(Balances::total_balance(&4), 40); + + assert_ok!(Council::submit_candidacy(Origin::signed(2), 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![true, false], 0)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_err!(Council::present_winner(Origin::signed(4), 2.into(), 80, 0), "incorrect total"); + + assert_eq!(Balances::total_balance(&4), 38); + }); + } + + #[test] + fn runners_up_should_be_kept() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert!(!Council::presentation_active()); + + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_ok!(Council::set_approvals(Origin::signed(6), vec![true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 2)); + assert_ok!(Council::set_approvals(Origin::signed(3), vec![false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(4), 3)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![false, false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 4)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, false, false, false, true], 0)); + + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert!(Council::presentation_active()); + assert_ok!(Council::present_winner(Origin::signed(4), 1.into(), 60, 0)); + assert_eq!(Council::leaderboard(), Some(vec![ + (0, 0), + (0, 0), + (0, 0), + (60, 1) + ])); + assert_ok!(Council::present_winner(Origin::signed(4), 3.into(), 30, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 4.into(), 40, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0)); + assert_eq!(Council::leaderboard(), Some(vec![ + (30, 3), + (40, 4), + (50, 5), + (60, 1) + ])); + + assert_ok!(Council::end_block(System::block_number())); + + assert!(!Council::presentation_active()); + assert_eq!(Council::active_council(), vec![(1, 11), (5, 11)]); + + assert!(!Council::is_a_candidate(&1)); + assert!(!Council::is_a_candidate(&5)); + assert!(!Council::is_a_candidate(&2)); + assert!(Council::is_a_candidate(&3)); + assert!(Council::is_a_candidate(&4)); + assert_eq!(Council::vote_index(), 1); + assert_eq!(Council::voter_last_active(2), Some(0)); + assert_eq!(Council::voter_last_active(3), Some(0)); + assert_eq!(Council::voter_last_active(4), Some(0)); + assert_eq!(Council::voter_last_active(5), Some(0)); + assert_eq!(Council::voter_last_active(6), Some(0)); + assert_eq!(Council::candidate_reg_info(3), Some((0, 2))); + assert_eq!(Council::candidate_reg_info(4), Some((0, 3))); + }); + } + + #[test] + fn second_tally_should_use_runners_up() { + with_externalities(&mut new_test_ext(false), || { + System::set_block_number(4); + assert_ok!(Council::submit_candidacy(Origin::signed(1), 0)); + assert_ok!(Council::set_approvals(Origin::signed(6), vec![true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(2), 1)); + assert_ok!(Council::set_approvals(Origin::signed(2), vec![false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(3), 2)); + assert_ok!(Council::set_approvals(Origin::signed(3), vec![false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(4), 3)); + assert_ok!(Council::set_approvals(Origin::signed(4), vec![false, false, false, true], 0)); + assert_ok!(Council::submit_candidacy(Origin::signed(5), 4)); + assert_ok!(Council::set_approvals(Origin::signed(5), vec![false, false, false, false, true], 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(6); + assert_ok!(Council::present_winner(Origin::signed(4), 1.into(), 60, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 3.into(), 30, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 4.into(), 40, 0)); + assert_ok!(Council::present_winner(Origin::signed(4), 5.into(), 50, 0)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(8); + assert_ok!(Council::set_approvals(Origin::signed(6), vec![false, false, true, false], 1)); + assert_ok!(Council::set_desired_seats(Origin::ROOT, 3)); + assert_ok!(Council::end_block(System::block_number())); + + System::set_block_number(10); + assert_ok!(Council::present_winner(Origin::signed(4), 3.into(), 90, 1)); + assert_ok!(Council::present_winner(Origin::signed(4), 4.into(), 40, 1)); + assert_ok!(Council::end_block(System::block_number())); + + assert!(!Council::presentation_active()); + assert_eq!(Council::active_council(), vec![(1, 11), (5, 11), (3, 15)]); + + assert!(!Council::is_a_candidate(&1)); + assert!(!Council::is_a_candidate(&2)); + assert!(!Council::is_a_candidate(&3)); + assert!(!Council::is_a_candidate(&5)); + assert!(Council::is_a_candidate(&4)); + assert_eq!(Council::vote_index(), 2); + assert_eq!(Council::voter_last_active(2), Some(0)); + assert_eq!(Council::voter_last_active(3), Some(0)); + assert_eq!(Council::voter_last_active(4), Some(0)); + assert_eq!(Council::voter_last_active(5), Some(0)); + assert_eq!(Council::voter_last_active(6), Some(1)); + + assert_eq!(Council::candidate_reg_info(4), Some((0, 3))); + }); + } +} diff --git a/runtime/council/src/voting.rs b/runtime/council/src/voting.rs new file mode 100644 index 00000000000..450eb034f12 --- /dev/null +++ b/runtime/council/src/voting.rs @@ -0,0 +1,505 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Council voting system. + +use rstd::prelude::*; +use rstd::borrow::Borrow; +use primitives::traits::{OnFinalise, Hash}; +use runtime_io::print; +use substrate_runtime_support::dispatch::Result; +use substrate_runtime_support::{StorageValue, StorageMap, IsSubType}; +use {system, democracy}; +use super::{Trait as CouncilTrait, Module as Council}; +use system::{ensure_signed, ensure_root}; + +pub trait Trait: CouncilTrait { + type Event: From> + Into<::Event>; +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn propose(origin, proposal: Box) -> Result; + fn vote(origin, proposal: T::Hash, approve: bool) -> Result; + fn veto(origin, proposal_hash: T::Hash) -> Result; + + fn set_cooloff_period(origin, blocks: T::BlockNumber) -> Result; + fn set_voting_period(origin, blocks: T::BlockNumber) -> Result; + } +} + +decl_storage! { + trait Store for Module as CouncilVoting { + pub CooloffPeriod get(cooloff_period): required T::BlockNumber; + pub VotingPeriod get(voting_period): required T::BlockNumber; + pub Proposals get(proposals): required Vec<(T::BlockNumber, T::Hash)>; // ordered by expiry. + pub ProposalOf get(proposal_of): map [ T::Hash => T::Proposal ]; + pub ProposalVoters get(proposal_voters): default map [ T::Hash => Vec ]; + pub CouncilVoteOf get(vote_of): map [ (T::Hash, T::AccountId) => bool ]; + pub VetoedProposal get(veto_of): map [ T::Hash => (T::BlockNumber, Vec) ]; + } +} + +/// An event in this module. +decl_event!( + pub enum Event with RawEvent + where ::Hash + { + /// A voting tally has happened for a referendum cancelation vote. + /// Last three are yes, no, abstain counts. + TallyCancelation(Hash, u32, u32, u32), + /// A voting tally has happened for a referendum vote. + /// Last three are yes, no, abstain counts. + TallyReferendum(Hash, u32, u32, u32), + } +); + +impl Module { + + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + pub fn is_vetoed>(proposal: B) -> bool { + Self::veto_of(proposal.borrow()) + .map(|(expiry, _): (T::BlockNumber, Vec)| >::block_number() < expiry) + .unwrap_or(false) + } + + pub fn will_still_be_councillor_at(who: &T::AccountId, n: T::BlockNumber) -> bool { + >::active_council().iter() + .find(|&&(ref a, _)| a == who) + .map(|&(_, expires)| expires > n) + .unwrap_or(false) + } + + pub fn is_councillor(who: &T::AccountId) -> bool { + >::active_council().iter() + .any(|&(ref a, _)| a == who) + } + + pub fn tally(proposal_hash: &T::Hash) -> (u32, u32, u32) { + Self::generic_tally(proposal_hash, |w: &T::AccountId, p: &T::Hash| Self::vote_of((*p, w.clone()))) + } + + // Dispatch + fn propose(origin: T::Origin, proposal: Box) -> Result { + let who = ensure_signed(origin)?; + + let expiry = >::block_number() + Self::voting_period(); + ensure!(Self::will_still_be_councillor_at(&who, expiry), "proposer would not be on council"); + + let proposal_hash = T::Hashing::hash_of(&proposal); + + ensure!(!>::exists(proposal_hash), "duplicate proposals not allowed"); + ensure!(!Self::is_vetoed(&proposal_hash), "proposal is vetoed"); + + let mut proposals = Self::proposals(); + proposals.push((expiry, proposal_hash)); + proposals.sort_by_key(|&(expiry, _)| expiry); + Self::set_proposals(&proposals); + + >::insert(proposal_hash, *proposal); + >::insert(proposal_hash, vec![who.clone()]); + >::insert((proposal_hash, who.clone()), true); + + Ok(()) + } + + fn vote(origin: T::Origin, proposal: T::Hash, approve: bool) -> Result { + let who = ensure_signed(origin)?; + + ensure!(Self::is_councillor(&who), "only councillors may vote on council proposals"); + + if Self::vote_of((proposal, who.clone())).is_none() { + >::mutate(proposal, |voters| voters.push(who.clone())); + } + >::insert((proposal, who), approve); + Ok(()) + } + + fn veto(origin: T::Origin, proposal_hash: T::Hash) -> Result { + let who = ensure_signed(origin)?; + + ensure!(Self::is_councillor(&who), "only councillors may veto council proposals"); + ensure!(>::exists(&proposal_hash), "proposal must exist to be vetoed"); + + let mut existing_vetoers = Self::veto_of(&proposal_hash) + .map(|pair| pair.1) + .unwrap_or_else(Vec::new); + let insert_position = existing_vetoers.binary_search(&who) + .err().ok_or("a councillor may not veto a proposal twice")?; + existing_vetoers.insert(insert_position, who); + Self::set_veto_of(&proposal_hash, >::block_number() + Self::cooloff_period(), existing_vetoers); + + Self::set_proposals(&Self::proposals().into_iter().filter(|&(_, h)| h != proposal_hash).collect::>()); + >::remove(proposal_hash); + >::remove(proposal_hash); + for (c, _) in >::active_council() { + >::remove((proposal_hash, c)); + } + Ok(()) + } + + fn set_cooloff_period(origin: T::Origin, blocks: T::BlockNumber) -> Result { + ensure_root(origin)?; + >::put(blocks); + Ok(()) + } + + fn set_voting_period(origin: T::Origin, blocks: T::BlockNumber) -> Result { + ensure_root(origin)?; + >::put(blocks); + Ok(()) + } + + // private + + + fn set_veto_of(proposal: &T::Hash, expiry: T::BlockNumber, vetoers: Vec) { + >::insert(proposal, (expiry, vetoers)); + } + + fn kill_veto_of(proposal: &T::Hash) { + >::remove(proposal); + } + + fn take_tally(proposal_hash: &T::Hash) -> (u32, u32, u32) { + Self::generic_tally(proposal_hash, |w: &T::AccountId, p: &T::Hash| >::take((*p, w.clone()))) + } + + fn generic_tally Option>(proposal_hash: &T::Hash, vote_of: F) -> (u32, u32, u32) { + let c = >::active_council(); + let (approve, reject) = c.iter() + .filter_map(|&(ref a, _)| vote_of(a, proposal_hash)) + .map(|approve| if approve { (1, 0) } else { (0, 1) }) + .fold((0, 0), |(a, b), (c, d)| (a + c, b + d)); + (approve, reject, c.len() as u32 - approve - reject) + } + + fn set_proposals(p: &Vec<(T::BlockNumber, T::Hash)>) { + >::put(p); + } + + fn take_proposal_if_expiring_at(n: T::BlockNumber) -> Option<(T::Proposal, T::Hash)> { + let proposals = Self::proposals(); + match proposals.first() { + Some(&(expiry, hash)) if expiry == n => { + // yes this is horrible, but fixing it will need substantial work in storage. + Self::set_proposals(&proposals[1..].to_vec()); + >::take(hash).map(|p| (p, hash)) /* defensive only: all queued proposal hashes must have associated proposals*/ + } + _ => None, + } + } + + fn end_block(now: T::BlockNumber) -> Result { + while let Some((proposal, proposal_hash)) = Self::take_proposal_if_expiring_at(now) { + let tally = Self::take_tally(&proposal_hash); + if let Some(&democracy::Call::cancel_referendum(ref_index)) = IsSubType::>::is_aux_sub_type(&proposal) { + Self::deposit_event(RawEvent::TallyCancelation(proposal_hash, tally.0, tally.1, tally.2)); + if let (_, 0, 0) = tally { + >::internal_cancel_referendum(ref_index); + } + } else { + Self::deposit_event(RawEvent::TallyReferendum(proposal_hash.clone(), tally.0, tally.1, tally.2)); + if tally.0 > tally.1 + tally.2 { + Self::kill_veto_of(&proposal_hash); + match tally { + (_, 0, 0) => >::internal_start_referendum(proposal, democracy::VoteThreshold::SuperMajorityAgainst).map(|_| ())?, + _ => >::internal_start_referendum(proposal, democracy::VoteThreshold::SimpleMajority).map(|_| ())?, + }; + } + } + } + Ok(()) + } +} + +impl OnFinalise for Module { + fn on_finalise(n: T::BlockNumber) { + if let Err(e) = Self::end_block(n) { + print("Guru meditation"); + print(e); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ::tests::*; + use ::tests::{Call, Origin}; + use substrate_runtime_support::Hashable; + use democracy::VoteThreshold; + + #[test] + fn basic_environment_works() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + assert_eq!(Balances::free_balance(&42), 0); + assert_eq!(CouncilVoting::cooloff_period(), 2); + assert_eq!(CouncilVoting::voting_period(), 1); + assert_eq!(CouncilVoting::will_still_be_councillor_at(&1, 1), true); + assert_eq!(CouncilVoting::will_still_be_councillor_at(&1, 10), false); + assert_eq!(CouncilVoting::will_still_be_councillor_at(&4, 10), false); + assert_eq!(CouncilVoting::is_councillor(&1), true); + assert_eq!(CouncilVoting::is_councillor(&4), false); + assert_eq!(CouncilVoting::proposals(), Vec::<(u64, H256)>::new()); + assert_eq!(CouncilVoting::proposal_voters(H256::default()), Vec::::new()); + assert_eq!(CouncilVoting::is_vetoed(&H256::default()), false); + assert_eq!(CouncilVoting::vote_of((H256::default(), 1)), None); + assert_eq!(CouncilVoting::tally(&H256::default()), (0, 0, 3)); + }); + } + + fn set_balance_proposal(value: u64) -> Call { + Call::Balances(balances::Call::set_balance(balances::address::Address::Id(42), value, 0)) + } + + fn cancel_referendum_proposal(id: u32) -> Call { + Call::Democracy(democracy::Call::cancel_referendum(id)) + } + + #[test] + fn referendum_cancellation_should_work_when_unanimous() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_ok!(Democracy::internal_start_referendum(proposal.clone(), VoteThreshold::SuperMajorityApprove), 0); + assert_eq!(Democracy::active_referendums(), vec![(0, 4, proposal, VoteThreshold::SuperMajorityApprove)]); + + let cancellation = cancel_referendum_proposal(0); + let hash = cancellation.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(cancellation))); + assert_ok!(CouncilVoting::vote(Origin::signed(2), hash, true)); + assert_ok!(CouncilVoting::vote(Origin::signed(3), hash, true)); + assert_eq!(CouncilVoting::proposals(), vec![(2, hash)]); + assert_ok!(CouncilVoting::end_block(System::block_number())); + + System::set_block_number(2); + assert_ok!(CouncilVoting::end_block(System::block_number())); + assert_eq!(Democracy::active_referendums(), vec![]); + assert_eq!(Balances::free_balance(&42), 0); + }); + } + + #[test] + fn referendum_cancellation_should_fail_when_not_unanimous() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_ok!(Democracy::internal_start_referendum(proposal.clone(), VoteThreshold::SuperMajorityApprove), 0); + + let cancellation = cancel_referendum_proposal(0); + let hash = cancellation.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(cancellation))); + assert_ok!(CouncilVoting::vote(Origin::signed(2), hash, true)); + assert_ok!(CouncilVoting::vote(Origin::signed(3), hash, false)); + assert_ok!(CouncilVoting::end_block(System::block_number())); + + System::set_block_number(2); + assert_ok!(CouncilVoting::end_block(System::block_number())); + assert_eq!(Democracy::active_referendums(), vec![(0, 4, proposal, VoteThreshold::SuperMajorityApprove)]); + }); + } + + #[test] + fn referendum_cancellation_should_fail_when_abstentions() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_ok!(Democracy::internal_start_referendum(proposal.clone(), VoteThreshold::SuperMajorityApprove), 0); + + let cancellation = cancel_referendum_proposal(0); + let hash = cancellation.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(cancellation))); + assert_ok!(CouncilVoting::vote(Origin::signed(2), hash, true)); + assert_ok!(CouncilVoting::end_block(System::block_number())); + + System::set_block_number(2); + assert_ok!(CouncilVoting::end_block(System::block_number())); + assert_eq!(Democracy::active_referendums(), vec![(0, 4, proposal, VoteThreshold::SuperMajorityApprove)]); + }); + } + + #[test] + fn veto_should_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash = proposal.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::veto(Origin::signed(2), hash)); + assert_eq!(CouncilVoting::proposals().len(), 0); + assert_eq!(Democracy::active_referendums().len(), 0); + }); + } + + #[test] + fn double_veto_should_not_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash = proposal.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::veto(Origin::signed(2), hash)); + + System::set_block_number(3); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_noop!(CouncilVoting::veto(Origin::signed(2), hash), "a councillor may not veto a proposal twice"); + }); + } + + #[test] + fn retry_in_cooloff_should_not_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash = proposal.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::veto(Origin::signed(2), hash)); + + System::set_block_number(2); + assert_noop!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone())), "proposal is vetoed"); + }); + } + + #[test] + fn retry_after_cooloff_should_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash = proposal.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::veto(Origin::signed(2), hash)); + + System::set_block_number(3); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::vote(Origin::signed(2), hash, false)); + assert_ok!(CouncilVoting::vote(Origin::signed(3), hash, true)); + assert_ok!(CouncilVoting::end_block(System::block_number())); + + System::set_block_number(4); + assert_ok!(CouncilVoting::end_block(System::block_number())); + assert_eq!(CouncilVoting::proposals().len(), 0); + assert_eq!(Democracy::active_referendums(), vec![(0, 7, set_balance_proposal(42), VoteThreshold::SimpleMajority)]); + }); + } + + #[test] + fn alternative_double_veto_should_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash = proposal.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::veto(Origin::signed(2), hash)); + + System::set_block_number(3); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::veto(Origin::signed(3), hash)); + assert_eq!(CouncilVoting::proposals().len(), 0); + assert_eq!(Democracy::active_referendums().len(), 0); + }); + } + + #[test] + fn simple_propose_should_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + let hash = proposal.blake2_256().into(); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_eq!(CouncilVoting::proposals().len(), 1); + assert_eq!(CouncilVoting::proposal_voters(&hash), vec![1]); + assert_eq!(CouncilVoting::vote_of((hash, 1)), Some(true)); + assert_eq!(CouncilVoting::tally(&hash), (1, 0, 2)); + }); + } + + #[test] + fn unvoted_proposal_should_expire_without_action() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_eq!(CouncilVoting::tally(&proposal.blake2_256().into()), (1, 0, 2)); + assert_ok!(CouncilVoting::end_block(System::block_number())); + + System::set_block_number(2); + assert_ok!(CouncilVoting::end_block(System::block_number())); + assert_eq!(CouncilVoting::proposals().len(), 0); + assert_eq!(Democracy::active_referendums().len(), 0); + }); + } + + #[test] + fn unanimous_proposal_should_expire_with_biased_referendum() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::vote(Origin::signed(2), proposal.blake2_256().into(), true)); + assert_ok!(CouncilVoting::vote(Origin::signed(3), proposal.blake2_256().into(), true)); + assert_eq!(CouncilVoting::tally(&proposal.blake2_256().into()), (3, 0, 0)); + assert_ok!(CouncilVoting::end_block(System::block_number())); + + System::set_block_number(2); + assert_ok!(CouncilVoting::end_block(System::block_number())); + assert_eq!(CouncilVoting::proposals().len(), 0); + assert_eq!(Democracy::active_referendums(), vec![(0, 5, proposal, VoteThreshold::SuperMajorityAgainst)]); + }); + } + + #[test] + fn majority_proposal_should_expire_with_unbiased_referendum() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_ok!(CouncilVoting::vote(Origin::signed(2), proposal.blake2_256().into(), true)); + assert_ok!(CouncilVoting::vote(Origin::signed(3), proposal.blake2_256().into(), false)); + assert_eq!(CouncilVoting::tally(&proposal.blake2_256().into()), (2, 1, 0)); + assert_ok!(CouncilVoting::end_block(System::block_number())); + + System::set_block_number(2); + assert_ok!(CouncilVoting::end_block(System::block_number())); + assert_eq!(CouncilVoting::proposals().len(), 0); + assert_eq!(Democracy::active_referendums(), vec![(0, 5, proposal, VoteThreshold::SimpleMajority)]); + }); + } + + #[test] + fn propose_by_public_should_not_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_noop!(CouncilVoting::propose(Origin::signed(4), Box::new(proposal)), "proposer would not be on council"); + }); + } + + #[test] + fn vote_by_public_should_not_work() { + with_externalities(&mut new_test_ext(true), || { + System::set_block_number(1); + let proposal = set_balance_proposal(42); + assert_ok!(CouncilVoting::propose(Origin::signed(1), Box::new(proposal.clone()))); + assert_noop!(CouncilVoting::vote(Origin::signed(4), proposal.blake2_256().into(), true), "only councillors may vote on council proposals"); + }); + } +} diff --git a/runtime/democracy/Cargo.toml b/runtime/democracy/Cargo.toml new file mode 100644 index 00000000000..88a94e6ba6a --- /dev/null +++ b/runtime/democracy/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "substrate-runtime-democracy" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +safe-mix = { version = "1.0", default_features = false} +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-runtime-balances = { path = "../balances", default_features = false } +substrate-runtime-consensus = { path = "../consensus", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "safe-mix/std", + "substrate-codec/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-consensus/std", + "substrate-runtime-balances/std", + "substrate-runtime-system/std", +] diff --git a/runtime/democracy/src/lib.rs b/runtime/democracy/src/lib.rs new file mode 100644 index 00000000000..12f2e7e7aa4 --- /dev/null +++ b/runtime/democracy/src/lib.rs @@ -0,0 +1,681 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Democratic system: Handles administration of general stakeholder voting. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +extern crate substrate_primitives; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_codec_derive; +#[macro_use] +extern crate substrate_runtime_std as rstd; +#[macro_use] +extern crate substrate_runtime_support; + +extern crate substrate_codec as codec; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_primitives as primitives; +extern crate substrate_runtime_balances as balances; +extern crate substrate_runtime_system as system; + +use rstd::prelude::*; +use rstd::result; +use primitives::traits::{Zero, OnFinalise, As, MaybeSerializeDebug}; +use substrate_runtime_support::{StorageValue, StorageMap, Parameter, Dispatchable, IsSubType}; +use substrate_runtime_support::dispatch::Result; +use system::{ensure_signed, ensure_root}; + +#[cfg(any(feature = "std", test))] +use std::collections::HashMap; + +mod vote_threshold; +pub use vote_threshold::{Approved, VoteThreshold}; + +/// A proposal index. +pub type PropIndex = u32; +/// A referendum index. +pub type ReferendumIndex = u32; + +pub trait Trait: balances::Trait + Sized { + type Proposal: Parameter + Dispatchable + IsSubType> + MaybeSerializeDebug; + + type Event: From> + Into<::Event>; +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn propose(origin, proposal: Box, value: T::Balance) -> Result; + fn second(origin, proposal: PropIndex) -> Result; + fn vote(origin, ref_index: ReferendumIndex, approve_proposal: bool) -> Result; + + fn start_referendum(origin, proposal: Box, vote_threshold: VoteThreshold) -> Result; + fn cancel_referendum(origin, ref_index: ReferendumIndex) -> Result; + } +} + +decl_storage! { + trait Store for Module as Democracy { + + /// The number of (public) proposals that have been made so far. + pub PublicPropCount get(public_prop_count): default PropIndex; + /// The public proposals. Unsorted. + pub PublicProps get(public_props): default Vec<(PropIndex, T::Proposal, T::AccountId)>; + /// Those who have locked a deposit. + pub DepositOf get(deposit_of): map [ PropIndex => (T::Balance, Vec) ]; + /// How often (in blocks) new public referenda are launched. + pub LaunchPeriod get(launch_period): required T::BlockNumber; + /// The minimum amount to be used as a deposit for a public referendum proposal. + pub MinimumDeposit get(minimum_deposit): required T::Balance; + + /// How often (in blocks) to check for new votes. + pub VotingPeriod get(voting_period): required T::BlockNumber; + + /// The next free referendum index, aka the number of referendums started so far. + pub ReferendumCount get(referendum_count): required ReferendumIndex; + /// The next referendum index that should be tallied. + pub NextTally get(next_tally): required ReferendumIndex; + /// Information concerning any given referendum. + pub ReferendumInfoOf get(referendum_info): map [ ReferendumIndex => (T::BlockNumber, T::Proposal, VoteThreshold) ]; + + /// Get the voters for the current proposal. + pub VotersFor get(voters_for): default map [ ReferendumIndex => Vec ]; + + /// Get the vote, if Some, of `who`. + pub VoteOf get(vote_of): map [ (ReferendumIndex, T::AccountId) => bool ]; + } +} + +decl_event!( + /// An event in this module. + pub enum Event with RawEvent + where ::Balance, ::AccountId + { + Tabled(PropIndex, Balance, Vec), + Started(ReferendumIndex, VoteThreshold), + Passed(ReferendumIndex), + NotPassed(ReferendumIndex), + Cancelled(ReferendumIndex), + Executed(ReferendumIndex, bool), + } +); + +impl Module { + + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + // exposed immutables. + + /// Get the amount locked in support of `proposal`; `None` if proposal isn't a valid proposal + /// index. + pub fn locked_for(proposal: PropIndex) -> Option { + Self::deposit_of(proposal).map(|(d, l)| d * T::Balance::sa(l.len() as u64)) + } + + /// Return true if `ref_index` is an on-going referendum. + pub fn is_active_referendum(ref_index: ReferendumIndex) -> bool { + >::exists(ref_index) + } + + /// Get all referendums currently active. + pub fn active_referendums() -> Vec<(ReferendumIndex, T::BlockNumber, T::Proposal, VoteThreshold)> { + let next = Self::next_tally(); + let last = Self::referendum_count(); + (next..last).into_iter() + .filter_map(|i| Self::referendum_info(i).map(|(n, p, t)| (i, n, p, t))) + .collect() + } + + /// Get all referendums ready for tally at block `n`. + pub fn maturing_referendums_at(n: T::BlockNumber) -> Vec<(ReferendumIndex, T::BlockNumber, T::Proposal, VoteThreshold)> { + let next = Self::next_tally(); + let last = Self::referendum_count(); + (next..last).into_iter() + .filter_map(|i| Self::referendum_info(i).map(|(n, p, t)| (i, n, p, t))) + .take_while(|&(_, block_number, _, _)| block_number == n) + .collect() + } + + /// Get the voters for the current proposal. + pub fn tally(ref_index: ReferendumIndex) -> (T::Balance, T::Balance) { + Self::voters_for(ref_index).iter() + .map(|a| (>::total_balance(a), Self::vote_of((ref_index, a.clone())).unwrap_or(false)/*defensive only: all items come from `voters`; for an item to be in `voters` there must be a vote registered; qed*/)) + .map(|(bal, vote)| if vote { (bal, Zero::zero()) } else { (Zero::zero(), bal) }) + .fold((Zero::zero(), Zero::zero()), |(a, b), (c, d)| (a + c, b + d)) + } + + // dispatching. + + /// Propose a sensitive action to be taken. + fn propose(origin: T::Origin, proposal: Box, value: T::Balance) -> Result { + let who = ensure_signed(origin)?; + ensure!(value >= Self::minimum_deposit(), "value too low"); + >::reserve(&who, value) + .map_err(|_| "proposer's balance too low")?; + + let index = Self::public_prop_count(); + >::put(index + 1); + >::insert(index, (value, vec![who.clone()])); + + let mut props = Self::public_props(); + props.push((index, (*proposal).clone(), who)); + >::put(props); + Ok(()) + } + + /// Propose a sensitive action to be taken. + fn second(origin: T::Origin, proposal: PropIndex) -> Result { + let who = ensure_signed(origin)?; + let mut deposit = Self::deposit_of(proposal) + .ok_or("can only second an existing proposal")?; + >::reserve(&who, deposit.0) + .map_err(|_| "seconder's balance too low")?; + deposit.1.push(who); + >::insert(proposal, deposit); + Ok(()) + } + + /// Vote in a referendum. If `approve_proposal` is true, the vote is to enact the proposal; + /// false would be a vote to keep the status quo. + fn vote(origin: T::Origin, ref_index: ReferendumIndex, approve_proposal: bool) -> Result { + let who = ensure_signed(origin)?; + ensure!(Self::is_active_referendum(ref_index), "vote given for invalid referendum."); + ensure!(!>::total_balance(&who).is_zero(), + "transactor must have balance to signal approval."); + if !>::exists(&(ref_index, who.clone())) { + >::mutate(ref_index, |voters| voters.push(who.clone())); + } + >::insert(&(ref_index, who), approve_proposal); + Ok(()) + } + + /// Start a referendum. + fn start_referendum(origin: T::Origin, proposal: Box, vote_threshold: VoteThreshold) -> Result { + ensure_root(origin)?; + Self::inject_referendum( + >::block_number() + Self::voting_period(), + *proposal, + vote_threshold + ).map(|_| ()) + } + + /// Remove a referendum. + fn cancel_referendum(origin: T::Origin, ref_index: ReferendumIndex) -> Result { + ensure_root(origin)?; + Self::clear_referendum(ref_index); + Ok(()) + } + + // exposed mutables. + + /// Start a referendum. Can be called directly by the council. + pub fn internal_start_referendum(proposal: T::Proposal, vote_threshold: VoteThreshold) -> result::Result { + >::inject_referendum(>::block_number() + >::voting_period(), proposal, vote_threshold) + } + + /// Remove a referendum. Can be called directly by the council. + pub fn internal_cancel_referendum(ref_index: ReferendumIndex) { + Self::deposit_event(RawEvent::Cancelled(ref_index)); + >::clear_referendum(ref_index); + } + + // private. + + /// Start a referendum + fn inject_referendum( + end: T::BlockNumber, + proposal: T::Proposal, + vote_threshold: VoteThreshold + ) -> result::Result { + let ref_index = Self::referendum_count(); + if ref_index > 0 && Self::referendum_info(ref_index - 1).map(|i| i.0 > end).unwrap_or(false) { + Err("Cannot inject a referendum that ends earlier than preceeding referendum")? + } + + >::put(ref_index + 1); + >::insert(ref_index, (end, proposal, vote_threshold)); + Self::deposit_event(RawEvent::Started(ref_index, vote_threshold)); + Ok(ref_index) + } + + /// Remove all info on a referendum. + fn clear_referendum(ref_index: ReferendumIndex) { + >::remove(ref_index); + >::remove(ref_index); + for v in Self::voters_for(ref_index) { + >::remove((ref_index, v)); + } + } + + /// Current era is ending; we should finish up any proposals. + fn end_block(now: T::BlockNumber) -> Result { + // pick out another public referendum if it's time. + if (now % Self::launch_period()).is_zero() { + let mut public_props = Self::public_props(); + if let Some((winner_index, _)) = public_props.iter() + .enumerate() + .max_by_key(|x| Self::locked_for((x.1).0).unwrap_or_else(Zero::zero)/*defensive only: All current public proposals have an amount locked*/) + { + let (prop_index, proposal, _) = public_props.swap_remove(winner_index); + >::put(public_props); + + if let Some((deposit, depositors)) = >::take(prop_index) {//: (T::Balance, Vec) = + // refund depositors + for d in &depositors { + >::unreserve(d, deposit); + } + Self::deposit_event(RawEvent::Tabled(prop_index, deposit, depositors)); + Self::inject_referendum(now + Self::voting_period(), proposal, VoteThreshold::SuperMajorityApprove)?; + } + } + } + + // tally up votes for any expiring referenda. + for (index, _, proposal, vote_threshold) in Self::maturing_referendums_at(now) { + let (approve, against) = Self::tally(index); + let total_issuance = >::total_issuance(); + Self::clear_referendum(index); + if vote_threshold.approved(approve, against, total_issuance) { + Self::deposit_event(RawEvent::Passed(index)); + let ok = proposal.dispatch(system::RawOrigin::Root.into()).is_ok(); + Self::deposit_event(RawEvent::Executed(index, ok)); + } else { + Self::deposit_event(RawEvent::NotPassed(index)); + } + >::put(index + 1); + } + Ok(()) + } +} + +impl OnFinalise for Module { + fn on_finalise(n: T::BlockNumber) { + if let Err(e) = Self::end_block(n) { + runtime_io::print(e); + } + } +} + +#[cfg(any(feature = "std", test))] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + pub launch_period: T::BlockNumber, + pub voting_period: T::BlockNumber, + pub minimum_deposit: T::Balance, +} + +#[cfg(any(feature = "std", test))] +impl GenesisConfig { + pub fn new() -> Self { + GenesisConfig { + launch_period: T::BlockNumber::sa(1), + voting_period: T::BlockNumber::sa(1), + minimum_deposit: T::Balance::sa(1), + } + } +} + +#[cfg(any(feature = "std", test))] +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + launch_period: T::BlockNumber::sa(1000), + voting_period: T::BlockNumber::sa(1000), + minimum_deposit: T::Balance::sa(0), + } + } +} + +#[cfg(any(feature = "std", test))] +impl primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> ::std::result::Result, Vec>, String> { + use codec::Encode; + + Ok(map![ + Self::hash(>::key()).to_vec() => self.launch_period.encode(), + Self::hash(>::key()).to_vec() => self.voting_period.encode(), + Self::hash(>::key()).to_vec() => self.minimum_deposit.encode(), + Self::hash(>::key()).to_vec() => (0 as ReferendumIndex).encode(), + Self::hash(>::key()).to_vec() => (0 as ReferendumIndex).encode(), + Self::hash(>::key()).to_vec() => (0 as PropIndex).encode() + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use runtime_io::with_externalities; + use substrate_primitives::{H256, Blake2Hasher}; + use primitives::BuildStorage; + use primitives::traits::{BlakeTwo256}; + use primitives::testing::{Digest, Header}; + + impl_outer_origin! { + pub enum Origin for Test {} + } + + impl_outer_dispatch! { + pub enum Call where origin: Origin { + Balances, + Democracy, + } + } + + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] + pub struct Test; + impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); + } + impl balances::Trait for Test { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = (); + type EnsureAccountLiquid = (); + type Event = (); + } + impl Trait for Test { + type Proposal = Call; + type Event = (); + } + + fn new_test_ext() -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(balances::GenesisConfig::{ + balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], + transaction_base_fee: 0, + transaction_byte_fee: 0, + existential_deposit: 0, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + }.build_storage().unwrap()); + t.extend(GenesisConfig::{ + launch_period: 1, + voting_period: 1, + minimum_deposit: 1, + }.build_storage().unwrap()); + t.into() + } + + type System = system::Module; + type Balances = balances::Module; + type Democracy = Module; + + #[test] + fn params_should_work() { + with_externalities(&mut new_test_ext(), || { + assert_eq!(Democracy::launch_period(), 1); + assert_eq!(Democracy::voting_period(), 1); + assert_eq!(Democracy::minimum_deposit(), 1); + assert_eq!(Democracy::referendum_count(), 0); + assert_eq!(Balances::free_balance(&42), 0); + assert_eq!(Balances::total_issuance(), 210); + }); + } + + fn set_balance_proposal(value: u64) -> Call { + Call::Balances(balances::Call::set_balance(balances::address::Address::Id(42), value, 0)) + } + + fn propose_set_balance(who: u64, value: u64, locked: u64) -> super::Result { + Democracy::propose(Origin::signed(who), Box::new(set_balance_proposal(value)), locked) + } + + #[test] + fn locked_for_should_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_ok!(propose_set_balance(1, 2, 2)); + assert_ok!(propose_set_balance(1, 4, 4)); + assert_ok!(propose_set_balance(1, 3, 3)); + assert_eq!(Democracy::locked_for(0), Some(2)); + assert_eq!(Democracy::locked_for(1), Some(4)); + assert_eq!(Democracy::locked_for(2), Some(3)); + }); + } + + #[test] + fn single_proposal_should_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_ok!(propose_set_balance(1, 2, 1)); + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + System::set_block_number(2); + let r = 0; + assert_ok!(Democracy::vote(Origin::signed(1), r, true)); + + assert_eq!(Democracy::referendum_count(), 1); + assert_eq!(Democracy::voters_for(r), vec![1]); + assert_eq!(Democracy::vote_of((r, 1)), Some(true)); + assert_eq!(Democracy::tally(r), (10, 0)); + + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + assert_eq!(Balances::free_balance(&42), 2); + }); + } + + #[test] + fn deposit_for_proposals_should_be_taken() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_ok!(propose_set_balance(1, 2, 5)); + assert_ok!(Democracy::second(Origin::signed(2), 0)); + assert_ok!(Democracy::second(Origin::signed(5), 0)); + assert_ok!(Democracy::second(Origin::signed(5), 0)); + assert_ok!(Democracy::second(Origin::signed(5), 0)); + assert_eq!(Balances::free_balance(&1), 5); + assert_eq!(Balances::free_balance(&2), 15); + assert_eq!(Balances::free_balance(&5), 35); + }); + } + + #[test] + fn deposit_for_proposals_should_be_returned() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_ok!(propose_set_balance(1, 2, 5)); + assert_ok!(Democracy::second(Origin::signed(2), 0)); + assert_ok!(Democracy::second(Origin::signed(5), 0)); + assert_ok!(Democracy::second(Origin::signed(5), 0)); + assert_ok!(Democracy::second(Origin::signed(5), 0)); + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + assert_eq!(Balances::free_balance(&1), 10); + assert_eq!(Balances::free_balance(&2), 20); + assert_eq!(Balances::free_balance(&5), 50); + }); + } + + #[test] + fn proposal_with_deposit_below_minimum_should_not_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_noop!(propose_set_balance(1, 2, 0), "value too low"); + }); + } + + #[test] + fn poor_proposer_should_not_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_noop!(propose_set_balance(1, 2, 11), "proposer\'s balance too low"); + }); + } + + #[test] + fn poor_seconder_should_not_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_ok!(propose_set_balance(2, 2, 11)); + assert_noop!(Democracy::second(Origin::signed(1), 0), "seconder\'s balance too low"); + }); + } + + #[test] + fn runners_up_should_come_after() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(0); + assert_ok!(propose_set_balance(1, 2, 2)); + assert_ok!(propose_set_balance(1, 4, 4)); + assert_ok!(propose_set_balance(1, 3, 3)); + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + System::set_block_number(1); + assert_ok!(Democracy::vote(Origin::signed(1), 0, true)); + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + assert_eq!(Balances::free_balance(&42), 4); + + System::set_block_number(2); + assert_ok!(Democracy::vote(Origin::signed(1), 1, true)); + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + assert_eq!(Balances::free_balance(&42), 3); + + System::set_block_number(3); + assert_ok!(Democracy::vote(Origin::signed(1), 2, true)); + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + }); + } + + #[test] + fn simple_passing_should_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + let r = Democracy::inject_referendum(1, set_balance_proposal(2), VoteThreshold::SuperMajorityApprove).unwrap(); + assert_ok!(Democracy::vote(Origin::signed(1), r, true)); + + assert_eq!(Democracy::voters_for(r), vec![1]); + assert_eq!(Democracy::vote_of((r, 1)), Some(true)); + assert_eq!(Democracy::tally(r), (10, 0)); + + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + assert_eq!(Balances::free_balance(&42), 2); + }); + } + + #[test] + fn cancel_referendum_should_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + let r = Democracy::inject_referendum(1, set_balance_proposal(2), VoteThreshold::SuperMajorityApprove).unwrap(); + assert_ok!(Democracy::vote(Origin::signed(1), r, true)); + assert_ok!(Democracy::cancel_referendum(Origin::ROOT, r)); + + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + assert_eq!(Balances::free_balance(&42), 0); + }); + } + + #[test] + fn simple_failing_should_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + let r = Democracy::inject_referendum(1, set_balance_proposal(2), VoteThreshold::SuperMajorityApprove).unwrap(); + assert_ok!(Democracy::vote(Origin::signed(1), r, false)); + + assert_eq!(Democracy::voters_for(r), vec![1]); + assert_eq!(Democracy::vote_of((r, 1)), Some(false)); + assert_eq!(Democracy::tally(r), (0, 10)); + + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + assert_eq!(Balances::free_balance(&42), 0); + }); + } + + #[test] + fn controversial_voting_should_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + let r = Democracy::inject_referendum(1, set_balance_proposal(2), VoteThreshold::SuperMajorityApprove).unwrap(); + assert_ok!(Democracy::vote(Origin::signed(1), r, true)); + assert_ok!(Democracy::vote(Origin::signed(2), r, false)); + assert_ok!(Democracy::vote(Origin::signed(3), r, false)); + assert_ok!(Democracy::vote(Origin::signed(4), r, true)); + assert_ok!(Democracy::vote(Origin::signed(5), r, false)); + assert_ok!(Democracy::vote(Origin::signed(6), r, true)); + + assert_eq!(Democracy::tally(r), (110, 100)); + + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + assert_eq!(Balances::free_balance(&42), 2); + }); + } + + #[test] + fn controversial_low_turnout_voting_should_work() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + let r = Democracy::inject_referendum(1, set_balance_proposal(2), VoteThreshold::SuperMajorityApprove).unwrap(); + assert_ok!(Democracy::vote(Origin::signed(5), r, false)); + assert_ok!(Democracy::vote(Origin::signed(6), r, true)); + + assert_eq!(Democracy::tally(r), (60, 50)); + + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + assert_eq!(Balances::free_balance(&42), 0); + }); + } + + #[test] + fn passing_low_turnout_voting_should_work() { + with_externalities(&mut new_test_ext(), || { + assert_eq!(Balances::free_balance(&42), 0); + assert_eq!(Balances::total_issuance(), 210); + + System::set_block_number(1); + let r = Democracy::inject_referendum(1, set_balance_proposal(2), VoteThreshold::SuperMajorityApprove).unwrap(); + assert_ok!(Democracy::vote(Origin::signed(4), r, true)); + assert_ok!(Democracy::vote(Origin::signed(5), r, false)); + assert_ok!(Democracy::vote(Origin::signed(6), r, true)); + + assert_eq!(Democracy::tally(r), (100, 50)); + + assert_eq!(Democracy::end_block(System::block_number()), Ok(())); + + assert_eq!(Balances::free_balance(&42), 2); + }); + } +} diff --git a/runtime/democracy/src/vote_threshold.rs b/runtime/democracy/src/vote_threshold.rs new file mode 100644 index 00000000000..ce461492220 --- /dev/null +++ b/runtime/democracy/src/vote_threshold.rs @@ -0,0 +1,97 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Voting thresholds. + +use primitives::traits::{Zero, IntegerSquareRoot}; +use rstd::ops::{Add, Mul, Div, Rem}; + +/// A means of determining if a vote is past pass threshold. +#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +pub enum VoteThreshold { + /// A supermajority of approvals is needed to pass this vote. + SuperMajorityApprove, + /// A supermajority of rejects is needed to fail this vote. + SuperMajorityAgainst, + /// A simple majority of approvals is needed to pass this vote. + SimpleMajority, +} + +pub trait Approved { + /// Given `approve` votes for and `against` votes against from a total electorate size of + /// `electorate` (`electorate - (approve + against)` are abstainers), then returns true if the + /// overall outcome is in favour of approval. + fn approved(&self, approve: Balance, against: Balance, electorate: Balance) -> bool; +} + +/// Return `true` iff `n1 / d1 < n2 / d2`. `d1` and `d2` may not be zero. +fn compare_rationals + Div + Rem + Ord + Copy>(mut n1: T, mut d1: T, mut n2: T, mut d2: T) -> bool { + // Uses a continued fractional representation for a non-overflowing compare. + // Detailed at https://janmr.com/blog/2014/05/comparing-rational-numbers-without-overflow/. + loop { + let q1 = n1 / d1; + let q2 = n2 / d2; + if q1 < q2 { + return true; + } + if q2 < q1 { + return false; + } + let r1 = n1 % d1; + let r2 = n2 % d2; + if r2.is_zero() { + return false; + } + if r1.is_zero() { + return true; + } + n1 = d2; + n2 = d1; + d1 = r2; + d2 = r1; + } +} + +impl + Mul + Div + Rem + Copy> Approved for VoteThreshold { + /// Given `approve` votes for and `against` votes against from a total electorate size of + /// `electorate` (`electorate - (approve + against)` are abstainers), then returns true if the + /// overall outcome is in favour of approval. + fn approved(&self, approve: Balance, against: Balance, electorate: Balance) -> bool { + let voters = approve + against; + let sqrt_voters = voters.integer_sqrt(); + let sqrt_electorate = electorate.integer_sqrt(); + if sqrt_voters.is_zero() { return false; } + match *self { + VoteThreshold::SuperMajorityApprove => + compare_rationals(against, sqrt_voters, approve, sqrt_electorate), + VoteThreshold::SuperMajorityAgainst => + compare_rationals(against, sqrt_electorate, approve, sqrt_voters), + VoteThreshold::SimpleMajority => approve > against, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn should_work() { + assert_eq!(VoteThreshold::SuperMajorityApprove.approved(60, 50, 210), false); + assert_eq!(VoteThreshold::SuperMajorityApprove.approved(100, 50, 210), true); + } +} diff --git a/runtime/example/Cargo.toml b/runtime/example/Cargo.toml new file mode 100644 index 00000000000..66da7e9355e --- /dev/null +++ b/runtime/example/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "substrate-runtime-example" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } +substrate-runtime-balances = { path = "../balances", default_features = false } + +[features] +default = ["std"] +std = [ + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-balances/std", + "serde/std", + "serde_derive", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-system/std", +] diff --git a/runtime/example/src/lib.rs b/runtime/example/src/lib.rs new file mode 100644 index 00000000000..8247106e384 --- /dev/null +++ b/runtime/example/src/lib.rs @@ -0,0 +1,415 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! The Example: A simple example of a runtime module demonstrating +//! concepts, APIs and structures common to most runtime modules. + +// Ensure we're `no_std` when compiling for Wasm. +#![cfg_attr(not(feature = "std"), no_std)] + +// Assert macros used in tests. +#[cfg_attr(feature = "std", macro_use)] +extern crate substrate_runtime_std; + +// Needed for tests (`with_externalities`). +#[cfg(test)] +extern crate substrate_runtime_io as runtime_io; + +// Needed for the set of mock primitives used in our tests. +#[cfg(test)] +extern crate substrate_primitives; + +// Needed for deriving `Serialize` and `Deserialize` for various types. +// We only implement the serde traits for std builds - they're unneeded +// in the wasm runtime. +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +// Needed for deriving `Encode` and `Decode` for `RawEvent`. +#[macro_use] +extern crate substrate_codec_derive; +extern crate substrate_codec as codec; + +// Needed for type-safe access to storage DB. +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +// Needed for various traits. In our case, `OnFinalise`. +extern crate substrate_runtime_primitives as runtime_primitives; +// `system` module provides us with all sorts of useful stuff and macros +// depend on it being around. +extern crate substrate_runtime_system as system; +// `balances` module is needed for our little example. It's not required in +// general (though if you want your module to be able to work with tokens, then you +// might find it useful). +extern crate substrate_runtime_balances as balances; + +use runtime_primitives::traits::OnFinalise; +use runtime_support::{StorageValue, dispatch::Result}; +use system::{ensure_signed, ensure_root}; + +/// Our module's configuration trait. All our types and consts go in here. If the +/// module is dependent on specific other modules, then their configuration traits +/// should be added to our implied traits list. +/// +/// `system::Trait` should always be included in our implied traits. +pub trait Trait: balances::Trait { + /// The overarching event type. + type Event: From> + Into<::Event>; +} + +// The module declaration. This states the entry points that we handle. The +// macro takes care of the marshalling of arguments and dispatch. +// +// Anyone can have these functions execute by signing and submitting +// an extrinsic. Ensure that calls into each of these execute in a time, memory and +// using storage space proportional to any costs paid for by the caller or otherwise the +// difficulty of forcing the call to happen. +// +// Generally you'll want to split these into three groups: +// - Public calls that are signed by an external account. +// - Root calls that are allowed to be made only by the governance system. +// - Inherent calls that are allowed to be made only by the block authors and validators. +// +// Information about where this dispatch initiated from is provided as the first argument +// "origin". As such functions must always look like: +// +// `fn foo(origin, bar: Bar, baz: Baz) -> Result = 0;` +// +// The `Result` is required as part of the syntax (and expands to the conventional dispatch +// result of `Result<(), &'static str>`). +// +// When you come to `impl` them later in the module, you must specify the full type for `origin`: +// +// `fn foo(origin: T::Origin, bar: Bar, baz: Baz) { ... }` +// +// There are three entries in the `system::Origin` enum that correspond +// to the above bullets: `::Signed(AccountId)`, `::Root` and `::Inherent`. You should always match +// against them as the first thing you do in your function. There are three convenience calls +// in system that do the matching for you and return a convenient result: `ensure_signed`, +// `ensure_root` and `ensure_inherent`. +decl_module! { + // Simple declaration of the `Module` type. Lets the macro know what its working on. + pub struct Module for enum Call where origin: T::Origin { + /// This is your public interface. Be extremely careful. + /// This is just a simple example of how to interact with the module from the external + /// world. + fn accumulate_dummy(origin, increase_by: T::Balance) -> Result; + + fn accumulate_foo(origin, increase_by: T::Balance) -> Result; + + /// A privileged call; in this case it resets our dummy value to something new. + fn set_dummy(origin, new_dummy: T::Balance) -> Result; + } +} + +/// An event in this module. Events are simple means of reporting specific conditions and +/// circumstances that have happened that users, Dapps and/or chain explorers would find +/// interesting and otherwise difficult to detect. +decl_event!( + pub enum Event with RawEvent + where ::Balance + { + // Just a normal `enum`, here's a dummy event to ensure it compiles. + /// Dummy event, just here so there's a generic type that's used. + Dummy(B), + } +); + +decl_storage! { + // A macro for the Storage trait, and its implementation, for this module. + // This allows for type-safe usage of the Substrate storage database, so you can + // keep things around between blocks. + trait Store for Module as Example { + // Any storage declarations of the form: + // `pub? Name get(getter_name)? : [required | default]? ;` + // where `` is either: + // - `Type` (a basic value item); or + // - `map [ KeyType => ValueType ]` (a map item). + // + // Note that there are two optional modifiers for the storage type declaration. + // - `Foo: u32`: + // - `Foo::put(1); Foo::get()` returns `Some(1)`; + // - `Foo::kill(); Foo::get()` returns `None`. + // - `Foo: required u32`: + // - `Foo::put(1); Foo::get()` returns `1`; + // - `Foo::kill(); Foo::get()` panics. + // - `Foo: default u32`: + // - `Foo::put(1); Foo::get()` returns `1`; + // - `Foo::kill(); Foo::get()` returns `0` (u32::default()). + // e.g. Foo: u32; + // e.g. pub Bar get(bar): default map [ T::AccountId => Vec<(T::Balance, u64)> ]; + // + // For basic value items, you'll get a type which implements + // `runtime_support::StorageValue`. For map items, you'll get a type which + // implements `runtime_support::StorageMap`. + // + // If they have a getter (`get(getter_name)`), then your module will come + // equipped with `fn getter_name() -> Type` for basic value items or + // `fn getter_name(key: KeyType) -> ValueType` for map items. + Dummy get(dummy): T::Balance; + + // this one uses the default, we'll demonstrate the usage of 'mutate' API. + Foo get(foo): default T::Balance; + } +} + +// The main implementation block for the module. Functions here fall into three broad +// categories: +// - Implementations of dispatch functions. The dispatch code generated by the module macro +// expects each of its functions to be implemented. +// - Public interface. These are functions that are `pub` and generally fall into inspector +// functions that do not write to storage and operation functions that do. +// - Private functions. These are your usual private utilities unavailable to other modules. +impl Module { + /// Deposit one of this module's events. + // TODO: move into `decl_module` macro. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + // Implement Calls and add public immutables and private mutables. + + // Implement dispatched function `accumulate_dummy`. This just increases the value + // of `Dummy` by `increase_by`. + // + // Since this is a dispatched function there are two extremely important things to + // remember: + // + // - MUST NOT PANIC: Under no circumstances (save, perhaps, storage getting into an + // irreparably damaged state) must this function panic. + // - NO SIDE-EFFECTS ON ERROR: This function must either complete totally (and return + // `Ok(())` or it must have no side-effects on storage and return `Err('Some reason')`. + // + // The first is relatively easy to audit for - just ensure all panickers are removed from + // logic that executes in production (which you do anyway, right?!). To ensure the second + // is followed, you should do all tests for validity at the top of your function. This + // is stuff like checking the sender (`origin`) or that state is such that the operation + // makes sense. + // + // Once you've determined that it's all good, then enact the operation and change storage. + // If you can't be certain that the operation will succeed without substantial computation + // then you have a classic blockchain attack scenario. The normal way of managing this is + // to attach a bond to the operation. As the first major alteration of storage, reserve + // some value from the sender's account (`Balances` module has a `reserve` function for + // exactly this scenario). This amount should be enough to cover any costs of the + // substantial execution in case it turns out that you can't proceed with the operation. + // + // If it eventually transpires that the operation is fine and, therefore, that the + // expense of the checks should be borne by the network, then you can refund the reserved + // deposit. If, however, the operation turns out to be invalid and the computation is + // wasted, then you can burn it or repatriate elsewhere. + // + // Security bonds ensure that attackers can't game it by ensuring that anyone interacting + // with the system either progresses it or pays for the trouble of faffing around with + // no progress. + // + // If you don't respect these rules, it is likely that your chain will be attackable. + fn accumulate_dummy(origin: T::Origin, increase_by: T::Balance) -> Result { + // This is a public call, so we ensure that the origin is some signed account. + let _sender = ensure_signed(origin)?; + + // Read the value of dummy from storage. + // let dummy = Self::dummy(); + // Will also work using the `::get` on the storage item type itself: + // let dummy = >::get(); + + // Calculate the new value. + // let new_dummy = dummy.map_or(increase_by, |dummy| dummy + increase_by); + + // Put the new value into storage. + // >::put(new_dummy); + // Will also work with a reference: + // >::put(&new_dummy); + + // Here's the new one of read and then modify the value. + >::mutate(|dummy| { + let new_dummy = dummy.map_or(increase_by, |dummy| dummy + increase_by); + *dummy = Some(new_dummy); + }); + + // Let's deposit an event to let the outside world know this happened. + Self::deposit_event(RawEvent::Dummy(increase_by)); + + // All good. + Ok(()) + } + + fn accumulate_foo(origin: T::Origin, increase_by: T::Balance) -> Result { + let _sender = ensure_signed(origin)?; + + // Because Foo has 'default', the type of 'foo' in closure is the raw type instead of an Option<> type. + >::mutate(|foo| *foo = *foo + increase_by); + + Ok(()) + } + + // Implementation of a privileged call. This doesn't have an `origin` parameter because + // it's not (directly) from an extrinsic, but rather the system as a whole has decided + // to execute it. Different runtimes have different reasons for allow privileged + // calls to be executed - we don't need to care why. Because it's privileged, we can + // assume it's a one-off operation and substantial processing/storage/memory can be used + // without worrying about gameability or attack scenarios. + fn set_dummy(origin: T::Origin, new_value: T::Balance) -> Result { + // This is a privileged call, so we ensure that the origin is "Root". + ensure_root(origin)?; + + // Put the new value into storage. + >::put(new_value); + + // All good. + Ok(()) + } +} + +// This trait expresses what should happen when the block is finalised. +impl OnFinalise for Module { + fn on_finalise(_: T::BlockNumber) { + // Anything that needs to be done at the end of the block. + // We just kill our dummy storage item. + >::kill(); + } +} + +#[cfg(feature = "std")] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +/// The genesis block configuration type. This is a simple default-capable struct that +/// contains any fields with which this module can be configured at genesis time. +pub struct GenesisConfig { + /// A value with which to initialise the Dummy storage item. + pub dummy: T::Balance, + pub foo: T::Balance, +} + +#[cfg(feature = "std")] +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + dummy: Default::default(), + foo: Default::default(), + } + } +} + +// This expresses the specific key/value pairs that must be placed in storage in order +// to initialise the module and properly reflect the configuration. +// +// Ideally this would re-use the `::put` logic in the storage item type for introducing +// the values into the `StorageMap` (which is just a `HashMap, Vec>`). That +// is not yet in place, though, so for now we do everything "manually", using `hash`, +// `::key()` and `.to_vec()` for the key and `.encode()` for the value. +#[cfg(feature = "std")] +impl runtime_primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> ::std::result::Result { + use codec::Encode; + Ok(map![ + Self::hash(>::key()).to_vec() => self.dummy.encode(), + Self::hash(>::key()).to_vec() => self.foo.encode() + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use runtime_io::with_externalities; + use substrate_primitives::{H256, Blake2Hasher}; + use runtime_primitives::BuildStorage; + use runtime_primitives::traits::{BlakeTwo256}; + + // The testing primitives are very useful for avoiding having to work with signatures + // or public keys. `u64` is used as the `AccountId` and no `Signature`s are requried. + use runtime_primitives::testing::{Digest, Header}; + + impl_outer_origin! { + pub enum Origin for Test {} + } + + // For testing the module, we construct most of a mock runtime. This means + // first constructing a configuration type (`Test`) which `impl`s each of the + // configuration traits of modules we want to use. + #[derive(Clone, Eq, PartialEq)] + pub struct Test; + impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); + } + impl balances::Trait for Test { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = (); + type EnsureAccountLiquid = (); + type Event = (); + } + impl Trait for Test { + type Event = (); + } + type Example = Module; + + // This function basically just builds a genesis storage key/value store according to + // our desired mockup. + fn new_test_ext() -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + // We use default for brevity, but you can configure as desired if needed. + t.extend(balances::GenesisConfig::::default().build_storage().unwrap()); + t.extend(GenesisConfig::{ + dummy: 42, + foo: 24, + }.build_storage().unwrap()); + t.into() + } + + #[test] + fn it_works_for_optional_value() { + with_externalities(&mut new_test_ext(), || { + // Check that GenesisBuilder works properly. + assert_eq!(Example::dummy(), Some(42)); + + // Check that accumulate works when we have Some value in Dummy already. + assert_ok!(Example::accumulate_dummy(Origin::signed(1), 27)); + assert_eq!(Example::dummy(), Some(69)); + + // Check that finalising the block removes Dummy from storage. + >::on_finalise(1); + assert_eq!(Example::dummy(), None); + + // Check that accumulate works when we Dummy has None in it. + assert_ok!(Example::accumulate_dummy(Origin::signed(1), 42)); + assert_eq!(Example::dummy(), Some(42)); + }); + } + + #[test] + fn it_works_for_default_value() { + with_externalities(&mut new_test_ext(), || { + assert_eq!(Example::foo(), 24); + assert_ok!(Example::accumulate_foo(Origin::signed(1), 1)); + assert_eq!(Example::foo(), 25); + }); + } +} diff --git a/runtime/executive/Cargo.toml b/runtime/executive/Cargo.toml new file mode 100644 index 00000000000..2cebd50692e --- /dev/null +++ b/runtime/executive/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "substrate-runtime-executive" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } + +[dev-dependencies] +substrate-primitives = { path = "../../core/primitives" } +substrate-runtime-balances = { path = "../balances" } +substrate-codec-derive = { path = "../../core/codec/derive" } + +[features] +default = ["std"] +std = [ + "substrate-runtime-std/std", + "substrate-runtime-support/std", + "serde/std", + "serde_derive", + "substrate-codec/std", + "substrate-runtime-primitives/std", + "substrate-runtime-io/std", + "substrate-runtime-system/std", +] diff --git a/runtime/executive/src/lib.rs b/runtime/executive/src/lib.rs new file mode 100644 index 00000000000..9b7e736f6d9 --- /dev/null +++ b/runtime/executive/src/lib.rs @@ -0,0 +1,359 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Executive: Handles all of the top-level stuff; essentially just executing blocks/extrinsics. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; +#[cfg(test)] +#[macro_use] +extern crate serde_derive; + +#[cfg(test)] +#[macro_use] +extern crate substrate_codec_derive; + +#[cfg_attr(test, macro_use)] +extern crate substrate_runtime_support as runtime_support; + +extern crate substrate_runtime_std as rstd; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_codec as codec; +extern crate substrate_runtime_primitives as primitives; +extern crate substrate_runtime_system as system; + +#[cfg(test)] +#[macro_use] +extern crate hex_literal; + +#[cfg(test)] +extern crate substrate_primitives; + +#[cfg(test)] +extern crate substrate_runtime_balances as balances; + +use rstd::prelude::*; +use rstd::marker::PhantomData; +use rstd::result; +use primitives::traits::{self, Header, Zero, One, Checkable, Applyable, CheckEqual, OnFinalise, + MakePayment, Hash}; +use codec::{Codec, Encode}; +use system::extrinsics_root; +use primitives::{ApplyOutcome, ApplyError}; + +mod internal { + pub enum ApplyError { + BadSignature(&'static str), + Stale, + Future, + CantPay, + } + + pub enum ApplyOutcome { + Success, + Fail(&'static str), + } +} + +pub struct Executive< + System, + Block, + Lookup, + Payment, + Finalisation, +>(PhantomData<(System, Block, Lookup, Payment, Finalisation)>); + +impl< + Address, + System: system::Trait, + Block: traits::Block, + Lookup: traits::Lookup, + Payment: MakePayment, + Finalisation: OnFinalise, +> Executive where + Block::Extrinsic: Checkable Result> + Codec, + Result>>::Checked: Applyable +{ + /// Start the execution of a particular block. + pub fn initialise_block(header: &System::Header) { + >::initialise(header.number(), header.parent_hash(), header.extrinsics_root()); + } + + fn initial_checks(block: &Block) { + let header = block.header(); + + // check parent_hash is correct. + let n = header.number().clone(); + assert!( + n > System::BlockNumber::zero() && >::block_hash(n - System::BlockNumber::one()) == *header.parent_hash(), + "Parent hash should be valid." + ); + + // check transaction trie root represents the transactions. + let xts_root = extrinsics_root::(&block.extrinsics()); + header.extrinsics_root().check_equal(&xts_root); + assert!(header.extrinsics_root() == &xts_root, "Transaction trie root must be valid."); + } + + /// Actually execute all transitioning for `block`. + pub fn execute_block(block: Block) { + Self::initialise_block(block.header()); + + // any initial checks + Self::initial_checks(&block); + + // execute transactions + let (header, extrinsics) = block.deconstruct(); + extrinsics.into_iter().for_each(Self::apply_extrinsic_no_note); + + // post-transactional book-keeping. + >::note_finished_extrinsics(); + Finalisation::on_finalise(*header.number()); + + // any final checks + Self::final_checks(&header); + } + + /// Finalise the block - it is up the caller to ensure that all header fields are valid + /// except state-root. + pub fn finalise_block() -> System::Header { + >::note_finished_extrinsics(); + Finalisation::on_finalise(>::block_number()); + + // setup extrinsics + >::derive_extrinsics(); + >::finalise() + } + + /// Apply extrinsic outside of the block execution function. + /// This doesn't attempt to validate anything regarding the block, but it builds a list of uxt + /// hashes. + pub fn apply_extrinsic(uxt: Block::Extrinsic) -> result::Result { + let encoded = uxt.encode(); + let encoded_len = encoded.len(); + >::note_extrinsic(encoded); + match Self::apply_extrinsic_no_note_with_len(uxt, encoded_len) { + Ok(internal::ApplyOutcome::Success) => Ok(ApplyOutcome::Success), + Ok(internal::ApplyOutcome::Fail(_)) => Ok(ApplyOutcome::Fail), + Err(internal::ApplyError::CantPay) => Err(ApplyError::CantPay), + Err(internal::ApplyError::BadSignature(_)) => Err(ApplyError::BadSignature), + Err(internal::ApplyError::Stale) => Err(ApplyError::Stale), + Err(internal::ApplyError::Future) => Err(ApplyError::Future), + } + } + + /// Apply an extrinsic inside the block execution function. + fn apply_extrinsic_no_note(uxt: Block::Extrinsic) { + let l = uxt.encode().len(); + match Self::apply_extrinsic_no_note_with_len(uxt, l) { + Ok(internal::ApplyOutcome::Success) => (), + Ok(internal::ApplyOutcome::Fail(e)) => runtime_io::print(e), + Err(internal::ApplyError::CantPay) => panic!("All extrinsics should have sender able to pay their fees"), + Err(internal::ApplyError::BadSignature(_)) => panic!("All extrinsics should be properly signed"), + Err(internal::ApplyError::Stale) | Err(internal::ApplyError::Future) => panic!("All extrinsics should have the correct nonce"), + } + } + + /// Actually apply an extrinsic given its `encoded_len`; this doesn't note its hash. + fn apply_extrinsic_no_note_with_len(uxt: Block::Extrinsic, encoded_len: usize) -> result::Result { + // Verify the signature is good. + let xt = uxt.check_with(Lookup::lookup).map_err(internal::ApplyError::BadSignature)?; + + if let Some(sender) = xt.sender() { + // check index + let expected_index = >::account_nonce(sender); + if xt.index() != &expected_index { return Err( + if xt.index() < &expected_index { internal::ApplyError::Stale } else { internal::ApplyError::Future } + ) } + + // pay any fees. + Payment::make_payment(sender, encoded_len).map_err(|_| internal::ApplyError::CantPay)?; + + // AUDIT: Under no circumstances may this function panic from here onwards. + + // increment nonce in storage + >::inc_account_nonce(sender); + } + + // decode parameters and dispatch + let r = xt.apply(); + + >::note_applied_extrinsic(&r); + + r.map(|_| internal::ApplyOutcome::Success).or_else(|e| Ok(internal::ApplyOutcome::Fail(e))) + } + + fn final_checks(header: &System::Header) { + // check digest + assert!(header.digest() == &>::digest()); + + // remove temporaries. + >::finalise(); + + // check storage root. + let storage_root = System::Hashing::storage_root(); + header.state_root().check_equal(&storage_root); + assert!(header.state_root() == &storage_root, "Storage root must match that calculated."); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use balances::Call; + use runtime_io::with_externalities; + use substrate_primitives::{H256, Blake2Hasher}; + use primitives::BuildStorage; + use primitives::traits::{Header as HeaderT, BlakeTwo256, Lookup}; + use primitives::testing::{Digest, Header, Block}; + use system; + + struct NullLookup; + impl Lookup for NullLookup { + type Source = u64; + type Target = u64; + fn lookup(s: Self::Source) -> Result { + Ok(s) + } + } + + impl_outer_origin! { + pub enum Origin for Runtime { + } + } + + impl_outer_event!{ + pub enum MetaEvent for Runtime { + balances + } + } + + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] + pub struct Runtime; + impl system::Trait for Runtime { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = substrate_primitives::H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = MetaEvent; + } + impl balances::Trait for Runtime { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = (); + type EnsureAccountLiquid = (); + type Event = MetaEvent; + } + + type TestXt = primitives::testing::TestXt>; + type Executive = super::Executive, NullLookup, balances::Module, ()>; + + #[test] + fn balance_transfer_dispatch_works() { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(balances::GenesisConfig:: { + balances: vec![(1, 111)], + transaction_base_fee: 10, + transaction_byte_fee: 0, + existential_deposit: 0, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + }.build_storage().unwrap()); + let xt = primitives::testing::TestXt(Some(1), 0, Call::transfer(2.into(), 69)); + let mut t = runtime_io::TestExternalities::from(t); + with_externalities(&mut t, || { + Executive::initialise_block(&Header::new(1, H256::default(), H256::default(), [69u8; 32].into(), Digest::default())); + Executive::apply_extrinsic(xt).unwrap(); + assert_eq!(>::total_balance(&1), 32); + assert_eq!(>::total_balance(&2), 69); + }); + } + + fn new_test_ext() -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(balances::GenesisConfig::::default().build_storage().unwrap()); + t.into() + } + + #[test] + fn block_import_works() { + with_externalities(&mut new_test_ext(), || { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: hex!("d1d3da2b1efb1a6ef740b8cdef52e4cf3c6dade6f8a360969fd7ef0034c53b54").into(), + extrinsics_root: hex!("45b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c0").into(), + digest: Digest { logs: vec![], }, + }, + extrinsics: vec![], + }); + }); + } + + #[test] + #[should_panic] + fn block_import_of_bad_state_root_fails() { + with_externalities(&mut new_test_ext(), || { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: [0u8; 32].into(), + extrinsics_root: hex!("45b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c0").into(), + digest: Digest { logs: vec![], }, + }, + extrinsics: vec![], + }); + }); + } + + #[test] + #[should_panic] + fn block_import_of_bad_extrinsic_root_fails() { + with_externalities(&mut new_test_ext(), || { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: hex!("d1d3da2b1efb1a6ef740b8cdef52e4cf3c6dade6f8a360969fd7ef0034c53b54").into(), + extrinsics_root: [0u8; 32].into(), + digest: Digest { logs: vec![], }, + }, + extrinsics: vec![], + }); + }); + } + + #[test] + fn bad_extrinsic_not_inserted() { + let mut t = new_test_ext(); + let xt = primitives::testing::TestXt(Some(1), 42, Call::transfer(33.into(), 69)); + with_externalities(&mut t, || { + Executive::initialise_block(&Header::new(1, H256::default(), H256::default(), [69u8; 32].into(), Digest::default())); + assert!(Executive::apply_extrinsic(xt).is_err()); + assert_eq!(>::extrinsic_index(), Some(0)); + }); + } +} diff --git a/runtime/primitives/Cargo.toml b/runtime/primitives/Cargo.toml new file mode 100644 index 00000000000..f67577664c5 --- /dev/null +++ b/runtime/primitives/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "substrate-runtime-primitives" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +num-traits = { version = "0.2", default_features = false } +integer-sqrt = { git = "https://github.com/paritytech/integer-sqrt-rs.git", branch = "master" } +serde = { version = "1.0", optional = true } +serde_derive = { version = "1.0", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +log = {version = "0.3", optional = true } + +[dev-dependencies] +serde_json = "1.0" + +[features] +default = ["std"] +std = [ + "num-traits/std", + "serde", + "serde_derive", + "log", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-codec/std", + "substrate-primitives/std", +] diff --git a/runtime/primitives/src/bft.rs b/runtime/primitives/src/bft.rs new file mode 100644 index 00000000000..17eb54ab105 --- /dev/null +++ b/runtime/primitives/src/bft.rs @@ -0,0 +1,195 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Message formats for the BFT consensus layer. + +use rstd::prelude::*; +use codec::{Decode, Encode, Input, Output}; +use substrate_primitives::{AuthorityId, Signature}; + +/// Type alias for extracting message type from block. +pub type ActionFor = Action::Hash>; + +/// Actions which can be taken during the BFT process. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub enum Action { + /// Proposal of a block candidate. + #[codec(index = "1")] + Propose(u32, Block), + /// Proposal header of a block candidate. Accompanies any proposal, + /// but is used for misbehavior reporting since blocks themselves are big. + #[codec(index = "2")] + ProposeHeader(u32, H), + /// Preparation to commit for a candidate. + #[codec(index = "3")] + Prepare(u32, H), + /// Vote to commit to a candidate. + #[codec(index = "4")] + Commit(u32, H), + /// Vote to advance round after inactive primary. + #[codec(index = "5")] + AdvanceRound(u32), +} + +/// Type alias for extracting message type from block. +pub type MessageFor = Message::Hash>; + +/// Messages exchanged between participants in the BFT consensus. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct Message { + /// The parent header hash this action is relative to. + pub parent: Hash, + /// The action being broadcasted. + pub action: Action, +} + +/// Justification of a block. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct Justification { + /// The round consensus was reached in. + pub round_number: u32, + /// The hash of the header justified. + pub hash: H, + /// The signatures and signers of the hash. + pub signatures: Vec<(AuthorityId, Signature)> +} + +// single-byte code to represent misbehavior kind. +#[repr(i8)] +enum MisbehaviorCode { + /// BFT: double prepare. + BftDoublePrepare = 0x11, + /// BFT: double commit. + BftDoubleCommit = 0x12, +} + +impl MisbehaviorCode { + fn from_i8(x: i8) -> Option { + match x { + 0x11 => Some(MisbehaviorCode::BftDoublePrepare), + 0x12 => Some(MisbehaviorCode::BftDoubleCommit), + _ => None, + } + } +} + +/// Misbehavior kinds. +#[derive(Clone, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub enum MisbehaviorKind { + /// BFT: double prepare. + BftDoublePrepare(u32, (Hash, Signature), (Hash, Signature)), + /// BFT: double commit. + BftDoubleCommit(u32, (Hash, Signature), (Hash, Signature)), +} + +impl Encode for MisbehaviorKind { + fn encode_to(&self, dest: &mut T) { + match *self { + MisbehaviorKind::BftDoublePrepare(ref round, (ref h_a, ref s_a), (ref h_b, ref s_b)) => { + dest.push(&(MisbehaviorCode::BftDoublePrepare as i8)); + dest.push(round); + dest.push(h_a); + dest.push(s_a); + dest.push(h_b); + dest.push(s_b); + } + MisbehaviorKind::BftDoubleCommit(ref round, (ref h_a, ref s_a), (ref h_b, ref s_b)) => { + dest.push(&(MisbehaviorCode::BftDoubleCommit as i8)); + dest.push(round); + dest.push(h_a); + dest.push(s_a); + dest.push(h_b); + dest.push(s_b); + } + } + } +} +impl Decode for MisbehaviorKind { + fn decode(input: &mut I) -> Option { + Some(match i8::decode(input).and_then(MisbehaviorCode::from_i8)? { + MisbehaviorCode::BftDoublePrepare => { + MisbehaviorKind::BftDoublePrepare( + u32::decode(input)?, + (Hash::decode(input)?, Signature::decode(input)?), + (Hash::decode(input)?, Signature::decode(input)?), + ) + } + MisbehaviorCode::BftDoubleCommit => { + MisbehaviorKind::BftDoubleCommit( + u32::decode(input)?, + (Hash::decode(input)?, Signature::decode(input)?), + (Hash::decode(input)?, Signature::decode(input)?), + ) + } + }) + } +} + + +/// A report of misbehavior by an authority. +#[derive(Clone, PartialEq, Eq, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct MisbehaviorReport { + /// The parent hash of the block where the misbehavior occurred. + pub parent_hash: Hash, + /// The parent number of the block where the misbehavior occurred. + pub parent_number: Number, + /// The authority who misbehavior. + pub target: AuthorityId, + /// The misbehavior kind. + pub misbehavior: MisbehaviorKind, +} + +#[cfg(test)] +mod test { + use super::*; + use substrate_primitives::H256; + + #[test] + fn misbehavior_report_roundtrip() { + let report = MisbehaviorReport:: { + parent_hash: [0; 32].into(), + parent_number: 999, + target: [1; 32].into(), + misbehavior: MisbehaviorKind::BftDoubleCommit( + 511, + ([2; 32].into(), [3; 64].into()), + ([4; 32].into(), [5; 64].into()), + ), + }; + + let encoded = report.encode(); + assert_eq!(MisbehaviorReport::::decode(&mut &encoded[..]).unwrap(), report); + + let report = MisbehaviorReport:: { + parent_hash: [0; 32].into(), + parent_number: 999, + target: [1; 32].into(), + misbehavior: MisbehaviorKind::BftDoublePrepare( + 511, + ([2; 32].into(), [3; 64].into()), + ([4; 32].into(), [5; 64].into()), + ), + }; + + let encoded = report.encode(); + assert_eq!(MisbehaviorReport::::decode(&mut &encoded[..]).unwrap(), report); + } +} diff --git a/runtime/primitives/src/generic/block.rs b/runtime/primitives/src/generic/block.rs new file mode 100644 index 00000000000..40538a392c3 --- /dev/null +++ b/runtime/primitives/src/generic/block.rs @@ -0,0 +1,105 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Generic implementation of a block and associated items. + +#[cfg(feature = "std")] +use std::fmt; + +use rstd::prelude::*; +use codec::Codec; +use traits::{self, Member, Block as BlockT, Header as HeaderT}; +use bft::Justification; + +/// Something to identify a block. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug, Serialize))] +#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +pub enum BlockId { + /// Identify by block header hash. + Hash(<::Header as HeaderT>::Hash), + /// Identify by block number. + Number(<::Header as HeaderT>::Number), +} + +impl BlockId { + /// Create a block ID from a hash. + pub fn hash(hash: Block::Hash) -> Self { + BlockId::Hash(hash) + } + + /// Create a block ID from a number. + pub fn number(number: ::Number) -> Self { + BlockId::Number(number) + } +} + +impl Copy for BlockId {} + +#[cfg(feature = "std")] +impl fmt::Display for BlockId { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{:?}", self) + } +} + +/// Abstraction over a substrate block. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +pub struct Block { + /// The block header. + pub header: Header, + /// The accompanying extrinsics. + pub extrinsics: Vec, +} + +impl traits::Block for Block +where + Header: HeaderT, + Extrinsic: Member + Codec, +{ + type Extrinsic = Extrinsic; + type Header = Header; + type Hash = ::Hash; + + fn header(&self) -> &Self::Header { + &self.header + } + fn extrinsics(&self) -> &[Self::Extrinsic] { + &self.extrinsics[..] + } + fn deconstruct(self) -> (Self::Header, Vec) { + (self.header, self.extrinsics) + } + fn new(header: Self::Header, extrinsics: Vec) -> Self { + Block { header, extrinsics } + } +} + +/// Abstraction over a substrate block and justification. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +pub struct SignedBlock { + /// Full block. + pub block: Block, + /// Block header justification. + pub justification: Justification, +} diff --git a/runtime/primitives/src/generic/checked_extrinsic.rs b/runtime/primitives/src/generic/checked_extrinsic.rs new file mode 100644 index 00000000000..e5aee32a02c --- /dev/null +++ b/runtime/primitives/src/generic/checked_extrinsic.rs @@ -0,0 +1,59 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Generic implementation of an extrinsic that has passed the verification +//! stage. + +use runtime_support::Dispatchable; +use traits::{self, Member, SimpleArithmetic, MaybeDisplay}; + +/// Definition of something that the external world might want to say; its +/// existence implies that it has been checked and is good, particularly with +/// regards to the signature. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +pub struct CheckedExtrinsic { + /// Who this purports to be from, if anyone (note this is not a signature). + pub signed: Option, + /// The number of extrinsics have come before from the same signer. + pub index: Index, + /// The function that should be called. + pub function: Call, +} + +impl traits::Applyable + for CheckedExtrinsic +where + AccountId: Member + MaybeDisplay, + Index: Member + MaybeDisplay + SimpleArithmetic, + Call: Member + Dispatchable, + ::Origin: From> +{ + type Index = Index; + type AccountId = AccountId; + + fn index(&self) -> &Self::Index { + &self.index + } + + fn sender(&self) -> Option<&Self::AccountId> { + self.signed.as_ref() + } + + fn apply(self) -> Result<(), &'static str> { + self.function.dispatch(self.signed.into()) + } +} diff --git a/runtime/primitives/src/generic/digest.rs b/runtime/primitives/src/generic/digest.rs new file mode 100644 index 00000000000..0616ec8914e --- /dev/null +++ b/runtime/primitives/src/generic/digest.rs @@ -0,0 +1,150 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Generic implementation of a digest. + +use rstd::prelude::*; + +use codec::{Decode, Encode, Codec, Input}; +use traits::{self, Member, DigestItem as DigestItemT}; + +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct Digest { + pub logs: Vec, +} + +impl Default for Digest { + fn default() -> Self { + Digest { logs: Vec::new(), } + } +} + +impl traits::Digest for Digest where + Item: DigestItemT + Codec +{ + type Item = Item; + + fn logs(&self) -> &[Self::Item] { + &self.logs + } + + fn push(&mut self, item: Self::Item) { + self.logs.push(item); + } +} + +/// Digest item that is able to encode/decode 'system' digest items and +/// provide opaque access to other items. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub enum DigestItem { + /// System digest item announcing that authorities set has been changed + /// in the block. Contains the new set of authorities. + AuthoritiesChange(Vec), + /// Any 'non-system' digest item, opaque to the native code. + Other(Vec), +} + +/// A 'referencing view' for digest item. Does not own its contents. Used by +/// final runtime implementations for encoding/decoding its log items. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum DigestItemRef<'a, AuthorityId: 'a> { + /// Reference to `DigestItem::AuthoritiesChange`. + AuthoritiesChange(&'a [AuthorityId]), + /// Reference to `DigestItem::Other`. + Other(&'a Vec), +} + +/// Type of the digest item. Used to gain explicit control over `DigestItem` encoding +/// process. We need an explicit control, because final runtimes are encoding their own +/// digest items using `DigestItemRef` type and we can't auto-derive `Decode` +/// trait for `DigestItemRef`. +#[repr(u32)] +#[derive(Encode, Decode)] +enum DigestItemType { + Other = 0, + AuthoritiesChange, +} + +impl DigestItem { + /// Returns Some if `self` is a `DigestItem::Other`. + pub fn as_other(&self) -> Option<&Vec> { + match *self { + DigestItem::Other(ref v) => Some(v), + _ => None, + } + } + + /// Returns a 'referencing view' for this digest item. + fn dref<'a>(&'a self) -> DigestItemRef<'a, AuthorityId> { + match *self { + DigestItem::AuthoritiesChange(ref v) => DigestItemRef::AuthoritiesChange(v), + DigestItem::Other(ref v) => DigestItemRef::Other(v), + } + } +} + +impl traits::DigestItem for DigestItem { + type AuthorityId = AuthorityId; + + fn as_authorities_change(&self) -> Option<&[Self::AuthorityId]> { + match *self { + DigestItem::AuthoritiesChange(ref authorities) => Some(authorities), + _ => None, + } + } +} + +impl Encode for DigestItem { + fn encode(&self) -> Vec { + self.dref().encode() + } +} + +impl Decode for DigestItem { + fn decode(input: &mut I) -> Option { + let item_type: DigestItemType = Decode::decode(input)?; + match item_type { + DigestItemType::AuthoritiesChange => Some(DigestItem::AuthoritiesChange( + Decode::decode(input)?, + )), + DigestItemType::Other => Some(DigestItem::Other( + Decode::decode(input)?, + )), + } + } +} + +impl<'a, AuthorityId: Encode> Encode for DigestItemRef<'a, AuthorityId> { + fn encode(&self) -> Vec { + let mut v = Vec::new(); + + match *self { + DigestItemRef::AuthoritiesChange(authorities) => { + DigestItemType::AuthoritiesChange.encode_to(&mut v); + authorities.encode_to(&mut v); + }, + DigestItemRef::Other(val) => { + DigestItemType::Other.encode_to(&mut v); + val.encode_to(&mut v); + }, + } + + v + } +} diff --git a/runtime/primitives/src/generic/header.rs b/runtime/primitives/src/generic/header.rs new file mode 100644 index 00000000000..8425a17bb7b --- /dev/null +++ b/runtime/primitives/src/generic/header.rs @@ -0,0 +1,167 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Generic implementation of a block header. + +#[cfg(feature = "std")] +use serde::{Deserialize, Deserializer}; + +use codec::{Decode, Encode, Codec, Input, Output}; +use traits::{self, Member, SimpleArithmetic, SimpleBitOps, MaybeDisplay, + Hash as HashT, DigestItem as DigestItemT}; +use generic::Digest; + +/// Abstraction over a block header for a substrate chain. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug, Serialize))] +#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +pub struct Header { + /// The parent hash. + pub parent_hash: ::Output, + /// The block number. + pub number: Number, + /// The state trie merkle root + pub state_root: ::Output, + /// The merkle root of the extrinsics. + pub extrinsics_root: ::Output, + /// A chain-specific digest of data useful for light clients or referencing auxiliary data. + pub digest: Digest, +} + +// Hack to work around the fact that deriving deserialize doesn't work nicely with +// the `hashing` trait used as a parameter. +// dummy struct that uses the hash type directly. +// https://github.com/serde-rs/serde/issues/1296 +#[cfg(feature = "std")] +#[serde(rename_all = "camelCase")] +#[derive(Deserialize)] +struct DeserializeHeader { + parent_hash: H, + number: N, + state_root: H, + extrinsics_root: H, + digest: Digest, +} + +#[cfg(feature = "std")] +impl From> for Header { + fn from(other: DeserializeHeader) -> Self { + Header { + parent_hash: other.parent_hash, + number: other.number, + state_root: other.state_root, + extrinsics_root: other.extrinsics_root, + digest: other.digest, + } + } +} + +#[cfg(feature = "std")] +impl<'a, Number: 'a, Hash: 'a + HashT, DigestItem: 'a> Deserialize<'a> for Header where + Number: Deserialize<'a>, + Hash::Output: Deserialize<'a>, + DigestItem: Deserialize<'a>, +{ + fn deserialize>(de: D) -> Result { + DeserializeHeader::::deserialize(de).map(Into::into) + } +} + +// TODO [ToDr] Issue with bounds +impl Decode for Header where + Number: Decode, + Hash: HashT, + Hash::Output: Decode, + DigestItem: DigestItemT + Decode, +{ + fn decode(input: &mut I) -> Option { + Some(Header { + parent_hash: Decode::decode(input)?, + number: Decode::decode(input)?, + state_root: Decode::decode(input)?, + extrinsics_root: Decode::decode(input)?, + digest: Decode::decode(input)?, + }) + } +} + +impl Encode for Header where + Number: Encode, + Hash: HashT, + Hash::Output: Encode, + DigestItem: DigestItemT + Encode, +{ + fn encode_to(&self, dest: &mut T) { + dest.push(&self.parent_hash); + dest.push(&self.number); + dest.push(&self.state_root); + dest.push(&self.extrinsics_root); + dest.push(&self.digest); + } +} + +impl traits::Header for Header where + Number: Member + ::rstd::hash::Hash + Copy + Codec + MaybeDisplay + SimpleArithmetic + Codec, + Hash: HashT, + DigestItem: DigestItemT + Codec, + Hash::Output: Default + ::rstd::hash::Hash + Copy + Member + MaybeDisplay + SimpleBitOps + Codec, + { + type Number = Number; + type Hash = ::Output; + type Hashing = Hash; + type Digest = Digest; + + fn number(&self) -> &Self::Number { &self.number } + fn set_number(&mut self, num: Self::Number) { self.number = num } + + fn extrinsics_root(&self) -> &Self::Hash { &self.extrinsics_root } + fn set_extrinsics_root(&mut self, root: Self::Hash) { self.extrinsics_root = root } + + fn state_root(&self) -> &Self::Hash { &self.state_root } + fn set_state_root(&mut self, root: Self::Hash) { self.state_root = root } + + fn parent_hash(&self) -> &Self::Hash { &self.parent_hash } + fn set_parent_hash(&mut self, hash: Self::Hash) { self.parent_hash = hash } + + fn digest(&self) -> &Self::Digest { &self.digest } + fn set_digest(&mut self, digest: Self::Digest) { self.digest = digest } + + fn new( + number: Self::Number, + extrinsics_root: Self::Hash, + state_root: Self::Hash, + parent_hash: Self::Hash, + digest: Self::Digest + ) -> Self { + Header { + number, extrinsics_root: extrinsics_root, state_root, parent_hash, digest + } + } +} + +impl Header where + Number: Member + ::rstd::hash::Hash + Copy + Codec + MaybeDisplay + SimpleArithmetic + Codec, + Hash: HashT, + DigestItem: DigestItemT + Codec, + Hash::Output: Default + ::rstd::hash::Hash + Copy + Member + MaybeDisplay + SimpleBitOps + Codec, + { + /// Convenience helper for computing the hash of the header without having + /// to import the trait. + pub fn hash(&self) -> Hash::Output { + Hash::hash_of(self) + } +} diff --git a/runtime/primitives/src/generic/mod.rs b/runtime/primitives/src/generic/mod.rs new file mode 100644 index 00000000000..23907d9f6cb --- /dev/null +++ b/runtime/primitives/src/generic/mod.rs @@ -0,0 +1,33 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Generic implementations of Extrinsic/Header/Block. +// end::description[] + +mod unchecked_extrinsic; +mod checked_extrinsic; +mod header; +mod block; +mod digest; +#[cfg(test)] +mod tests; + +pub use self::unchecked_extrinsic::UncheckedExtrinsic; +pub use self::checked_extrinsic::CheckedExtrinsic; +pub use self::header::Header; +pub use self::block::{Block, SignedBlock, BlockId}; +pub use self::digest::{Digest, DigestItem, DigestItemRef}; diff --git a/runtime/primitives/src/generic/tests.rs b/runtime/primitives/src/generic/tests.rs new file mode 100644 index 00000000000..6d22357d190 --- /dev/null +++ b/runtime/primitives/src/generic/tests.rs @@ -0,0 +1,105 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tests for the generic implementations of Extrinsic/Header/Block. + +use codec::{Decode, Encode}; +use substrate_primitives::{H256, H512}; +use super::{Digest, Header, DigestItem, UncheckedExtrinsic}; + +type Block = super::Block< + Header>, + UncheckedExtrinsic, +>; + +#[test] +fn block_roundtrip_serialization() { + let block: Block = Block { + header: Header { + parent_hash: [0u8; 32].into(), + number: 100_000, + state_root: [1u8; 32].into(), + extrinsics_root: [2u8; 32].into(), + digest: Digest { logs: vec![ + DigestItem::Other::(vec![1, 2, 3]), + DigestItem::Other::(vec![4, 5, 6]), + ] }, + }, + extrinsics: vec![ + UncheckedExtrinsic::new_signed( + 0, + 100, + [255u8; 32].into(), + H512::from([0u8; 64]).into() + ), + UncheckedExtrinsic::new_signed( + 100, + 99, + [128u8; 32].into(), + H512::from([255u8; 64]).into() + ) + ] + }; + + { + let encoded = ::serde_json::to_vec(&block).unwrap(); + let decoded: Block = ::serde_json::from_slice(&encoded).unwrap(); + + assert_eq!(block, decoded); + } + { + let encoded = block.encode(); + let decoded = Block::decode(&mut &encoded[..]).unwrap(); + + assert_eq!(block, decoded); + } +} + +#[test] +fn system_digest_item_encoding() { + let item = DigestItem::AuthoritiesChange::(vec![10, 20, 30]); + let encoded = item.encode(); + assert_eq!(encoded, vec![ + // type = DigestItemType::AuthoritiesChange + 1, + // number of items in athorities set + 3, 0, 0, 0, + // authorities + 10, 0, 0, 0, + 20, 0, 0, 0, + 30, 0, 0, 0, + ]); + + let decoded: DigestItem = Decode::decode(&mut &encoded[..]).unwrap(); + assert_eq!(item, decoded); +} + +#[test] +fn non_system_digest_item_encoding() { + let item = DigestItem::Other::(vec![10, 20, 30]); + let encoded = item.encode(); + assert_eq!(encoded, vec![ + // type = DigestItemType::Other + 0, + // length of other data + 3, 0, 0, 0, + // authorities + 10, 20, 30, + ]); + + let decoded: DigestItem = Decode::decode(&mut &encoded[..]).unwrap(); + assert_eq!(item, decoded); +} \ No newline at end of file diff --git a/runtime/primitives/src/generic/unchecked_extrinsic.rs b/runtime/primitives/src/generic/unchecked_extrinsic.rs new file mode 100644 index 00000000000..c7e3693fbf1 --- /dev/null +++ b/runtime/primitives/src/generic/unchecked_extrinsic.rs @@ -0,0 +1,159 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Generic implementation of an unchecked (pre-verification) extrinsic. + +#[cfg(feature = "std")] +use std::fmt; + +use rstd::prelude::*; +use codec::{Decode, Encode, Input}; +use traits::{self, Member, SimpleArithmetic, MaybeDisplay}; +use super::CheckedExtrinsic; + +/// A extrinsic right from the external world. This is unchecked and so +/// can contain a signature. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +pub struct UncheckedExtrinsic { + /// The signature and address, if this is a signed extrinsic. + pub signature: Option<(Address, Signature)>, + /// The number of extrinsics have come before from the same signer. + pub index: Index, + /// The function that should be called. + pub function: Call, +} + +impl UncheckedExtrinsic { + /// New instance of a signed extrinsic aka "transaction". + pub fn new_signed(index: Index, function: Call, signed: Address, signature: Signature) -> Self { + UncheckedExtrinsic { + signature: Some((signed, signature)), + index, + function, + } + } + + /// New instance of an unsigned extrinsic aka "inherent". + pub fn new_unsigned(index: Index, function: Call) -> Self { + UncheckedExtrinsic { + signature: None, + index, + function, + } + } + + /// `true` if there is a signature. + pub fn is_signed(&self) -> bool { + self.signature.is_some() + } +} + +impl traits::Checkable + for UncheckedExtrinsic +where + Address: Member + MaybeDisplay, + Index: Encode + Member + MaybeDisplay + SimpleArithmetic, + Call: Encode + Member, + Signature: Member + traits::Verify, + AccountId: Member + MaybeDisplay, + ThisLookup: FnOnce(Address) -> Result, +{ + type Checked = CheckedExtrinsic; + + fn check_with(self, lookup: ThisLookup) -> Result { + Ok(match self.signature { + Some((signed, signature)) => { + let payload = (self.index, self.function); + let signed = lookup(signed)?; + if !::verify_encoded_lazy(&signature, &payload, &signed) { + return Err("bad signature in extrinsic") + } + CheckedExtrinsic { + signed: Some(signed), + index: payload.0, + function: payload.1, + } + } + None => CheckedExtrinsic { + signed: None, + index: self.index, + function: self.function, + }, + }) + } +} + +impl Decode + for UncheckedExtrinsic +where + Address: Decode, + Signature: Decode, + Index: Decode, + Call: Decode, +{ + fn decode(input: &mut I) -> Option { + // This is a little more complicated than usual since the binary format must be compatible + // with substrate's generic `Vec` type. Basically this just means accepting that there + // will be a prefix of u32, which has the total number of bytes following (we don't need + // to use this). + let _length_do_not_remove_me_see_above: u32 = Decode::decode(input)?; + + Some(UncheckedExtrinsic { + signature: Decode::decode(input)?, + index: Decode::decode(input)?, + function: Decode::decode(input)?, + }) + } +} + +impl Encode + for UncheckedExtrinsic +where + Address: Encode, + Signature: Encode, + Index: Encode, + Call: Encode, +{ + fn encode(&self) -> Vec { + let mut v = Vec::new(); + + // need to prefix with the total length as u32 to ensure it's binary comptible with + // Vec. we'll make room for it here, then overwrite once we know the length. + v.extend(&[0u8; 4]); + + self.signature.encode_to(&mut v); + self.index.encode_to(&mut v); + self.function.encode_to(&mut v); + + let length = (v.len() - 4) as u32; + length.using_encoded(|s| v[0..4].copy_from_slice(s)); + + v + } +} + +/// TODO: use derive when possible. +#[cfg(feature = "std")] +impl fmt::Debug for UncheckedExtrinsic where + Address: fmt::Debug, + Index: fmt::Debug, + Call: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "UncheckedExtrinsic({:?}, {:?}, {:?})", self.signature.as_ref().map(|x| &x.0), self.function, self.index) + } +} diff --git a/runtime/primitives/src/lib.rs b/runtime/primitives/src/lib.rs new file mode 100644 index 00000000000..e9bddc3286d --- /dev/null +++ b/runtime/primitives/src/lib.rs @@ -0,0 +1,434 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! System manager: Handles all of the top-level stuff; executing block/transaction, setting code +//! and depositing logs. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[cfg(feature = "std")] +#[macro_use] +extern crate log; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate num_traits; +extern crate integer_sqrt; +extern crate substrate_runtime_std as rstd; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_support as runtime_support; +extern crate substrate_codec as codec; +extern crate substrate_primitives; + +#[cfg(test)] +extern crate serde_json; + +#[cfg(feature = "std")] +use std::collections::HashMap; + +use rstd::prelude::*; +use substrate_primitives::hash::{H256, H512}; + +#[cfg(feature = "std")] +use substrate_primitives::hexdisplay::ascii_format; + +#[cfg(feature = "std")] +pub mod testing; + +pub mod traits; +pub mod generic; +pub mod bft; + +use traits::{Verify, Lazy}; + +#[cfg(feature = "std")] +pub use serde::{Serialize, de::DeserializeOwned}; + +/// A set of key value pairs for storage. +#[cfg(feature = "std")] +pub type StorageMap = HashMap, Vec>; + +/// Complex storage builder stuff. +#[cfg(feature = "std")] +pub trait BuildStorage { + fn hash(data: &[u8]) -> [u8; 16] { + let r = runtime_io::twox_128(data); + trace!(target: "build_storage", "{} <= {}", substrate_primitives::hexdisplay::HexDisplay::from(&r), ascii_format(data)); + r + } + fn build_storage(self) -> Result; +} + +#[cfg(feature = "std")] +impl BuildStorage for StorageMap { + fn build_storage(self) -> Result { + Ok(self) + } +} + +/// Permill is parts-per-million (i.e. after multiplying by this, divide by 1000000). +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +#[derive(Encode, Decode, Default, Copy, Clone, PartialEq, Eq)] +pub struct Permill(u32); + +// TODO: impl Mul for N where N: As +impl Permill { + pub fn times + ::rstd::ops::Mul + ::rstd::ops::Div>(self, b: N) -> N { + // TODO: handle overflows + b * >::sa(self.0 as usize) / >::sa(1000000) + } + + pub fn from_millionths(x: u32) -> Permill { Permill(x) } + + pub fn from_percent(x: u32) -> Permill { Permill(x * 10_000) } + + #[cfg(feature = "std")] + pub fn from_fraction(x: f64) -> Permill { Permill((x * 1_000_000.0) as u32) } +} + +#[cfg(feature = "std")] +impl From for Permill { + fn from(x: f64) -> Permill { + Permill::from_fraction(x) + } +} + +#[cfg(feature = "std")] +impl From for Permill { + fn from(x: f32) -> Permill { + Permill::from_fraction(x as f64) + } +} + +/// Ed25519 signature verify. +#[derive(Eq, PartialEq, Clone, Default, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct Ed25519Signature(pub H512); + +impl Verify for Ed25519Signature { + type Signer = H256; + fn verify>(&self, mut msg: L, signer: &Self::Signer) -> bool { + runtime_io::ed25519_verify(&(self.0).0, msg.get(), &signer.0[..]) + } +} + +impl From for Ed25519Signature { + fn from(h: H512) -> Ed25519Signature { + Ed25519Signature(h) + } +} + +#[derive(Eq, PartialEq, Clone, Copy, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize))] +#[repr(u8)] +/// Outcome of a valid extrinsic application. Capable of being sliced. +pub enum ApplyOutcome { + /// Successful application (extrinsic reported no issue). + Success = 0, + /// Failed application (extrinsic was probably a no-op other than fees). + Fail = 1, +} + +impl codec::Encode for ApplyOutcome { + fn using_encoded R>(&self, f: F) -> R { + f(&[*self as u8]) + } +} + +#[derive(Eq, PartialEq, Clone, Copy, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize))] +#[repr(u8)] +/// Reason why an extrinsic couldn't be applied (i.e. invalid extrinsic). +pub enum ApplyError { + /// Bad signature. + BadSignature = 0, + /// Nonce too low. + Stale = 1, + /// Nonce too high. + Future = 2, + /// Sending account had too low a balance. + CantPay = 3, +} + +impl codec::Encode for ApplyError { + fn using_encoded R>(&self, f: F) -> R { + f(&[*self as u8]) + } +} + +/// Result from attempt to apply an extrinsic. +pub type ApplyResult = Result; + +/// Verify a signature on an encoded value in a lazy manner. This can be +/// an optimization if the signature scheme has an "unsigned" escape hash. +pub fn verify_encoded_lazy(sig: &V, item: &T, signer: &V::Signer) -> bool { + // The `Lazy` trait expresses something like `X: FnMut &'a T>`. + // unfortunately this is a lifetime relationship that can't + // be expressed without generic associated types, better unification of HRTBs in type position, + // and some kind of integration into the Fn* traits. + struct LazyEncode { + inner: F, + encoded: Option>, + } + + impl Vec> traits::Lazy<[u8]> for LazyEncode { + fn get(&mut self) -> &[u8] { + self.encoded.get_or_insert_with(&self.inner).as_slice() + } + } + + sig.verify( + LazyEncode { inner: || item.encode(), encoded: None }, + signer, + ) +} + +#[macro_export] +macro_rules! __impl_outer_config_types { + ($concrete:ident $config:ident $snake:ident $($rest:ident)*) => { + #[cfg(any(feature = "std", test))] + pub type $config = $snake::GenesisConfig<$concrete>; + __impl_outer_config_types! {$concrete $($rest)*} + }; + ($concrete:ident) => () +} + +#[macro_export] +/// Implement the output "meta" module configuration struct. +macro_rules! impl_outer_config { + ( pub struct $main:ident for $concrete:ident { $( $config:ident => $snake:ident, )* } ) => { + __impl_outer_config_types! { $concrete $( $config $snake )* } + #[cfg(any(feature = "std", test))] + #[derive(Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + #[serde(deny_unknown_fields)] + pub struct $main { + $( + pub $snake: Option<$config>, + )* + } + #[cfg(any(feature = "std", test))] + impl $crate::BuildStorage for $main { + fn build_storage(self) -> ::std::result::Result<$crate::StorageMap, String> { + let mut s = $crate::StorageMap::new(); + $( + if let Some(extra) = self.$snake { + s.extend(extra.build_storage()?); + } + )* + Ok(s) + } + } + } +} + +/// Generates enum that contains all possible log entries for the runtime. +/// Every individual module of the runtime that is mentioned, must +/// expose a `Log` and `RawLog` enums. +/// +/// Generated enum is binary-compatible with and could be interpreted +/// as `generic::DigestItem`. +/// +/// Requires `use runtime_primitives::generic;` to be used. +/// +/// Runtime requirements: +/// 1) binary representation of all supported 'system' log items should stay +/// the same. Otherwise, the native code will be unable to read log items +/// generated by previous runtime versions +/// 2) the support of 'system' log items should never be dropped by runtime. +/// Otherwise, native code will lost its ability to read items of this type +/// even if they were generated by the versions which have supported these +/// items. +#[macro_export] +macro_rules! impl_outer_log { + ( + $(#[$attr:meta])* + pub enum $name:ident ($internal:ident: DigestItem<$( $genarg:ty ),*>) for $trait:ident { + $( $module:ident($( $item:ident ),*) ),* + } + ) => { + /// Wrapper for all possible log entries for the `$trait` runtime. Provides binary-compatible + /// `Encode`/`Decode` implementations with the corresponding `generic::DigestItem`. + #[derive(Clone, PartialEq, Eq)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + $(#[$attr])* + #[allow(non_camel_case_types)] + pub struct $name($internal); + + /// All possible log entries for the `$trait` runtime. `Encode`/`Decode` implementations + /// are auto-generated => it is not binary-compatible with `generic::DigestItem`. + #[derive(Clone, PartialEq, Eq, Encode, Decode)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + $(#[$attr])* + #[allow(non_camel_case_types)] + enum $internal { + $( + $module($module::Log<$trait>), + )* + } + + impl $name { + /// Try to convert `$name` into `generic::DigestItemRef`. Returns Some when + /// `self` is a 'system' log && it has been marked as 'system' in macro call. + /// Otherwise, None is returned. + #[allow(unreachable_patterns)] + fn dref<'a>(&'a self) -> Option> { + match self.0 { + $($( + $internal::$module($module::RawLog::$item(ref v)) => + Some(generic::DigestItemRef::$item(v)), + )*)* + _ => None, + } + } + } + + impl From> for $name { + /// Converts `generic::DigestItem` into `$name`. If `generic::DigestItem` represents + /// a system item which is supported by the runtime, it is returned. + /// Otherwise we expect a `Other` log item. Trying to convert from anything other + /// will lead to panic in runtime, since the runtime does not supports this 'system' + /// log item. + #[allow(unreachable_patterns)] + fn from(gen: generic::DigestItem<$($genarg),*>) -> Self { + match gen { + $($( + generic::DigestItem::$item(value) => + $name($internal::$module($module::RawLog::$item(value))), + )*)* + _ => gen.as_other() + .and_then(|value| Decode::decode(&mut &value[..])) + .map($name) + .expect("not allowed to fail in runtime"), + } + } + } + + impl Decode for $name { + /// `generic::DigestItem` binray compatible decode. + fn decode(input: &mut I) -> Option { + let gen: generic::DigestItem<$($genarg),*> = Decode::decode(input)?; + Some($name::from(gen)) + } + } + + impl Encode for $name { + /// `generic::DigestItem` binray compatible encode. + fn encode(&self) -> Vec { + match self.dref() { + Some(dref) => dref.encode(), + None => { + let gen: generic::DigestItem<$($genarg),*> = generic::DigestItem::Other(self.0.encode()); + gen.encode() + }, + } + } + } + + $( + impl From<$module::Log<$trait>> for $name { + /// Converts single module log item into `$name`. + fn from(x: $module::Log<$trait>) -> Self { + $name(x.into()) + } + } + + impl From<$module::Log<$trait>> for $internal { + /// Converts single module log item into `$internal`. + fn from(x: $module::Log<$trait>) -> Self { + $internal::$module(x) + } + } + )* + }; +} + +#[cfg(test)] +mod tests { + use codec::{Encode, Decode, Input}; + + pub trait RuntimeT { + type AuthorityId; + } + + pub struct Runtime; + + impl RuntimeT for Runtime { + type AuthorityId = u64; + } + + #[test] + fn impl_outer_log_works() { + mod a { + use super::RuntimeT; + pub type Log = RawLog<::AuthorityId>; + + #[derive(Serialize, Deserialize, Debug, Encode, Decode, PartialEq, Eq, Clone)] + pub enum RawLog { A1(AuthorityId), AuthoritiesChange(Vec), A3(AuthorityId) } + } + + mod b { + use super::RuntimeT; + pub type Log = RawLog<::AuthorityId>; + + #[derive(Serialize, Deserialize, Debug, Encode, Decode, PartialEq, Eq, Clone)] + pub enum RawLog { B1(AuthorityId), B2(AuthorityId) } + } + + use super::generic; // required before macro invocation + + // TODO try to avoid redundant brackets: a(AuthoritiesChange), b + impl_outer_log! { + pub enum Log(InternalLog: DigestItem) for Runtime { + a(AuthoritiesChange), b() + } + } + + // encode/decode regular item + let b1: Log = b::RawLog::B1::(777).into(); + let encoded_b1 = b1.encode(); + let decoded_b1: Log = Decode::decode(&mut &encoded_b1[..]).unwrap(); + assert_eq!(b1, decoded_b1); + + // encode/decode system item + let auth_change: Log = a::RawLog::AuthoritiesChange::(vec![100, 200, 300]).into(); + let encoded_auth_change = auth_change.encode(); + let decoded_auth_change: Log = Decode::decode(&mut &encoded_auth_change[..]).unwrap(); + assert_eq!(auth_change, decoded_auth_change); + + // interpret regular item using `generic::DigestItem` + let generic_b1: generic::DigestItem = Decode::decode(&mut &encoded_b1[..]).unwrap(); + match generic_b1 { + generic::DigestItem::Other(_) => (), + _ => panic!("unexpected generic_b1: {:?}", generic_b1), + } + + // interpret system item using `generic::DigestItem` + let generic_auth_change: generic::DigestItem = Decode::decode(&mut &encoded_auth_change[..]).unwrap(); + match generic_auth_change { + generic::DigestItem::AuthoritiesChange(authorities) => assert_eq!(authorities, vec![100, 200, 300]), + _ => panic!("unexpected generic_auth_change: {:?}", generic_auth_change), + } + } +} diff --git a/runtime/primitives/src/testing.rs b/runtime/primitives/src/testing.rs new file mode 100644 index 00000000000..764828bfe56 --- /dev/null +++ b/runtime/primitives/src/testing.rs @@ -0,0 +1,138 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Testing utilities. + +use serde::{Serialize, de::DeserializeOwned}; +use std::fmt::Debug; +use codec::Codec; +use runtime_support::Dispatchable; +use traits::{self, Checkable, Applyable, BlakeTwo256}; + +pub use substrate_primitives::H256; + +#[derive(Default, PartialEq, Eq, Clone, Serialize, Deserialize, Debug, Encode, Decode)] +pub struct Digest { + pub logs: Vec, +} + +impl traits::Digest for Digest { + type Item = u64; + + fn logs(&self) -> &[Self::Item] { + &self.logs + } + + fn push(&mut self, item: Self::Item) { + self.logs.push(item); + } +} + +impl traits::DigestItem for () { + type AuthorityId = (); +} + +impl traits::DigestItem for u64 { + type AuthorityId = (); +} + +#[derive(PartialEq, Eq, Clone, Serialize, Deserialize, Debug, Encode, Decode)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct Header { + pub parent_hash: H256, + pub number: u64, + pub state_root: H256, + pub extrinsics_root: H256, + pub digest: Digest, +} + +impl traits::Header for Header { + type Number = u64; + type Hashing = BlakeTwo256; + type Hash = H256; + type Digest = Digest; + + fn number(&self) -> &Self::Number { &self.number } + fn set_number(&mut self, num: Self::Number) { self.number = num } + + fn extrinsics_root(&self) -> &Self::Hash { &self.extrinsics_root } + fn set_extrinsics_root(&mut self, root: Self::Hash) { self.extrinsics_root = root } + + fn state_root(&self) -> &Self::Hash { &self.state_root } + fn set_state_root(&mut self, root: Self::Hash) { self.state_root = root } + + fn parent_hash(&self) -> &Self::Hash { &self.parent_hash } + fn set_parent_hash(&mut self, hash: Self::Hash) { self.parent_hash = hash } + + fn digest(&self) -> &Self::Digest { &self.digest } + fn set_digest(&mut self, digest: Self::Digest) { self.digest = digest } + + fn new( + number: Self::Number, + extrinsics_root: Self::Hash, + state_root: Self::Hash, + parent_hash: Self::Hash, + digest: Self::Digest + ) -> Self { + Header { + number, extrinsics_root: extrinsics_root, state_root, parent_hash, digest + } + } +} + +#[derive(PartialEq, Eq, Clone, Serialize, Deserialize, Debug, Encode, Decode)] +pub struct Block { + pub header: Header, + pub extrinsics: Vec, +} + +impl traits::Block for Block { + type Extrinsic = Xt; + type Header = Header; + type Hash =

::Hash; + + fn header(&self) -> &Self::Header { + &self.header + } + fn extrinsics(&self) -> &[Self::Extrinsic] { + &self.extrinsics[..] + } + fn deconstruct(self) -> (Self::Header, Vec) { + (self.header, self.extrinsics) + } + fn new(header: Self::Header, extrinsics: Vec) -> Self { + Block { header, extrinsics } + } +} + +#[derive(PartialEq, Eq, Clone, Serialize, Deserialize, Debug, Encode, Decode)] +pub struct TestXt(pub Option, pub u64, pub Call); + +impl Checkable for TestXt { + type Checked = Self; + fn check_with(self, _: Context) -> Result { Ok(self) } +} +impl Applyable for TestXt where + Call: Sized + Send + Sync + Clone + Eq + Dispatchable + Codec + Debug + Serialize + DeserializeOwned, + ::Origin: From> +{ + type AccountId = u64; + type Index = u64; + fn sender(&self) -> Option<&u64> { self.0.as_ref() } + fn index(&self) -> &u64 { &self.1 } + fn apply(self) -> Result<(), &'static str> { self.2.dispatch(self.0.into()) } +} diff --git a/runtime/primitives/src/traits.rs b/runtime/primitives/src/traits.rs new file mode 100644 index 00000000000..32aa47a16f9 --- /dev/null +++ b/runtime/primitives/src/traits.rs @@ -0,0 +1,462 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Primitives for the runtime modules. + +use rstd::prelude::*; +use rstd::{self, result}; +use runtime_io; +#[cfg(feature = "std")] use std::fmt::{Debug, Display}; +#[cfg(feature = "std")] use serde::{Serialize, de::DeserializeOwned}; +use substrate_primitives; +use substrate_primitives::Blake2Hasher; +use codec::{Codec, Encode}; +pub use integer_sqrt::IntegerSquareRoot; +pub use num_traits::{Zero, One, Bounded}; +pub use num_traits::ops::checked::{CheckedAdd, CheckedSub, CheckedMul, CheckedDiv}; +use rstd::ops::{Add, Sub, Mul, Div, Rem, AddAssign, SubAssign, MulAssign, DivAssign, + RemAssign, Shl, Shr}; + +/// A lazy value. +pub trait Lazy { + fn get(&mut self) -> &T; +} + +impl<'a> Lazy<[u8]> for &'a [u8] { + fn get(&mut self) -> &[u8] { &**self } +} + +/// Means of signature verification. +pub trait Verify { + /// Type of the signer. + type Signer; + /// Verify a signature. Return `true` if signature is valid for the value. + fn verify>(&self, msg: L, signer: &Self::Signer) -> bool; +} + +/// Some sort of check on the origin is performed by this object. +pub trait EnsureOrigin { + type Success; + fn ensure_origin(o: OuterOrigin) -> Result; +} + +/// Means of changing one type into another in a manner dependent on the source type. +pub trait Lookup { + /// Type to lookup from. + type Source; + /// Type to lookup into. + type Target; + /// Attempt a lookup. + fn lookup(s: Self::Source) -> result::Result; +} + +/// Simple payment making trait, operating on a single generic `AccountId` type. +pub trait MakePayment { + /// Make some sort of payment concerning `who` for an extrinsic (transaction) of encoded length + /// `encoded_len` bytes. Return true iff the payment was successful. + fn make_payment(who: &AccountId, encoded_len: usize) -> Result<(), &'static str>; +} + +impl MakePayment for () { + fn make_payment(_: &T, _: usize) -> Result<(), &'static str> { Ok(()) } +} + +/// Extensible conversion trait. Generic over both source and destination types. +pub trait Convert { + /// Make conversion. + fn convert(a: A) -> B; +} + +/// Simple trait similar to `Into`, except that it can be used to convert numerics between each +/// other. +pub trait As { + /// Convert forward (ala `Into::into`). + fn as_(self) -> T; + /// Convert backward (ala `From::from`). + fn sa(T) -> Self; +} + +macro_rules! impl_numerics { + ( $( $t:ty ),* ) => { + $( + impl_numerics!($t: u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize,); + )* + }; + ( $f:ty : $t:ty, $( $rest:ty, )* ) => { + impl As<$t> for $f { + fn as_(self) -> $t { self as $t } + fn sa(t: $t) -> Self { t as Self } + } + impl_numerics!($f: $( $rest, )*); + }; + ( $f:ty : ) => {} +} + +impl_numerics!(u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize); + +pub struct Identity; +impl Convert for Identity { + fn convert(a: T) -> T { a } +} +impl Convert for () { + fn convert(_: T) -> () { () } +} + +pub trait RefInto { + fn ref_into(&self) -> &T; +} +impl RefInto for T { + fn ref_into(&self) -> &T { &self } +} + +pub trait SimpleArithmetic: + Zero + One + IntegerSquareRoot + As + + Add + AddAssign + + Sub + SubAssign + + Mul + MulAssign + + Div + DivAssign + + Rem + RemAssign + + Shl + Shr + + CheckedAdd + + CheckedSub + + CheckedMul + + CheckedDiv + + PartialOrd + Ord +{} +impl + + Add + AddAssign + + Sub + SubAssign + + Mul + MulAssign + + Div + DivAssign + + Rem + RemAssign + + Shl + Shr + + CheckedAdd + + CheckedSub + + CheckedMul + + CheckedDiv + + PartialOrd + Ord +> SimpleArithmetic for T {} + +/// Trait for things that can be clear (have no bits set). For numeric types, essentially the same +/// as `Zero`. +pub trait Clear { + /// True iff no bits are set. + fn is_clear(&self) -> bool; + + /// Return the value of Self that is clear. + fn clear() -> Self; +} + +impl Clear for T { + fn is_clear(&self) -> bool { *self == Self::clear() } + fn clear() -> Self { Default::default() } +} + +pub trait SimpleBitOps: + Sized + Clear + + rstd::ops::BitOr + + rstd::ops::BitAnd +{} +impl + + rstd::ops::BitAnd +> SimpleBitOps for T {} + +/// The block finalisation trait. Implementing this lets you express what should happen +/// for your module when the block is ending. +pub trait OnFinalise { + /// The block is being finalised. Implement to have something happen. + fn on_finalise(_n: BlockNumber) {} +} + +impl OnFinalise for () {} + +macro_rules! tuple_impl { + ($one:ident,) => { + impl> OnFinalise for ($one,) { + fn on_finalise(n: Number) { + $one::on_finalise(n); + } + } + }; + ($first:ident, $($rest:ident,)+) => { + impl< + Number: Copy, + $first: OnFinalise, + $($rest: OnFinalise),+ + > OnFinalise for ($first, $($rest),+) { + fn on_finalise(n: Number) { + $first::on_finalise(n); + $($rest::on_finalise(n);)+ + } + } + tuple_impl!($($rest,)+); + } +} + +#[allow(non_snake_case)] +tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z,); + +/// Abstraction around hashing +pub trait Hash: 'static + MaybeSerializeDebug + Clone + Eq + PartialEq { // Stupid bug in the Rust compiler believes derived + // traits must be fulfilled by all type parameters. + /// The hash type produced. + type Output: Member + AsRef<[u8]>; + + /// Produce the hash of some byte-slice. + fn hash(s: &[u8]) -> Self::Output; + + /// Produce the hash of some codec-encodable value. + fn hash_of(s: &S) -> Self::Output { + Encode::using_encoded(s, Self::hash) + } + + /// Produce the patricia-trie root of a mapping from indices to byte slices. + fn enumerated_trie_root(items: &[&[u8]]) -> Self::Output; + + /// Iterator-based version of `enumerated_trie_root`. + fn ordered_trie_root< + I: IntoIterator, + A: AsRef<[u8]> + >(input: I) -> Self::Output; + + /// The Patricia tree root of the given mapping as an iterator. + fn trie_root< + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]> + >(input: I) -> Self::Output; + + /// Acquire the global storage root. + fn storage_root() -> Self::Output; +} + +/// Blake2-256 Hash implementation. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +pub struct BlakeTwo256; + +impl Hash for BlakeTwo256 { + type Output = substrate_primitives::H256; + fn hash(s: &[u8]) -> Self::Output { + runtime_io::blake2_256(s).into() + } + fn enumerated_trie_root(items: &[&[u8]]) -> Self::Output { + runtime_io::enumerated_trie_root::(items).into() + } + fn trie_root< + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]> + >(input: I) -> Self::Output { + runtime_io::trie_root::(input).into() + } + fn ordered_trie_root< + I: IntoIterator, + A: AsRef<[u8]> + >(input: I) -> Self::Output { + runtime_io::ordered_trie_root::(input).into() + } + fn storage_root() -> Self::Output { + runtime_io::storage_root().into() + } +} + +/// Something that can be checked for equality and printed out to a debug channel if bad. +pub trait CheckEqual { + fn check_equal(&self, other: &Self); +} + +impl CheckEqual for substrate_primitives::H256 { + #[cfg(feature = "std")] + fn check_equal(&self, other: &Self) { + use substrate_primitives::hexdisplay::HexDisplay; + if &self.0 != &other.0 { + println!("Hash: given={}, expected={}", HexDisplay::from(&self.0), HexDisplay::from(&other.0)); + } + } + + #[cfg(not(feature = "std"))] + fn check_equal(&self, other: &Self) { + if self != other { + runtime_io::print("Hash not equal"); + runtime_io::print(&self.0[..]); + runtime_io::print(&other.0[..]); + } + } +} + +#[cfg(feature = "std")] +pub trait MaybeSerializeDebugButNotDeserialize: Serialize + Debug {} +#[cfg(feature = "std")] +impl MaybeSerializeDebugButNotDeserialize for T {} + +#[cfg(not(feature = "std"))] +pub trait MaybeSerializeDebugButNotDeserialize {} +#[cfg(not(feature = "std"))] +impl MaybeSerializeDebugButNotDeserialize for T {} + +#[cfg(feature = "std")] +pub trait MaybeSerializeDebug: Serialize + DeserializeOwned + Debug {} +#[cfg(feature = "std")] +impl MaybeSerializeDebug for T {} + +#[cfg(not(feature = "std"))] +pub trait MaybeSerializeDebug {} +#[cfg(not(feature = "std"))] +impl MaybeSerializeDebug for T {} + +#[cfg(feature = "std")] +pub trait MaybeDisplay: Display {} +#[cfg(feature = "std")] +impl MaybeDisplay for T {} + +#[cfg(not(feature = "std"))] +pub trait MaybeDisplay {} +#[cfg(not(feature = "std"))] +impl MaybeDisplay for T {} + +pub trait Member: Send + Sync + Sized + MaybeSerializeDebug + Eq + PartialEq + Clone + 'static {} +impl Member for T {} + +/// Something which fulfills the abstract idea of a Substrate header. It has types for a `Number`, +/// a `Hash` and a `Digest`. It provides access to an `extrinsics_root`, `state_root` and +/// `parent_hash`, as well as a `digest` and a block `number`. +/// +/// You can also create a `new` one from those fields. +pub trait Header: Clone + Send + Sync + Codec + Eq + MaybeSerializeDebug + 'static { + type Number: Member + ::rstd::hash::Hash + Copy + MaybeDisplay + SimpleArithmetic + Codec; + type Hash: Member + ::rstd::hash::Hash + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]>; + type Hashing: Hash; + type Digest: Digest; + + fn new( + number: Self::Number, + extrinsics_root: Self::Hash, + state_root: Self::Hash, + parent_hash: Self::Hash, + digest: Self::Digest + ) -> Self; + + fn number(&self) -> &Self::Number; + fn set_number(&mut self, Self::Number); + + fn extrinsics_root(&self) -> &Self::Hash; + fn set_extrinsics_root(&mut self, Self::Hash); + + fn state_root(&self) -> &Self::Hash; + fn set_state_root(&mut self, Self::Hash); + + fn parent_hash(&self) -> &Self::Hash; + fn set_parent_hash(&mut self, Self::Hash); + + fn digest(&self) -> &Self::Digest; + fn set_digest(&mut self, Self::Digest); + + fn hash(&self) -> Self::Hash { + ::hash_of(self) + } +} + +/// Something which fulfills the abstract idea of a Substrate block. It has types for an +/// `Extrinsic` piece of information as well as a `Header`. +/// +/// You can get an iterator over each of the `extrinsics` and retrieve the `header`. +pub trait Block: Clone + Send + Sync + Codec + Eq + MaybeSerializeDebug + 'static { + type Extrinsic: Member + Codec; + type Header: Header; + type Hash: Member + ::rstd::hash::Hash + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]>; + + fn header(&self) -> &Self::Header; + fn extrinsics(&self) -> &[Self::Extrinsic]; + fn deconstruct(self) -> (Self::Header, Vec); + fn new(header: Self::Header, extrinsics: Vec) -> Self; + fn hash(&self) -> Self::Hash { + <::Hashing as Hash>::hash_of(self.header()) + } +} + +/// Extract the hashing type for a block. +pub type HashFor = <::Header as Header>::Hashing; +/// Extract the number type for a block. +pub type NumberFor = <::Header as Header>::Number; + +/// A "checkable" piece of information, used by the standard Substrate Executive in order to +/// check the validity of a piece of extrinsic information, usually by verifying the signature. +/// Implement for pieces of information that require some additional context `Context` in order to be +/// checked. +pub trait Checkable: Sized { + /// Returned if `check_with` succeeds. + type Checked; + + fn check_with(self, context: Context) -> Result; +} + +/// A "checkable" piece of information, used by the standard Substrate Executive in order to +/// check the validity of a piece of extrinsic information, usually by verifying the signature. +/// Implement for pieces of information that don't require additional context in order to be +/// checked. +pub trait BlindCheckable: Sized { + /// Returned if `check` succeeds. + type Checked; + + fn check(self) -> Result; +} + +// Every `BlindCheckable` is also a `Checkable` for arbitrary `Context`. +impl Checkable for T { + type Checked = ::Checked; + fn check_with(self, _: Context) -> Result { + BlindCheckable::check(self) + } +} + +/// An "executable" piece of information, used by the standard Substrate Executive in order to +/// enact a piece of extrinsic information by marshalling and dispatching to a named functioon +/// call. +/// +/// Also provides information on to whom this information is attributable and an index that allows +/// each piece of attributable information to be disambiguated. +pub trait Applyable: Sized + Send + Sync { + type AccountId: Member + MaybeDisplay; + type Index: Member + MaybeDisplay + SimpleArithmetic; + fn index(&self) -> &Self::Index; + fn sender(&self) -> Option<&Self::AccountId>; + fn apply(self) -> Result<(), &'static str>; +} + +/// Something that acts like a `Digest` - it can have `Log`s `push`ed onto it and these `Log`s are +/// each `Codec`. +pub trait Digest: Member + Default { + type Item: DigestItem; + fn logs(&self) -> &[Self::Item]; + fn push(&mut self, item: Self::Item); +} + +/// Single digest item. Could be any type that implements `Member` and provides methods +/// for casting member to 'system' log items, known to substrate. +/// +/// If the runtime does not supports some 'system' items, use `()` as a stub. +pub trait DigestItem: Member { + type AuthorityId; + + /// Returns Some if the entry is the `AuthoritiesChange` entry. + fn as_authorities_change(&self) -> Option<&[Self::AuthorityId]> { + None + } +} diff --git a/runtime/session/Cargo.toml b/runtime/session/Cargo.toml new file mode 100644 index 00000000000..9cffec07b8d --- /dev/null +++ b/runtime/session/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "substrate-runtime-session" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +safe-mix = { version = "1.0", default_features = false} +substrate-keyring = { path = "../../core/keyring", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-runtime-consensus = { path = "../consensus", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } +substrate-runtime-timestamp = { path = "../timestamp", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "safe-mix/std", + "substrate-keyring", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-consensus/std", + "substrate-runtime-system/std", + "substrate-runtime-timestamp/std" +] diff --git a/runtime/session/src/lib.rs b/runtime/session/src/lib.rs new file mode 100644 index 00000000000..007dd43729f --- /dev/null +++ b/runtime/session/src/lib.rs @@ -0,0 +1,467 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Session manager: is told the validators and allows them to manage their session keys for the +//! consensus module. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[cfg(any(feature = "std", test))] +extern crate substrate_keyring as keyring; + +#[cfg(any(feature = "std", test))] +extern crate substrate_primitives; + +#[cfg_attr(feature = "std", macro_use)] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_codec as codec; +extern crate substrate_runtime_primitives as primitives; +extern crate substrate_runtime_consensus as consensus; +extern crate substrate_runtime_system as system; +extern crate substrate_runtime_timestamp as timestamp; + +use rstd::prelude::*; +use primitives::traits::{Zero, One, OnFinalise, Convert, As}; +use runtime_support::{StorageValue, StorageMap}; +use runtime_support::dispatch::Result; +use system::{ensure_signed, ensure_root}; + +#[cfg(any(feature = "std", test))] +use std::collections::HashMap; + +/// A session has changed. +pub trait OnSessionChange { + /// Session has changed. + fn on_session_change(time_elapsed: T, should_reward: bool); +} + +impl OnSessionChange for () { + fn on_session_change(_: T, _: bool) {} +} + +pub trait Trait: timestamp::Trait { + type ConvertAccountIdToSessionKey: Convert; + type OnSessionChange: OnSessionChange; + type Event: From> + Into<::Event>; +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn set_key(origin, key: T::SessionKey) -> Result; + + fn set_length(origin, new: T::BlockNumber) -> Result; + fn force_new_session(origin, apply_rewards: bool) -> Result; + } +} + +/// An event in this module. +decl_event!( + pub enum Event with RawEvent + where ::BlockNumber + { + /// New session has happened. Note that the argument is the session index, not the block + /// number as the type might suggest. + NewSession(BlockNumber), + } +); + +decl_storage! { + trait Store for Module as Session { + + /// The current set of validators. + pub Validators get(validators): required Vec; + /// Current length of the session. + pub SessionLength get(length): required T::BlockNumber; + /// Current index of the session. + pub CurrentIndex get(current_index): required T::BlockNumber; + /// Timestamp when current session started. + pub CurrentStart get(current_start): required T::Moment; + + /// New session is being forced is this entry exists; in which case, the boolean value is whether + /// the new session should be considered a normal rotation (rewardable) or exceptional (slashable). + pub ForcingNewSession get(forcing_new_session): bool; + /// Block at which the session length last changed. + LastLengthChange: T::BlockNumber; + /// The next key for a given validator. + NextKeyFor: map [ T::AccountId => T::SessionKey ]; + /// The next session length. + NextSessionLength: T::BlockNumber; + } +} + +impl Module { + + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + /// The number of validators currently. + pub fn validator_count() -> u32 { + >::get().len() as u32 // TODO: can probably optimised + } + + /// The last length change, if there was one, zero if not. + pub fn last_length_change() -> T::BlockNumber { + >::get().unwrap_or_else(T::BlockNumber::zero) + } + + /// Sets the session key of `_validator` to `_key`. This doesn't take effect until the next + /// session. + fn set_key(origin: T::Origin, key: T::SessionKey) -> Result { + let who = ensure_signed(origin)?; + // set new value for next session + >::insert(who, key); + Ok(()) + } + + /// Set a new era length. Won't kick in until the next era change (at current length). + fn set_length(origin: T::Origin, new: T::BlockNumber) -> Result { + ensure_root(origin)?; + >::put(new); + Ok(()) + } + + /// Forces a new session. + pub fn force_new_session(origin: T::Origin, apply_rewards: bool) -> Result { + ensure_root(origin)?; + Self::apply_force_new_session(apply_rewards) + } + + // INTERNAL API (available to other runtime modules) + + /// Forces a new session, no origin. + pub fn apply_force_new_session(apply_rewards: bool) -> Result { + >::put(apply_rewards); + Ok(()) + } + + /// Set the current set of validators. + /// + /// Called by `staking::next_era()` only. `next_session` should be called after this in order to + /// update the session keys to the next validator set. + pub fn set_validators(new: &[T::AccountId]) { + >::put(&new.to_vec()); // TODO: optimise. + >::set_authorities( + &new.iter().cloned().map(T::ConvertAccountIdToSessionKey::convert).collect::>() + ); + } + + /// Hook to be called after transaction processing. + pub fn check_rotate_session(block_number: T::BlockNumber) { + // do this last, after the staking system has had chance to switch out the authorities for the + // new set. + // check block number and call next_session if necessary. + let is_final_block = ((block_number - Self::last_length_change()) % Self::length()).is_zero(); + let (should_end_session, apply_rewards) = >::take() + .map_or((is_final_block, is_final_block), |apply_rewards| (true, apply_rewards)); + if should_end_session { + Self::rotate_session(is_final_block, apply_rewards); + } + } + + /// Move onto next session: register the new authority set. + pub fn rotate_session(is_final_block: bool, apply_rewards: bool) { + let now = >::get(); + let time_elapsed = now.clone() - Self::current_start(); + let session_index = >::get() + One::one(); + + Self::deposit_event(RawEvent::NewSession(session_index)); + + // Increment current session index. + >::put(session_index); + >::put(now); + + // Enact era length change. + let len_changed = if let Some(next_len) = >::take() { + >::put(next_len); + true + } else { + false + }; + if len_changed || !is_final_block { + let block_number = >::block_number(); + >::put(block_number); + } + + T::OnSessionChange::on_session_change(time_elapsed, apply_rewards); + + // Update any changes in session keys. + Self::validators().iter().enumerate().for_each(|(i, v)| { + if let Some(n) = >::take(v) { + >::set_authority(i as u32, &n); + } + }); + } + + /// Get the time that should have elapsed over a session if everything was working perfectly. + pub fn ideal_session_duration() -> T::Moment { + let block_period = >::block_period(); + let session_length = >::sa(Self::length()); + session_length * block_period + } + + /// Number of blocks remaining in this session, not counting this one. If the session is + /// due to rotate at the end of this block, then it will return 0. If the just began, then + /// it will return `Self::length() - 1`. + pub fn blocks_remaining() -> T::BlockNumber { + let length = Self::length(); + let length_minus_1 = length - One::one(); + let block_number = >::block_number(); + length_minus_1 - (block_number - Self::last_length_change() + length_minus_1) % length + } +} + +impl OnFinalise for Module { + fn on_finalise(n: T::BlockNumber) { + Self::check_rotate_session(n); + } +} + +#[cfg(any(feature = "std", test))] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + pub session_length: T::BlockNumber, + pub validators: Vec, +} + +#[cfg(any(feature = "std", test))] +impl Default for GenesisConfig { + fn default() -> Self { + use primitives::traits::As; + GenesisConfig { + session_length: T::BlockNumber::sa(1000), + validators: vec![], + } + } +} + +#[cfg(any(feature = "std", test))] +impl primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> ::std::result::Result, Vec>, String> { + + use codec::Encode; + use primitives::traits::As; + Ok(map![ + Self::hash(>::key()).to_vec() => self.session_length.encode(), + Self::hash(>::key()).to_vec() => T::BlockNumber::sa(0).encode(), + Self::hash(>::key()).to_vec() => T::Moment::zero().encode(), + Self::hash(>::key()).to_vec() => self.validators.encode() + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use runtime_io::with_externalities; + use substrate_primitives::{H256, Blake2Hasher}; + use primitives::BuildStorage; + use primitives::traits::{Identity, BlakeTwo256}; + use primitives::testing::{Digest, Header}; + + impl_outer_origin!{ + pub enum Origin for Test {} + } + + #[derive(Clone, Eq, PartialEq)] + pub struct Test; + impl consensus::Trait for Test { + const NOTE_OFFLINE_POSITION: u32 = 1; + type Log = u64; + type SessionKey = u64; + type OnOfflineValidator = (); + } + impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); + } + impl timestamp::Trait for Test { + const TIMESTAMP_SET_POSITION: u32 = 0; + type Moment = u64; + } + impl Trait for Test { + type ConvertAccountIdToSessionKey = Identity; + type OnSessionChange = (); + type Event = (); + } + + type System = system::Module; + type Consensus = consensus::Module; + type Session = Module; + + fn new_test_ext() -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(consensus::GenesisConfig::{ + code: vec![], + authorities: vec![1, 2, 3], + }.build_storage().unwrap()); + t.extend(timestamp::GenesisConfig::{ + period: 5, + }.build_storage().unwrap()); + t.extend(GenesisConfig::{ + session_length: 2, + validators: vec![1, 2, 3], + }.build_storage().unwrap()); + t.into() + } + + #[test] + fn simple_setup_should_work() { + with_externalities(&mut new_test_ext(), || { + assert_eq!(Consensus::authorities(), vec![1, 2, 3]); + assert_eq!(Session::length(), 2); + assert_eq!(Session::validators(), vec![1, 2, 3]); + }); + } + + #[test] + fn should_work_with_early_exit() { + with_externalities(&mut new_test_ext(), || { + System::set_block_number(1); + assert_ok!(Session::set_length(Origin::ROOT, 10)); + assert_eq!(Session::blocks_remaining(), 1); + Session::check_rotate_session(1); + + System::set_block_number(2); + assert_eq!(Session::blocks_remaining(), 0); + Session::check_rotate_session(2); + assert_eq!(Session::length(), 10); + + System::set_block_number(7); + assert_eq!(Session::current_index(), 1); + assert_eq!(Session::blocks_remaining(), 5); + assert_ok!(Session::force_new_session(Origin::ROOT, false)); + Session::check_rotate_session(7); + + System::set_block_number(8); + assert_eq!(Session::current_index(), 2); + assert_eq!(Session::blocks_remaining(), 9); + Session::check_rotate_session(8); + + System::set_block_number(17); + assert_eq!(Session::current_index(), 2); + assert_eq!(Session::blocks_remaining(), 0); + Session::check_rotate_session(17); + + System::set_block_number(18); + assert_eq!(Session::current_index(), 3); + }); + } + + #[test] + fn session_length_change_should_work() { + with_externalities(&mut new_test_ext(), || { + // Block 1: Change to length 3; no visible change. + System::set_block_number(1); + assert_ok!(Session::set_length(Origin::ROOT, 3)); + Session::check_rotate_session(1); + assert_eq!(Session::length(), 2); + assert_eq!(Session::current_index(), 0); + + // Block 2: Length now changed to 3. Index incremented. + System::set_block_number(2); + assert_ok!(Session::set_length(Origin::ROOT, 3)); + Session::check_rotate_session(2); + assert_eq!(Session::length(), 3); + assert_eq!(Session::current_index(), 1); + + // Block 3: Length now changed to 3. Index incremented. + System::set_block_number(3); + Session::check_rotate_session(3); + assert_eq!(Session::length(), 3); + assert_eq!(Session::current_index(), 1); + + // Block 4: Change to length 2; no visible change. + System::set_block_number(4); + assert_ok!(Session::set_length(Origin::ROOT, 2)); + Session::check_rotate_session(4); + assert_eq!(Session::length(), 3); + assert_eq!(Session::current_index(), 1); + + // Block 5: Length now changed to 2. Index incremented. + System::set_block_number(5); + Session::check_rotate_session(5); + assert_eq!(Session::length(), 2); + assert_eq!(Session::current_index(), 2); + + // Block 6: No change. + System::set_block_number(6); + Session::check_rotate_session(6); + assert_eq!(Session::length(), 2); + assert_eq!(Session::current_index(), 2); + + // Block 7: Next index. + System::set_block_number(7); + Session::check_rotate_session(7); + assert_eq!(Session::length(), 2); + assert_eq!(Session::current_index(), 3); + }); + } + + #[test] + fn session_change_should_work() { + with_externalities(&mut new_test_ext(), || { + // Block 1: No change + System::set_block_number(1); + Session::check_rotate_session(1); + assert_eq!(Consensus::authorities(), vec![1, 2, 3]); + + // Block 2: Session rollover, but no change. + System::set_block_number(2); + Session::check_rotate_session(2); + assert_eq!(Consensus::authorities(), vec![1, 2, 3]); + + // Block 3: Set new key for validator 2; no visible change. + System::set_block_number(3); + assert_ok!(Session::set_key(Origin::signed(2), 5)); + assert_eq!(Consensus::authorities(), vec![1, 2, 3]); + + Session::check_rotate_session(3); + assert_eq!(Consensus::authorities(), vec![1, 2, 3]); + + // Block 4: Session rollover, authority 2 changes. + System::set_block_number(4); + Session::check_rotate_session(4); + assert_eq!(Consensus::authorities(), vec![1, 5, 3]); + }); + } +} diff --git a/runtime/staking/Cargo.toml b/runtime/staking/Cargo.toml new file mode 100644 index 00000000000..84a14febae3 --- /dev/null +++ b/runtime/staking/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "substrate-runtime-staking" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +safe-mix = { version = "1.0", default_features = false} +substrate-keyring = { path = "../../core/keyring", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-sandbox = { path = "../../core/runtime-sandbox", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-runtime-balances = { path = "../balances", default_features = false } +substrate-runtime-consensus = { path = "../consensus", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } +substrate-runtime-session = { path = "../session", default_features = false } +substrate-runtime-timestamp = { path = "../timestamp", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "safe-mix/std", + "substrate-keyring", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-sandbox/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-balances/std", + "substrate-runtime-session/std", + "substrate-runtime-system/std", + "substrate-runtime-timestamp/std" +] diff --git a/runtime/staking/src/genesis_config.rs b/runtime/staking/src/genesis_config.rs new file mode 100644 index 00000000000..e2b50ca1229 --- /dev/null +++ b/runtime/staking/src/genesis_config.rs @@ -0,0 +1,77 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Build a staking genesis block. + +#![cfg(feature = "std")] + +use std::collections::HashMap; +use rstd::prelude::*; +use codec::Encode; +use runtime_support::StorageValue; +use primitives::traits::As; +use substrate_primitives::Blake2Hasher; +use {runtime_io, primitives}; +use super::{Trait, Intentions, CurrentEra, OfflineSlashGrace, MinimumValidatorCount, + BondingDuration, SessionsPerEra, ValidatorCount, SessionReward, OfflineSlash}; + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + pub sessions_per_era: T::BlockNumber, + pub current_era: T::BlockNumber, + pub intentions: Vec, + pub validator_count: u32, + pub minimum_validator_count: u32, + pub bonding_duration: T::BlockNumber, + pub session_reward: T::Balance, + pub offline_slash: T::Balance, + pub offline_slash_grace: u32, +} + +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + sessions_per_era: T::BlockNumber::sa(1000), + current_era: T::BlockNumber::sa(0), + intentions: vec![], + validator_count: 0, + minimum_validator_count: 0, + bonding_duration: T::BlockNumber::sa(1000), + session_reward: T::Balance::sa(0), + offline_slash: T::Balance::sa(0), + offline_slash_grace: 0, + } + } +} + +impl primitives::BuildStorage for GenesisConfig { + fn build_storage(self) -> ::std::result::Result, Vec>, String> { + let r: runtime_io::TestExternalities = map![ + Self::hash(>::key()).to_vec() => self.intentions.encode(), + Self::hash(>::key()).to_vec() => self.sessions_per_era.encode(), + Self::hash(>::key()).to_vec() => self.validator_count.encode(), + Self::hash(>::key()).to_vec() => self.minimum_validator_count.encode(), + Self::hash(>::key()).to_vec() => self.bonding_duration.encode(), + Self::hash(>::key()).to_vec() => self.current_era.encode(), + Self::hash(>::key()).to_vec() => self.session_reward.encode(), + Self::hash(>::key()).to_vec() => self.offline_slash.encode(), + Self::hash(>::key()).to_vec() => self.offline_slash_grace.encode() + ]; + Ok(r.into()) + } +} diff --git a/runtime/staking/src/lib.rs b/runtime/staking/src/lib.rs new file mode 100644 index 00000000000..f9737d6a3c2 --- /dev/null +++ b/runtime/staking/src/lib.rs @@ -0,0 +1,576 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + + + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Staking manager: Periodically determines the best set of validators. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +#[cfg_attr(feature = "std", macro_use)] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_codec as codec; +extern crate substrate_primitives; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_primitives as primitives; +extern crate substrate_runtime_balances as balances; +extern crate substrate_runtime_consensus as consensus; +extern crate substrate_runtime_sandbox as sandbox; +extern crate substrate_runtime_session as session; +extern crate substrate_runtime_system as system; +extern crate substrate_runtime_timestamp as timestamp; + +use rstd::prelude::*; +use runtime_support::{Parameter, StorageValue, StorageMap}; +use runtime_support::dispatch::Result; +use session::OnSessionChange; +use primitives::traits::{Zero, One, Bounded, OnFinalise, + As, Lookup}; +use balances::{address::Address, OnDilution}; +use system::{ensure_root, ensure_signed}; + +mod mock; + +mod tests; +mod genesis_config; + +#[cfg(feature = "std")] +pub use genesis_config::GenesisConfig; + +const DEFAULT_MINIMUM_VALIDATOR_COUNT: usize = 4; + +#[derive(PartialEq, Clone)] +#[cfg_attr(test, derive(Debug))] +pub enum LockStatus { + Liquid, + LockedUntil(BlockNumber), + Bonded, +} + +/// Preference of what happens on a slash event. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +pub struct ValidatorPrefs { + /// Validator should ensure this many more slashes than is necessary before being unstaked. + pub unstake_threshold: u32, + // Reward that validator takes up-front; only the rest is split between themself and nominators. + pub validator_payment: Balance, +} + +impl Default for ValidatorPrefs { + fn default() -> Self { + ValidatorPrefs { + unstake_threshold: 3, + validator_payment: Default::default(), + } + } +} + +pub trait Trait: balances::Trait + session::Trait { + /// Some tokens minted. + type OnRewardMinted: OnDilution<::Balance>; + + /// The overarching event type. + type Event: From> + Into<::Event>; +} + +decl_module! { + #[cfg_attr(feature = "std", serde(bound(deserialize = "T::Balance: ::serde::de::DeserializeOwned")))] + pub struct Module for enum Call where origin: T::Origin { + fn stake(origin) -> Result; + fn unstake(origin, intentions_index: u32) -> Result; + fn nominate(origin, target: Address) -> Result; + fn unnominate(origin, target_index: u32) -> Result; + fn register_preferences(origin, intentions_index: u32, prefs: ValidatorPrefs) -> Result; + + fn set_sessions_per_era(origin, new: T::BlockNumber) -> Result; + fn set_bonding_duration(origin, new: T::BlockNumber) -> Result; + fn set_validator_count(origin, new: u32) -> Result; + fn force_new_era(origin, apply_rewards: bool) -> Result; + fn set_offline_slash_grace(origin, new: u32) -> Result; + } +} + +/// An event in this module. +decl_event!( + pub enum Event with RawEvent + where ::Balance, ::AccountId + { + /// All validators have been rewarded by the given balance. + Reward(Balance), + /// One validator (and their nominators) has been given a offline-warning (they're still + /// within their grace). The accrued number of slashes is recorded, too. + OfflineWarning(AccountId, u32), + /// One validator (and their nominators) has been slashed by the given amount. + OfflineSlash(AccountId, Balance), + } +); + +pub type PairOf = (T, T); + +decl_storage! { + trait Store for Module as Staking { + + /// The ideal number of staking participants. + pub ValidatorCount get(validator_count): required u32; + /// Minimum number of staking participants before emergency conditions are imposed. + pub MinimumValidatorCount: u32; + /// The length of a staking era in sessions. + pub SessionsPerEra get(sessions_per_era): required T::BlockNumber; + /// Maximum reward, per validator, that is provided per acceptable session. + pub SessionReward get(session_reward): required T::Balance; + /// Slash, per validator that is taken for the first time they are found to be offline. + pub OfflineSlash get(offline_slash): required T::Balance; + /// Number of instances of offline reports before slashing begins for validators. + pub OfflineSlashGrace get(offline_slash_grace): default u32; + /// The length of the bonding duration in blocks. + pub BondingDuration get(bonding_duration): required T::BlockNumber; + + /// The current era index. + pub CurrentEra get(current_era): required T::BlockNumber; + /// Preferences that a validator has. + pub ValidatorPreferences get(validator_preferences): default map [ T::AccountId => ValidatorPrefs ]; + /// All the accounts with a desire to stake. + pub Intentions get(intentions): default Vec; + /// All nominator -> nominee relationships. + pub Nominating get(nominating): map [ T::AccountId => T::AccountId ]; + /// Nominators for a particular account. + pub NominatorsFor get(nominators_for): default map [ T::AccountId => Vec ]; + /// Nominators for a particular account that is in action right now. + pub CurrentNominatorsFor get(current_nominators_for): default map [ T::AccountId => Vec ]; + /// The next value of sessions per era. + pub NextSessionsPerEra get(next_sessions_per_era): T::BlockNumber; + /// The session index at which the era length last changed. + pub LastEraLengthChange get(last_era_length_change): default T::BlockNumber; + + /// The highest and lowest staked validator slashable balances. + pub StakeRange get(stake_range): default PairOf; + + /// The block at which the `who`'s funds become entirely liquid. + pub Bondage get(bondage): default map [ T::AccountId => T::BlockNumber ]; + /// The number of times a given validator has been reported offline. This gets decremented by one each era that passes. + pub SlashCount get(slash_count): default map [ T::AccountId => u32 ]; + + /// We are forcing a new era. + pub ForcingNewEra get(forcing_new_era): (); + } +} + +impl Module { + + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + // PUBLIC IMMUTABLES + + /// MinimumValidatorCount getter, introduces a default. + pub fn minimum_validator_count() -> usize { + >::get().map(|v| v as usize).unwrap_or(DEFAULT_MINIMUM_VALIDATOR_COUNT) + } + + /// The length of a staking era in blocks. + pub fn era_length() -> T::BlockNumber { + Self::sessions_per_era() * >::length() + } + + /// Balance of a (potential) validator that includes all nominators. + pub fn nomination_balance(who: &T::AccountId) -> T::Balance { + Self::nominators_for(who).iter() + .map(>::total_balance) + .fold(Zero::zero(), |acc, x| acc + x) + } + + /// The total balance that can be slashed from an account. + pub fn slashable_balance(who: &T::AccountId) -> T::Balance { + Self::nominators_for(who).iter() + .map(>::total_balance) + .fold(>::total_balance(who), |acc, x| acc + x) + } + + /// The block at which the `who`'s funds become entirely liquid. + pub fn unlock_block(who: &T::AccountId) -> LockStatus { + match Self::bondage(who) { + i if i == T::BlockNumber::max_value() => LockStatus::Bonded, + i if i <= >::block_number() => LockStatus::Liquid, + i => LockStatus::LockedUntil(i), + } + } + + // PUBLIC DISPATCH + + /// Declare the desire to stake for the transactor. + /// + /// Effects will be felt at the beginning of the next era. + fn stake(origin: T::Origin) -> Result { + let who = ensure_signed(origin)?; + ensure!(Self::nominating(&who).is_none(), "Cannot stake if already nominating."); + let mut intentions = >::get(); + // can't be in the list twice. + ensure!(intentions.iter().find(|&t| t == &who).is_none(), "Cannot stake if already staked."); + + >::insert(&who, T::BlockNumber::max_value()); + intentions.push(who); + >::put(intentions); + Ok(()) + } + + /// Retract the desire to stake for the transactor. + /// + /// Effects will be felt at the beginning of the next era. + fn unstake(origin: T::Origin, intentions_index: u32) -> Result { + let who = ensure_signed(origin)?; + // unstake fails in degenerate case of having too few existing staked parties + if Self::intentions().len() <= Self::minimum_validator_count() { + return Err("cannot unstake when there are too few staked participants") + } + Self::apply_unstake(&who, intentions_index as usize) + } + + fn nominate(origin: T::Origin, target: Address) -> Result { + let who = ensure_signed(origin)?; + let target = >::lookup(target)?; + + ensure!(Self::nominating(&who).is_none(), "Cannot nominate if already nominating."); + ensure!(Self::intentions().iter().find(|&t| t == &who).is_none(), "Cannot nominate if already staked."); + + // update nominators_for + let mut t = Self::nominators_for(&target); + t.push(who.clone()); + >::insert(&target, t); + + // update nominating + >::insert(&who, &target); + + // Update bondage + >::insert(&who, T::BlockNumber::max_value()); + + Ok(()) + } + + /// Will panic if called when source isn't currently nominating target. + /// Updates Nominating, NominatorsFor and NominationBalance. + fn unnominate(origin: T::Origin, target_index: u32) -> Result { + let source = ensure_signed(origin)?; + let target_index = target_index as usize; + + let target = >::get(&source).ok_or("Account must be nominating")?; + + let mut t = Self::nominators_for(&target); + if t.get(target_index) != Some(&source) { + return Err("Invalid target index") + } + + // Ok - all valid. + + // update nominators_for + t.swap_remove(target_index); + >::insert(&target, t); + + // update nominating + >::remove(&source); + + // update bondage + >::insert(source, >::block_number() + Self::bonding_duration()); + Ok(()) + } + + /// Set the given account's preference for slashing behaviour should they be a validator. + /// + /// An error (no-op) if `Self::intentions()[intentions_index] != origin`. + fn register_preferences( + origin: T::Origin, + intentions_index: u32, + prefs: ValidatorPrefs + ) -> Result { + let who = ensure_signed(origin)?; + + if Self::intentions().get(intentions_index as usize) != Some(&who) { + return Err("Invalid index") + } + + >::insert(who, prefs); + + Ok(()) + } + + // PRIV DISPATCH + + /// Set the number of sessions in an era. + fn set_sessions_per_era(origin: T::Origin, new: T::BlockNumber) -> Result { + ensure_root(origin)?; + >::put(&new); + Ok(()) + } + + /// The length of the bonding duration in eras. + fn set_bonding_duration(origin: T::Origin, new: T::BlockNumber) -> Result { + ensure_root(origin)?; + >::put(&new); + Ok(()) + } + + /// The length of a staking era in sessions. + fn set_validator_count(origin: T::Origin, new: u32) -> Result { + ensure_root(origin)?; + >::put(&new); + Ok(()) + } + + /// Force there to be a new era. This also forces a new session immediately after. + /// `apply_rewards` should be true for validators to get the session reward. + fn force_new_era(origin: T::Origin, apply_rewards: bool) -> Result { + ensure_root(origin)?; + Self::apply_force_new_era(apply_rewards) + } + + // Just force_new_era without origin check. + fn apply_force_new_era(apply_rewards: bool) -> Result { + >::put(()); + >::apply_force_new_session(apply_rewards) + } + + + /// Set the offline slash grace period. + fn set_offline_slash_grace(origin: T::Origin, new: u32) -> Result { + ensure_root(origin)?; + >::put(&new); + Ok(()) + } + + // PUBLIC MUTABLES (DANGEROUS) + + /// Slash a given validator by a specific amount. Removes the slash from their balance by preference, + /// and reduces the nominators' balance if needed. + fn slash_validator(v: &T::AccountId, slash: T::Balance) { + // skip the slash in degenerate case of having only 4 staking participants despite having a larger + // desired number of validators (validator_count). + if Self::intentions().len() <= Self::minimum_validator_count() { + return + } + + if let Some(rem) = >::slash(v, slash) { + let noms = Self::current_nominators_for(v); + let total = noms.iter().map(>::total_balance).fold(T::Balance::zero(), |acc, x| acc + x); + if !total.is_zero() { + let safe_mul_rational = |b| b * rem / total;// TODO: avoid overflow + for n in noms.iter() { + let _ = >::slash(n, safe_mul_rational(>::total_balance(n))); // best effort - not much that can be done on fail. + } + } + } + } + + /// Reward a given validator by a specific amount. Add the reward to their, and their nominators' + /// balance, pro-rata. + fn reward_validator(who: &T::AccountId, reward: T::Balance) { + let off_the_table = reward.min(Self::validator_preferences(who).validator_payment); + let reward = reward - off_the_table; + let validator_cut = if reward.is_zero() { + Zero::zero() + } else { + let noms = Self::current_nominators_for(who); + let total = noms.iter() + .map(>::total_balance) + .fold(>::total_balance(who), |acc, x| acc + x) + .max(One::one()); + let safe_mul_rational = |b| b * reward / total;// TODO: avoid overflow + for n in noms.iter() { + let _ = >::reward(n, safe_mul_rational(>::total_balance(n))); + } + safe_mul_rational(>::total_balance(who)) + }; + let _ = >::reward(who, validator_cut + off_the_table); + } + + /// Actually carry out the unstake operation. + /// Assumes `intentions()[intentions_index] == who`. + fn apply_unstake(who: &T::AccountId, intentions_index: usize) -> Result { + let mut intentions = Self::intentions(); + if intentions.get(intentions_index) != Some(who) { + return Err("Invalid index"); + } + intentions.swap_remove(intentions_index); + >::put(intentions); + >::remove(who); + >::remove(who); + >::insert(who, >::block_number() + Self::bonding_duration()); + Ok(()) + } + + /// Get the reward for the session, assuming it ends with this block. + fn this_session_reward(actual_elapsed: T::Moment) -> T::Balance { + let ideal_elapsed = >::ideal_session_duration(); + let per65536: u64 = (T::Moment::sa(65536u64) * ideal_elapsed.clone() / actual_elapsed.max(ideal_elapsed)).as_(); + Self::session_reward() * T::Balance::sa(per65536) / T::Balance::sa(65536u64) + } + + /// Session has just changed. We need to determine whether we pay a reward, slash and/or + /// move to a new era. + fn new_session(actual_elapsed: T::Moment, should_reward: bool) { + if should_reward { + // apply good session reward + let reward = Self::this_session_reward(actual_elapsed); + let validators = >::validators(); + for v in validators.iter() { + Self::reward_validator(v, reward); + } + Self::deposit_event(RawEvent::Reward(reward)); + let total_minted = reward * >::sa(validators.len()); + let total_rewarded_stake = Self::stake_range().0 * >::sa(validators.len()); + T::OnRewardMinted::on_dilution(total_minted, total_rewarded_stake); + } + + let session_index = >::current_index(); + if >::take().is_some() + || ((session_index - Self::last_era_length_change()) % Self::sessions_per_era()).is_zero() + { + Self::new_era(); + } + } + + /// The era has changed - enact new staking set. + /// + /// NOTE: This always happens immediately before a session change to ensure that new validators + /// get a chance to set their session keys. + fn new_era() { + // Increment current era. + >::put(&(>::get() + One::one())); + + // Enact era length change. + if let Some(next_spe) = Self::next_sessions_per_era() { + if next_spe != Self::sessions_per_era() { + >::put(&next_spe); + >::put(&>::current_index()); + } + } + + // evaluate desired staking amounts and nominations and optimise to find the best + // combination of validators, then use session::internal::set_validators(). + // for now, this just orders would-be stakers by their balances and chooses the top-most + // >::get() of them. + // TODO: this is not sound. this should be moved to an off-chain solution mechanism. + let mut intentions = Self::intentions() + .into_iter() + .map(|v| (Self::slashable_balance(&v), v)) + .collect::>(); + + // Avoid reevaluate validator set if it would leave us with fewer than the minimum + // needed validators + if intentions.len() < Self::minimum_validator_count() { + return + } + + intentions.sort_unstable_by(|&(ref b1, _), &(ref b2, _)| b2.cmp(&b1)); + + >::put( + if !intentions.is_empty() { + let n = >::get() as usize; + (intentions[0].0, intentions[n - 1].0) + } else { + (Zero::zero(), Zero::zero()) + } + ); + let vals = &intentions.into_iter() + .map(|(_, v)| v) + .take(>::get() as usize) + .collect::>(); + for v in >::validators().iter() { + >::remove(v); + let slash_count = >::take(v); + if slash_count > 1 { + >::insert(v, slash_count - 1); + } + } + for v in vals.iter() { + >::insert(v, Self::nominators_for(v)); + } + >::set_validators(vals); + } +} + +impl OnFinalise for Module { + fn on_finalise(_n: T::BlockNumber) { + } +} + +impl OnSessionChange for Module { + fn on_session_change(elapsed: T::Moment, should_reward: bool) { + Self::new_session(elapsed, should_reward); + } +} + +impl balances::EnsureAccountLiquid for Module { + fn ensure_account_liquid(who: &T::AccountId) -> Result { + if Self::bondage(who) <= >::block_number() { + Ok(()) + } else { + Err("cannot transfer illiquid funds") + } + } +} + +impl balances::OnFreeBalanceZero for Module { + fn on_free_balance_zero(who: &T::AccountId) { + >::remove(who); + } +} + +impl consensus::OnOfflineValidator for Module { + fn on_offline_validator(validator_index: usize) { + let v = >::validators()[validator_index].clone(); + let slash_count = Self::slash_count(&v); + >::insert(v.clone(), slash_count + 1); + let grace = Self::offline_slash_grace(); + + let event = if slash_count >= grace { + let instances = slash_count - grace; + let slash = Self::offline_slash() << instances; + let next_slash = slash << 1u32; + let _ = Self::slash_validator(&v, slash); + if instances >= Self::validator_preferences(&v).unstake_threshold + || Self::slashable_balance(&v) < next_slash + { + if let Some(pos) = Self::intentions().into_iter().position(|x| &x == &v) { + Self::apply_unstake(&v, pos) + .expect("pos derived correctly from Self::intentions(); \ + apply_unstake can only fail if pos wrong; \ + Self::intentions() doesn't change; qed"); + } + let _ = Self::apply_force_new_era(false); + } + RawEvent::OfflineSlash(v, slash) + } else { + RawEvent::OfflineWarning(v, slash_count) + }; + Self::deposit_event(event); + } +} diff --git a/runtime/staking/src/mock.rs b/runtime/staking/src/mock.rs new file mode 100644 index 00000000000..7dda4fe7ebe --- /dev/null +++ b/runtime/staking/src/mock.rs @@ -0,0 +1,126 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Test utilities + +#![cfg(test)] + +use primitives::BuildStorage; +use primitives::traits::{Identity}; +use primitives::testing::{Digest, Header}; +use substrate_primitives::{H256, Blake2Hasher}; +use runtime_io; +use {GenesisConfig, Module, Trait, consensus, session, system, timestamp, balances}; + +impl_outer_origin!{ + pub enum Origin for Test {} +} + +// Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub struct Test; +impl consensus::Trait for Test { + const NOTE_OFFLINE_POSITION: u32 = 1; + type Log = u64; + type SessionKey = u64; + type OnOfflineValidator = (); +} +impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = ::primitives::traits::BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); +} +impl balances::Trait for Test { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = Staking; + type EnsureAccountLiquid = Staking; + type Event = (); +} +impl session::Trait for Test { + type ConvertAccountIdToSessionKey = Identity; + type OnSessionChange = Staking; + type Event = (); +} +impl timestamp::Trait for Test { + const TIMESTAMP_SET_POSITION: u32 = 0; + type Moment = u64; +} +impl Trait for Test { + type OnRewardMinted = (); + type Event = (); +} + +pub fn new_test_ext(ext_deposit: u64, session_length: u64, sessions_per_era: u64, current_era: u64, monied: bool, reward: u64) -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + let balance_factor = if ext_deposit > 0 { + 256 + } else { + 1 + }; + t.extend(consensus::GenesisConfig::{ + code: vec![], + authorities: vec![], + }.build_storage().unwrap()); + t.extend(session::GenesisConfig::{ + session_length, + validators: vec![10, 20], + }.build_storage().unwrap()); + t.extend(balances::GenesisConfig::{ + balances: if monied { + if reward > 0 { + vec![(1, 10 * balance_factor), (2, 20 * balance_factor), (3, 30 * balance_factor), (4, 40 * balance_factor), (10, balance_factor), (20, balance_factor)] + } else { + vec![(1, 10 * balance_factor), (2, 20 * balance_factor), (3, 30 * balance_factor), (4, 40 * balance_factor)] + } + } else { + vec![(10, balance_factor), (20, balance_factor)] + }, + transaction_base_fee: 0, + transaction_byte_fee: 0, + existential_deposit: ext_deposit, + transfer_fee: 0, + creation_fee: 0, + reclaim_rebate: 0, + }.build_storage().unwrap()); + t.extend(GenesisConfig::{ + sessions_per_era, + current_era, + intentions: vec![10, 20], + validator_count: 2, + minimum_validator_count: 0, + bonding_duration: sessions_per_era * session_length * 3, + session_reward: reward, + offline_slash: if monied { 20 } else { 0 }, + offline_slash_grace: 0, + }.build_storage().unwrap()); + t.extend(timestamp::GenesisConfig::{ + period: 5 + }.build_storage().unwrap()); + t.into() +} + +pub type System = system::Module; +pub type Balances = balances::Module; +pub type Session = session::Module; +pub type Timestamp = timestamp::Module; +pub type Staking = Module; diff --git a/runtime/staking/src/tests.rs b/runtime/staking/src/tests.rs new file mode 100644 index 00000000000..66da1ed1737 --- /dev/null +++ b/runtime/staking/src/tests.rs @@ -0,0 +1,502 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tests for the module. + +#![cfg(test)] + +use super::*; +use consensus::OnOfflineValidator; +use runtime_io::with_externalities; +use mock::{Balances, Session, Staking, System, Timestamp, Test, new_test_ext, Origin}; + +#[test] +fn note_null_offline_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + assert_eq!(Staking::offline_slash_grace(), 0); + assert_eq!(Staking::slash_count(&10), 0); + assert_eq!(Balances::free_balance(&10), 1); + ::system::ExtrinsicIndex::::put(1); + assert_eq!(Staking::slash_count(&10), 0); + assert_eq!(Balances::free_balance(&10), 1); + assert!(Staking::forcing_new_era().is_none()); + }); +} + +#[test] +fn note_offline_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + Balances::set_free_balance(&10, 70); + assert_eq!(Staking::offline_slash_grace(), 0); + assert_eq!(Staking::slash_count(&10), 0); + assert_eq!(Balances::free_balance(&10), 70); + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + assert_eq!(Staking::slash_count(&10), 1); + assert_eq!(Balances::free_balance(&10), 50); + assert!(Staking::forcing_new_era().is_none()); + }); +} + +#[test] +fn note_offline_exponent_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + Balances::set_free_balance(&10, 150); + assert_eq!(Staking::offline_slash_grace(), 0); + assert_eq!(Staking::slash_count(&10), 0); + assert_eq!(Balances::free_balance(&10), 150); + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + assert_eq!(Staking::slash_count(&10), 1); + assert_eq!(Balances::free_balance(&10), 130); + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + assert_eq!(Staking::slash_count(&10), 2); + assert_eq!(Balances::free_balance(&10), 90); + assert!(Staking::forcing_new_era().is_none()); + }); +} + +#[test] +fn note_offline_grace_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + Balances::set_free_balance(&10, 70); + Balances::set_free_balance(&20, 70); + assert_ok!(Staking::set_offline_slash_grace(Origin::ROOT, 1)); + assert_eq!(Staking::offline_slash_grace(), 1); + + assert_eq!(Staking::slash_count(&10), 0); + assert_eq!(Balances::free_balance(&10), 70); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + assert_eq!(Staking::slash_count(&10), 1); + assert_eq!(Balances::free_balance(&10), 70); + assert_eq!(Staking::slash_count(&20), 0); + assert_eq!(Balances::free_balance(&20), 70); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + Staking::on_offline_validator(1); + assert_eq!(Staking::slash_count(&10), 2); + assert_eq!(Balances::free_balance(&10), 50); + assert_eq!(Staking::slash_count(&20), 1); + assert_eq!(Balances::free_balance(&20), 70); + assert!(Staking::forcing_new_era().is_none()); + }); +} + +#[test] +fn note_offline_force_unstake_session_change_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + Balances::set_free_balance(&10, 70); + Balances::set_free_balance(&20, 70); + assert_ok!(Staking::stake(Origin::signed(1))); + + assert_eq!(Staking::slash_count(&10), 0); + assert_eq!(Balances::free_balance(&10), 70); + assert_eq!(Staking::intentions(), vec![10, 20, 1]); + assert_eq!(Session::validators(), vec![10, 20]); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + assert_eq!(Balances::free_balance(&10), 50); + assert_eq!(Staking::slash_count(&10), 1); + assert_eq!(Staking::intentions(), vec![10, 20, 1]); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + assert_eq!(Staking::intentions(), vec![1, 20]); + assert_eq!(Balances::free_balance(&10), 10); + assert!(Staking::forcing_new_era().is_some()); + }); +} + +#[test] +fn note_offline_auto_unstake_session_change_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + Balances::set_free_balance(&10, 7000); + Balances::set_free_balance(&20, 7000); + assert_ok!(Staking::register_preferences(Origin::signed(10), 0, ValidatorPrefs { unstake_threshold: 1, validator_payment: 0 })); + + assert_eq!(Staking::intentions(), vec![10, 20]); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + Staking::on_offline_validator(1); + assert_eq!(Balances::free_balance(&10), 6980); + assert_eq!(Balances::free_balance(&20), 6980); + assert_eq!(Staking::intentions(), vec![10, 20]); + assert!(Staking::forcing_new_era().is_none()); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + Staking::on_offline_validator(1); + assert_eq!(Balances::free_balance(&10), 6940); + assert_eq!(Balances::free_balance(&20), 6940); + assert_eq!(Staking::intentions(), vec![20]); + assert!(Staking::forcing_new_era().is_some()); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(1); + assert_eq!(Balances::free_balance(&10), 6940); + assert_eq!(Balances::free_balance(&20), 6860); + assert_eq!(Staking::intentions(), vec![20]); + + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(1); + assert_eq!(Balances::free_balance(&10), 6940); + assert_eq!(Balances::free_balance(&20), 6700); + assert_eq!(Staking::intentions(), vec![0u64; 0]); + }); +} + + +#[test] +fn rewards_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + assert_eq!(Staking::era_length(), 9); + assert_eq!(Staking::sessions_per_era(), 3); + assert_eq!(Staking::last_era_length_change(), 0); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::current_index(), 0); + assert_eq!(Balances::total_balance(&10), 1); + + System::set_block_number(3); + Timestamp::set_timestamp(15); // on time. + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::current_index(), 1); + assert_eq!(Balances::total_balance(&10), 11); + System::set_block_number(6); + Timestamp::set_timestamp(31); // a little late + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::current_index(), 2); + assert_eq!(Balances::total_balance(&10), 20); // less reward + System::set_block_number(9); + Timestamp::set_timestamp(50); // very late + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 1); + assert_eq!(Session::current_index(), 3); + assert_eq!(Balances::total_balance(&10), 27); // much less reward + }); +} + +#[test] +fn slashing_should_work() { + with_externalities(&mut new_test_ext(0, 3, 3, 0, true, 10), || { + assert_eq!(Staking::era_length(), 9); + assert_eq!(Staking::sessions_per_era(), 3); + assert_eq!(Staking::last_era_length_change(), 0); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::current_index(), 0); + assert_eq!(Balances::total_balance(&10), 1); + + System::set_block_number(3); + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::current_index(), 1); + assert_eq!(Balances::total_balance(&10), 11); + + System::set_block_number(6); + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::current_index(), 2); + assert_eq!(Balances::total_balance(&10), 21); + + System::set_block_number(7); + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + Staking::on_offline_validator(1); + assert_eq!(Balances::total_balance(&10), 1); + }); +} + + + +#[test] +fn staking_should_work() { + with_externalities(&mut new_test_ext(0, 1, 2, 0, true, 0), || { + + assert_eq!(Staking::era_length(), 2); + assert_eq!(Staking::validator_count(), 2); + assert_eq!(Session::validators(), vec![10, 20]); + + assert_ok!(Staking::set_bonding_duration(Origin::ROOT, 2)); + assert_eq!(Staking::bonding_duration(), 2); + + // Block 1: Add three validators. No obvious change. + System::set_block_number(1); + assert_ok!(Staking::stake(Origin::signed(1))); + assert_ok!(Staking::stake(Origin::signed(2))); + assert_ok!(Staking::stake(Origin::signed(4))); + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::validators(), vec![10, 20]); + + // Block 2: New validator set now. + System::set_block_number(2); + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 1); + assert_eq!(Session::validators(), vec![4, 2]); + + // Block 3: Unstake highest, introduce another staker. No change yet. + System::set_block_number(3); + assert_ok!(Staking::stake(Origin::signed(3))); + assert_ok!(Staking::unstake(Origin::signed(4), Staking::intentions().iter().position(|&x| x == 4).unwrap() as u32)); + assert_eq!(Staking::current_era(), 1); + Session::check_rotate_session(System::block_number()); + + // Block 4: New era - validators change. + System::set_block_number(4); + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 2); + assert_eq!(Session::validators(), vec![3, 2]); + + // Block 5: Transfer stake from highest to lowest. No change yet. + System::set_block_number(5); + assert_ok!(Balances::transfer(Origin::signed(4), 1.into(), 40)); + Session::check_rotate_session(System::block_number()); + + // Block 6: Lowest now validator. + System::set_block_number(6); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::validators(), vec![1, 3]); + + // Block 7: Unstake three. No change yet. + System::set_block_number(7); + assert_ok!(Staking::unstake(Origin::signed(3), Staking::intentions().iter().position(|&x| x == 3).unwrap() as u32)); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::validators(), vec![1, 3]); + + // Block 8: Back to one and two. + System::set_block_number(8); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::validators(), vec![1, 2]); + }); +} + +#[test] +fn nominating_and_rewards_should_work() { + with_externalities(&mut new_test_ext(0, 1, 1, 0, true, 10), || { + assert_eq!(Staking::era_length(), 1); + assert_eq!(Staking::validator_count(), 2); + assert_eq!(Staking::bonding_duration(), 3); + assert_eq!(Session::validators(), vec![10, 20]); + + System::set_block_number(1); + assert_ok!(Staking::stake(Origin::signed(1))); + assert_ok!(Staking::stake(Origin::signed(2))); + assert_ok!(Staking::stake(Origin::signed(3))); + assert_ok!(Staking::nominate(Origin::signed(4), 1.into())); + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 1); + assert_eq!(Session::validators(), vec![1, 3]); // 4 + 1, 3 + assert_eq!(Balances::total_balance(&1), 10); + assert_eq!(Balances::total_balance(&2), 20); + assert_eq!(Balances::total_balance(&3), 30); + assert_eq!(Balances::total_balance(&4), 40); + + System::set_block_number(2); + assert_ok!(Staking::unnominate(Origin::signed(4), 0)); + Session::check_rotate_session(System::block_number()); + assert_eq!(Staking::current_era(), 2); + assert_eq!(Session::validators(), vec![3, 2]); + assert_eq!(Balances::total_balance(&1), 12); + assert_eq!(Balances::total_balance(&2), 20); + assert_eq!(Balances::total_balance(&3), 40); + assert_eq!(Balances::total_balance(&4), 48); + + System::set_block_number(3); + assert_ok!(Staking::stake(Origin::signed(4))); + assert_ok!(Staking::unstake(Origin::signed(3), Staking::intentions().iter().position(|&x| x == 3).unwrap() as u32)); + assert_ok!(Staking::nominate(Origin::signed(3), 1.into())); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::validators(), vec![1, 4]); + assert_eq!(Balances::total_balance(&1), 12); + assert_eq!(Balances::total_balance(&2), 30); + assert_eq!(Balances::total_balance(&3), 50); + assert_eq!(Balances::total_balance(&4), 48); + + System::set_block_number(4); + Session::check_rotate_session(System::block_number()); + assert_eq!(Balances::total_balance(&1), 13); + assert_eq!(Balances::total_balance(&2), 30); + assert_eq!(Balances::total_balance(&3), 58); + assert_eq!(Balances::total_balance(&4), 58); + }); +} + +#[test] +fn rewards_with_off_the_table_should_work() { + with_externalities(&mut new_test_ext(0, 1, 1, 0, true, 10), || { + System::set_block_number(1); + assert_ok!(Staking::stake(Origin::signed(1))); + assert_ok!(Staking::nominate(Origin::signed(2), 1.into())); + assert_ok!(Staking::stake(Origin::signed(3))); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::validators(), vec![1, 3]); // 1 + 2, 3 + assert_eq!(Balances::total_balance(&1), 10); + assert_eq!(Balances::total_balance(&2), 20); + assert_eq!(Balances::total_balance(&3), 30); + + System::set_block_number(2); + assert_ok!(Staking::register_preferences(Origin::signed(1), Staking::intentions().into_iter().position(|i| i == 1).unwrap() as u32, ValidatorPrefs { unstake_threshold: 3, validator_payment: 4 })); + Session::check_rotate_session(System::block_number()); + assert_eq!(Balances::total_balance(&1), 16); + assert_eq!(Balances::total_balance(&2), 24); + assert_eq!(Balances::total_balance(&3), 40); + }); +} + +#[test] +fn nominating_slashes_should_work() { + with_externalities(&mut new_test_ext(0, 2, 2, 0, true, 10), || { + assert_eq!(Staking::era_length(), 4); + assert_eq!(Staking::validator_count(), 2); + assert_eq!(Staking::bonding_duration(), 12); + assert_eq!(Session::validators(), vec![10, 20]); + + System::set_block_number(2); + Session::check_rotate_session(System::block_number()); + + Timestamp::set_timestamp(15); + System::set_block_number(4); + assert_ok!(Staking::stake(Origin::signed(1))); + assert_ok!(Staking::stake(Origin::signed(3))); + assert_ok!(Staking::nominate(Origin::signed(2), 3.into())); + assert_ok!(Staking::nominate(Origin::signed(4), 1.into())); + Session::check_rotate_session(System::block_number()); + + assert_eq!(Staking::current_era(), 1); + assert_eq!(Session::validators(), vec![1, 3]); // 1 + 4, 3 + 2 + assert_eq!(Balances::total_balance(&1), 10); + assert_eq!(Balances::total_balance(&2), 20); + assert_eq!(Balances::total_balance(&3), 30); + assert_eq!(Balances::total_balance(&4), 40); + + System::set_block_number(5); + ::system::ExtrinsicIndex::::put(1); + Staking::on_offline_validator(0); + Staking::on_offline_validator(1); + assert_eq!(Balances::total_balance(&1), 0); + assert_eq!(Balances::total_balance(&2), 20); + assert_eq!(Balances::total_balance(&3), 10); + assert_eq!(Balances::total_balance(&4), 30); + }); +} + +#[test] +fn double_staking_should_fail() { + with_externalities(&mut new_test_ext(0, 1, 2, 0, true, 0), || { + System::set_block_number(1); + assert_ok!(Staking::stake(Origin::signed(1))); + assert_noop!(Staking::stake(Origin::signed(1)), "Cannot stake if already staked."); + assert_noop!(Staking::nominate(Origin::signed(1), 1.into()), "Cannot nominate if already staked."); + assert_ok!(Staking::nominate(Origin::signed(2), 1.into())); + assert_noop!(Staking::stake(Origin::signed(2)), "Cannot stake if already nominating."); + assert_noop!(Staking::nominate(Origin::signed(2), 1.into()), "Cannot nominate if already nominating."); + }); +} + +#[test] +fn staking_eras_work() { + with_externalities(&mut new_test_ext(0, 1, 2, 0, true, 0), || { + assert_eq!(Staking::era_length(), 2); + assert_eq!(Staking::sessions_per_era(), 2); + assert_eq!(Staking::last_era_length_change(), 0); + assert_eq!(Staking::current_era(), 0); + assert_eq!(Session::current_index(), 0); + + // Block 1: No change. + System::set_block_number(1); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::current_index(), 1); + assert_eq!(Staking::sessions_per_era(), 2); + assert_eq!(Staking::last_era_length_change(), 0); + assert_eq!(Staking::current_era(), 0); + + // Block 2: Simple era change. + System::set_block_number(2); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::current_index(), 2); + assert_eq!(Staking::sessions_per_era(), 2); + assert_eq!(Staking::last_era_length_change(), 0); + assert_eq!(Staking::current_era(), 1); + + // Block 3: Schedule an era length change; no visible changes. + System::set_block_number(3); + assert_ok!(Staking::set_sessions_per_era(Origin::ROOT, 3)); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::current_index(), 3); + assert_eq!(Staking::sessions_per_era(), 2); + assert_eq!(Staking::last_era_length_change(), 0); + assert_eq!(Staking::current_era(), 1); + + // Block 4: Era change kicks in. + System::set_block_number(4); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::current_index(), 4); + assert_eq!(Staking::sessions_per_era(), 3); + assert_eq!(Staking::last_era_length_change(), 4); + assert_eq!(Staking::current_era(), 2); + + // Block 5: No change. + System::set_block_number(5); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::current_index(), 5); + assert_eq!(Staking::sessions_per_era(), 3); + assert_eq!(Staking::last_era_length_change(), 4); + assert_eq!(Staking::current_era(), 2); + + // Block 6: No change. + System::set_block_number(6); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::current_index(), 6); + assert_eq!(Staking::sessions_per_era(), 3); + assert_eq!(Staking::last_era_length_change(), 4); + assert_eq!(Staking::current_era(), 2); + + // Block 7: Era increment. + System::set_block_number(7); + Session::check_rotate_session(System::block_number()); + assert_eq!(Session::current_index(), 7); + assert_eq!(Staking::sessions_per_era(), 3); + assert_eq!(Staking::last_era_length_change(), 4); + assert_eq!(Staking::current_era(), 3); + }); +} + +#[test] +fn staking_balance_transfer_when_bonded_should_not_work() { + with_externalities(&mut new_test_ext(0, 1, 3, 1, false, 0), || { + Balances::set_free_balance(&1, 111); + assert_ok!(Staking::stake(Origin::signed(1))); + assert_noop!(Balances::transfer(Origin::signed(1), 2.into(), 69), "cannot transfer illiquid funds"); + }); +} + +#[test] +fn deducting_balance_when_bonded_should_not_work() { + with_externalities(&mut new_test_ext(0, 1, 3, 1, false, 0), || { + Balances::set_free_balance(&1, 111); + >::insert(1, 2); + System::set_block_number(1); + assert_eq!(Staking::unlock_block(&1), LockStatus::LockedUntil(2)); + assert_noop!(Balances::reserve(&1, 69), "cannot transfer illiquid funds"); + }); +} diff --git a/runtime/support/Cargo.toml b/runtime/support/Cargo.toml new file mode 100644 index 00000000000..a718a24573f --- /dev/null +++ b/runtime/support/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-runtime-support" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = { version = "0.1.0", optional = true } +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-codec = { path = "../../core/codec", default_features = false } + +[dev-dependencies] +pretty_assertions = "0.5.1" +serde_json = { version = "1.0" } +substrate-codec-derive = { path = "../../core/codec/derive" } + +[features] +default = ["std"] +std = [ + "hex-literal", + "serde/std", + "serde_derive", + "substrate-primitives/std", + "substrate-runtime-io/std", + "substrate-codec/std", + "substrate-runtime-std/std", +] +nightly = [] +strict = [] diff --git a/runtime/support/README.adoc b/runtime/support/README.adoc new file mode 100644 index 00000000000..699235e5ae6 --- /dev/null +++ b/runtime/support/README.adoc @@ -0,0 +1,14 @@ + += Runtime Support + +.Summary +[source, toml] +---- +include::Cargo.toml[lines=2..5] +---- + +.Description +---- +include::src/lib.rs[tag=description] +---- + diff --git a/runtime/support/src/dispatch.rs b/runtime/support/src/dispatch.rs new file mode 100644 index 00000000000..90f6fd7a6d4 --- /dev/null +++ b/runtime/support/src/dispatch.rs @@ -0,0 +1,584 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Dispatch system. Just dispatches calls. + +pub use rstd::prelude::{Vec, Clone, Eq, PartialEq}; +#[cfg(feature = "std")] +pub use std::fmt; +pub use rstd::result; +#[cfg(feature = "std")] +use serde; +pub use codec::{Codec, Decode, Encode, Input, Output}; + +pub type Result = result::Result<(), &'static str>; + +pub trait Dispatchable { + type Origin; + type Trait; + fn dispatch(self, origin: Self::Origin) -> Result; +} + +#[cfg(feature = "std")] +pub trait Callable { + type Call: Dispatchable + Codec + ::serde::Serialize + Clone + PartialEq + Eq; +} +#[cfg(not(feature = "std"))] +pub trait Callable { + type Call: Dispatchable + Codec + Clone + PartialEq + Eq; +} + +// dirty hack to work around serde_derive issue +// https://github.com/rust-lang/rust/issues/51331 +pub type CallableCallFor = ::Call; + +#[cfg(feature = "std")] +pub trait Parameter: Codec + serde::Serialize + Clone + Eq + fmt::Debug {} + +#[cfg(feature = "std")] +impl Parameter for T where T: Codec + serde::Serialize + Clone + Eq + fmt::Debug {} + +#[cfg(not(feature = "std"))] +pub trait Parameter: Codec + Clone + Eq {} + +#[cfg(not(feature = "std"))] +impl Parameter for T where T: Codec + Clone + Eq {} + +/// Declare a struct for this module, then implement dispatch logic to create a pairing of several +/// dispatch traits and enums. +#[macro_export] +macro_rules! decl_module { + ( + $(#[$attr:meta])* + pub struct $mod_type:ident<$trait_instance:ident: $trait_name:ident> + for enum $call_type:ident where origin: $origin_type:ty {$( + $(#[doc = $doc_attr:tt])* + fn $fn_name:ident(origin + $( + , $param_name:ident : $param:ty + )* + ) -> $result:ty; + )*} + ) => { + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, Copy, PartialEq, Eq)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + // TODO: switching based on std feature is because of an issue in + // serde-derive for when we attempt to derive `Deserialize` on these types, + // in a situation where we've imported `substrate_runtime_support` as another name. + #[cfg(feature = "std")] + pub struct $mod_type<$trait_instance: $trait_name>(::std::marker::PhantomData<$trait_instance>); + + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, Copy, PartialEq, Eq)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + #[cfg(not(feature = "std"))] + pub struct $mod_type<$trait_instance: $trait_name>(::core::marker::PhantomData<$trait_instance>); + + #[cfg(feature = "std")] + $(#[$attr])* + #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] + pub enum $call_type<$trait_instance: $trait_name> { + __PhantomItem(::std::marker::PhantomData<$trait_instance>), + __OtherPhantomItem(::std::marker::PhantomData<$trait_instance>), + $( + #[allow(non_camel_case_types)] + $fn_name ( $( $param ),* ), + )* + } + + #[cfg(not(feature = "std"))] + $(#[$attr])* + #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] + pub enum $call_type<$trait_instance: $trait_name> { + __PhantomItem(::core::marker::PhantomData<$trait_instance>), + __OtherPhantomItem(::core::marker::PhantomData<$trait_instance>), + $( + #[allow(non_camel_case_types)] + $fn_name ( $( $param ),* ), + )* + } + + // manual implementation of clone/eq/partialeq because using derive erroneously requires + // clone/eq/partialeq from T. + impl<$trait_instance: $trait_name> $crate::dispatch::Clone + for $call_type<$trait_instance> + { + fn clone(&self) -> Self { + match *self { + $( + $call_type::$fn_name( $( ref $param_name ),* ) => + $call_type::$fn_name( $( $param_name.clone() ),* ) + ,)* + _ => unreachable!(), + } + } + } + impl<$trait_instance: $trait_name> $crate::dispatch::PartialEq + for $call_type<$trait_instance> + { + fn eq(&self, _other: &Self) -> bool { + match *self { + $( + $call_type::$fn_name( $( ref $param_name ),* ) => { + let self_params = ( $( $param_name, )* ); + if let $call_type::$fn_name( $( ref $param_name ),* ) = *_other { + self_params == ( $( $param_name, )* ) + } else { + match *_other { + $call_type::__PhantomItem(_) => unreachable!(), + $call_type::__OtherPhantomItem(_) => unreachable!(), + _ => false, + } + } + } + )* + _ => unreachable!(), + } + } + } + impl<$trait_instance: $trait_name> $crate::dispatch::Eq + for $call_type<$trait_instance> + {} + + #[cfg(feature = "std")] + impl<$trait_instance: $trait_name> $crate::dispatch::fmt::Debug + for $call_type<$trait_instance> + { + fn fmt(&self, _f: &mut $crate::dispatch::fmt::Formatter) -> $crate::dispatch::result::Result<(), $crate::dispatch::fmt::Error> { + match *self { + $( + $call_type::$fn_name( $( ref $param_name ),* ) => + write!(_f, "{}{:?}", + stringify!($fn_name), + ( $( $param_name.clone(), )* ) + ) + ,)* + _ => unreachable!(), + } + } + } + + impl<$trait_instance: $trait_name> $crate::dispatch::Decode for $call_type<$trait_instance> { + fn decode(input: &mut I) -> Option { + let _input_id = input.read_byte()?; + __impl_decode!(input; _input_id; 0; $call_type; $( fn $fn_name( $( $param_name ),* ); )*) + } + } + + impl<$trait_instance: $trait_name> $crate::dispatch::Encode for $call_type<$trait_instance> { + fn encode_to(&self, _dest: &mut W) { + __impl_encode!(_dest; *self; 0; $call_type; $( fn $fn_name( $( $param_name ),* ); )*); + if let $call_type::__PhantomItem(_) = *self { unreachable!() } + if let $call_type::__OtherPhantomItem(_) = *self { unreachable!() } + } + } + impl<$trait_instance: $trait_name> $crate::dispatch::Dispatchable + for $call_type<$trait_instance> + { + type Trait = $trait_instance; + type Origin = $origin_type; + fn dispatch(self, _origin: Self::Origin) -> $crate::dispatch::Result { + match self { + $( + $call_type::$fn_name( $( $param_name ),* ) => + <$mod_type<$trait_instance>>::$fn_name( _origin $(, $param_name )* ), + )* + _ => { panic!("__PhantomItem should never be used.") }, + } + } + } + impl<$trait_instance: $trait_name> $crate::dispatch::Callable + for $mod_type<$trait_instance> + { + type Call = $call_type<$trait_instance>; + } + + impl<$trait_instance: $trait_name> $mod_type<$trait_instance> { + pub fn dispatch>(d: D, origin: D::Origin) -> $crate::dispatch::Result { + d.dispatch(origin) + } + } + + __dispatch_impl_json_metadata! { + $mod_type $trait_instance $trait_name $call_type $origin_type + {$( $(#[doc = $doc_attr])* fn $fn_name(origin $(, $param_name : $param )*) -> $result; )*} + } + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_decode { + ( + $input:expr; + $input_id:expr; + $fn_id:expr; + $call_type:ident; + fn $fn_name:ident( + $( $param_name:ident ),* + ); + $($rest:tt)* + ) => { + { + if $input_id == ($fn_id) { + $( + let $param_name = $crate::dispatch::Decode::decode($input)?; + )* + return Some($call_type:: $fn_name( $( $param_name ),* )); + } + + __impl_decode!($input; $input_id; $fn_id + 1; $call_type; $($rest)*) + } + }; + ( + $input:expr; + $input_id:expr; + $fn_id:expr; + $call_type:ident; + ) => { + None + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_encode { + ( + $dest:expr; + $self:expr; + $fn_id:expr; + $call_type:ident; + fn $fn_name:ident( + $( $param_name:ident ),* + ); + $($rest:tt)* + ) => { + { + if let $call_type::$fn_name( + $( + ref $param_name + ),* + ) = $self { + $dest.push_byte(($fn_id) as u8); + $( + $param_name.encode_to($dest); + )* + } + + __impl_encode!($dest; $self; $fn_id + 1; $call_type; $($rest)*) + } + }; + ( + $dest:expr; + $self:expr; + $fn_id:expr; + $call_type:ident; + ) => {{}} +} + +pub trait IsSubType { + fn is_aux_sub_type(&self) -> Option<&::Call>; +} + +/// Implement a meta-dispatch module to dispatch to other dispatchers. +#[macro_export] +macro_rules! impl_outer_dispatch { + () => (); + ( + $(#[$attr:meta])* + pub enum $call_type:ident where origin: $origin:ty { + $( + $camelcase:ident, + )* + } + $( $rest:tt )* + ) => { + $(#[$attr])* + #[derive(Clone, PartialEq, Eq)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + pub enum $call_type { + $( + $camelcase ( $crate::dispatch::CallableCallFor<$camelcase> ) + ,)* + } + __impl_outer_dispatch_common! { $call_type, $($camelcase,)* } + impl $crate::dispatch::Dispatchable for $call_type { + type Origin = $origin; + type Trait = $call_type; + fn dispatch(self, origin: $origin) -> $crate::dispatch::Result { + match self { + $( + $call_type::$camelcase(call) => call.dispatch(origin), + )* + } + } + } + $( + impl $crate::dispatch::IsSubType<$camelcase> for $call_type { + fn is_aux_sub_type(&self) -> Option<&<$camelcase as $crate::dispatch::Callable>::Call> { + if let $call_type::$camelcase ( ref r ) = *self { + Some(r) + } else { + None + } + } + } + )* + impl_outer_dispatch!{ $($rest)* } + } +} + +/// Implement a meta-dispatch module to dispatch to other dispatchers. +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_outer_dispatch_common { + ( + $call_type:ident, $( $camelcase:ident, )* + ) => { + impl $crate::dispatch::Decode for $call_type { + fn decode(input: &mut I) -> Option { + let input_id = input.read_byte()?; + __impl_decode!(input; input_id; 0; $call_type; $( fn $camelcase ( outer_dispatch_param ); )*) + } + } + + impl $crate::dispatch::Encode for $call_type { + fn encode_to(&self, dest: &mut W) { + __impl_encode!(dest; *self; 0; $call_type; $( fn $camelcase( outer_dispatch_param ); )*) + } + } + + } +} + +/// Implement the `json_metadata` function. +#[macro_export] +#[doc(hidden)] +macro_rules! __dispatch_impl_json_metadata { + ( + $mod_type:ident $trait_instance:ident $trait_name:ident + $($rest:tt)* + ) => { + impl<$trait_instance: $trait_name> $mod_type<$trait_instance> { + pub fn json_metadata() -> &'static str { + concat!(r#"{ "name": ""#, stringify!($mod_type), r#"", "call": "#, + __call_to_json!($($rest)*), " }") + } + } + } +} + +/// Convert the list of calls into their JSON representation, joined by ",". +#[macro_export] +#[doc(hidden)] +macro_rules! __call_to_json { + // WITH AUX + ( + $call_type:ident $origin_type:ty + {$( + $(#[doc = $doc_attr:tt])* + fn $fn_name:ident(origin + $( + , $param_name:ident : $param:ty + )* + ) -> $result:ty; + )*} + ) => { + concat!( + r#"{ "name": ""#, stringify!($call_type), + r#"", "functions": {"#, + __functions_to_json!(""; 0; $origin_type; $( + fn $fn_name(origin $(, $param_name: $param )* ) -> $result; + __function_doc_to_json!(""; $($doc_attr)*); + )*), " } }" + ) + }; +} + +/// Convert a list of functions into their JSON representation, joined by ",". +#[macro_export] +#[doc(hidden)] +macro_rules! __functions_to_json { + // WITHOUT AUX + ( + $prefix_str:tt; + $fn_id:expr; + fn $fn_name:ident( + $($param_name:ident : $param:ty),* + ) -> $result:ty; + $fn_doc:expr; + $($rest:tt)* + ) => { + concat!($prefix_str, " ", + __function_to_json!( + fn $fn_name( + $($param_name : $param),* + ) -> $result; + $fn_doc; + $fn_id; + ), __functions_to_json!(","; $fn_id + 1; $($rest)*) + ) + }; + // WITH AUX + ( + $prefix_str:tt; + $fn_id:expr; + $origin_type:ty; + fn $fn_name:ident(origin + $( + , $param_name:ident : $param:ty + )* + ) -> $result:ty; + $fn_doc:expr; + $($rest:tt)* + ) => { + concat!($prefix_str, " ", + __function_to_json!( + fn $fn_name( + origin: $origin_type + $(, $param_name : $param)* + ) -> $result; + $fn_doc; + $fn_id; + ), __functions_to_json!(","; $fn_id + 1; $origin_type; $($rest)*) + ) + }; + // BASE CASE + ( + $prefix_str:tt; + $fn_id:expr; + $($origin_type:ty;)* + ) => { + "" + } +} + +/// Convert a function into its JSON representation. +#[macro_export] +#[doc(hidden)] +macro_rules! __function_to_json { + ( + fn $fn_name:ident( + $first_param_name:ident : $first_param:ty $(, $param_name:ident : $param:ty)* + ) -> $result:ty; + $fn_doc:tt; + $fn_id:expr; + ) => { + concat!( + r#"""#, stringify!($fn_id), r#"""#, + r#": { "name": ""#, stringify!($fn_name), + r#"", "params": [ "#, + concat!(r#"{ "name": ""#, stringify!($first_param_name), r#"", "type": ""#, stringify!($first_param), r#"" }"# ), + $( + concat!(r#", { "name": ""#, stringify!($param_name), r#"", "type": ""#, stringify!($param), r#"" }"# ), + )* + r#" ], "description": ["#, $fn_doc, " ] }" + ) + }; +} + +/// Convert a function documentation attribute into its JSON representation. +#[macro_export] +#[doc(hidden)] +macro_rules! __function_doc_to_json { + ( + $prefix_str:tt; + $doc_attr:tt + $($rest:tt)* + ) => { + concat!( + $prefix_str, r#" ""#, + $doc_attr, + r#"""#, + __function_doc_to_json!(","; $($rest)*) + ) + }; + ( + $prefix_str:tt; + ) => { + "" + } +} + +#[cfg(test)] +// Do not complain about unused `dispatch` and `dispatch_aux`. +#[allow(dead_code)] +mod tests { + use super::*; + use serde; + use serde_json; + + pub trait Trait { + type Origin; + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin { + /// Hi, this is a comment. + fn aux_0(origin) -> Result; + fn aux_1(origin, data: i32) -> Result; + fn aux_2(origin, data: i32, data2: String) -> Result; + } + } + + const EXPECTED_METADATA: &str = concat!( + r#"{ "name": "Module", "call": "#, + r#"{ "name": "Call", "functions": { "#, + r#""0": { "name": "aux_0", "params": [ "#, + r#"{ "name": "origin", "type": "T::Origin" }"#, + r#" ], "description": [ " Hi, this is a comment." ] }, "#, + r#""0 + 1": { "name": "aux_1", "params": [ "#, + r#"{ "name": "origin", "type": "T::Origin" }, "#, + r#"{ "name": "data", "type": "i32" }"#, + r#" ], "description": [ ] }, "#, + r#""0 + 1 + 1": { "name": "aux_2", "params": [ "#, + r#"{ "name": "origin", "type": "T::Origin" }, "#, + r#"{ "name": "data", "type": "i32" }, "#, + r#"{ "name": "data2", "type": "String" }"#, + r#" ], "description": [ ] }"#, + r#" } }"#, + r#" }"#, + ); + + impl Module { + fn aux_0(_: T::Origin) -> Result { + unreachable!() + } + + fn aux_1(_: T::Origin, _: i32) -> Result { + unreachable!() + } + + fn aux_2(_: T::Origin, _: i32, _: String) -> Result { + unreachable!() + } + } + + struct TraitImpl {} + + impl Trait for TraitImpl { + type Origin = u32; + } + + #[test] + fn module_json_metadata() { + let metadata = Module::::json_metadata(); + assert_eq!(EXPECTED_METADATA, metadata); + let _: serde::de::IgnoredAny = + serde_json::from_str(metadata).expect("Is valid json syntax"); + } +} diff --git a/runtime/support/src/event.rs b/runtime/support/src/event.rs new file mode 100644 index 00000000000..a9767f51bf7 --- /dev/null +++ b/runtime/support/src/event.rs @@ -0,0 +1,298 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +/// Implement an `Event`/`RawEvent` for a module. +#[macro_export] +macro_rules! decl_event { + ( + $(#[$attr:meta])* + pub enum Event<$( $evt_generic_param:ident )*> with RawEvent<$( $generic_param:ident ),*> + where $( <$generic:ident as $trait:path>::$trait_type:ident),* { + $( + $(#[doc = $doc_attr:tt])* + $event:ident( $( $param:path ),* ), + )* + } + ) => { + pub type Event<$( $evt_generic_param )*> = RawEvent<$( <$generic as $trait>::$trait_type ),*>; + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, PartialEq, Eq, Encode, Decode)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + $(#[$attr])* + pub enum RawEvent<$( $generic_param ),*> { + $( + $( #[doc = $doc_attr] )* + $event($( $param ),*), + )* + } + impl<$( $generic_param ),*> From> for () { + fn from(_: RawEvent<$( $generic_param ),*>) -> () { () } + } + impl<$( $generic_param ),*> RawEvent<$( $generic_param ),*> { + #[allow(dead_code)] + pub fn event_json_metadata() -> &'static str { + concat!( + "{", + __impl_event_json_metadata!(""; + $( + $event ( $( $param ),* ); + __function_doc_to_json!(""; $($doc_attr)*); + )* + ), + " }" + ) + } + } + }; + ( + $(#[$attr:meta])* + pub enum Event { + $( + $(#[doc = $doc_attr:tt])* + $event:ident, + )* + } + ) => { + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, PartialEq, Eq, Encode, Decode)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + $(#[$attr])* + pub enum Event { + $( + $( #[doc = $doc_attr] )* + $event, + )* + } + impl From for () { + fn from(_: Event) -> () { () } + } + impl Event { + #[allow(dead_code)] + pub fn event_json_metadata() -> &'static str { + concat!( + "{", + __impl_event_json_metadata!(""; + $( + $event; + __function_doc_to_json!(""; $($doc_attr)*); + )* + ), + " }" + ) + } + } + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_event_json_metadata { + ( + $prefix_str:expr; + $event:ident( $first_param:path $(, $param:path )* ); + $event_doc:expr; + $( $rest:tt )* + ) => { + concat!($prefix_str, " ", "\"", stringify!($event), r#"": { "params": [ ""#, + stringify!($first_param), "\"" + $(, concat!(", \"", stringify!($param), "\"") )*, r#" ], "description": ["#, + $event_doc, " ] }", + __impl_event_json_metadata!(","; $( $rest )*) + ) + }; + ( + $prefix_str:expr; + $event:ident; + $event_doc:expr; + $( $rest:tt )* + ) => { + concat!($prefix_str, " ", "\"", stringify!($event), + r#"": { "params": null, "description": ["#, $event_doc, " ] }", + __impl_event_json_metadata!(","; $( $rest )*) + ) + }; + ( + $prefix_str:expr; + ) => { + "" + } +} + +#[macro_export] +macro_rules! impl_outer_event { + ($(#[$attr:meta])* pub enum $name:ident for $runtime:ident { $( $module:ident ),* }) => { + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, PartialEq, Eq, Encode, Decode)] + #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] + $(#[$attr])* + #[allow(non_camel_case_types)] + pub enum $name { + system(system::Event), + $( + $module($module::Event<$runtime>), + )* + } + impl From for $name { + fn from(x: system::Event) -> Self { + $name::system(x) + } + } + $( + impl From<$module::Event<$runtime>> for $name { + fn from(x: $module::Event<$runtime>) -> Self { + $name::$module(x) + } + } + )* + __impl_outer_event_json_metadata!($runtime; $name; $( $module )*); + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_outer_event_json_metadata { + ( + $runtime:ident; + $event_name:ident; + $( $module:ident )* + ) => { + impl $runtime { + #[allow(dead_code)] + pub fn outer_event_json_metadata() -> (&'static str, &'static [(&'static str, fn() -> &'static str)]) { + static METADATA: &[(&str, fn() -> &'static str)] = &[ + ("system", system::Event::event_json_metadata) + $( + , ( + stringify!($module), + $module::Event::<$runtime>::event_json_metadata + ) + )* + ]; + ( + stringify!($event_name), + METADATA + ) + } + } + } +} + +#[cfg(test)] +#[allow(dead_code)] +mod tests { + use serde; + use serde_json; + + mod system { + pub trait Trait { + type Origin; + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + + decl_event!( + pub enum Event { + SystemEvent, + } + ); + } + + mod event_module { + pub trait Trait { + type Origin; + type Balance; + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + + decl_event!( + pub enum Event with RawEvent + where ::Balance + { + /// Hi, I am a comment. + TestEvent(Balance), + } + ); + } + + mod event_module2 { + pub trait Trait { + type Origin; + type Balance; + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + + decl_event!( + pub enum Event with RawEvent + where ::Balance + { + TestEvent(Balance), + } + ); + } + + #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode, Deserialize, Serialize)] + pub struct TestRuntime; + + impl_outer_event! { + pub enum TestEvent for TestRuntime { + event_module, event_module2 + } + } + + impl event_module::Trait for TestRuntime { + type Origin = u32; + type Balance = u32; + } + + impl event_module2::Trait for TestRuntime { + type Origin = u32; + type Balance = u32; + } + + impl system::Trait for TestRuntime { + type Origin = u32; + } + + const EXPECTED_METADATA: (&str, &[(&str, &str)]) = ( + "TestEvent", &[ + ("system", r#"{ "SystemEvent": { "params": null, "description": [ ] } }"#), + ("event_module", r#"{ "TestEvent": { "params": [ "Balance" ], "description": [ " Hi, I am a comment." ] } }"#), + ("event_module2", r#"{ "TestEvent": { "params": [ "Balance" ], "description": [ ] } }"#), + ] + ); + + #[test] + fn outer_event_json_metadata() { + let metadata = TestRuntime::outer_event_json_metadata(); + assert_eq!(EXPECTED_METADATA.0, metadata.0); + assert_eq!(EXPECTED_METADATA.1.len(), metadata.1.len()); + + for (expected, got) in EXPECTED_METADATA.1.iter().zip(metadata.1.iter()) { + assert_eq!(expected.0, got.0); + assert_eq!(expected.1, got.1()); + let _: serde::de::IgnoredAny = + serde_json::from_str(got.1()).expect(&format!("Is valid json syntax: {}", got.1())); + } + } +} diff --git a/runtime/support/src/hashable.rs b/runtime/support/src/hashable.rs new file mode 100644 index 00000000000..f069e366617 --- /dev/null +++ b/runtime/support/src/hashable.rs @@ -0,0 +1,38 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Hashable trait. + +use codec::Codec; +use runtime_io::{blake2_256, twox_128, twox_256}; + +pub trait Hashable: Sized { + fn blake2_256(&self) -> [u8; 32]; + fn twox_128(&self) -> [u8; 16]; + fn twox_256(&self) -> [u8; 32]; +} + +impl Hashable for T { + fn blake2_256(&self) -> [u8; 32] { + blake2_256(&self.encode()) + } + fn twox_128(&self) -> [u8; 16] { + twox_128(&self.encode()) + } + fn twox_256(&self) -> [u8; 32] { + twox_256(&self.encode()) + } +} diff --git a/runtime/support/src/lib.rs b/runtime/support/src/lib.rs new file mode 100644 index 00000000000..73099360dad --- /dev/null +++ b/runtime/support/src/lib.rs @@ -0,0 +1,191 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// tag::description[] +//! Support code for the runtime. +// end::description[] + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(feature = "std")] +extern crate serde; + +extern crate substrate_runtime_std as rstd; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_primitives as primitives; + +#[cfg(test)] +#[macro_use] +extern crate pretty_assertions; +#[cfg(test)] +#[macro_use] +extern crate serde_derive; +#[cfg(test)] +extern crate serde_json; +#[cfg(test)] +#[macro_use] +extern crate substrate_codec_derive; + +#[doc(hidden)] +pub extern crate substrate_codec as codec; +pub use self::storage::generator::Storage as GenericStorage; + +#[cfg(feature = "std")] +pub mod alloc { + pub use std::boxed; + pub use std::vec; +} + +#[macro_use] +pub mod dispatch; +#[macro_use] +pub mod storage; +mod hashable; +#[macro_use] +mod event; +#[macro_use] +pub mod metadata; + +pub use self::storage::{StorageVec, StorageList, StorageValue, StorageMap}; +pub use self::hashable::Hashable; +pub use self::dispatch::{Parameter, Dispatchable, Callable, IsSubType}; +pub use runtime_io::print; + + +#[macro_export] +macro_rules! fail { + ( $y:expr ) => {{ + return Err($y); + }} +} + +#[macro_export] +macro_rules! ensure { + ( $x:expr, $y:expr ) => {{ + if !$x { + fail!($y); + } + }} +} + +#[macro_export] +#[cfg(feature = "std")] +macro_rules! assert_noop { + ( $x:expr , $y:expr ) => { + let h = runtime_io::storage_root(); + assert_err!($x, $y); + assert_eq!(h, runtime_io::storage_root()); + } +} + +#[macro_export] +#[cfg(feature = "std")] +macro_rules! assert_err { + ( $x:expr , $y:expr ) => { + assert_eq!($x, Err($y)); + } +} + +#[macro_export] +#[cfg(feature = "std")] +macro_rules! assert_ok { + ( $x:expr ) => { + assert_eq!($x, Ok(())); + }; + ( $x:expr, $y:expr ) => { + assert_eq!($x, Ok($y)); + } +} + +/// The void type - it cannot exist. +// Oh rust, you crack me up... +#[derive(Clone, Eq, PartialEq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum Void {} + +#[macro_export] +macro_rules! impl_outer_origin { + ($(#[$attr:meta])* pub enum $name:ident for $trait:ident where system = $system:ident { $( $module:ident ),* }) => { + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. + #[derive(Clone, PartialEq, Eq)] + #[cfg_attr(feature = "std", derive(Debug))] + $(#[$attr])* + #[allow(non_camel_case_types)] + pub enum $name { + system($system::Origin<$trait>), + $( + $module($module::Origin), + )* + #[allow(dead_code)] + Void($crate::Void) + } + #[allow(dead_code)] + impl $name { + pub const INHERENT: Self = $name::system($system::RawOrigin::Inherent); + pub const ROOT: Self = $name::system($system::RawOrigin::Root); + pub fn signed(by: <$trait as $system::Trait>::AccountId) -> Self { + $name::system($system::RawOrigin::Signed(by)) + } + } + impl From<$system::Origin<$trait>> for $name { + fn from(x: $system::Origin<$trait>) -> Self { + $name::system(x) + } + } + impl Into>> for $name { + fn into(self) -> Option<$system::Origin<$trait>> { + if let $name::system(l) = self { + Some(l) + } else { + None + } + } + } + impl From::AccountId>> for $name { + fn from(x: Option<<$trait as $system::Trait>::AccountId>) -> Self { + <$system::Origin<$trait>>::from(x).into() + } + } + $( + impl From<$module::Origin> for $name { + fn from(x: $module::Origin) -> Self { + $name::$module(x) + } + } + impl Into> for $name { + fn into(self) -> Option<$module::Origin> { + if let $name::$module(l) = self { + Some(l) + } else { + None + } + } + } + )* + }; + ($(#[$attr:meta])* pub enum $name:ident for $trait:ident { $( $module:ident ),* }) => { + impl_outer_origin! { + $(#[$attr])* + pub enum $name for $trait where system = system { + $( $module ),* + } + } + } +} diff --git a/runtime/support/src/metadata.rs b/runtime/support/src/metadata.rs new file mode 100644 index 00000000000..12953871d32 --- /dev/null +++ b/runtime/support/src/metadata.rs @@ -0,0 +1,459 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use codec::{Encode, Output}; +#[cfg(feature = "std")] +use codec::{Decode, Input}; +use alloc; + +/// Make Box available on `std` and `no_std`. +pub type Box = alloc::boxed::Box; +/// Make Vec available on `std` and `no_std`. +pub type Vec = alloc::vec::Vec; + +/// Implements the json metadata support for the given runtime and all its modules. +/// +/// Example: +/// ```compile_fail +/// impl_json_metadata!(for RUNTIME_NAME with modules MODULE0, MODULE2, MODULE3 with Storage); +/// ``` +/// +/// In this example, just `MODULE3` implements the `Storage` trait. +#[macro_export] +macro_rules! impl_json_metadata { + ( + for $runtime:ident with modules + $( $rest:tt )* + ) => { + impl $runtime { + pub fn json_metadata() -> $crate::metadata::Vec<$crate::metadata::JSONMetadata> { + let events = Self::outer_event_json_metadata(); + __impl_json_metadata!($runtime; + $crate::metadata::JSONMetadata::Events { + name: events.0, + events: events.1, + }; + $( $rest )* + ) + } + } + } +} + +/// The metadata of a runtime encoded as JSON. +#[derive(Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum JSONMetadata { + Events { name: &'static str, events: &'static [(&'static str, fn() -> &'static str)] }, + Module { module: &'static str, prefix: &'static str }, + ModuleWithStorage { module: &'static str, prefix: &'static str, storage: &'static str } +} + +impl Encode for JSONMetadata { + fn encode_to(&self, dest: &mut W) { + match self { + JSONMetadata::Events { name, events } => { + 0i8.encode_to(dest); + name.encode_to(dest); + events.iter().fold(0u32, |count, _| count + 1).encode_to(dest); + events + .iter() + .map(|(module, data)| (module, data())) + .for_each(|val| val.encode_to(dest)); + }, + JSONMetadata::Module { module, prefix } => { + 1i8.encode_to(dest); + prefix.encode_to(dest); + module.encode_to(dest); + }, + JSONMetadata::ModuleWithStorage { module, prefix, storage } => { + 2i8.encode_to(dest); + prefix.encode_to(dest); + module.encode_to(dest); + storage.encode_to(dest); + } + } + } +} + +impl PartialEq for JSONMetadata { + fn eq(&self, other: &JSONMetadata) -> bool { + match (self, other) { + ( + JSONMetadata::Events { name: lname, events: left }, + JSONMetadata::Events { name: rname, events: right } + ) => { + lname == rname && left.iter().zip(right.iter()).fold(true, |res, (l, r)| { + res && l.0 == r.0 && l.1() == r.1() + }) + }, + ( + JSONMetadata::Module { prefix: lpre, module: lmod }, + JSONMetadata::Module { prefix: rpre, module: rmod } + ) => { + lpre == rpre && lmod == rmod + }, + ( + JSONMetadata::ModuleWithStorage { prefix: lpre, module: lmod, storage: lstore }, + JSONMetadata::ModuleWithStorage { prefix: rpre, module: rmod, storage: rstore } + ) => { + lpre == rpre && lmod == rmod && lstore == rstore + }, + _ => false, + } + } +} + +/// Utility struct for making `JSONMetadata` decodeable. +#[derive(Eq, PartialEq, Debug)] +#[cfg(feature = "std")] +pub enum JSONMetadataDecodable { + Events { name: String, events: Vec<(String, String)> }, + Module { module: String, prefix: String }, + ModuleWithStorage { module: String, prefix: String, storage: String } +} + +#[cfg(feature = "std")] +impl JSONMetadataDecodable { + /// Returns the instance as JSON string. + /// The first value of the tuple is the name of the metadata type and the second in the JSON string. + pub fn into_json_string(self) -> (&'static str, String) { + match self { + JSONMetadataDecodable::Events { name, events } => { + ( + "events", + format!( + r#"{{ "name": "{}", "events": {{ {} }} }}"#, name, + events.iter().enumerate() + .fold(String::from(""), |mut json, (i, (name, data))| { + if i > 0 { + json.push_str(", "); + } + json.push_str(&format!(r#""{}": {}"#, name, data)); + json + }) + ) + ) + }, + JSONMetadataDecodable::Module { prefix, module } => { + ("module", format!(r#"{{ "prefix": "{}", "module": {} }}"#, prefix, module)) + }, + JSONMetadataDecodable::ModuleWithStorage { prefix, module, storage } => { + ( + "moduleWithStorage", + format!( + r#"{{ "prefix": "{}", "module": {}, "storage": {} }}"#, + prefix, module, storage + ) + ) + } + } + } +} + +#[cfg(feature = "std")] +impl Decode for JSONMetadataDecodable { + fn decode(input: &mut I) -> Option { + i8::decode(input).and_then(|variant| { + match variant { + 0 => String::decode(input) + .and_then(|name| Vec::<(String, String)>::decode(input).map(|events| (name, events))) + .and_then(|(name, events)| Some(JSONMetadataDecodable::Events { name, events })), + 1 => String::decode(input) + .and_then(|prefix| String::decode(input).map(|v| (prefix, v))) + .and_then(|(prefix, module)| Some(JSONMetadataDecodable::Module { prefix, module })), + 2 => String::decode(input) + .and_then(|prefix| String::decode(input).map(|v| (prefix, v))) + .and_then(|(prefix, module)| String::decode(input).map(|v| (prefix, module, v))) + .and_then(|(prefix, module, storage)| Some(JSONMetadataDecodable::ModuleWithStorage { prefix, module, storage })), + _ => None, + } + }) + } +} + +#[cfg(test)] +impl PartialEq for JSONMetadataDecodable { + fn eq(&self, other: &JSONMetadata) -> bool { + match (self, other) { + ( + JSONMetadataDecodable::Events { name: lname, events: left }, + JSONMetadata::Events { name: rname, events: right } + ) => { + lname == rname && left.iter().zip(right.iter()).fold(true, |res, (l, r)| { + res && l.0 == r.0 && l.1 == r.1() + }) + }, + ( + JSONMetadataDecodable::Module { prefix: lpre, module: lmod }, + JSONMetadata::Module { prefix: rpre, module: rmod } + ) => { + lpre == rpre && lmod == rmod + }, + ( + JSONMetadataDecodable::ModuleWithStorage { prefix: lpre, module: lmod, storage: lstore }, + JSONMetadata::ModuleWithStorage { prefix: rpre, module: rmod, storage: rstore } + ) => { + lpre == rpre && lmod == rmod && lstore == rstore + }, + _ => false, + } + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_json_metadata { + ( + $runtime: ident; + $( $metadata:expr ),*; + $mod:ident::$module:ident, + $( $rest:tt )* + ) => { + __impl_json_metadata!( + $runtime; + $( $metadata, )* $crate::metadata::JSONMetadata::Module { + module: $mod::$module::<$runtime>::json_metadata(), prefix: stringify!($mod) + }; + $( $rest )* + ) + }; + ( + $runtime: ident; + $( $metadata:expr ),*; + $mod:ident::$module:ident + ) => { + __impl_json_metadata!( + $runtime; + $( $metadata, )* $crate::metadata::JSONMetadata::Module { + module: $mod::$module::<$runtime>::json_metadata(), prefix: stringify!($mod) + }; + ) + }; + ( + $runtime: ident; + $( $metadata:expr ),*; + $mod:ident::$module:ident with Storage, + $( $rest:tt )* + ) => { + __impl_json_metadata!( + $runtime; + $( $metadata, )* $crate::metadata::JSONMetadata::ModuleWithStorage { + module: $mod::$module::<$runtime>::json_metadata(), prefix: stringify!($mod), + storage: $mod::$module::<$runtime>::store_json_metadata() + }; + $( $rest )* + ) + }; + ( + $runtime: ident; + $( $metadata:expr ),*; + $mod:ident::$module:ident with Storage + ) => { + __impl_json_metadata!( + $runtime; + $( $metadata, )* $crate::metadata::JSONMetadata::ModuleWithStorage { + module: $mod::$module::<$runtime>::json_metadata(), prefix: stringify!($mod), + storage: $mod::$module::<$runtime>::store_json_metadata() + }; + ) + }; + ( + $runtime:ident; + $( $metadata:expr ),*; + ) => { + <[_]>::into_vec($crate::metadata::Box::new([ $( $metadata ),* ])) + }; +} + +#[cfg(test)] +// Do not complain about unused `dispatch` and `dispatch_aux`. +#[allow(dead_code)] +mod tests { + use super::*; + use serde; + use serde_json; + + mod system { + pub trait Trait { + type Origin; + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + + decl_event!( + pub enum Event { + SystemEvent, + } + ); + } + + mod event_module { + use dispatch::Result; + + pub trait Trait { + type Origin; + type Balance; + } + + decl_event!( + pub enum Event with RawEvent + where ::Balance + { + /// Hi, I am a comment. + TestEvent(Balance), + } + ); + + decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn aux_0(origin) -> Result; + } + } + + impl Module { + fn aux_0(_: T::Origin) -> Result { + unreachable!() + } + } + } + + mod event_module2 { + pub trait Trait { + type Origin; + type Balance; + } + + decl_event!( + pub enum Event with RawEvent + where ::Balance + { + TestEvent(Balance), + } + ); + + decl_module! { + pub struct ModuleWithStorage for enum Call where origin: T::Origin {} + } + + decl_storage! { + trait Store for ModuleWithStorage as TestStorage { + StorageMethod : u32; + } + } + } + + #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode, Deserialize, Serialize)] + pub struct TestRuntime; + + impl_outer_event! { + pub enum TestEvent for TestRuntime { + event_module, event_module2 + } + } + + impl event_module::Trait for TestRuntime { + type Origin = u32; + type Balance = u32; + } + + impl event_module2::Trait for TestRuntime { + type Origin = u32; + type Balance = u32; + } + + impl system::Trait for TestRuntime { + type Origin = u32; + } + + impl_json_metadata!( + for TestRuntime with modules + event_module::Module, + event_module2::ModuleWithStorage with Storage + ); + + fn system_event_json() -> &'static str { + r#"{ "SystemEvent": { "params": null, "description": [ ] } }"# + } + + fn event_module_event_json() -> &'static str { + r#"{ "TestEvent": { "params": [ "Balance" ], "description": [ " Hi, I am a comment." ] } }"# + } + + fn event_module2_event_json() -> &'static str { + r#"{ "TestEvent": { "params": [ "Balance" ], "description": [ ] } }"# + } + + const EXPECTED_METADATA: &[JSONMetadata] = &[ + JSONMetadata::Events { + name: "TestEvent", + events: &[ + ("system", system_event_json), + ("event_module", event_module_event_json), + ("event_module2", event_module2_event_json), + ] + }, + JSONMetadata::Module { + module: concat!( + r#"{ "name": "Module", "call": "#, + r#"{ "name": "Call", "functions": "#, + r#"{ "0": { "name": "aux_0", "params": [ "#, + r#"{ "name": "origin", "type": "T::Origin" } ], "#, + r#""description": [ ] } } } }"# + ), + prefix: "event_module" + }, + JSONMetadata::ModuleWithStorage { + module: r#"{ "name": "ModuleWithStorage", "call": { "name": "Call", "functions": { } } }"#, + prefix: "event_module2", + storage: concat!( + r#"{ "prefix": "TestStorage", "items": { "#, + r#""StorageMethod": { "description": [ ], "modifier": null, "type": "u32" }"#, + r#" } }"# + ) + } + ]; + + #[test] + fn runtime_json_metadata() { + let metadata = TestRuntime::json_metadata(); + assert_eq!(EXPECTED_METADATA, &metadata[..]); + } + + #[test] + fn json_metadata_encode_and_decode() { + let metadata = TestRuntime::json_metadata(); + let metadata_encoded = metadata.encode(); + let metadata_decoded = Vec::::decode(&mut &metadata_encoded[..]); + + assert_eq!(&metadata_decoded.unwrap()[..], &metadata[..]); + } + + #[test] + fn into_json_string_is_valid_json() { + let metadata = TestRuntime::json_metadata(); + let metadata_encoded = metadata.encode(); + let metadata_decoded = Vec::::decode(&mut &metadata_encoded[..]); + + for mdata in metadata_decoded.unwrap().into_iter() { + let json = mdata.into_json_string(); + let _: serde::de::IgnoredAny = + serde_json::from_str(&json.1).expect(&format!("Is valid json syntax: {}", json.1)); + } + } +} diff --git a/runtime/support/src/storage/generator.rs b/runtime/support/src/storage/generator.rs new file mode 100644 index 00000000000..7c820cc5cce --- /dev/null +++ b/runtime/support/src/storage/generator.rs @@ -0,0 +1,1659 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Strongly typed wrappers around values in storage. +//! +//! This crate exports a macro `storage_items!` and traits describing behavior of generated +//! structs. +//! +//! Three kinds of data types are currently supported: +//! - values +//! - maps +//! - lists +//! +//! # Examples: +//! +//! ```rust +//! #[macro_use] +//! extern crate substrate_runtime_support; +//! +//! type AuthorityId = [u8; 32]; +//! type Balance = u64; +//! pub type SessionKey = [u8; 32]; +//! +//! storage_items! { +//! // public value +//! pub Value: b"putd_key" => SessionKey; +//! // private map. +//! Balances: b"private_map:" => map [AuthorityId => Balance]; +//! // private list. +//! Authorities: b"auth:" => list [AuthorityId]; +//! } +//! +//!# fn main() { } +//! ``` + +use codec; +use rstd::vec::Vec; +#[doc(hidden)] +pub use rstd::borrow::Borrow; +#[doc(hidden)] +pub use rstd::marker::PhantomData; + +/// Abstraction around storage. +pub trait Storage { + /// true if the key exists in storage. + fn exists(&self, key: &[u8]) -> bool; + + /// Load the bytes of a key from storage. Can panic if the type is incorrect. + fn get(&self, key: &[u8]) -> Option; + + /// Load the bytes of a key from storage. Can panic if the type is incorrect. Will panic if + /// it's not there. + fn require(&self, key: &[u8]) -> T { self.get(key).expect("Required values must be in storage") } + + /// Load the bytes of a key from storage. Can panic if the type is incorrect. The type's + /// default is returned if it's not there. + fn get_or_default(&self, key: &[u8]) -> T { self.get(key).unwrap_or_default() } + + /// Put a value in under a key. + fn put(&self, key: &[u8], val: &T); + + /// Remove the bytes of a key from storage. + fn kill(&self, key: &[u8]); + + /// Take a value from storage, deleting it after reading. + fn take(&self, key: &[u8]) -> Option { + let value = self.get(key); + self.kill(key); + value + } + + /// Take a value from storage, deleting it after reading. + fn take_or_panic(&self, key: &[u8]) -> T { self.take(key).expect("Required values must be in storage") } + + /// Take a value from storage, deleting it after reading. + fn take_or_default(&self, key: &[u8]) -> T { self.take(key).unwrap_or_default() } +} + +/// A strongly-typed value kept in storage. +pub trait StorageValue { + /// The type that get/take returns. + type Query; + + /// Get the storage key. + fn key() -> &'static [u8]; + + /// true if the value is defined in storage. + fn exists(storage: &S) -> bool { + storage.exists(Self::key()) + } + + /// Load the value from the provided storage instance. + fn get(storage: &S) -> Self::Query; + + /// Take a value from storage, removing it afterwards. + fn take(storage: &S) -> Self::Query; + + /// Store a value under this key into the provided storage instance. + fn put(val: &T, storage: &S) { + storage.put(Self::key(), val) + } + + /// Mutate this value + fn mutate(f: F, storage: &S); + + /// Clear the storage value. + fn kill(storage: &S) { + storage.kill(Self::key()) + } +} + +/// A strongly-typed list in storage. +pub trait StorageList { + /// Get the prefix key in storage. + fn prefix() -> &'static [u8]; + + /// Get the key used to put the length field. + fn len_key() -> Vec; + + /// Get the storage key used to fetch a value at a given index. + fn key_for(index: u32) -> Vec; + + /// Read out all the items. + fn items(storage: &S) -> Vec; + + /// Set the current set of items. + fn set_items(items: &[T], storage: &S); + + /// Set the item at the given index. + fn set_item(index: u32, item: &T, storage: &S); + + /// Load the value at given index. Returns `None` if the index is out-of-bounds. + fn get(index: u32, storage: &S) -> Option; + + /// Load the length of the list + fn len(storage: &S) -> u32; + + /// Clear the list. + fn clear(storage: &S); +} + +/// A strongly-typed map in storage. +pub trait StorageMap { + /// The type that get/take returns. + type Query; + + /// Get the prefix key in storage. + fn prefix() -> &'static [u8]; + + /// Get the storage key used to fetch a value corresponding to a specific key. + fn key_for(x: &K) -> Vec; + + /// true if the value is defined in storage. + fn exists(key: &K, storage: &S) -> bool { + storage.exists(&Self::key_for(key)[..]) + } + + /// Load the value associated with the given key from the map. + fn get(key: &K, storage: &S) -> Self::Query; + + /// Take the value under a key. + fn take(key: &K, storage: &S) -> Self::Query; + + /// Store a value to be associated with the given key from the map. + fn insert(key: &K, val: &V, storage: &S) { + storage.put(&Self::key_for(key)[..], val); + } + + /// Remove the value under a key. + fn remove(key: &K, storage: &S) { + storage.kill(&Self::key_for(key)[..]); + } + + /// Mutate the value under a key. + fn mutate(key: &K, f: F, storage: &S); +} + +// TODO: Remove this in favour of `decl_storage` macro. +/// Declares strongly-typed wrappers around codec-compatible types in storage. +#[macro_export] +macro_rules! storage_items { + // simple values + ($name:ident : $key:expr => $ty:ty; $($t:tt)*) => { + __storage_items_internal!(() () (OPTION_TYPE Option<$ty>) (get) (take) $name: $key => $ty); + storage_items!($($t)*); + }; + (pub $name:ident : $key:expr => $ty:ty; $($t:tt)*) => { + __storage_items_internal!((pub) () (OPTION_TYPE Option<$ty>) (get) (take) $name: $key => $ty); + storage_items!($($t)*); + }; + ($name:ident : $key:expr => default $ty:ty; $($t:tt)*) => { + __storage_items_internal!(() () (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $key => $ty); + storage_items!($($t)*); + }; + (pub $name:ident : $key:expr => default $ty:ty; $($t:tt)*) => { + __storage_items_internal!((pub) () (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $key => $ty); + storage_items!($($t)*); + }; + ($name:ident : $key:expr => required $ty:ty; $($t:tt)*) => { + __storage_items_internal!(() () (RAW_TYPE $ty) (require) (take_or_panic) $name: $key => $ty); + storage_items!($($t)*); + }; + (pub $name:ident : $key:expr => required $ty:ty; $($t:tt)*) => { + __storage_items_internal!((pub) () (RAW_TYPE $ty) (require) (take_or_panic) $name: $key => $ty); + storage_items!($($t)*); + }; + + ($name:ident get($getfn:ident) : $key:expr => $ty:ty; $($t:tt)*) => { + __storage_items_internal!(() ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $name: $key => $ty); + storage_items!($($t)*); + }; + (pub $name:ident get($getfn:ident) : $key:expr => $ty:ty; $($t:tt)*) => { + __storage_items_internal!((pub) ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $name: $key => $ty); + storage_items!($($t)*); + }; + ($name:ident get($getfn:ident) : $key:expr => default $ty:ty; $($t:tt)*) => { + __storage_items_internal!(() ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $key => $ty); + storage_items!($($t)*); + }; + (pub $name:ident get($getfn:ident) : $key:expr => default $ty:ty; $($t:tt)*) => { + __storage_items_internal!((pub) ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $key => $ty); + storage_items!($($t)*); + }; + ($name:ident get($getfn:ident) : $key:expr => required $ty:ty; $($t:tt)*) => { + __storage_items_internal!(() ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $name: $key => $ty); + storage_items!($($t)*); + }; + (pub $name:ident get($getfn:ident) : $key:expr => required $ty:ty; $($t:tt)*) => { + __storage_items_internal!((pub) ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $name: $key => $ty); + storage_items!($($t)*); + }; + + // maps + ($name:ident : $prefix:expr => map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!(() () (OPTION_TYPE Option<$ty>) (get) (take) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + (pub $name:ident : $prefix:expr => map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!((pub) () (OPTION_TYPE Option<$ty>) (get) (take) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + ($name:ident : $prefix:expr => default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!(() () (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + (pub $name:ident : $prefix:expr => default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!((pub) () (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + ($name:ident : $prefix:expr => required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!(() () (RAW_TYPE $ty) (require) (take_or_panic) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + (pub $name:ident : $prefix:expr => required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!((pub) () (RAW_TYPE $ty) (require) (take_or_panic) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + + ($name:ident get($getfn:ident) : $prefix:expr => map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!(() ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + (pub $name:ident get($getfn:ident) : $prefix:expr => map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!((pub) ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + ($name:ident get($getfn:ident) : $prefix:expr => default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!(() ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + (pub $name:ident get($getfn:ident) : $prefix:expr => default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!((pub) ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + ($name:ident get($getfn:ident) : $prefix:expr => required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!(() ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + (pub $name:ident get($getfn:ident) : $prefix:expr => required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __storage_items_internal!((pub) ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $name: $prefix => map [$kty => $ty]); + storage_items!($($t)*); + }; + + + // lists + ($name:ident : $prefix:expr => list [$ty:ty]; $($t:tt)*) => { + __storage_items_internal!(() $name: $prefix => list [$ty]); + storage_items!($($t)*); + }; + (pub $name:ident : $prefix:expr => list [$ty:ty]; $($t:tt)*) => { + __storage_items_internal!((pub) $name: $prefix => list [$ty]); + storage_items!($($t)*); + }; + () => () +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __handle_wrap_internal { + (RAW_TYPE { $($raw:tt)* } { $($option:tt)* }) => { + $($raw)*; + }; + (OPTION_TYPE { $($raw:tt)* } { $($option:tt)* }) => { + $($option)*; + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __storage_items_internal { + // generator for values. + (($($vis:tt)*) ($get_fn:ident) ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $name:ident : $key:expr => $ty:ty) => { + __storage_items_internal!{ ($($vis)*) () ($wraptype $gettype) ($getter) ($taker) $name : $key => $ty } + pub fn $get_fn() -> $gettype { <$name as $crate::storage::generator::StorageValue<$ty>> :: get(&$crate::storage::RuntimeStorage) } + }; + (($($vis:tt)*) () ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $name:ident : $key:expr => $ty:ty) => { + $($vis)* struct $name; + + impl $crate::storage::generator::StorageValue<$ty> for $name { + type Query = $gettype; + + /// Get the storage key. + fn key() -> &'static [u8] { + $key + } + + /// Load the value from the provided storage instance. + fn get(storage: &S) -> Self::Query { + storage.$getter($key) + } + + /// Take a value from storage, removing it afterwards. + fn take(storage: &S) -> Self::Query { + storage.$taker($key) + } + + /// Mutate this value. + fn mutate(f: F, storage: &S) { + let mut val = >::get(storage); + + f(&mut val); + + __handle_wrap_internal!($wraptype { + // raw type case + >::put(&val, storage) + } { + // Option<> type case + match val { + Some(val) => >::put(&val, storage), + None => >::kill(storage), + } + }); + } + } + }; + // generator for maps. + (($($vis:tt)*) ($get_fn:ident) ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $name:ident : $prefix:expr => map [$kty:ty => $ty:ty]) => { + __storage_items_internal!{ ($($vis)*) () ($wraptype $gettype) ($getter) ($taker) $name : $prefix => map [$kty => $ty] } + pub fn $get_fn>(key: K) -> $gettype { + <$name as $crate::storage::generator::StorageMap<$kty, $ty>> :: get(key.borrow(), &$crate::storage::RuntimeStorage) + } + }; + (($($vis:tt)*) () ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $name:ident : $prefix:expr => map [$kty:ty => $ty:ty]) => { + $($vis)* struct $name; + + impl $crate::storage::generator::StorageMap<$kty, $ty> for $name { + type Query = $gettype; + + /// Get the prefix key in storage. + fn prefix() -> &'static [u8] { + $prefix + } + + /// Get the storage key used to fetch a value corresponding to a specific key. + fn key_for(x: &$kty) -> Vec { + let mut key = $prefix.to_vec(); + $crate::codec::Encode::encode_to(x, &mut key); + key + } + + /// Load the value associated with the given key from the map. + fn get(key: &$kty, storage: &S) -> Self::Query { + let key = <$name as $crate::storage::generator::StorageMap<$kty, $ty>>::key_for(key); + storage.$getter(&key[..]) + } + + /// Take the value, reading and removing it. + fn take(key: &$kty, storage: &S) -> Self::Query { + let key = <$name as $crate::storage::generator::StorageMap<$kty, $ty>>::key_for(key); + storage.$taker(&key[..]) + } + + /// Mutate the value under a key. + fn mutate(key: &$kty, f: F, storage: &S) { + let mut val = >::take(key, storage); + + f(&mut val); + + __handle_wrap_internal!($wraptype { + // raw type case + >::insert(key, &val, storage) + } { + // Option<> type case + match val { + Some(val) => >::insert(key, &val, storage), + None => >::remove(key, storage), + } + }); + } + } + }; + // generator for lists. + (($($vis:tt)*) $name:ident : $prefix:expr => list [$ty:ty]) => { + $($vis)* struct $name; + + impl $name { + fn clear_item(index: u32, storage: &S) { + if index < <$name as $crate::storage::generator::StorageList<$ty>>::len(storage) { + storage.kill(&<$name as $crate::storage::generator::StorageList<$ty>>::key_for(index)); + } + } + + fn set_len(count: u32, storage: &S) { + (count..<$name as $crate::storage::generator::StorageList<$ty>>::len(storage)).for_each(|i| $name::clear_item(i, storage)); + storage.put(&<$name as $crate::storage::generator::StorageList<$ty>>::len_key(), &count); + } + } + + impl $crate::storage::generator::StorageList<$ty> for $name { + /// Get the prefix key in storage. + fn prefix() -> &'static [u8] { + $prefix + } + + /// Get the key used to put the length field. + // TODO: concat macro should accept byte literals. + fn len_key() -> Vec { + let mut key = $prefix.to_vec(); + key.extend(b"len"); + key + } + + /// Get the storage key used to fetch a value at a given index. + fn key_for(index: u32) -> Vec { + let mut key = $prefix.to_vec(); + $crate::codec::Encode::encode_to(&index, &mut key); + key + } + + /// Read out all the items. + fn items(storage: &S) -> Vec<$ty> { + (0..<$name as $crate::storage::generator::StorageList<$ty>>::len(storage)) + .map(|i| <$name as $crate::storage::generator::StorageList<$ty>>::get(i, storage).expect("all items within length are set; qed")) + .collect() + } + + /// Set the current set of items. + fn set_items(items: &[$ty], storage: &S) { + $name::set_len(items.len() as u32, storage); + items.iter() + .enumerate() + .for_each(|(i, item)| <$name as $crate::storage::generator::StorageList<$ty>>::set_item(i as u32, item, storage)); + } + + fn set_item(index: u32, item: &$ty, storage: &S) { + if index < <$name as $crate::storage::generator::StorageList<$ty>>::len(storage) { + storage.put(&<$name as $crate::storage::generator::StorageList<$ty>>::key_for(index)[..], item); + } + } + + /// Load the value at given index. Returns `None` if the index is out-of-bounds. + fn get(index: u32, storage: &S) -> Option<$ty> { + storage.get(&<$name as $crate::storage::generator::StorageList<$ty>>::key_for(index)[..]) + } + + /// Load the length of the list. + fn len(storage: &S) -> u32 { + storage.get(&<$name as $crate::storage::generator::StorageList<$ty>>::len_key()).unwrap_or_default() + } + + /// Clear the list. + fn clear(storage: &S) { + for i in 0..<$name as $crate::storage::generator::StorageList<$ty>>::len(storage) { + $name::clear_item(i, storage); + } + + storage.kill(&<$name as $crate::storage::generator::StorageList<$ty>>::len_key()[..]) + } + } + }; +} + +// TODO: revisit this idiom once we get `type`s in `impl`s. +/*impl Module { + type Now = super::Now; +}*/ + +/// Declares strongly-typed wrappers around codec-compatible types in storage. +/// +/// For now we implement a convenience trait with pre-specialised associated types, one for each +/// storage item. This allows you to gain access to publicly visisible storage items from a +/// module type. Currently you must disambiguate by using `::Item` rather than +/// the simpler `Module::Item`. Hopefully the rust guys with fix this soon. +#[macro_export] +macro_rules! decl_storage { + ( + trait $storetype:ident for $modulename:ident<$traitinstance:ident: $traittype:ident> as $cratename:ident { + $($t:tt)* + } + ) => { + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + trait $storetype { + __decl_store_items!($($t)*); + } + impl<$traitinstance: $traittype> $storetype for $modulename<$traitinstance> { + __impl_store_items!($traitinstance $($t)*); + } + impl<$traitinstance: $traittype> $modulename<$traitinstance> { + __impl_store_fns!($traitinstance $($t)*); + __impl_store_json_metadata!($cratename; $($t)*); + } + }; + ( + pub trait $storetype:ident for $modulename:ident<$traitinstance:ident: $traittype:ident> as $cratename:ident { + $($t:tt)* + } + ) => { + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + pub trait $storetype { + __decl_store_items!($($t)*); + } + impl<$traitinstance: $traittype> $storetype for $modulename<$traitinstance> { + __impl_store_items!($traitinstance $($t)*); + } + impl<$traitinstance: $traittype> $modulename<$traitinstance> { + __impl_store_fns!($traitinstance $($t)*); + } + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __decl_storage_items { + // simple values + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident : $ty:ty; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) () (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident : $ty:ty; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) () (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident : default $ty:ty; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) () (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident : default $ty:ty; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) () (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident : required $ty:ty; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) () (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident : required $ty:ty; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) () (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: $ty); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + + // maps + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) () (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) () (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) () (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) () (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) () (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) () (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) ($getfn) (OPTION_TYPE Option<$ty>) (get) (take) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (get_or_default) (take_or_default) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!(() ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + ($cratename:ident $traittype:ident $traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_storage_item!((pub) ($traittype as $traitinstance) ($getfn) (RAW_TYPE $ty) (require) (take_or_panic) $cratename $name: map [$kty => $ty]); + __decl_storage_items!($cratename $traittype $traitinstance $($t)*); + }; + + // exit + ($cratename:ident $traittype:ident $traitinstance:ident) => () +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __decl_storage_item { + // generator for values. + (($($vis:tt)*) ($traittype:ident as $traitinstance:ident) ($get_fn:ident) ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $cratename:ident $name:ident : $ty:ty) => { + __decl_storage_item!{ ($($vis)*) ($traittype as $traitinstance) () ($wraptype $gettype) ($getter) ($taker) $cratename $name : $ty } + }; + (($($vis:tt)*) ($traittype:ident as $traitinstance:ident) () ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $cratename:ident $name:ident : $ty:ty) => { + $($vis)* struct $name<$traitinstance: $traittype>($crate::storage::generator::PhantomData<$traitinstance>); + + impl<$traitinstance: $traittype> $crate::storage::generator::StorageValue<$ty> for $name<$traitinstance> { + type Query = $gettype; + + /// Get the storage key. + fn key() -> &'static [u8] { + stringify!($cratename $name).as_bytes() + } + + /// Load the value from the provided storage instance. + fn get(storage: &S) -> Self::Query { + storage.$getter(<$name<$traitinstance> as $crate::storage::generator::StorageValue<$ty>>::key()) + } + + /// Take a value from storage, removing it afterwards. + fn take(storage: &S) -> Self::Query { + storage.$taker(<$name<$traitinstance> as $crate::storage::generator::StorageValue<$ty>>::key()) + } + + /// Mutate the value under a key. + fn mutate(f: F, storage: &S) { + let mut val = >::get(storage); + + f(&mut val); + + __handle_wrap_internal!($wraptype { + // raw type case + >::put(&val, storage) + } { + // Option<> type case + match val { + Some(val) => >::put(&val, storage), + None => >::kill(storage), + } + }) + } + } + }; + // generator for maps. + (($($vis:tt)*) ($traittype:ident as $traitinstance:ident) ($get_fn:ident) ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $cratename:ident $name:ident : map [$kty:ty => $ty:ty]) => { + __decl_storage_item!{ ($($vis)*) ($traittype as $traitinstance) () ($wraptype $gettype) ($getter) ($taker) $cratename $name : map [$kty => $ty] } + }; + (($($vis:tt)*) ($traittype:ident as $traitinstance:ident) () ($wraptype:ident $gettype:ty) ($getter:ident) ($taker:ident) $cratename:ident $name:ident : map [$kty:ty => $ty:ty]) => { + $($vis)* struct $name<$traitinstance: $traittype>($crate::storage::generator::PhantomData<$traitinstance>); + + impl<$traitinstance: $traittype> $crate::storage::generator::StorageMap<$kty, $ty> for $name<$traitinstance> { + type Query = $gettype; + + /// Get the prefix key in storage. + fn prefix() -> &'static [u8] { + stringify!($cratename $name).as_bytes() + } + + /// Get the storage key used to fetch a value corresponding to a specific key. + fn key_for(x: &$kty) -> Vec { + let mut key = <$name<$traitinstance> as $crate::storage::generator::StorageMap<$kty, $ty>>::prefix().to_vec(); + $crate::codec::Encode::encode_to(x, &mut key); + key + } + + /// Load the value associated with the given key from the map. + fn get(key: &$kty, storage: &S) -> Self::Query { + let key = <$name<$traitinstance> as $crate::storage::generator::StorageMap<$kty, $ty>>::key_for(key); + storage.$getter(&key[..]) + } + + /// Take the value, reading and removing it. + fn take(key: &$kty, storage: &S) -> Self::Query { + let key = <$name<$traitinstance> as $crate::storage::generator::StorageMap<$kty, $ty>>::key_for(key); + storage.$taker(&key[..]) + } + + /// Mutate the value under a key + fn mutate(key: &$kty, f: F, storage: &S) { + let mut val = >::take(key, storage); + + f(&mut val); + + __handle_wrap_internal!($wraptype { + >::insert(key, &val, storage); + } { + match val { + Some(val) => >::insert(key, &val, storage), + None => >::remove(key, storage), + } + }); + } + } + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __decl_store_items { + // simple values + ($(#[$doc:meta])* $name:ident : default $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident : default $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident : required $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident : required $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident : $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident : $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + + ($(#[$doc:meta])* $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + + // maps + ($(#[$doc:meta])* $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + + ($(#[$doc:meta])* $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + ($(#[$doc:meta])* pub $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __decl_store_item!($name); __decl_store_items!($($t)*); + }; + + // exit + () => () +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __decl_store_item { + ($name:ident) => { type $name; } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_store_fns { + // simple values + ($traitinstance:ident $(#[$doc:meta])* $name:ident : default $ty:ty; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : default $ty:ty; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : required $ty:ty; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : required $ty:ty; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : $ty:ty; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : $ty:ty; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) $ty); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) $ty); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) $ty); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) $ty); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn (Option<$ty>) $ty); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn (Option<$ty>) $ty); + __impl_store_fns!($traitinstance $($t)*); + }; + + // maps + ($traitinstance:ident $(#[$doc:meta])* $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fns!($traitinstance $($t)*); + }; + + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) map [$kty => $ty]); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) map [$kty => $ty]); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) map [$kty => $ty]); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn ($ty) map [$kty => $ty]); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn (Option<$ty>) map [$kty => $ty]); + __impl_store_fns!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_fn!($traitinstance $name $getfn (Option<$ty>) map [$kty => $ty]); + __impl_store_fns!($traitinstance $($t)*); + }; + + // exit + ($traitinstance:ident) => () +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_store_fn { + ($traitinstance:ident $name:ident $get_fn:ident ($gettype:ty) $ty:ty) => { + pub fn $get_fn() -> $gettype { + <$name<$traitinstance> as $crate::storage::generator::StorageValue<$ty>> :: get(&$crate::storage::RuntimeStorage) + } + }; + ($traitinstance:ident $name:ident $get_fn:ident ($gettype:ty) map [$kty:ty => $ty:ty]) => { + pub fn $get_fn>(key: K) -> $gettype { + <$name<$traitinstance> as $crate::storage::generator::StorageMap<$kty, $ty>> :: get(key.borrow(), &$crate::storage::RuntimeStorage) + } + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_store_items { + // simple values + ($traitinstance:ident $(#[$doc:meta])* $name:ident : default $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : default $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : required $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : required $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : default $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : required $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : $ty:ty; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + + // maps + ($traitinstance:ident $(#[$doc:meta])* $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : default map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : required map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + ($traitinstance:ident $(#[$doc:meta])* pub $name:ident get($getfn:ident) : map [$kty:ty => $ty:ty]; $($t:tt)*) => { + __impl_store_item!($name $traitinstance); + __impl_store_items!($traitinstance $($t)*); + }; + + // exit + ($traitinstance:ident) => () +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_store_item { + ($name:ident $traitinstance:ident) => { type $name = $name<$traitinstance>; } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __impl_store_json_metadata { + ( + $cratename:ident; + $($rest:tt)* + ) => { + pub fn store_json_metadata() -> &'static str { + concat!(r#"{ "prefix": ""#, stringify!($cratename), r#"", "items": {"#, + __store_functions_to_json!(""; $($rest)*), " } }") + } + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __store_functions_to_json { + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident : + default $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident : + default $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident : + required $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident : + required $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + // simple values + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident : + $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident : + $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident get($getfn:ident) : + default $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident get($getfn:ident) : + default $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident get($getfn:ident) : + required $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident get($getfn:ident) : + required $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident get($getfn:ident) : + $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident get($getfn:ident) : + $ty:ty; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + + // maps + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident : + default map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident : + default map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident : + required map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident : + required map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident : + map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident : + map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* $name:ident get($getfn:ident) : + default map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident get($getfn:ident) : + default map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), default + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* $name:ident get($getfn:ident) : + required map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident get($getfn:ident) : + required map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty), required + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + $name:ident get($getfn:ident) : + map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ( + $prefix_str:tt; + $(#[doc = $doc_attr:tt])* + pub $name:ident get($getfn:ident) : + map [$kty:ty => $ty:ty]; $($t:tt)* + ) => { + concat!( + __store_function_to_json!($prefix_str, + __function_doc_to_json!(""; $($doc_attr)*), + $name, __store_type_to_json!($kty, $ty) + ), + __store_functions_to_json!(","; $($t)*) + ) + }; + ($prefix_str:tt;) => { "" } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __store_function_to_json { + ($prefix_str:tt, $fn_doc:expr, $name:ident, $type:expr, $modifier:ident) => { + __store_function_to_json!($prefix_str; $fn_doc; $name; $type; + concat!("\"", stringify!($modifier), "\"")) + }; + ($prefix_str:tt, $fn_doc:expr, $name:ident, $type:expr) => { + __store_function_to_json!($prefix_str; $fn_doc; $name; $type; "null") + }; + ($prefix_str:tt; $fn_doc:expr; $name:ident; $type:expr; $modifier:expr) => { + concat!($prefix_str, " \"", stringify!($name), "\": { ", + r#""description": ["#, $fn_doc, " ], ", + r#""modifier": "#, $modifier, r#", "type": "#, $type, r#" }"# + ) + } +} + +#[macro_export] +#[doc(hidden)] +macro_rules! __store_type_to_json { + ($name:ty) => { + concat!("\"", stringify!($name), "\"") + }; + ($key: ty, $value:ty) => { + concat!(r#"{ "key": ""#, stringify!($key), r#"", "value": ""#, + stringify!($value), "\" }") + } +} + +#[cfg(test)] +// Do not complain about unused `dispatch` and `dispatch_aux`. +#[allow(dead_code)] +mod tests { + use std::collections::HashMap; + use std::cell::RefCell; + use codec::Codec; + use super::*; + use serde; + use serde_json; + + impl Storage for RefCell, Vec>> { + fn exists(&self, key: &[u8]) -> bool { + self.borrow_mut().get(key).is_some() + } + + fn get(&self, key: &[u8]) -> Option { + self.borrow_mut().get(key).map(|v| T::decode(&mut &v[..]).unwrap()) + } + + fn put(&self, key: &[u8], val: &T) { + self.borrow_mut().insert(key.to_owned(), val.encode()); + } + + fn kill(&self, key: &[u8]) { + self.borrow_mut().remove(key); + } + } + + storage_items! { + Value: b"a" => u32; + List: b"b:" => list [u64]; + Map: b"c:" => map [u32 => [u8; 32]]; + } + + #[test] + fn value() { + let storage = RefCell::new(HashMap::new()); + assert!(Value::get(&storage).is_none()); + Value::put(&100_000, &storage); + assert_eq!(Value::get(&storage), Some(100_000)); + Value::kill(&storage); + assert!(Value::get(&storage).is_none()); + } + + #[test] + fn list() { + let storage = RefCell::new(HashMap::new()); + assert_eq!(List::len(&storage), 0); + assert!(List::items(&storage).is_empty()); + + List::set_items(&[0, 2, 4, 6, 8], &storage); + assert_eq!(List::items(&storage), &[0, 2, 4, 6, 8]); + assert_eq!(List::len(&storage), 5); + + List::set_item(2, &10, &storage); + assert_eq!(List::items(&storage), &[0, 2, 10, 6, 8]); + assert_eq!(List::len(&storage), 5); + + List::clear(&storage); + assert_eq!(List::len(&storage), 0); + assert!(List::items(&storage).is_empty()); + } + + #[test] + fn map() { + let storage = RefCell::new(HashMap::new()); + assert!(Map::get(&5, &storage).is_none()); + Map::insert(&5, &[1; 32], &storage); + assert_eq!(Map::get(&5, &storage), Some([1; 32])); + assert_eq!(Map::take(&5, &storage), Some([1; 32])); + assert!(Map::get(&5, &storage).is_none()); + assert!(Map::get(&999, &storage).is_none()); + } + + pub trait Trait { + type Origin; + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + + decl_storage! { + trait Store for Module as TestStorage { + /// Hello, this is doc! + U32 : u32; + GETU32 get(u32_getter): u32; + pub PUBU32 : u32; + pub GETPUBU32 get(pub_u32_getter): u32; + U32Default : default u32; + GETU32Default get(get_u32_default): default u32; + pub PUBU32Default : default u32; + pub GETPUBU32Default get(pub_get_u32_default): default u32; + U32Required : required u32; + GETU32Required get(get_u32_required): required u32; + pub PUBU32Required : required u32; + pub GETPUBU32Required get(pub_get_u32_required): required u32; + + MAPU32 : map [ u32 => String ]; + /// Hello, this is doc! + /// Hello, this is doc 2! + GETMAPU32 get(map_u32_getter): map [ u32 => String ]; + pub PUBMAPU32 : map [ u32 => String ]; + pub GETPUBMAPU32 get(map_pub_u32_getter): map [ u32 => String ]; + MAPU32Default : default map [ u32 => String ]; + GETMAPU32Default get(map_get_u32_default): default map [ u32 => String ]; + pub PUBMAPU32Default : default map [ u32 => String ]; + pub GETPUBMAPU32Default get(map_pub_get_u32_default): default map [ u32 => String ]; + MAPU32Required : required map [ u32 => String ]; + GETMAPU32Required get(map_get_u32_required): required map [ u32 => String ]; + pub PUBMAPU32Required : required map [ u32 => String ]; + pub GETPUBMAPU32Required get(map_pub_get_u32_required): required map [ u32 => String ]; + + } + } + + struct TraitImpl {} + + impl Trait for TraitImpl { + type Origin = u32; + } + + const EXPECTED_METADATA: &str = concat!( + r#"{ "prefix": "TestStorage", "items": { "#, + r#""U32": { "description": [ " Hello, this is doc!" ], "modifier": null, "type": "u32" }, "#, + r#""GETU32": { "description": [ ], "modifier": null, "type": "u32" }, "#, + r#""PUBU32": { "description": [ ], "modifier": null, "type": "u32" }, "#, + r#""GETPUBU32": { "description": [ ], "modifier": null, "type": "u32" }, "#, + r#""U32Default": { "description": [ ], "modifier": "default", "type": "u32" }, "#, + r#""GETU32Default": { "description": [ ], "modifier": "default", "type": "u32" }, "#, + r#""PUBU32Default": { "description": [ ], "modifier": "default", "type": "u32" }, "#, + r#""GETPUBU32Default": { "description": [ ], "modifier": "default", "type": "u32" }, "#, + r#""U32Required": { "description": [ ], "modifier": "required", "type": "u32" }, "#, + r#""GETU32Required": { "description": [ ], "modifier": "required", "type": "u32" }, "#, + r#""PUBU32Required": { "description": [ ], "modifier": "required", "type": "u32" }, "#, + r#""GETPUBU32Required": { "description": [ ], "modifier": "required", "type": "u32" }, "#, + r#""MAPU32": { "description": [ ], "modifier": null, "type": { "key": "u32", "value": "String" } }, "#, + r#""GETMAPU32": { "description": [ " Hello, this is doc!", " Hello, this is doc 2!" ], "modifier": null, "type": { "key": "u32", "value": "String" } }, "#, + r#""PUBMAPU32": { "description": [ ], "modifier": null, "type": { "key": "u32", "value": "String" } }, "#, + r#""GETPUBMAPU32": { "description": [ ], "modifier": null, "type": { "key": "u32", "value": "String" } }, "#, + r#""MAPU32Default": { "description": [ ], "modifier": "default", "type": { "key": "u32", "value": "String" } }, "#, + r#""GETMAPU32Default": { "description": [ ], "modifier": "default", "type": { "key": "u32", "value": "String" } }, "#, + r#""PUBMAPU32Default": { "description": [ ], "modifier": "default", "type": { "key": "u32", "value": "String" } }, "#, + r#""GETPUBMAPU32Default": { "description": [ ], "modifier": "default", "type": { "key": "u32", "value": "String" } }, "#, + r#""MAPU32Required": { "description": [ ], "modifier": "required", "type": { "key": "u32", "value": "String" } }, "#, + r#""GETMAPU32Required": { "description": [ ], "modifier": "required", "type": { "key": "u32", "value": "String" } }, "#, + r#""PUBMAPU32Required": { "description": [ ], "modifier": "required", "type": { "key": "u32", "value": "String" } }, "#, + r#""GETPUBMAPU32Required": { "description": [ ], "modifier": "required", "type": { "key": "u32", "value": "String" } }"#, + " } }" + ); + + #[test] + fn store_json_metadata() { + let metadata = Module::::store_json_metadata(); + assert_eq!(EXPECTED_METADATA, metadata); + let _: serde::de::IgnoredAny = + serde_json::from_str(metadata).expect("Is valid json syntax"); + } +} + +#[cfg(test)] +// Do not complain about unused `dispatch` and `dispatch_aux`. +#[allow(dead_code)] +mod test2 { + pub trait Trait { + type Origin; + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + + type PairOf = (T, T); + + decl_storage! { + trait Store for Module as TestStorage { + SingleDef : default u32; + PairDef : default PairOf; + Single : u32; + Pair : (u32, u32); + } + } + + struct TraitImpl {} + + impl Trait for TraitImpl { + type Origin = u32; + } +} diff --git a/runtime/support/src/storage/mod.rs b/runtime/support/src/storage/mod.rs new file mode 100644 index 00000000000..7b5665bf9bc --- /dev/null +++ b/runtime/support/src/storage/mod.rs @@ -0,0 +1,609 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Stuff to do with the runtime's storage. + +use rstd::prelude::*; +use rstd::borrow::Borrow; +use runtime_io::{self, twox_128}; +use codec::{Codec, Decode, KeyedVec, Input}; + +#[macro_use] +pub mod generator; + +// TODO: consider using blake256 to avoid possible preimage attack. + +struct IncrementalInput<'a> { + key: &'a [u8], + pos: usize, +} + +impl<'a> Input for IncrementalInput<'a> { + fn read(&mut self, into: &mut [u8]) -> usize { + let len = runtime_io::read_storage(self.key, into, self.pos).unwrap_or(0); + let read = ::rstd::cmp::min(len, into.len()); + self.pos += read; + read + } +} + + /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. +pub fn get(key: &[u8]) -> Option { + let key = twox_128(key); + runtime_io::read_storage(&key[..], &mut [0; 0][..], 0).map(|_| { + let mut input = IncrementalInput { + key: &key[..], + pos: 0, + }; + Decode::decode(&mut input).expect("storage is not null, therefore must be a valid type") + }) +} + +/// Return the value of the item in storage under `key`, or the type's default if there is no +/// explicit entry. +pub fn get_or_default(key: &[u8]) -> T { + get(key).unwrap_or_else(Default::default) +} + +/// Return the value of the item in storage under `key`, or `default_value` if there is no +/// explicit entry. +pub fn get_or(key: &[u8], default_value: T) -> T { + get(key).unwrap_or(default_value) +} + +/// Return the value of the item in storage under `key`, or `default_value()` if there is no +/// explicit entry. +pub fn get_or_else T>(key: &[u8], default_value: F) -> T { + get(key).unwrap_or_else(default_value) +} + +/// Put `value` in storage under `key`. +pub fn put(key: &[u8], value: &T) { + value.using_encoded(|slice| runtime_io::set_storage(&twox_128(key)[..], slice)); +} + +/// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. +pub fn take(key: &[u8]) -> Option { + let r = get(key); + if r.is_some() { + kill(key); + } + r +} + +/// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage, +/// the default for its type. +pub fn take_or_default(key: &[u8]) -> T { + take(key).unwrap_or_else(Default::default) +} + +/// Return the value of the item in storage under `key`, or `default_value` if there is no +/// explicit entry. Ensure there is no explicit entry on return. +pub fn take_or(key: &[u8], default_value: T) -> T { + take(key).unwrap_or(default_value) +} + +/// Return the value of the item in storage under `key`, or `default_value()` if there is no +/// explicit entry. Ensure there is no explicit entry on return. +pub fn take_or_else T>(key: &[u8], default_value: F) -> T { + take(key).unwrap_or_else(default_value) +} + +/// Check to see if `key` has an explicit entry in storage. +pub fn exists(key: &[u8]) -> bool { + runtime_io::exists_storage(&twox_128(key)[..]) +} + +/// Ensure `key` has no explicit entry in storage. +pub fn kill(key: &[u8]) { + runtime_io::clear_storage(&twox_128(key)[..]); +} + +/// Get a Vec of bytes from storage. +pub fn get_raw(key: &[u8]) -> Option> { + runtime_io::storage(&twox_128(key)[..]) +} + +/// Put a raw byte slice into storage. +pub fn put_raw(key: &[u8], value: &[u8]) { + runtime_io::set_storage(&twox_128(key)[..], value) +} + +/// The underlying runtime storage. +pub struct RuntimeStorage; + +impl ::GenericStorage for RuntimeStorage { + fn exists(&self, key: &[u8]) -> bool { + super::storage::exists(key) + } + + /// Load the bytes of a key from storage. Can panic if the type is incorrect. + fn get(&self, key: &[u8]) -> Option { + super::storage::get(key) + } + + /// Put a value in under a key. + fn put(&self, key: &[u8], val: &T) { + super::storage::put(key, val) + } + + /// Remove the bytes of a key from storage. + fn kill(&self, key: &[u8]) { + super::storage::kill(key) + } + + /// Take a value from storage, deleting it after reading. + fn take(&self, key: &[u8]) -> Option { + super::storage::take(key) + } +} + +/// A trait for working with macro-generated storage values under the substrate storage API. +pub trait StorageValue { + /// The type that get/take return. + type Query; + + /// Get the storage key. + fn key() -> &'static [u8]; + + /// Does the value (explicitly) exist in storage? + fn exists() -> bool; + + /// Load the value from the provided storage instance. + fn get() -> Self::Query; + + /// Store a value under this key into the provided storage instance. + fn put>(val: Arg); + + /// Mutate the value + fn mutate(f: F); + + /// Clear the storage value. + fn kill(); + + /// Take a value from storage, removing it afterwards. + fn take() -> Self::Query; +} + +impl StorageValue for U where U: generator::StorageValue { + type Query = U::Query; + + fn key() -> &'static [u8] { + >::key() + } + fn exists() -> bool { + U::exists(&RuntimeStorage) + } + fn get() -> Self::Query { + U::get(&RuntimeStorage) + } + fn put>(val: Arg) { + U::put(val.borrow(), &RuntimeStorage) + } + fn mutate(f: F) { + U::mutate(f, &RuntimeStorage) + } + fn kill() { + U::kill(&RuntimeStorage) + } + fn take() -> Self::Query { + U::take(&RuntimeStorage) + } +} + +/// A strongly-typed list in storage. +pub trait StorageList { + /// Get the prefix key in storage. + fn prefix() -> &'static [u8]; + + /// Get the key used to store the length field. + fn len_key() -> Vec; + + /// Get the storage key used to fetch a value at a given index. + fn key_for(index: u32) -> Vec; + + /// Read out all the items. + fn items() -> Vec; + + /// Set the current set of items. + fn set_items(items: &[T]); + + /// Set the item at the given index. + fn set_item>(index: u32, val: Arg); + + /// Load the value at given index. Returns `None` if the index is out-of-bounds. + fn get(index: u32) -> Option; + + /// Load the length of the list + fn len() -> u32; + + /// Clear the list. + fn clear(); +} + +impl StorageList for U where U: generator::StorageList { + fn prefix() -> &'static [u8] { + >::prefix() + } + + fn len_key() -> Vec { + >::len_key() + } + + fn key_for(index: u32) -> Vec { + >::key_for(index) + } + + fn items() -> Vec { + U::items(&RuntimeStorage) + } + + fn set_items(items: &[T]) { + U::set_items(items, &RuntimeStorage) + } + + fn set_item>(index: u32, val: Arg) { + U::set_item(index, val.borrow(), &RuntimeStorage) + } + + fn get(index: u32) -> Option { + U::get(index, &RuntimeStorage) + } + + fn len() -> u32 { + U::len(&RuntimeStorage) + } + + fn clear() { + U::clear(&RuntimeStorage) + } +} + +/// A strongly-typed map in storage. +pub trait StorageMap { + /// The type that get/take return. + type Query; + + /// Get the prefix key in storage. + fn prefix() -> &'static [u8]; + + /// Get the storage key used to fetch a value corresponding to a specific key. + fn key_for>(key: KeyArg) -> Vec; + + /// Does the value (explicitly) exist in storage? + fn exists>(key: KeyArg) -> bool; + + /// Load the value associated with the given key from the map. + fn get>(key: KeyArg) -> Self::Query; + + /// Store a value to be associated with the given key from the map. + fn insert, ValArg: Borrow>(key: KeyArg, val: ValArg); + + /// Remove the value under a key. + fn remove>(key: KeyArg); + + /// Mutate the value under a key. + fn mutate, F: FnOnce(&mut Self::Query)>(key: KeyArg, f: F); + + /// Take the value under a key. + fn take>(key: KeyArg) -> Self::Query; +} + +impl StorageMap for U where U: generator::StorageMap { + type Query = U::Query; + + fn prefix() -> &'static [u8] { + >::prefix() + } + + fn key_for>(key: KeyArg) -> Vec { + >::key_for(key.borrow()) + } + + fn exists>(key: KeyArg) -> bool { + U::exists(key.borrow(), &RuntimeStorage) + } + + fn get>(key: KeyArg) -> Self::Query { + U::get(key.borrow(), &RuntimeStorage) + } + + fn insert, ValArg: Borrow>(key: KeyArg, val: ValArg) { + U::insert(key.borrow(), val.borrow(), &RuntimeStorage) + } + + fn remove>(key: KeyArg) { + U::remove(key.borrow(), &RuntimeStorage) + } + + fn mutate, F: FnOnce(&mut Self::Query)>(key: KeyArg, f: F) { + U::mutate(key.borrow(), f, &RuntimeStorage) + } + + fn take>(key: KeyArg) -> Self::Query { + U::take(key.borrow(), &RuntimeStorage) + } +} + +/// A trait to conveniently store a vector of storable data. +pub trait StorageVec { + type Item: Default + Sized + Codec; + const PREFIX: &'static [u8]; + + /// Get the current set of items. + fn items() -> Vec { + (0..Self::count()).into_iter().map(Self::item).collect() + } + + /// Set the current set of items. + fn set_items(items: I) + where + I: IntoIterator, + T: Borrow, + { + let mut count: u32 = 0; + + for i in items.into_iter() { + put(&count.to_keyed_vec(Self::PREFIX), i.borrow()); + count = count.checked_add(1).expect("exceeded runtime storage capacity"); + } + + Self::set_count(count); + } + + /// Push an item. + fn push(item: &Self::Item) { + let len = Self::count(); + put(&len.to_keyed_vec(Self::PREFIX), item); + Self::set_count(len + 1); + } + + fn set_item(index: u32, item: &Self::Item) { + if index < Self::count() { + put(&index.to_keyed_vec(Self::PREFIX), item); + } + } + + fn clear_item(index: u32) { + if index < Self::count() { + kill(&index.to_keyed_vec(Self::PREFIX)); + } + } + + fn item(index: u32) -> Self::Item { + get_or_default(&index.to_keyed_vec(Self::PREFIX)) + } + + fn set_count(count: u32) { + (count..Self::count()).for_each(Self::clear_item); + put(&b"len".to_keyed_vec(Self::PREFIX), &count); + } + + fn count() -> u32 { + get_or_default(&b"len".to_keyed_vec(Self::PREFIX)) + } +} + +pub mod unhashed { + use rstd::borrow::Borrow; + use super::{runtime_io, Codec, Decode, KeyedVec, Vec, IncrementalInput}; + + /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. + pub fn get(key: &[u8]) -> Option { + runtime_io::read_storage(key, &mut [0; 0][..], 0).map(|_| { + let mut input = IncrementalInput { + key, + pos: 0, + }; + Decode::decode(&mut input).expect("storage is not null, therefore must be a valid type") + }) + } + + /// Return the value of the item in storage under `key`, or the type's default if there is no + /// explicit entry. + pub fn get_or_default(key: &[u8]) -> T { + get(key).unwrap_or_else(Default::default) + } + + /// Return the value of the item in storage under `key`, or `default_value` if there is no + /// explicit entry. + pub fn get_or(key: &[u8], default_value: T) -> T { + get(key).unwrap_or(default_value) + } + + /// Return the value of the item in storage under `key`, or `default_value()` if there is no + /// explicit entry. + pub fn get_or_else T>(key: &[u8], default_value: F) -> T { + get(key).unwrap_or_else(default_value) + } + + /// Put `value` in storage under `key`. + pub fn put(key: &[u8], value: &T) { + value.using_encoded(|slice| runtime_io::set_storage(key, slice)); + } + + /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. + pub fn take(key: &[u8]) -> Option { + let r = get(key); + if r.is_some() { + kill(key); + } + r + } + + /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage, + /// the default for its type. + pub fn take_or_default(key: &[u8]) -> T { + take(key).unwrap_or_else(Default::default) + } + + /// Return the value of the item in storage under `key`, or `default_value` if there is no + /// explicit entry. Ensure there is no explicit entry on return. + pub fn take_or(key: &[u8], default_value: T) -> T { + take(key).unwrap_or(default_value) + } + + /// Return the value of the item in storage under `key`, or `default_value()` if there is no + /// explicit entry. Ensure there is no explicit entry on return. + pub fn take_or_else T>(key: &[u8], default_value: F) -> T { + take(key).unwrap_or_else(default_value) + } + + /// Check to see if `key` has an explicit entry in storage. + pub fn exists(key: &[u8]) -> bool { + runtime_io::read_storage(key, &mut [0;0][..], 0).is_some() + } + + /// Ensure `key` has no explicit entry in storage. + pub fn kill(key: &[u8]) { + runtime_io::clear_storage(key); + } + + /// Ensure keys with the given `prefix` have no entries in storage. + pub fn kill_prefix(prefix: &[u8]) { + runtime_io::clear_prefix(prefix); + } + + /// Get a Vec of bytes from storage. + pub fn get_raw(key: &[u8]) -> Option> { + runtime_io::storage(key) + } + + /// Put a raw byte slice into storage. + pub fn put_raw(key: &[u8], value: &[u8]) { + runtime_io::set_storage(key, value) + } + + /// A trait to conveniently store a vector of storable data. + pub trait StorageVec { + type Item: Default + Sized + Codec; + const PREFIX: &'static [u8]; + + /// Get the current set of items. + fn items() -> Vec { + (0..Self::count()).into_iter().map(Self::item).collect() + } + + /// Set the current set of items. + fn set_items(items: I) + where + I: IntoIterator, + T: Borrow, + { + let mut count: u32 = 0; + + for i in items.into_iter() { + put(&count.to_keyed_vec(Self::PREFIX), i.borrow()); + count = count.checked_add(1).expect("exceeded runtime storage capacity"); + } + + Self::set_count(count); + } + + fn set_item(index: u32, item: &Self::Item) { + if index < Self::count() { + put(&index.to_keyed_vec(Self::PREFIX), item); + } + } + + fn clear_item(index: u32) { + if index < Self::count() { + kill(&index.to_keyed_vec(Self::PREFIX)); + } + } + + fn item(index: u32) -> Self::Item { + get_or_default(&index.to_keyed_vec(Self::PREFIX)) + } + + fn set_count(count: u32) { + (count..Self::count()).for_each(Self::clear_item); + put(&b"len".to_keyed_vec(Self::PREFIX), &count); + } + + fn count() -> u32 { + get_or_default(&b"len".to_keyed_vec(Self::PREFIX)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use runtime_io::{twox_128, TestExternalities, with_externalities}; + + #[test] + fn integers_can_be_stored() { + let mut t = TestExternalities::new(); + with_externalities(&mut t, || { + let x = 69u32; + put(b":test", &x); + let y: u32 = get(b":test").unwrap(); + assert_eq!(x, y); + }); + with_externalities(&mut t, || { + let x = 69426942i64; + put(b":test", &x); + let y: i64 = get(b":test").unwrap(); + assert_eq!(x, y); + }); + } + + #[test] + fn bools_can_be_stored() { + let mut t = TestExternalities::new(); + with_externalities(&mut t, || { + let x = true; + put(b":test", &x); + let y: bool = get(b":test").unwrap(); + assert_eq!(x, y); + }); + + with_externalities(&mut t, || { + let x = false; + put(b":test", &x); + let y: bool = get(b":test").unwrap(); + assert_eq!(x, y); + }); + } + + #[test] + fn vecs_can_be_retrieved() { + let mut t = TestExternalities::new(); + with_externalities(&mut t, || { + runtime_io::set_storage(&twox_128(b":test"), b"\x0b\0\0\0Hello world"); + let x = b"Hello world".to_vec(); + let y = get::>(b":test").unwrap(); + assert_eq!(x, y); + + }); + } + + #[test] + fn vecs_can_be_stored() { + let mut t = TestExternalities::new(); + let x = b"Hello world".to_vec(); + + with_externalities(&mut t, || { + put(b":test", &x); + }); + + with_externalities(&mut t, || { + let y: Vec = get(b":test").unwrap(); + assert_eq!(x, y); + }); + } +} diff --git a/runtime/system/Cargo.toml b/runtime/system/Cargo.toml new file mode 100644 index 00000000000..2991c2a00fe --- /dev/null +++ b/runtime/system/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-runtime-system" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +safe-mix = { version = "1.0", default_features = false} +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "safe-mix/std", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", +] diff --git a/runtime/system/src/lib.rs b/runtime/system/src/lib.rs new file mode 100644 index 00000000000..ae8ceff6a60 --- /dev/null +++ b/runtime/system/src/lib.rs @@ -0,0 +1,426 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! System manager: Handles lowest level stuff like depositing logs, basic set up and take down of +//! temporary storage entries, access to old block hashes. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(any(feature = "std", test))] +extern crate substrate_primitives; + +#[cfg_attr(any(feature = "std", test), macro_use)] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_codec as codec; +extern crate substrate_runtime_io as runtime_io; +extern crate substrate_runtime_primitives as primitives; +extern crate safe_mix; + +use rstd::prelude::*; +use primitives::traits::{self, CheckEqual, SimpleArithmetic, SimpleBitOps, Zero, One, Bounded, + Hash, Member, MaybeDisplay, EnsureOrigin}; +use runtime_support::{StorageValue, StorageMap, Parameter}; +use safe_mix::TripletMix; + +#[cfg(any(feature = "std", test))] +use rstd::marker::PhantomData; +#[cfg(any(feature = "std", test))] +use codec::Encode; + +#[cfg(any(feature = "std", test))] +use runtime_io::{twox_128, TestExternalities, Blake2Hasher}; + +/// Compute the extrinsics root of a list of extrinsics. +pub fn extrinsics_root(extrinsics: &[E]) -> H::Output { + extrinsics_data_root::(extrinsics.iter().map(codec::Encode::encode).collect()) +} + +/// Compute the extrinsics root of a list of extrinsics. +pub fn extrinsics_data_root(xts: Vec>) -> H::Output { + let xts = xts.iter().map(Vec::as_slice).collect::>(); + H::enumerated_trie_root(&xts) +} + +pub trait Trait: Eq + Clone { + type Origin: Into>> + From>; + type Index: Parameter + Member + Default + MaybeDisplay + SimpleArithmetic + Copy; + type BlockNumber: Parameter + Member + MaybeDisplay + SimpleArithmetic + Default + Bounded + Copy + rstd::hash::Hash; + type Hash: Parameter + Member + MaybeDisplay + SimpleBitOps + Default + Copy + CheckEqual + rstd::hash::Hash + AsRef<[u8]>; + type Hashing: Hash; + type Digest: Parameter + Member + Default + traits::Digest; + type AccountId: Parameter + Member + MaybeDisplay + Ord + Default; + type Header: Parameter + traits::Header< + Number = Self::BlockNumber, + Hash = Self::Hash, + Digest = Self::Digest + >; + type Event: Parameter + Member + From; +} + +pub type DigestItemOf = <::Digest as traits::Digest>::Item; + +decl_module! { + pub struct Module for enum Call where origin: T::Origin {} +} + +/// A phase of a block's execution. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(Serialize, PartialEq, Eq, Clone, Debug))] +pub enum Phase { + /// Applying an extrinsic. + ApplyExtrinsic(u32), + /// The end. + Finalization, +} + +/// Record of an event happening. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(Serialize, PartialEq, Eq, Clone, Debug))] +pub struct EventRecord { + /// The phase of the block it happened in. + pub phase: Phase, + /// The event itself. + pub event: E, +} + +/// Event for the system module. +decl_event!( + pub enum Event { + /// An extrinsic completed successfully. + ExtrinsicSuccess, + /// An extrinsic failed. + ExtrinsicFailed, + } +); + +/// Origin for the system module. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum RawOrigin { + /// The system itself ordained this dispatch to happen: this is the highest privilege level. + Root, + /// It is signed by some public key and we provide the AccountId. + Signed(AccountId), + /// It is signed by nobody but included and agreed upon by the validators anyway: it's "inherently" true. + Inherent, +} + +impl From> for RawOrigin { + fn from(s: Option) -> RawOrigin { + match s { + Some(who) => RawOrigin::Signed(who), + None => RawOrigin::Inherent, + } + } +} + +/// Exposed trait-generic origin type. +pub type Origin = RawOrigin<::AccountId>; + +decl_storage! { + trait Store for Module as System { + + pub AccountNonce get(account_nonce): default map [ T::AccountId => T::Index ]; + + ExtrinsicCount: u32; + pub BlockHash get(block_hash): required map [ T::BlockNumber => T::Hash ]; + pub ExtrinsicIndex get(extrinsic_index): u32; + ExtrinsicData get(extrinsic_data): required map [ u32 => Vec ]; + RandomSeed get(random_seed): required T::Hash; + /// The current block number being processed. Set by `execute_block`. + Number get(block_number): required T::BlockNumber; + ParentHash get(parent_hash): required T::Hash; + ExtrinsicsRoot get(extrinsics_root): required T::Hash; + Digest get(digest): default T::Digest; + + Events get(events): default Vec>; + } +} + +pub struct EnsureRoot(::rstd::marker::PhantomData); +impl>>, AccountId> EnsureOrigin for EnsureRoot { + type Success = (); + fn ensure_origin(o: O) -> Result { + ensure_root(o) + } +} + +/// Ensure that the origin `o` represents a signed extrinsic (i.e. transaction). +/// Returns `Ok` with the account that signed the extrinsic or an `Err` otherwise. +pub fn ensure_signed(o: OuterOrigin) -> Result + where OuterOrigin: Into>> +{ + match o.into() { + Some(RawOrigin::Signed(t)) => Ok(t), + _ => Err("bad origin: expected to be a signed origin"), + } +} + +/// Ensure that the origin `o` represents the root. Returns `Ok` or an `Err` otherwise. +pub fn ensure_root(o: OuterOrigin) -> Result<(), &'static str> + where OuterOrigin: Into>> +{ + match o.into() { + Some(RawOrigin::Root) => Ok(()), + _ => Err("bad origin: expected to be a root origin"), + } +} + +/// Ensure that the origin `o` represents an unsigned extrinsic. Returns `Ok` or an `Err` otherwise. +pub fn ensure_inherent(o: OuterOrigin) -> Result<(), &'static str> + where OuterOrigin: Into>> +{ + match o.into() { + Some(RawOrigin::Inherent) => Ok(()), + _ => Err("bad origin: expected to be an inherent origin"), + } +} + +impl Module { + /// Start the execution of a particular block. + pub fn initialise(number: &T::BlockNumber, parent_hash: &T::Hash, txs_root: &T::Hash) { + // populate environment. + >::put(number); + >::put(parent_hash); + >::insert(*number - One::one(), parent_hash); + >::put(txs_root); + >::put(Self::calculate_random()); + >::put(0u32); + >::kill(); + } + + /// Remove temporary "environment" entries in storage. + pub fn finalise() -> T::Header { + >::kill(); + >::kill(); + + let number = >::take(); + let parent_hash = >::take(); + let digest = >::take(); + let extrinsics_root = >::take(); + let storage_root = T::Hashing::storage_root(); + + // > stays to be inspected by the client. + + ::new(number, extrinsics_root, storage_root, parent_hash, digest) + } + + /// Deposits a log and ensures it matches the blocks log data. + pub fn deposit_log(item: ::Item) { + let mut l = >::get(); + traits::Digest::push(&mut l, item); + >::put(l); + } + + /// Deposits an event onto this block's event record. + pub fn deposit_event(event: T::Event) { + let phase = >::get().map_or(Phase::Finalization, |c| Phase::ApplyExtrinsic(c)); + let mut events = Self::events(); + events.push(EventRecord { phase, event }); + >::put(events); + } + + /// Calculate the current block's random seed. + fn calculate_random() -> T::Hash { + assert!(Self::block_number() > Zero::zero(), "Block number may never be zero"); + (0..81) + .scan( + Self::block_number() - One::one(), + |c, _| { if *c > Zero::zero() { *c -= One::one() }; Some(*c) + }) + .map(Self::block_hash) + .triplet_mix() + } + + /// Get the basic externalities for this module, useful for tests. + #[cfg(any(feature = "std", test))] + pub fn externalities() -> TestExternalities { + map![ + twox_128(&>::key_for(T::BlockNumber::zero())).to_vec() => [69u8; 32].encode(), // TODO: replace with Hash::default().encode + twox_128(>::key()).to_vec() => T::BlockNumber::one().encode(), + twox_128(>::key()).to_vec() => [69u8; 32].encode(), // TODO: replace with Hash::default().encode + twox_128(>::key()).to_vec() => T::Hash::default().encode() + ] + } + + /// Set the block number to something in particular. Can be used as an alternative to + /// `initialise` for tests that don't need to bother with the other environment entries. + #[cfg(any(feature = "std", test))] + pub fn set_block_number(n: T::BlockNumber) { + >::put(n); + } + + /// Set the parent hash number to something in particular. Can be used as an alternative to + /// `initialise` for tests that don't need to bother with the other environment entries. + #[cfg(any(feature = "std", test))] + pub fn set_parent_hash(n: T::Hash) { + >::put(n); + } + + /// Set the random seed to something in particular. Can be used as an alternative to + /// `initialise` for tests that don't need to bother with the other environment entries. + #[cfg(any(feature = "std", test))] + pub fn set_random_seed(seed: T::Hash) { + >::put(seed); + } + + /// Increment a particular account's nonce by 1. + pub fn inc_account_nonce(who: &T::AccountId) { + >::insert(who, Self::account_nonce(who) + T::Index::one()); + } + + /// Note what the extrinsic data of the current extrinsic index is. If this is called, then + /// ensure `derive_extrinsics` is also called before block-building is completed. + pub fn note_extrinsic(encoded_xt: Vec) { + >::insert(>::get().unwrap_or_default(), encoded_xt); + } + + /// To be called immediately after an extrinsic has been applied. + pub fn note_applied_extrinsic(r: &Result<(), &'static str>) { + Self::deposit_event(match r { + Ok(_) => Event::ExtrinsicSuccess, + Err(_) => Event::ExtrinsicFailed, + }.into()); + >::put(>::get().unwrap_or_default() + 1u32); + } + + /// To be called immediately after `note_applied_extrinsic` of the last extrinsic of the block + /// has been called. + pub fn note_finished_extrinsics() { + >::put(>::get().unwrap_or_default()); + >::kill(); + } + + /// Remove all extrinsics data and save the extrinsics trie root. + pub fn derive_extrinsics() { + let extrinsics = (0..>::get().unwrap_or_default()).map(>::take).collect(); + let xts_root = extrinsics_data_root::(extrinsics); + >::put(xts_root); + } +} + +#[cfg(any(feature = "std", test))] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig(PhantomData); + +#[cfg(any(feature = "std", test))] +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig(PhantomData) + } +} + +#[cfg(any(feature = "std", test))] +impl primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> Result { + use codec::Encode; + + Ok(map![ + Self::hash(&>::key_for(T::BlockNumber::zero())).to_vec() => [69u8; 32].encode(), + Self::hash(>::key()).to_vec() => 1u64.encode(), + Self::hash(>::key()).to_vec() => [69u8; 32].encode(), + Self::hash(>::key()).to_vec() => [0u8; 32].encode(), + Self::hash(>::key()).to_vec() => [0u8; 4].encode() + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use runtime_io::with_externalities; + use substrate_primitives::H256; + use primitives::BuildStorage; + use primitives::traits::BlakeTwo256; + use primitives::testing::{Digest, Header}; + + impl_outer_origin!{ + pub enum Origin for Test where system = super {} + } + + #[derive(Clone, Eq, PartialEq)] + pub struct Test; + impl Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = u16; + } + + impl From for u16 { + fn from(e: Event) -> u16 { + match e { + Event::ExtrinsicSuccess => 100, + Event::ExtrinsicFailed => 101, + } + } + } + + type System = Module; + + + + fn new_test_ext() -> runtime_io::TestExternalities { + GenesisConfig::::default().build_storage().unwrap().into() + } + + #[test] + fn deposit_event_should_work() { + with_externalities(&mut new_test_ext(), || { + System::initialise(&1, &[0u8; 32].into(), &[0u8; 32].into()); + System::note_finished_extrinsics(); + System::deposit_event(1u16); + System::finalise(); + assert_eq!(System::events(), vec![EventRecord { phase: Phase::Finalization, event: 1u16 }]); + + System::initialise(&2, &[0u8; 32].into(), &[0u8; 32].into()); + System::deposit_event(42u16); + System::note_applied_extrinsic(&Ok(())); + System::note_applied_extrinsic(&Err("")); + System::note_finished_extrinsics(); + System::deposit_event(3u16); + System::finalise(); + assert_eq!(System::events(), vec![ + EventRecord { phase: Phase::ApplyExtrinsic(0), event: 42u16 }, + EventRecord { phase: Phase::ApplyExtrinsic(0), event: 100u16 }, + EventRecord { phase: Phase::ApplyExtrinsic(1), event: 101u16 }, + EventRecord { phase: Phase::Finalization, event: 3u16 } + ]); + }); + } +} diff --git a/runtime/timestamp/Cargo.toml b/runtime/timestamp/Cargo.toml new file mode 100644 index 00000000000..b8c42308a79 --- /dev/null +++ b/runtime/timestamp/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "substrate-runtime-timestamp" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } +substrate-runtime-consensus = { path = "../consensus", default_features = false } + +[features] +default = ["std"] +std = [ + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-consensus/std", + "serde/std", + "serde_derive", + "substrate-codec/std", + "substrate-primitives/std", + "substrate-runtime-system/std", +] diff --git a/runtime/timestamp/src/lib.rs b/runtime/timestamp/src/lib.rs new file mode 100644 index 00000000000..b586dae19b9 --- /dev/null +++ b/runtime/timestamp/src/lib.rs @@ -0,0 +1,210 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Timestamp manager: just handles the current timestamp. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg_attr(any(feature = "std", test), macro_use)] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +#[cfg(any(feature = "std", test))] +extern crate substrate_runtime_io as runtime_io; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[cfg(test)] +extern crate substrate_primitives; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_runtime_system as system; +extern crate substrate_runtime_consensus as consensus; +extern crate substrate_codec as codec; + +use runtime_support::{StorageValue, Parameter}; +use runtime_support::dispatch::Result; +use runtime_primitives::traits::{OnFinalise, SimpleArithmetic, As, Zero}; +use system::ensure_inherent; + +pub trait Trait: consensus::Trait + system::Trait { + // the position of the required timestamp-set extrinsic. + const TIMESTAMP_SET_POSITION: u32; + + type Moment: Parameter + Default + SimpleArithmetic + As; +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn set(origin, now: T::Moment) -> Result; + } +} + +decl_storage! { + trait Store for Module as Timestamp { + pub Now get(now): required T::Moment; + /// The minimum (and advised) period between blocks. + pub BlockPeriod get(block_period): required T::Moment; + + /// Did the timestamp get updated in this block? + DidUpdate: default bool; + } +} + +impl Module { + pub fn get() -> T::Moment { + Self::now() + } + + /// Set the current time. + fn set(origin: T::Origin, now: T::Moment) -> Result { + ensure_inherent(origin)?; + assert!(!::DidUpdate::exists(), "Timestamp must be updated only once in the block"); + assert!( + >::extrinsic_index() == Some(T::TIMESTAMP_SET_POSITION), + "Timestamp extrinsic must be at position {} in the block", + T::TIMESTAMP_SET_POSITION + ); + assert!( + Self::now().is_zero() || now >= Self::now() + Self::block_period(), + "Timestamp but increment by at least between sequential blocks" + ); + ::Now::put(now); + ::DidUpdate::put(true); + Ok(()) + } + + /// Set the timestamp to something in particular. Only used for tests. + #[cfg(any(feature = "std", test))] + pub fn set_timestamp(now: T::Moment) { + ::Now::put(now); + } +} + +impl OnFinalise for Module { + fn on_finalise(_n: T::BlockNumber) { + assert!(::DidUpdate::take(), "Timestamp must be updated once in the block"); + } +} + +#[cfg(any(feature = "std", test))] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct GenesisConfig { + pub period: T::Moment, +} + +#[cfg(any(feature = "std", test))] +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + period: T::Moment::sa(5), + } + } +} + +#[cfg(any(feature = "std", test))] +impl runtime_primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> ::std::result::Result { + use codec::Encode; + Ok(map![ + Self::hash(>::key()).to_vec() => self.period.encode(), + Self::hash(>::key()).to_vec() => T::Moment::sa(0).encode() + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use runtime_io::with_externalities; + use substrate_primitives::H256; + use runtime_primitives::BuildStorage; + use runtime_primitives::traits::{BlakeTwo256}; + use runtime_primitives::testing::{Digest, Header}; + + impl_outer_origin! { + pub enum Origin for Test {} + } + + #[derive(Clone, Eq, PartialEq)] + pub struct Test; + impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); + } + impl consensus::Trait for Test { + const NOTE_OFFLINE_POSITION: u32 = 1; + type Log = u64; + type SessionKey = u64; + type OnOfflineValidator = (); + } + impl Trait for Test { + const TIMESTAMP_SET_POSITION: u32 = 0; + type Moment = u64; + } + type Timestamp = Module; + + #[test] + fn timestamp_works() { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(GenesisConfig:: { period: 0 }.build_storage().unwrap()); + let mut t = runtime_io::TestExternalities::from(t); + with_externalities(&mut t, || { + Timestamp::set_timestamp(42); + assert_ok!(Timestamp::dispatch(Call::set(69), Origin::INHERENT)); + assert_eq!(Timestamp::now(), 69); + }); + } + + #[test] + #[should_panic(expected = "Timestamp must be updated only once in the block")] + fn double_timestamp_should_fail() { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(GenesisConfig:: { period: 5 }.build_storage().unwrap()); + let mut t = runtime_io::TestExternalities::from(t); + with_externalities(&mut t, || { + Timestamp::set_timestamp(42); + assert_ok!(Timestamp::dispatch(Call::set(69), Origin::INHERENT)); + let _ = Timestamp::dispatch(Call::set(70), Origin::INHERENT); + }); + } + + #[test] + #[should_panic(expected = "Timestamp but increment by at least between sequential blocks")] + fn block_period_is_enforced() { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(GenesisConfig:: { period: 5 }.build_storage().unwrap()); + let mut t = runtime_io::TestExternalities::from(t); + with_externalities(&mut t, || { + Timestamp::set_timestamp(42); + let _ = Timestamp::dispatch(Call::set(46), Origin::INHERENT); + }); + } +} diff --git a/runtime/treasury/Cargo.toml b/runtime/treasury/Cargo.toml new file mode 100644 index 00000000000..221439917b9 --- /dev/null +++ b/runtime/treasury/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "substrate-runtime-treasury" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +hex-literal = "0.1.0" +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-io = { path = "../../core/runtime-io", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } +substrate-runtime-primitives = { path = "../primitives", default_features = false } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-primitives = { path = "../../core/primitives", default_features = false } +substrate-runtime-system = { path = "../system", default_features = false } +substrate-runtime-balances = { path = "../balances", default_features = false } + +[features] +default = ["std"] +std = [ + "substrate-runtime-std/std", + "substrate-runtime-io/std", + "substrate-runtime-support/std", + "substrate-runtime-primitives/std", + "substrate-runtime-balances/std", + "serde/std", + "serde_derive", + "substrate-codec/std", + "substrate-codec-derive/std", + "substrate-primitives/std", + "substrate-runtime-system/std", +] diff --git a/runtime/treasury/src/lib.rs b/runtime/treasury/src/lib.rs new file mode 100644 index 00000000000..1edf33a5630 --- /dev/null +++ b/runtime/treasury/src/lib.rs @@ -0,0 +1,546 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! The Treasury: Keeps account of the taxed cash and handles its deployment. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg_attr(feature = "std", macro_use)] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_runtime_support as runtime_support; + +#[cfg(feature = "std")] +extern crate substrate_runtime_io as runtime_io; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_codec as codec; +#[cfg(test)] +extern crate substrate_primitives; +extern crate substrate_runtime_primitives as runtime_primitives; +extern crate substrate_runtime_system as system; +extern crate substrate_runtime_balances as balances; + +use rstd::prelude::*; +use runtime_support::{StorageValue, StorageMap}; +use runtime_support::dispatch::Result; +use runtime_primitives::{Permill, traits::{OnFinalise, Zero, EnsureOrigin}}; +use balances::OnDilution; +use system::{ensure_signed, ensure_root}; + +/// Our module's configuration trait. All our types and consts go in here. If the +/// module is dependent on specific other modules, then their configuration traits +/// should be added to our implied traits list. +/// +/// `system::Trait` should always be included in our implied traits. +pub trait Trait: balances::Trait { + /// Origin from which approvals must come. + type ApproveOrigin: EnsureOrigin; + + /// Origin from which rejections must come. + type RejectOrigin: EnsureOrigin; + + /// The overarching event type. + type Event: From> + Into<::Event>; +} + +type ProposalIndex = u32; + +// The module declaration. This states the entry points that we handle. The +// macro takes care of the marshalling of arguments and dispatch. +decl_module! { + // Simple declaration of the `Module` type. Lets the macro know what its working on. + pub struct Module for enum Call where origin: T::Origin { + // Put forward a suggestion for spending. A deposit proportional to the value + // is reserved and slashed if the proposal is rejected. It is returned once the + // proposal is awarded. + fn propose_spend(origin, value: T::Balance, beneficiary: T::AccountId) -> Result; + + // Set the balance of funds available to spend. + fn set_pot(origin, new_pot: T::Balance) -> Result; + + // (Re-)configure this module. + fn configure(origin, proposal_bond: Permill, proposal_bond_minimum: T::Balance, spend_period: T::BlockNumber, burn: Permill) -> Result; + + // Reject a proposed spend. The original deposit will be slashed. + fn reject_proposal(origin, roposal_id: ProposalIndex) -> Result; + + // Approve a proposal. At a later time, the proposal will be allocated to the beneficiary + // and the original deposit will be returned. + fn approve_proposal(origin, proposal_id: ProposalIndex) -> Result; + } +} + +/// A spending proposal. +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] +#[derive(Encode, Decode, Clone, PartialEq, Eq)] +pub struct Proposal { + proposer: AccountId, + value: Balance, + beneficiary: AccountId, + bond: Balance, +} + +decl_storage! { + trait Store for Module as Treasury { + // Config... + + /// Proportion of funds that should be bonded in order to place a proposal. An accepted + /// proposal gets these back. A rejected proposal doesn't. + ProposalBond get(proposal_bond): required Permill; + + /// Minimum amount of funds that should be placed ina deposit for making a proposal. + ProposalBondMinimum get(proposal_bond_minimum): required T::Balance; + + /// Period between successive spends. + SpendPeriod get(spend_period): required T::BlockNumber; + + /// Percentage of spare funds (if any) that are burnt per spend period. + Burn get(burn): required Permill; + + // State... + + /// Total funds available to this module for spending. + Pot get(pot): default T::Balance; + + /// Number of proposals that have been made. + ProposalCount get(proposal_count): default ProposalIndex; + + /// Proposals that have been made. + Proposals get(proposals): map [ ProposalIndex => Proposal ]; + + /// Proposal indices that have been approved but not yet awarded. + Approvals get(approvals): default Vec; + } +} + +/// An event in this module. +decl_event!( + pub enum Event with RawEvent + where ::Balance, ::AccountId + { + /// New proposal. + Proposed(ProposalIndex), + /// We have ended a spend period and will now allocate funds. + Spending(Balance), + /// Some funds have been allocated. + Awarded(ProposalIndex, Balance, AccountId), + /// Some of our funds have been burnt. + Burnt(Balance), + /// Spending has finished; this is the amount that rolls over until next spend. + Rollover(Balance), + } +); + +impl Module { + /// Deposit one of this module's events. + fn deposit_event(event: Event) { + >::deposit_event(::Event::from(event).into()); + } + + // Implement Calls and add public immutables and private mutables. + + fn propose_spend(origin: T::Origin, value: T::Balance, beneficiary: T::AccountId) -> Result { + let proposer = ensure_signed(origin)?; + + let bond = Self::calculate_bond(value); + >::reserve(&proposer, bond) + .map_err(|_| "Proposer's balance too low")?; + + let c = Self::proposal_count(); + >::put(c + 1); + >::insert(c, Proposal { proposer, value, beneficiary, bond }); + + Self::deposit_event(RawEvent::Proposed(c)); + + Ok(()) + } + + fn reject_proposal(origin: T::Origin, proposal_id: ProposalIndex) -> Result { + T::RejectOrigin::ensure_origin(origin)?; + + let proposal = >::take(proposal_id).ok_or("No proposal at that index")?; + + let value = proposal.bond; + let _ = >::slash_reserved(&proposal.proposer, value); + + Ok(()) + } + + fn approve_proposal(origin: T::Origin, proposal_id: ProposalIndex) -> Result { + T::ApproveOrigin::ensure_origin(origin)?; + + ensure!(>::exists(proposal_id), "No proposal at that index"); + + >::mutate(|v| v.push(proposal_id)); + + Ok(()) + } + + fn set_pot(origin: T::Origin, new_pot: T::Balance) -> Result { + ensure_root(origin)?; + // Put the new value into storage. + >::put(new_pot); + + // All good. + Ok(()) + } + + fn configure( + origin: T::Origin, + proposal_bond: Permill, + proposal_bond_minimum: T::Balance, + spend_period: T::BlockNumber, + burn: Permill + ) -> Result { + ensure_root(origin)?; + >::put(proposal_bond); + >::put(proposal_bond_minimum); + >::put(spend_period); + >::put(burn); + Ok(()) + } + + /// The needed bond for a proposal whose spend is `value`. + fn calculate_bond(value: T::Balance) -> T::Balance { + Self::proposal_bond_minimum().max(Self::proposal_bond().times(value)) + } + + // Spend some money! + fn spend_funds() { + let mut budget_remaining = Self::pot(); + Self::deposit_event(RawEvent::Spending(budget_remaining)); + + let mut missed_any = false; + let remaining_approvals: Vec<_> = >::get().into_iter().filter(|&index| { + // Should always be true, but shouldn't panic if false or we're screwed. + if let Some(p) = Self::proposals(index) { + if p.value <= budget_remaining { + budget_remaining -= p.value; + >::remove(index); + + // return their deposit. + let _ = >::unreserve(&p.proposer, p.bond); + + // provide the allocation. + >::increase_free_balance_creating(&p.beneficiary, p.value); + + Self::deposit_event(RawEvent::Awarded(index, p.value, p.beneficiary)); + false + } else { + missed_any = true; + true + } + } else { + false + } + }).collect(); + >::put(remaining_approvals); + + if !missed_any { + // burn some proportion of the remaining budget if we run a surplus. + let burn = Self::burn().times(budget_remaining); + budget_remaining -= burn; + Self::deposit_event(RawEvent::Burnt(burn)) + } + + Self::deposit_event(RawEvent::Rollover(budget_remaining)); + + >::put(budget_remaining); + } +} + +impl OnDilution for Module { + fn on_dilution(minted: T::Balance, portion: T::Balance) { + // Mint extra funds for the treasury to keep the ratio of portion to total_issuance equal + // pre dilution and post-dilution. + if !minted.is_zero() && !portion.is_zero() { + let total_issuance = >::total_issuance(); + let funding = (total_issuance - portion) / portion * minted; + >::mutate(|x| *x += funding); + } + } +} + +impl OnFinalise for Module { + fn on_finalise(n: T::BlockNumber) { + // Check to see if we should spend some funds! + if (n % Self::spend_period()).is_zero() { + Self::spend_funds(); + } + } +} + +#[cfg(feature = "std")] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +/// The genesis block configuration type. This is a simple default-capable struct that +/// contains any fields with which this module can be configured at genesis time. +pub struct GenesisConfig { + pub proposal_bond: Permill, + pub proposal_bond_minimum: T::Balance, + pub spend_period: T::BlockNumber, + pub burn: Permill, +} + +#[cfg(feature = "std")] +impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + proposal_bond: Default::default(), + proposal_bond_minimum: Default::default(), + spend_period: runtime_primitives::traits::One::one(), + burn: Default::default(), + } + } +} + +#[cfg(feature = "std")] +impl runtime_primitives::BuildStorage for GenesisConfig +{ + fn build_storage(self) -> ::std::result::Result { + use codec::Encode; + Ok(map![ + Self::hash(>::key()).to_vec() => self.proposal_bond.encode(), + Self::hash(>::key()).to_vec() => self.proposal_bond_minimum.encode(), + Self::hash(>::key()).to_vec() => self.spend_period.encode(), + Self::hash(>::key()).to_vec() => self.burn.encode() + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use runtime_io::with_externalities; + use substrate_primitives::{H256, Blake2Hasher}; + use runtime_primitives::BuildStorage; + use runtime_primitives::traits::{BlakeTwo256}; + use runtime_primitives::testing::{Digest, Header}; + + impl_outer_origin! { + pub enum Origin for Test {} + } + + #[derive(Clone, Eq, PartialEq)] + pub struct Test; + impl system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; + type Event = (); + } + impl balances::Trait for Test { + type Balance = u64; + type AccountIndex = u64; + type OnFreeBalanceZero = (); + type EnsureAccountLiquid = (); + type Event = (); + } + impl Trait for Test { + type ApproveOrigin = system::EnsureRoot; + type RejectOrigin = system::EnsureRoot; + type Event = (); + } + type Balances = balances::Module; + type Treasury = Module; + + fn new_test_ext() -> runtime_io::TestExternalities { + let mut t = system::GenesisConfig::::default().build_storage().unwrap(); + t.extend(balances::GenesisConfig::{ + balances: vec![(0, 100), (1, 99), (2, 1)], + transaction_base_fee: 0, + transaction_byte_fee: 0, + transfer_fee: 0, + creation_fee: 0, + existential_deposit: 0, + reclaim_rebate: 0, + }.build_storage().unwrap()); + t.extend(GenesisConfig::{ + proposal_bond: Permill::from_percent(5), + proposal_bond_minimum: 1, + spend_period: 2, + burn: Permill::from_percent(50), + }.build_storage().unwrap()); + t.into() + } + + #[test] + fn genesis_config_works() { + with_externalities(&mut new_test_ext(), || { + assert_eq!(Treasury::proposal_bond(), Permill::from_percent(5)); + assert_eq!(Treasury::proposal_bond_minimum(), 1); + assert_eq!(Treasury::spend_period(), 2); + assert_eq!(Treasury::burn(), Permill::from_percent(50)); + assert_eq!(Treasury::pot(), 0); + assert_eq!(Treasury::proposal_count(), 0); + }); + } + + #[test] + fn minting_works() { + with_externalities(&mut new_test_ext(), || { + // Check that accumulate works when we have Some value in Dummy already. + Treasury::on_dilution(100, 100); + assert_eq!(Treasury::pot(), 100); + }); + } + + #[test] + fn spend_proposal_takes_min_deposit() { + with_externalities(&mut new_test_ext(), || { + assert_ok!(Treasury::propose_spend(Origin::signed(0), 1, 3)); + assert_eq!(Balances::free_balance(&0), 99); + assert_eq!(Balances::reserved_balance(&0), 1); + }); + } + + #[test] + fn spend_proposal_takes_proportional_deposit() { + with_externalities(&mut new_test_ext(), || { + assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); + assert_eq!(Balances::free_balance(&0), 95); + assert_eq!(Balances::reserved_balance(&0), 5); + }); + } + + #[test] + fn spend_proposal_fails_when_proposer_poor() { + with_externalities(&mut new_test_ext(), || { + assert_noop!(Treasury::propose_spend(Origin::signed(2), 100, 3), "Proposer's balance too low"); + }); + } + + #[test] + fn accepted_spend_proposal_ignored_outside_spend_period() { + with_externalities(&mut new_test_ext(), || { + Treasury::on_dilution(100, 100); + + assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); + assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + + >::on_finalise(1); + assert_eq!(Balances::free_balance(&3), 0); + assert_eq!(Treasury::pot(), 100); + }); + } + + #[test] + fn unused_pot_should_diminish() { + with_externalities(&mut new_test_ext(), || { + Treasury::on_dilution(100, 100); + + >::on_finalise(2); + assert_eq!(Treasury::pot(), 50); + }); + } + + #[test] + fn rejected_spend_proposal_ignored_on_spend_period() { + with_externalities(&mut new_test_ext(), || { + Treasury::on_dilution(100, 100); + + assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); + assert_ok!(Treasury::reject_proposal(Origin::ROOT, 0)); + + >::on_finalise(2); + assert_eq!(Balances::free_balance(&3), 0); + assert_eq!(Treasury::pot(), 50); + }); + } + + #[test] + fn reject_already_rejected_spend_proposal_fails() { + with_externalities(&mut new_test_ext(), || { + Treasury::on_dilution(100, 100); + + assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); + assert_ok!(Treasury::reject_proposal(Origin::ROOT, 0)); + assert_noop!(Treasury::reject_proposal(Origin::ROOT, 0), "No proposal at that index"); + }); + } + + #[test] + fn reject_non_existant_spend_proposal_fails() { + with_externalities(&mut new_test_ext(), || { + assert_noop!(Treasury::reject_proposal(Origin::ROOT, 0), "No proposal at that index"); + }); + } + + #[test] + fn accept_non_existant_spend_proposal_fails() { + with_externalities(&mut new_test_ext(), || { + assert_noop!(Treasury::approve_proposal(Origin::ROOT, 0), "No proposal at that index"); + }); + } + + #[test] + fn accept_already_rejected_spend_proposal_fails() { + with_externalities(&mut new_test_ext(), || { + Treasury::on_dilution(100, 100); + + assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); + assert_ok!(Treasury::reject_proposal(Origin::ROOT, 0)); + assert_noop!(Treasury::approve_proposal(Origin::ROOT, 0), "No proposal at that index"); + }); + } + + #[test] + fn accepted_spend_proposal_enacted_on_spend_period() { + with_externalities(&mut new_test_ext(), || { + Treasury::on_dilution(100, 100); + + assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); + assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + + >::on_finalise(2); + assert_eq!(Balances::free_balance(&3), 100); + assert_eq!(Treasury::pot(), 0); + }); + } + + #[test] + fn pot_underflow_should_not_diminish() { + with_externalities(&mut new_test_ext(), || { + Treasury::on_dilution(100, 100); + + assert_ok!(Treasury::propose_spend(Origin::signed(0), 150, 3)); + assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + + >::on_finalise(2); + assert_eq!(Treasury::pot(), 100); + + Treasury::on_dilution(100, 100); + >::on_finalise(4); + assert_eq!(Balances::free_balance(&3), 150); + assert_eq!(Treasury::pot(), 25); + }); + } +} diff --git a/runtime/version/Cargo.toml b/runtime/version/Cargo.toml new file mode 100644 index 00000000000..10f91e9677c --- /dev/null +++ b/runtime/version/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "substrate-runtime-version" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +serde = { version = "1.0", default_features = false } +serde_derive = { version = "1.0", optional = true } +substrate-codec = { path = "../../core/codec", default_features = false } +substrate-codec-derive = { path = "../../core/codec/derive", default_features = false } +substrate-runtime-std = { path = "../../core/runtime-std", default_features = false } +substrate-runtime-support = { path = "../support", default_features = false } + +[features] +default = ["std"] +std = [ + "serde/std", + "serde_derive", + "substrate-codec/std", + "substrate-runtime-std/std", + "substrate-runtime-support/std", +] diff --git a/runtime/version/src/lib.rs b/runtime/version/src/lib.rs new file mode 100644 index 00000000000..632351b7631 --- /dev/null +++ b/runtime/version/src/lib.rs @@ -0,0 +1,130 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Version module for runtime; Provide a function that returns runtime version. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate serde; + +#[cfg(feature = "std")] +#[macro_use] +extern crate serde_derive; + +#[allow(unused_imports)] +#[macro_use] +extern crate substrate_runtime_std as rstd; + +#[macro_use] +extern crate substrate_codec_derive; + +extern crate substrate_codec as codec; + +#[cfg(feature = "std")] +use std::fmt; + +#[cfg(feature = "std")] +pub type VersionString = ::std::borrow::Cow<'static, str>; +#[cfg(not(feature = "std"))] +pub type VersionString = &'static str; + +#[cfg(feature = "std")] +#[macro_export] +macro_rules! ver_str { + ( $y:expr ) => {{ ::std::borrow::Cow::Borrowed($y) }} +} + +#[cfg(not(feature = "std"))] +#[macro_export] +macro_rules! ver_str { + ( $y:expr ) => {{ $y }} +} + +/// Runtime version. +/// This should not be thought of as classic Semver (major/minor/tiny). +/// This triplet have different semantics and mis-interpretation could cause problems. +/// In particular: bug fixes should result in an increment of `spec_version` and possibly `authoring_version`, +/// absolutely not `impl_version` since they change the semantics of the runtime. +#[derive(Clone, PartialEq, Eq, Encode)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize, Decode))] +pub struct RuntimeVersion { + /// Identifies the different Substrate runtimes. There'll be at least polkadot and node. + /// A different on-chain spec_name to that of the native runtime would normally result + /// in node not attempting to sync or author blocks. + pub spec_name: VersionString, + + /// Name of the implementation of the spec. This is of little consequence for the node + /// and serves only to differentiate code of different implementation teams. For this + /// codebase, it will be parity-polkadot. If there were a non-Rust implementation of the + /// Polkadot runtime (e.g. C++), then it would identify itself with an accordingly different + /// `impl_name`. + pub impl_name: VersionString, + + /// `authoring_version` is the version of the authorship interface. An authoring node + /// will not attempt to author blocks unless this is equal to its native runtime. + pub authoring_version: u32, + + /// Version of the runtime specification. A full-node will not attempt to use its native + /// runtime in substitute for the on-chain Wasm runtime unless all of `spec_name`, + /// `spec_version` and `authoring_version` are the same between Wasm and native. + pub spec_version: u32, + + /// Version of the implementation of the specification. Nodes are free to ignore this; it + /// serves only as an indication that the code is different; as long as the other two versions + /// are the same then while the actual code may be different, it is nonetheless required to + /// do the same thing. + /// Non-consensus-breaking optimisations are about the only changes that could be made which + /// would result in only the `impl_version` changing. + pub impl_version: u32, +} + +// TODO: remove this after PoC-2 +#[cfg(feature = "std")] +impl Default for RuntimeVersion { + fn default() -> RuntimeVersion { + RuntimeVersion { + spec_name: ver_str!("polkadot"), + impl_name: ver_str!("parity-polkadot"), + authoring_version: 0, + spec_version: 0, + impl_version: 0, + } + } +} + +#[cfg(feature = "std")] +impl fmt::Display for RuntimeVersion { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}-{}:{}({}-{})", self.spec_name, self.spec_version, self.authoring_version, self.impl_name, self.impl_version) + } +} + +#[cfg(feature = "std")] +impl RuntimeVersion { + /// Check if this version matches other version for calling into runtime. + pub fn can_call_with(&self, other: &RuntimeVersion) -> bool { + self.spec_version == other.spec_version && + self.spec_name == other.spec_name && + self.authoring_version == other.authoring_version + } + + /// Check if this version matches other version for authoring blocks. + pub fn can_author_with(&self, other: &RuntimeVersion) -> bool { + self.authoring_version == other.authoring_version && + self.spec_name == other.spec_name + } +} -- GitLab