From da8b91ae7b8ab8fc9a04eb8bda397a6e7b59d92c Mon Sep 17 00:00:00 2001 From: cheme <emericchevalier.pro@gmail.com> Date: Fri, 2 Aug 2019 19:51:59 +0200 Subject: [PATCH] Trie simplification. (#2815) * switch to simple codec, trie broken for now * Actualy use trie_root_noext * align some hash, failing test on EMCH comment * Fix trie code over layout instead of hash, revert legacy code for legacy mainnet ?? * stub behind LayOut * fix no_std * temp solution for legacy trie behind feature legacy-key in various crate * use remote project * rc client db need prefix * update trie deps * bum spec runtime version * Removing legacy as default. * Switch mode to non legacy. * bump runtime version * Remove legacy trie compatibility features. * fix warning * bump version * change hash on new test. * Move dependency (#11 trie PR) patched to a parity repo. Bench reverted to correct hasher. Some renaming and doc improvments. * ChildBitmap renaming to BitMap. * Renaming of LayOut to Layout. * formatting. * Removing abreviation such as _ix nb_ or bm. * Update deps and apply renaming 'Buff' -> 'Buffer'. * Align to latest trie crates naming changes. * Update trie dependency. * Update trie dependency. * change block_import test hash * update trie deps (trie use new scale codec but it does not seems to be an issue). * update to use latest trie version (no mgmt of multiple radix). * tabify * Restoring test to 10 000. * Use published crate, trie bench is currently down until publishing (require another pr to update version). * Update trie-bench. --- substrate/Cargo.lock | 163 ++++--- substrate/core/client/Cargo.toml | 2 +- substrate/core/client/db/Cargo.toml | 2 +- substrate/core/client/db/src/lib.rs | 89 ++-- substrate/core/client/src/cht.rs | 3 +- substrate/core/client/src/client.rs | 4 +- substrate/core/client/src/genesis.rs | 5 +- substrate/core/client/src/in_mem.rs | 6 +- substrate/core/client/src/light/fetcher.rs | 9 +- substrate/core/executor/Cargo.toml | 2 +- substrate/core/executor/src/wasm_executor.rs | 7 +- substrate/core/keyring/Cargo.toml | 4 +- substrate/core/primitives/Cargo.toml | 4 +- substrate/core/sr-io/Cargo.toml | 2 +- substrate/core/sr-io/with_std.rs | 7 +- substrate/core/state-machine/Cargo.toml | 9 +- substrate/core/state-machine/src/backend.rs | 9 +- substrate/core/state-machine/src/basic.rs | 8 +- .../src/changes_trie/changes_iterator.rs | 4 +- .../state-machine/src/changes_trie/mod.rs | 9 +- .../state-machine/src/changes_trie/storage.rs | 8 +- substrate/core/state-machine/src/ext.rs | 11 +- substrate/core/state-machine/src/lib.rs | 5 +- .../state-machine/src/overlayed_changes.rs | 3 +- .../core/state-machine/src/proving_backend.rs | 38 +- substrate/core/state-machine/src/testing.rs | 2 +- .../core/state-machine/src/trie_backend.rs | 19 +- .../state-machine/src/trie_backend_essence.rs | 60 +-- substrate/core/test-client/Cargo.toml | 2 +- substrate/core/test-runtime/Cargo.toml | 4 +- substrate/core/test-runtime/src/lib.rs | 27 +- substrate/core/trie/Cargo.toml | 14 +- substrate/core/trie/benches/bench.rs | 6 +- substrate/core/trie/src/lib.rs | 427 +++++++++--------- substrate/core/trie/src/node_codec.rs | 244 +++++++--- substrate/core/trie/src/node_header.rs | 121 +++-- substrate/core/trie/src/trie_stream.rs | 119 +++-- substrate/node-template/Cargo.toml | 2 +- substrate/node/executor/Cargo.toml | 2 +- substrate/node/executor/src/lib.rs | 4 +- substrate/node/runtime/src/lib.rs | 4 +- substrate/srml/executive/src/lib.rs | 2 +- substrate/srml/session/src/historical.rs | 6 +- 43 files changed, 890 insertions(+), 588 deletions(-) diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index adb20893d87..8810fc7ebb9 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -217,6 +217,11 @@ name = "bitvec" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bitvec" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "blake2" version = "0.8.0" @@ -302,6 +307,11 @@ name = "bumpalo" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "byte-slice-cast" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byte-tools" version = "0.2.0" @@ -685,19 +695,6 @@ dependencies = [ "syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "derive_more" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "difference" version = "2.0.0" @@ -1118,12 +1115,12 @@ dependencies = [ [[package]] name = "hash-db" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "hash256-std-hasher" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1485,11 +1482,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "keccak-hasher" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash256-std-hasher 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2070,10 +2067,10 @@ dependencies = [ [[package]] name = "memory-db" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2309,7 +2306,7 @@ dependencies = [ "substrate-state-machine 2.0.0", "substrate-test-client 2.0.0", "substrate-trie 2.0.0", - "trie-root 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "wabt 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2404,7 +2401,7 @@ dependencies = [ "substrate-service 2.0.0", "substrate-transaction-pool 2.0.0", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "vergen 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2632,6 +2629,18 @@ dependencies = [ "unsigned-varint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "parity-scale-codec" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", + "bitvec 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-slice-cast 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", + "vecarray 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "parity-send-wrapper" version = "0.1.0" @@ -3570,7 +3579,7 @@ name = "sr-io" version = "2.0.0" dependencies = [ "environmental 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "libsecp256k1 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 4.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4131,12 +4140,12 @@ dependencies = [ [[package]] name = "strum" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "strum_macros" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4247,7 +4256,7 @@ dependencies = [ "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures-preview 0.3.0-alpha.17 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", "kvdb-memorydb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", @@ -4274,7 +4283,7 @@ name = "substrate-client-db" version = "2.0.0" dependencies = [ "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", "kvdb-memorydb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", "kvdb-rocksdb 0.1.4 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)", @@ -4463,7 +4472,7 @@ version = "2.0.0" dependencies = [ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "libsecp256k1 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4545,8 +4554,8 @@ version = "2.0.0" dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "sr-primitives 2.0.0", - "strum 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "strum_macros 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strum 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strum_macros 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-primitives 2.0.0", ] @@ -4668,8 +4677,8 @@ dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash256-std-hasher 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "impl-serde 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4830,7 +4839,7 @@ dependencies = [ name = "substrate-state-machine" version = "2.0.0" dependencies = [ - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4839,8 +4848,8 @@ dependencies = [ "substrate-panic-handler 2.0.0", "substrate-primitives 2.0.0", "substrate-trie 2.0.0", - "trie-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4869,7 +4878,7 @@ name = "substrate-test-client" version = "2.0.0" dependencies = [ "futures-preview 0.3.0-alpha.17 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 4.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "sr-primitives 2.0.0", "substrate-client 2.0.0", @@ -4887,6 +4896,7 @@ version = "2.0.0" dependencies = [ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 4.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", "sr-io 2.0.0", @@ -4910,7 +4920,7 @@ dependencies = [ "substrate-test-runtime-client 2.0.0", "substrate-trie 2.0.0", "substrate-wasm-builder-runner 1.0.2", - "trie-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4962,17 +4972,17 @@ name = "substrate-trie" version = "2.0.0" dependencies = [ "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memory-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 4.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "substrate-primitives 2.0.0", - "trie-bench 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-standardmap 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-bench 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -5389,26 +5399,26 @@ dependencies = [ [[package]] name = "trie-bench" -version = "0.14.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memory-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-codec 4.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-standardmap 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "trie-db" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5416,19 +5426,19 @@ dependencies = [ [[package]] name = "trie-root" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "trie-standardmap" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -5578,6 +5588,16 @@ name = "vec_map" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "vecarray" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "parity-codec 4.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", + "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "vergen" version = "3.0.4" @@ -5971,6 +5991,7 @@ dependencies = [ "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" "checksum bitmask 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5da9b3d9f6f585199287a473f4f8dfab6566cf827d15c00c219f53c645687ead" "checksum bitvec 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b67491e1cc6f37da6c4415cd743cb8d2e2c65388acc91ca3094a054cbf3cbd0c" +"checksum bitvec 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9633b74910e1870f50f5af189b08487195cdb83c0e27a71d6f64d5e09dd0538b" "checksum blake2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91721a6330935673395a0607df4d49a9cb90ae12d259f1b3e0a3f6e1d486872e" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum block-buffer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1339a1042f5d9f295737ad4d9a6ab6bf81c84a933dba110b9200cd6d1448b814" @@ -5981,6 +6002,7 @@ dependencies = [ "checksum bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e0a692f1c740e7e821ca71a22cf99b9b2322dfa94d10f71443befb1797b3946a" "checksum build-helper 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bdce191bf3fa4995ce948c8c83b4640a1745457a149e73c6db75b4ffe36aad5f" "checksum bumpalo 2.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2cd43d82f27d68911e6ee11ee791fb248f138f5d69424dc02e098d4f152b0b05" +"checksum byte-slice-cast 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7cbcbf18128ec71d8d4a0d054461ec59fff5b75b7d10a4c9b7c7cb1a379c3e77" "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" "checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855" @@ -6024,7 +6046,6 @@ dependencies = [ "checksum curve25519-dalek 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5d4b820e8711c211745880150f5fac78ab07d6e3851d8ce9f5a02cedc199174c" "checksum data-encoding 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f4f47ca1860a761136924ddd2422ba77b2ea54fe8cc75b9040804a0d9d32ad97" "checksum derive_more 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6d944ac6003ed268757ef1ee686753b57efc5fcf0ebe7b64c9fc81e7e32ff839" -"checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe" "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" "checksum digest 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e5b29bf156f3f4b3c4f610a25ff69370616ae6e0657d416de22645483e72af0a" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" @@ -6073,8 +6094,8 @@ dependencies = [ "checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" "checksum globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "925aa2cac82d8834e2b2a4415b6f6879757fb5c0928fc445ae76461a12eed8f2" "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" -"checksum hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c4a2710506bcc28e53b6d48d9686b233a31ad831597da7de91e6112a2fc8f260" -"checksum hash256-std-hasher 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff4a5dcbaf4fe8977852851d137546bcad8679c9582f170032ca35b30701138e" +"checksum hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32c87fec93c4a2d264483ef843ac1930ae7c7999d97d73721305a5188b4c23a4" +"checksum hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16293646125e09e5bc216d9f73fa81ab31c4f97007d56c036bbf15a58e970540" "checksum hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3bae29b6653b3412c2e71e9d486db9f9df5d701941d86683005efb9f2d28e3da" "checksum hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8e04cb7a5051270ef3fa79f8c7604d581ecfa73d520e74f554e45541c4b5881a" "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" @@ -6112,7 +6133,7 @@ dependencies = [ "checksum jsonrpc-server-utils 12.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7aac8e0029d19582b68c9fd498d18bdcf0846612c968acc93b6e5ae67eea4e0" "checksum jsonrpc-ws-server 12.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "698fee4fcaf09a5927b7e39dd8a8136a102b343cebacaa351fc4def01a050a5b" "checksum keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" -"checksum keccak-hasher 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3767172fe16797c41f975f12f38247964ace8e5e1a2539b82d5e19f9106b1cb9" +"checksum keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bf18164fd7ce989041f8fc4a1ae72a8bd1bec3575f2aeaf1d4968fc053aabef" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum kvdb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)" = "<none>" "checksum kvdb-memorydb 0.1.0 (git+https://github.com/paritytech/parity-common?rev=b0317f649ab2c665b7987b8475878fc4d2e1f81d)" = "<none>" @@ -6157,7 +6178,7 @@ dependencies = [ "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" "checksum memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce6075db033bbbb7ee5a0bbd3a3186bbae616f57fb001c485c7ff77955f8177f" -"checksum memory-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "896b24d1a9850e7a25b070d552f311cbb8735214456efa222dcc4c431073c215" +"checksum memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a688133a81c915553c1dd9c3e859949f43a854cb8f8773e690e849b53b1f89f0" "checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" "checksum merlin 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "66448a173ad394ef5ebf734efa724f3644dcffda083b1e89979da4461ddac079" "checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" @@ -6195,6 +6216,7 @@ dependencies = [ "checksum parity-codec-derive 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "00a486fd383382ddcb2de928364b1f82571c1e48274fc43b7667a4738ee4056c" "checksum parity-multiaddr 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "045b3c7af871285146300da35b1932bb6e4639b66c7c98e85d06a32cbc4e8fa7" "checksum parity-multihash 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "df3a17dc27848fd99e4f87eb0f8c9baba6ede0a6d555400c850ca45254ef4ce3" +"checksum parity-scale-codec 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b72212671aded3ca515379ea16aa774289e06cd595c7e31f06afdc48d6516e3" "checksum parity-send-wrapper 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aa9777aa91b8ad9dd5aaa04a9b6bcb02c7f1deb952fca5a66034d5e63afc5c6f" "checksum parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2005637ccf93dbb60c85081ccaaf3f945f573da48dcc79f27f9646caa3ec1dc" "checksum parity-wasm 0.31.3 (registry+https://github.com/rust-lang/crates.io-index)" = "511379a8194230c2395d2f5fa627a5a7e108a9f976656ce723ae68fca4097bfc" @@ -6305,8 +6327,8 @@ dependencies = [ "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "16c2cdbf9cc375f15d1b4141bc48aeef444806655cd0e904207edc8d68d86ed7" "checksum structopt-derive 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "53010261a84b37689f9ed7d395165029f9cc7abb9f56bbfe86bee2597ed25107" -"checksum strum 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1810e25f576e7ffce1ff5243b37066da5ded0310b3274c20baaeccb1145b2806" -"checksum strum_macros 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "572a2f4e53dd4c3483fd79e5cc10ddd773a3acb1169bbfe8762365e107110579" +"checksum strum 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e5d1c33039533f051704951680f1adfd468fd37ac46816ded0d9ee068e60f05f" +"checksum strum_macros 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "47cd23f5c7dee395a00fa20135e2ec0fffcdfa151c56182966d7a3261343432e" "checksum substrate-bip39 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d69ace596e9ca97837cc41f8edcfc4e0a997f227d5fc153d1010b60a0fe9acda" "checksum substrate-wasm-builder-runner 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f52ecbff6cc3d6e5c6401828e15937b680f459d6803ce238f01fe615bc40d071" "checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" @@ -6345,10 +6367,10 @@ dependencies = [ "checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" "checksum toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b8c96d7873fa7ef8bdeb3a9cda3ac48389b4154f32b9803b4bc26220b677b039" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" -"checksum trie-bench 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "401abff5ad06075d2c65d1eedeaaa70616d0df268f3186a82cf1aa2d798977d7" -"checksum trie-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1803d8ff63ec3743bee883aacf3df74c524ffab188d9abebe18ded4da0dcd5d4" -"checksum trie-root 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "226f4b2e7bc6a71172ffe7f137385cf63833de7c684059dde7520ddbf1fb04f4" -"checksum trie-standardmap 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b65b79aee5dcdcc7247fdd811f7e26b47e65ecc17f776ecf5db8e8fd46db3b54" +"checksum trie-bench 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a1861db0e69cc3d650083ca1e70e6f5aeb871491409abc0efaf321dff48df24a" +"checksum trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b65d609ae631d808c6c1cc23a622733d5a0b66a7d67e9f5cd5171562a1f4cb5" +"checksum trie-root 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b0eaa64e50d686c89e6d4817ed33cb18cfa249e9071b7918b18ecfacc7867" +"checksum trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64fda153c00484d640bc91334624be22ead0e5baca917d9fd53ff29bdebcf9b2" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum trybuild 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f2e8e773ac21d176ee05243456b9f1a942cd1a586dab188ced05b8e8d58dc635" "checksum twofish 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d261e83e727c8e2dbb75dacac67c36e35db36a958ee504f2164fc052434e1" @@ -6370,6 +6392,7 @@ dependencies = [ "checksum utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9d50aa7650df78abf942826607c62468ce18d9019673d4a2ebe1865dbb96ffde" "checksum vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "33dd455d0f96e90a75803cfeb7f948768c08d70a6de9a8d2362461935698bf95" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum vecarray 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d4d68a73b7d7d950c6558b6009e9fba229fb67562bda9fd02198f614f4ecf83f" "checksum vergen 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6aba5e34f93dc7051dfad05b98a18e9156f27e7b431fe1d2398cb6061c0a1dba" "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" diff --git a/substrate/core/client/Cargo.toml b/substrate/core/client/Cargo.toml index 704f1ebccb5..6847e68d53b 100644 --- a/substrate/core/client/Cargo.toml +++ b/substrate/core/client/Cargo.toml @@ -17,7 +17,7 @@ state-machine = { package = "substrate-state-machine", path = "../state-machine" keyring = { package = "substrate-keyring", path = "../keyring", optional = true } trie = { package = "substrate-trie", path = "../trie", optional = true } substrate-telemetry = { path = "../telemetry", optional = true } -hash-db = { version = "0.14.0", default-features = false } +hash-db = { version = "0.15.0", default-features = false } kvdb = { git = "https://github.com/paritytech/parity-common", optional = true, rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" } parity-codec = { version = "4.1.1", default-features = false, features = ["derive"] } primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } diff --git a/substrate/core/client/db/Cargo.toml b/substrate/core/client/db/Cargo.toml index 202b21558a3..5c1a0de55cb 100644 --- a/substrate/core/client/db/Cargo.toml +++ b/substrate/core/client/db/Cargo.toml @@ -12,7 +12,7 @@ kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true } kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" } linked-hash-map = "0.5" -hash-db = { version = "0.14.0" } +hash-db = { version = "0.15.0" } primitives = { package = "substrate-primitives", path = "../../primitives" } sr-primitives = { path = "../../sr-primitives" } client = { package = "substrate-client", path = "../../client" } diff --git a/substrate/core/client/db/src/lib.rs b/substrate/core/client/db/src/lib.rs index 906404e8fcb..b22ffc50b5b 100644 --- a/substrate/core/client/db/src/lib.rs +++ b/substrate/core/client/db/src/lib.rs @@ -43,7 +43,7 @@ use client::blockchain::HeaderBackend; use client::ExecutionStrategies; use client::backend::{StorageCollection, ChildStorageCollection}; use parity_codec::{Decode, Encode}; -use hash_db::Hasher; +use hash_db::{Hasher, Prefix}; use kvdb::{KeyValueDB, DBTransaction}; use trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; use parking_lot::{Mutex, RwLock}; @@ -507,7 +507,7 @@ struct StorageDb<Block: BlockT> { } impl<Block: BlockT> state_machine::Storage<Blake2Hasher> for StorageDb<Block> { - fn get(&self, key: &H256, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H256, prefix: Prefix) -> Result<Option<DBValue>, String> { let key = prefixed_key::<Blake2Hasher>(key, prefix); self.state_db.get(&key, self).map(|r| r.map(|v| DBValue::from_slice(&v))) .map_err(|e| format!("Database backend error: {:?}", e)) @@ -535,7 +535,7 @@ impl DbGenesisStorage { } impl state_machine::Storage<Blake2Hasher> for DbGenesisStorage { - fn get(&self, _key: &H256, _prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, _key: &H256, _prefix: Prefix) -> Result<Option<DBValue>, String> { Ok(None) } } @@ -675,7 +675,7 @@ impl<Block> state_machine::ChangesTrieStorage<Blake2Hasher, NumberFor<Block>> where Block: BlockT<Hash=H256>, { - fn get(&self, key: &H256, _prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H256, _prefix: Prefix) -> Result<Option<DBValue>, String> { self.db.get(columns::CHANGES_TRIE, &key[..]) .map_err(|err| format!("{}", err)) } @@ -1428,7 +1428,7 @@ where Block: BlockT<Hash=H256> {} #[cfg(test)] mod tests { - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; use super::*; use crate::columns; use client::backend::Backend as BTrait; @@ -1654,7 +1654,7 @@ mod tests { op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap(); - key = op.db_updates.insert(&[], b"hello"); + key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); op.set_block_data( header, Some(vec![]), @@ -1663,8 +1663,10 @@ mod tests { ).unwrap(); backend.commit_operation(op).unwrap(); - - assert_eq!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().unwrap(), &b"hello"[..]); + assert_eq!(backend.storage.db.get( + columns::STATE, + &trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX) + ).unwrap().unwrap(), &b"hello"[..]); hash }; @@ -1688,8 +1690,8 @@ mod tests { ).0.into(); let hash = header.hash(); - op.db_updates.insert(&[], b"hello"); - op.db_updates.remove(&key, &[]); + op.db_updates.insert(EMPTY_PREFIX, b"hello"); + op.db_updates.remove(&key, EMPTY_PREFIX); op.set_block_data( header, Some(vec![]), @@ -1698,8 +1700,10 @@ mod tests { ).unwrap(); backend.commit_operation(op).unwrap(); - - assert_eq!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().unwrap(), &b"hello"[..]); + assert_eq!(backend.storage.db.get( + columns::STATE, + &trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX) + ).unwrap().unwrap(), &b"hello"[..]); hash }; @@ -1723,7 +1727,7 @@ mod tests { ).0.into(); let hash = header.hash(); - op.db_updates.remove(&key, &[]); + op.db_updates.remove(&key, EMPTY_PREFIX); op.set_block_data( header, Some(vec![]), @@ -1733,7 +1737,11 @@ mod tests { backend.commit_operation(op).unwrap(); - assert!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().is_some()); + + assert!(backend.storage.db.get( + columns::STATE, + &trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX) + ).unwrap().is_some()); hash }; @@ -1764,14 +1772,19 @@ mod tests { ).unwrap(); backend.commit_operation(op).unwrap(); - - assert!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().is_none()); + assert!(backend.storage.db.get( + columns::STATE, + &trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX) + ).unwrap().is_none()); } backend.finalize_block(BlockId::Number(1), None).unwrap(); backend.finalize_block(BlockId::Number(2), None).unwrap(); backend.finalize_block(BlockId::Number(3), None).unwrap(); - assert!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().is_none()); + assert!(backend.storage.db.get( + columns::STATE, + &trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX) + ).unwrap().is_none()); } #[test] @@ -1789,7 +1802,7 @@ mod tests { assert_eq!(backend.changes_tries_storage.root(&anchor, block), Ok(Some(changes_root))); for (key, (val, _)) in changes_trie_update.drain() { - assert_eq!(backend.changes_trie_storage().unwrap().get(&key, &[]), Ok(Some(val))); + assert_eq!(backend.changes_trie_storage().unwrap().get(&key, EMPTY_PREFIX), Ok(Some(val))); } }; @@ -1915,23 +1928,23 @@ mod tests { let mut tx = DBTransaction::new(); backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 12); backend.storage.db.write(tx).unwrap(); - assert!(backend.changes_tries_storage.get(&root1, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root3, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root4, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root5, &[]).unwrap().is_some()); - assert!(backend.changes_tries_storage.get(&root6, &[]).unwrap().is_some()); - assert!(backend.changes_tries_storage.get(&root7, &[]).unwrap().is_some()); - assert!(backend.changes_tries_storage.get(&root8, &[]).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root1, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root3, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root4, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root5, EMPTY_PREFIX).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root6, EMPTY_PREFIX).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root7, EMPTY_PREFIX).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root8, EMPTY_PREFIX).unwrap().is_some()); // now simulate finalization of block#16, causing prune of tries at #5..#8 let mut tx = DBTransaction::new(); backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 16); backend.storage.db.write(tx).unwrap(); - assert!(backend.changes_tries_storage.get(&root5, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root6, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root7, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root8, &[]).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root5, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root6, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root7, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root8, EMPTY_PREFIX).unwrap().is_none()); // now "change" pruning mode to archive && simulate finalization of block#20 // => no changes tries are pruned, because we never prune in archive mode @@ -1939,10 +1952,10 @@ mod tests { let mut tx = DBTransaction::new(); backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 20); backend.storage.db.write(tx).unwrap(); - assert!(backend.changes_tries_storage.get(&root9, &[]).unwrap().is_some()); - assert!(backend.changes_tries_storage.get(&root10, &[]).unwrap().is_some()); - assert!(backend.changes_tries_storage.get(&root11, &[]).unwrap().is_some()); - assert!(backend.changes_tries_storage.get(&root12, &[]).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root9, EMPTY_PREFIX).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root10, EMPTY_PREFIX).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root11, EMPTY_PREFIX).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root12, EMPTY_PREFIX).unwrap().is_some()); } #[test] @@ -1981,15 +1994,15 @@ mod tests { let mut tx = DBTransaction::new(); backend.changes_tries_storage.prune(&config, &mut tx, block5, 5); backend.storage.db.write(tx).unwrap(); - assert!(backend.changes_tries_storage.get(&root1, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root1, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_some()); // now simulate finalization of block#6, causing prune of tries at #2 let mut tx = DBTransaction::new(); backend.changes_tries_storage.prune(&config, &mut tx, block6, 6); backend.storage.db.write(tx).unwrap(); - assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_none()); - assert!(backend.changes_tries_storage.get(&root3, &[]).unwrap().is_some()); + assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_none()); + assert!(backend.changes_tries_storage.get(&root3, EMPTY_PREFIX).unwrap().is_some()); } #[test] diff --git a/substrate/core/client/src/cht.rs b/substrate/core/client/src/cht.rs index 56803a98277..b3282009237 100644 --- a/substrate/core/client/src/cht.rs +++ b/substrate/core/client/src/cht.rs @@ -78,7 +78,8 @@ pub fn compute_root<Header, Hasher, I>( Hasher::Out: Ord, I: IntoIterator<Item=ClientResult<Option<Header::Hash>>>, { - Ok(trie::trie_root::<Hasher, _, _, _>( + use trie::TrieConfiguration; + Ok(trie::trie_types::Layout::<Hasher>::trie_root( build_pairs::<Header, I>(cht_size, cht_num, hashes)? )) } diff --git a/substrate/core/client/src/client.rs b/substrate/core/client/src/client.rs index a056ec6c7b9..64519b60ddc 100644 --- a/substrate/core/client/src/client.rs +++ b/substrate/core/client/src/client.rs @@ -57,7 +57,7 @@ use state_machine::{ ChangesTrieRootsStorage, ChangesTrieStorage, key_changes, key_changes_proof, OverlayedChanges, NeverOffchainExt, }; -use hash_db::Hasher; +use hash_db::{Hasher, Prefix}; use crate::backend::{ self, BlockImportOperation, PrunableStateChangesTrieStorage, @@ -614,7 +614,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where } impl<'a, Block: BlockT> ChangesTrieStorage<Blake2Hasher, NumberFor<Block>> for AccessedRootsRecorder<'a, Block> { - fn get(&self, key: &H256, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H256, prefix: Prefix) -> Result<Option<DBValue>, String> { self.storage.get(key, prefix) } } diff --git a/substrate/core/client/src/genesis.rs b/substrate/core/client/src/genesis.rs index 719661e344f..1a3ab4c1fa3 100644 --- a/substrate/core/client/src/genesis.rs +++ b/substrate/core/client/src/genesis.rs @@ -71,11 +71,12 @@ mod tests { state_root: Hash, txs: Vec<Transfer> ) -> (Vec<u8>, Hash) { - use trie::ordered_trie_root; + use trie::{TrieConfiguration, trie_types::Layout}; let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::<Vec<_>>(); - let extrinsics_root = ordered_trie_root::<Blake2Hasher, _, _>(transactions.iter().map(Encode::encode)).into(); + let iter = transactions.iter().map(Encode::encode); + let extrinsics_root = Layout::<Blake2Hasher>::ordered_trie_root(iter).into(); let mut header = Header { parent_hash, diff --git a/substrate/core/client/src/in_mem.rs b/substrate/core/client/src/in_mem.rs index dd56c76a2be..d392fbe9b21 100644 --- a/substrate/core/client/src/in_mem.rs +++ b/substrate/core/client/src/in_mem.rs @@ -25,7 +25,7 @@ use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor} use sr_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay}; use state_machine::backend::{Backend as StateBackend, InMemory}; use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId}; -use hash_db::Hasher; +use hash_db::{Hasher, Prefix}; use trie::MemoryDB; use consensus::well_known_cache_keys::Id as CacheKeyId; @@ -755,8 +755,8 @@ impl<Block, H> state_machine::ChangesTrieStorage<H, NumberFor<Block>> for Change Block: BlockT, H: Hasher, { - fn get(&self, _key: &H::Out, _prefix: &[u8]) -> Result<Option<state_machine::DBValue>, String> { - Err("Dummy implementation".into()) + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<state_machine::DBValue>, String> { + self.0.get(key, prefix) } } diff --git a/substrate/core/client/src/light/fetcher.rs b/substrate/core/client/src/light/fetcher.rs index e788b3594be..b83907c932c 100644 --- a/substrate/core/client/src/light/fetcher.rs +++ b/substrate/core/client/src/light/fetcher.rs @@ -21,7 +21,7 @@ use std::collections::BTreeMap; use std::marker::PhantomData; use std::future::Future; -use hash_db::{HashDB, Hasher}; +use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use parity_codec::{Decode, Encode}; use primitives::{ChangesTrieConfiguration, convert_hash}; use sr_primitives::traits::{ @@ -333,7 +333,7 @@ impl<E, H, B: BlockT, S: BlockchainStorage<B>, F> LightDataChecker<E, H, B, S, F // we share the storage for multiple checks, do it here let mut cht_root = H::Out::default(); cht_root.as_mut().copy_from_slice(local_cht_root.as_ref()); - if !storage.contains(&cht_root, &[]) { + if !storage.contains(&cht_root, EMPTY_PREFIX) { return Err(ClientError::InvalidCHTProof.into()); } @@ -610,8 +610,9 @@ pub mod tests { } fn header_with_computed_extrinsics_root(extrinsics: Vec<Extrinsic>) -> Header { - let extrinsics_root = - trie::ordered_trie_root::<Blake2Hasher, _, _>(extrinsics.iter().map(Encode::encode)); + use trie::{TrieConfiguration, trie_types::Layout}; + let iter = extrinsics.iter().map(Encode::encode); + let extrinsics_root = Layout::<Blake2Hasher>::ordered_trie_root(iter); // only care about `extrinsics_root` Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default()) diff --git a/substrate/core/executor/Cargo.toml b/substrate/core/executor/Cargo.toml index 23205e1e402..1f9f3947ac9 100644 --- a/substrate/core/executor/Cargo.toml +++ b/substrate/core/executor/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Parity Technologies <admin@parity.io>"] edition = "2018" [dependencies] -derive_more = "0.15.0" +derive_more = "0.14.0" parity-codec = "4.1.1" runtime_io = { package = "sr-io", path = "../sr-io" } primitives = { package = "substrate-primitives", path = "../primitives" } diff --git a/substrate/core/executor/src/wasm_executor.rs b/substrate/core/executor/src/wasm_executor.rs index d7a8640e32d..b776a520d7b 100644 --- a/substrate/core/executor/src/wasm_executor.rs +++ b/substrate/core/executor/src/wasm_executor.rs @@ -32,7 +32,7 @@ use primitives::offchain; use primitives::hexdisplay::HexDisplay; use primitives::sandbox as sandbox_primitives; use primitives::{H256, Blake2Hasher}; -use trie::ordered_trie_root; +use trie::{TrieConfiguration, trie_types::Layout}; use crate::sandbox; use crate::allocator; use log::trace; @@ -528,7 +528,7 @@ impl_function_executor!(this: FunctionExecutor<'e, E>, ) ) .collect::<Result<Vec<_>>>()?; - let r = ordered_trie_root::<Blake2Hasher, _, _>(values.into_iter()); + let r = Layout::<Blake2Hasher>::ordered_trie_root(values.into_iter()); this.memory.set(result, &r[..]) .map_err(|_| "Invalid attempt to set memory in ext_blake2_256_enumerated_trie_root")?; Ok(()) @@ -1629,10 +1629,11 @@ mod tests { #[test] fn enumerated_trie_root_should_work() { let mut ext = TestExternalities::<Blake2Hasher>::default(); + let trie_input = vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()]; let test_code = WASM_BINARY; assert_eq!( WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_enumerated_trie_root", &[]).unwrap(), - ordered_trie_root::<Blake2Hasher, _, _>(vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()].iter()).as_fixed_bytes().encode() + Layout::<Blake2Hasher>::ordered_trie_root(trie_input.iter()).as_fixed_bytes().encode() ); } diff --git a/substrate/core/keyring/Cargo.toml b/substrate/core/keyring/Cargo.toml index 84d6f765b74..1a58dc133e3 100644 --- a/substrate/core/keyring/Cargo.toml +++ b/substrate/core/keyring/Cargo.toml @@ -8,5 +8,5 @@ edition = "2018" primitives = { package = "substrate-primitives", path = "../primitives" } sr-primitives = { path = "../sr-primitives" } lazy_static = { version = "1.0" } -strum = "0.14.0" -strum_macros = "0.14.0" +strum = "0.15.0" +strum_macros = "0.15.0" diff --git a/substrate/core/primitives/Cargo.toml b/substrate/core/primitives/Cargo.toml index fc18251f403..d0cdf34141b 100644 --- a/substrate/core/primitives/Cargo.toml +++ b/substrate/core/primitives/Cargo.toml @@ -14,8 +14,8 @@ byteorder = { version = "1.3.1", default-features = false } primitive-types = { version = "0.4.0", default-features = false, features = ["codec"] } impl-serde = { version = "0.1", optional = true } wasmi = { version = "0.5.0", optional = true } -hash-db = { version = "0.14.0", default-features = false } -hash256-std-hasher = { version = "0.14.0", default-features = false } +hash-db = { version = "0.15.0", default-features = false } +hash256-std-hasher = { version = "0.15.0", default-features = false } ed25519-dalek = { version = "1.0.0-pre.1", optional = true } base58 = { version = "0.1", optional = true } blake2-rfc = { version = "0.2.18", optional = true } diff --git a/substrate/core/sr-io/Cargo.toml b/substrate/core/sr-io/Cargo.toml index 0172d3ac757..a1032457214 100644 --- a/substrate/core/sr-io/Cargo.toml +++ b/substrate/core/sr-io/Cargo.toml @@ -12,7 +12,7 @@ rustc_version = "0.2" rstd = { package = "sr-std", path = "../sr-std", default-features = false } primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } codec = { package = "parity-codec", version = "4.1.1", default-features = false } -hash-db = { version = "0.14.0", default-features = false } +hash-db = { version = "0.15.0", default-features = false } libsecp256k1 = { version = "0.2.1", optional = true } tiny-keccak = { version = "1.4.2", optional = true } environmental = { version = "1.0.1", optional = true } diff --git a/substrate/core/sr-io/with_std.rs b/substrate/core/sr-io/with_std.rs index 8c70efdcd81..94136735931 100644 --- a/substrate/core/sr-io/with_std.rs +++ b/substrate/core/sr-io/with_std.rs @@ -29,6 +29,7 @@ pub use substrate_state_machine::{ use environmental::environmental; use primitives::{offchain, hexdisplay::HexDisplay, H256}; +use trie::{TrieConfiguration, trie_types::Layout}; #[cfg(feature = "std")] use std::collections::HashMap; @@ -169,7 +170,7 @@ impl StorageApi for () { H: Hasher, H::Out: Ord, { - trie::ordered_trie_root::<H, _, _>(input.iter()) + Layout::<H>::ordered_trie_root(input) } fn trie_root<H, I, A, B>(input: I) -> H::Out @@ -180,7 +181,7 @@ impl StorageApi for () { H: Hasher, H::Out: Ord, { - trie::trie_root::<H, _, _, _>(input) + Layout::<H>::trie_root(input) } fn ordered_trie_root<H, I, A>(input: I) -> H::Out @@ -190,7 +191,7 @@ impl StorageApi for () { H: Hasher, H::Out: Ord, { - trie::ordered_trie_root::<H, _, _>(input) + Layout::<H>::ordered_trie_root(input) } } diff --git a/substrate/core/state-machine/Cargo.toml b/substrate/core/state-machine/Cargo.toml index bcb471d5306..c7a15bf9572 100644 --- a/substrate/core/state-machine/Cargo.toml +++ b/substrate/core/state-machine/Cargo.toml @@ -8,9 +8,9 @@ edition = "2018" [dependencies] log = "0.4" parking_lot = "0.8.0" -hash-db = "0.14.0" -trie-db = "0.14.0" -trie-root = "0.14.0" +hash-db = "0.15.0" +trie-db = "0.15.0" +trie-root = "0.15.0" trie = { package = "substrate-trie", path = "../trie" } primitives = { package = "substrate-primitives", path = "../primitives" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } @@ -19,3 +19,6 @@ num-traits = "0.2" [dev-dependencies] hex-literal = "0.2.0" + +[features] +default = [] diff --git a/substrate/core/state-machine/src/backend.rs b/substrate/core/state-machine/src/backend.rs index c86c802bfbd..795393f3313 100644 --- a/substrate/core/state-machine/src/backend.rs +++ b/substrate/core/state-machine/src/backend.rs @@ -24,7 +24,8 @@ use log::warn; use hash_db::Hasher; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::TrieBackendStorage; -use trie::{TrieDBMut, TrieMut, MemoryDB, trie_root, child_trie_root, default_child_trie_root}; +use trie::{TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration}; +use trie::trie_types::{TrieDBMut, Layout}; /// A state backend is used to read state data and can have changes committed /// to it. @@ -315,7 +316,7 @@ impl<H: Hasher> Backend<H> for InMemory<H> { .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); - let root = trie_root::<H, _, _, _>(existing_pairs.chain(transaction.iter().cloned()) + let root = Layout::<H>::trie_root(existing_pairs.chain(transaction.iter().cloned()) .collect::<HashMap<_, _>>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -338,7 +339,7 @@ impl<H: Hasher> Backend<H> for InMemory<H> { .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); - let root = child_trie_root::<H, _, _, _>( + let root = child_trie_root::<Layout<H>, _, _, _>( &storage_key, existing_pairs.chain(transaction.iter().cloned()) .collect::<HashMap<_, _>>() @@ -348,7 +349,7 @@ impl<H: Hasher> Backend<H> for InMemory<H> { let full_transaction = transaction.into_iter().map(|(k, v)| (Some(storage_key.clone()), k, v)).collect(); - let is_default = root == default_child_trie_root::<H>(&storage_key); + let is_default = root == default_child_trie_root::<Layout<H>>(&storage_key); (root, is_default, full_transaction) } diff --git a/substrate/core/state-machine/src/basic.rs b/substrate/core/state-machine/src/basic.rs index 0ed25980602..53ab7316713 100644 --- a/substrate/core/state-machine/src/basic.rs +++ b/substrate/core/state-machine/src/basic.rs @@ -20,7 +20,8 @@ use std::collections::HashMap; use std::iter::FromIterator; use crate::backend::{Backend, InMemory}; use hash_db::Hasher; -use trie::trie_root; +use trie::TrieConfiguration; +use trie::trie_types::Layout; use primitives::offchain; use primitives::storage::well_known_keys::is_child_storage_key; use super::{ChildStorageKey, Externalities}; @@ -149,7 +150,7 @@ impl<H: Hasher> Externalities<H> for BasicExternalities where H::Out: Ord { fn chain_id(&self) -> u64 { 42 } fn storage_root(&mut self) -> H::Out { - trie_root::<H, _, _, _>(self.top.clone()) + Layout::<H>::trie_root(self.top.clone()) } fn child_storage_root(&mut self, storage_key: ChildStorageKey<H>) -> Vec<u8> { @@ -186,7 +187,8 @@ mod tests { ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); - const ROOT: [u8; 32] = hex!("0b41e488cccbd67d1f1089592c2c235f5c5399b053f7fe9152dd4b5f279914cd"); + const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa"); + assert_eq!(ext.storage_root(), H256::from(ROOT)); } diff --git a/substrate/core/state-machine/src/changes_trie/changes_iterator.rs b/substrate/core/state-machine/src/changes_trie/changes_iterator.rs index ee4b9cc19b8..66d09c98d4b 100644 --- a/substrate/core/state-machine/src/changes_trie/changes_iterator.rs +++ b/substrate/core/state-machine/src/changes_trie/changes_iterator.rs @@ -20,7 +20,7 @@ use std::cell::RefCell; use std::collections::VecDeque; use parity_codec::{Decode, Encode}; -use hash_db::{HashDB, Hasher}; +use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use num_traits::One; use trie::{Recorder, MemoryDB}; use crate::changes_trie::{AnchorBlockId, Configuration, RootsStorage, Storage, BlockNumber}; @@ -115,7 +115,7 @@ pub fn key_changes_proof_check<S: RootsStorage<H, Number>, H: Hasher, Number: Bl let mut proof_db = MemoryDB::<H>::default(); for item in proof { - proof_db.insert(&[], &item); + proof_db.insert(EMPTY_PREFIX, &item); } let proof_db = InMemoryStorage::with_db(proof_db); diff --git a/substrate/core/state-machine/src/changes_trie/mod.rs b/substrate/core/state-machine/src/changes_trie/mod.rs index ab36eb6423b..48ba9eac951 100644 --- a/substrate/core/state-machine/src/changes_trie/mod.rs +++ b/substrate/core/state-machine/src/changes_trie/mod.rs @@ -46,14 +46,15 @@ pub use self::storage::InMemoryStorage; pub use self::changes_iterator::{key_changes, key_changes_proof, key_changes_proof_check}; pub use self::prune::{prune, oldest_non_pruned_trie}; -use hash_db::Hasher; +use hash_db::{Hasher, Prefix}; use crate::backend::Backend; use num_traits::{One, Zero}; use parity_codec::{Decode, Encode}; use primitives; use crate::changes_trie::build::prepare_input; use crate::overlayed_changes::OverlayedChanges; -use trie::{MemoryDB, TrieDBMut, TrieMut, DBValue}; +use trie::{MemoryDB, DBValue, TrieMut}; +use trie::trie_types::TrieDBMut; /// Changes that are made outside of extrinsics are marked with this index; pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff; @@ -108,7 +109,7 @@ pub trait RootsStorage<H: Hasher, Number: BlockNumber>: Send + Sync { /// Changes trie storage. Provides access to trie roots and trie nodes. pub trait Storage<H: Hasher, Number: BlockNumber>: RootsStorage<H, Number> { /// Get a trie node. - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String>; + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>; } /// Changes trie storage -> trie backend essence adapter. @@ -117,7 +118,7 @@ pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage<H> for TrieBackendStorageAdapter<'a, H, N> { type Overlay = trie::MemoryDB<H>; - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> { self.0.get(key, prefix) } } diff --git a/substrate/core/state-machine/src/changes_trie/storage.rs b/substrate/core/state-machine/src/changes_trie/storage.rs index 8da20525153..bb2256235a1 100644 --- a/substrate/core/state-machine/src/changes_trie/storage.rs +++ b/substrate/core/state-machine/src/changes_trie/storage.rs @@ -17,7 +17,7 @@ //! Changes trie storage utilities. use std::collections::BTreeMap; -use hash_db::Hasher; +use hash_db::{Hasher, Prefix}; use trie::DBValue; use trie::MemoryDB; use parking_lot::RwLock; @@ -101,7 +101,7 @@ impl<H: Hasher, Number: BlockNumber> InMemoryStorage<H, Number> { pub fn remove_from_storage(&self, keys: &HashSet<H::Out>) { let mut data = self.data.write(); for key in keys { - data.mdb.remove_and_purge(key, &[]); + data.mdb.remove_and_purge(key, hash_db::EMPTY_PREFIX); } } @@ -132,7 +132,7 @@ impl<H: Hasher, Number: BlockNumber> RootsStorage<H, Number> for InMemoryStorage } impl<H: Hasher, Number: BlockNumber> Storage<H, Number> for InMemoryStorage<H, Number> { - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> { MemoryDB::<H>::get(&self.data.read().mdb, key, prefix) } } @@ -151,7 +151,7 @@ impl<'a, H, Number, S> TrieBackendStorage<H> for TrieBackendAdapter<'a, H, Numbe { type Overlay = MemoryDB<H>; - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> { self.storage.get(key, prefix) } } diff --git a/substrate/core/state-machine/src/ext.rs b/substrate/core/state-machine/src/ext.rs index b910ddcaf6b..7f0fa1de3f0 100644 --- a/substrate/core/state-machine/src/ext.rs +++ b/substrate/core/state-machine/src/ext.rs @@ -25,6 +25,7 @@ use hash_db::Hasher; use primitives::offchain; use primitives::storage::well_known_keys::is_child_storage_key; use trie::{MemoryDB, default_child_trie_root}; +use trie::trie_types::Layout; const EXT_NOT_ALLOWED_TO_FAIL: &str = "Externalities not allowed to fail within runtime"; @@ -297,7 +298,7 @@ where self .storage(storage_key.as_ref()) .unwrap_or( - default_child_trie_root::<H>(storage_key.as_ref()) + default_child_trie_root::<Layout<H>>(storage_key.as_ref()) ) } else { let storage_key = storage_key.as_ref(); @@ -394,8 +395,10 @@ mod tests { let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let backend = TestBackend::default(); let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None); + let root = hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").into(); + assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), - Some(hex!("5b829920b9c8d554a19ee2a1ba593c4f2ee6fc32822d083e04236d693e8358d5").into())); + Some(root)); } #[test] @@ -405,7 +408,9 @@ mod tests { let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let backend = TestBackend::default(); let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None); + let root = hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").into(); + assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), - Some(hex!("bcf494e41e29a15c9ae5caa053fe3cb8b446ee3e02a254efbdec7a19235b76e4").into())); + Some(root)); } } diff --git a/substrate/core/state-machine/src/lib.rs b/substrate/core/state-machine/src/lib.rs index 7b2eb07819e..d2ffc23b781 100644 --- a/substrate/core/state-machine/src/lib.rs +++ b/substrate/core/state-machine/src/lib.rs @@ -38,7 +38,8 @@ mod trie_backend; mod trie_backend_essence; use overlayed_changes::OverlayedChangeSet; -pub use trie::{TrieMut, TrieDBMut, DBValue, MemoryDB}; +pub use trie::{TrieMut, DBValue, MemoryDB}; +pub use trie::trie_types::{Layout, TrieDBMut}; pub use testing::TestExternalities; pub use basic::BasicExternalities; pub use ext::Ext; @@ -71,7 +72,7 @@ pub struct ChildStorageKey<'a, H: Hasher> { impl<'a, H: Hasher> ChildStorageKey<'a, H> { fn new(storage_key: Cow<'a, [u8]>) -> Option<Self> { - if !trie::is_child_trie_key_valid::<H>(&storage_key) { + if !trie::is_child_trie_key_valid::<Layout<H>>(&storage_key) { return None; } diff --git a/substrate/core/state-machine/src/overlayed_changes.rs b/substrate/core/state-machine/src/overlayed_changes.rs index c6d7ab89d3b..34bf8ca3c5d 100644 --- a/substrate/core/state-machine/src/overlayed_changes.rs +++ b/substrate/core/state-machine/src/overlayed_changes.rs @@ -372,7 +372,8 @@ mod tests { Some(&changes_trie_storage), crate::NeverOffchainExt::new(), ); - const ROOT: [u8; 32] = hex!("0b41e488cccbd67d1f1089592c2c235f5c5399b053f7fe9152dd4b5f279914cd"); + const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa"); + assert_eq!(ext.storage_root(), H256::from(ROOT)); } diff --git a/substrate/core/state-machine/src/proving_backend.rs b/substrate/core/state-machine/src/proving_backend.rs index 19f779e067b..5862c19e8d7 100644 --- a/substrate/core/state-machine/src/proving_backend.rs +++ b/substrate/core/state-machine/src/proving_backend.rs @@ -18,13 +18,13 @@ use std::{cell::RefCell, rc::Rc}; use log::debug; -use hash_db::Hasher; -use hash_db::HashDB; +use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; use trie::{ - MemoryDB, PrefixedMemoryDB, TrieError, default_child_trie_root, + MemoryDB, PrefixedMemoryDB, default_child_trie_root, read_trie_value_with, read_child_trie_value_with, record_all_keys }; pub use trie::Recorder; +pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; @@ -50,11 +50,21 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> let map_e = |e| format!("Trie lookup error: {}", e); - read_trie_value_with::<H, _, Ephemeral<S, H>>(&eph, self.backend.root(), key, &mut *self.proof_recorder).map_err(map_e) + read_trie_value_with::<Layout<H>, _, Ephemeral<S, H>>( + &eph, + self.backend.root(), + key, + &mut *self.proof_recorder + ).map_err(map_e) } - pub fn child_storage(&mut self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, String> { - let root = self.storage(storage_key)?.unwrap_or(default_child_trie_root::<H>(storage_key)); + pub fn child_storage( + &mut self, + storage_key: &[u8], + key: &[u8] + ) -> Result<Option<Vec<u8>>, String> { + let root = self.storage(storage_key)? + .unwrap_or(default_child_trie_root::<Layout<H>>(storage_key)); let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new( @@ -64,7 +74,13 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value_with(storage_key, &eph, &root, key, &mut *self.proof_recorder).map_err(map_e) + read_child_trie_value_with::<Layout<H>, _, _>( + storage_key, + &eph, + &root, + key, + &mut *self.proof_recorder + ).map_err(map_e) } pub fn record_all_keys(&mut self) { @@ -76,7 +92,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> let mut iter = move || -> Result<(), Box<TrieError<H::Out>>> { let root = self.backend.root(); - record_all_keys::<H, _>(&eph, root, &mut *self.proof_recorder) + record_all_keys::<Layout<H>, _>(&eph, root, &mut *self.proof_recorder) }; if let Err(e) = iter() { @@ -199,7 +215,7 @@ where { let db = create_proof_check_backend_storage(proof); - if db.contains(&root, &[]) { + if db.contains(&root, EMPTY_PREFIX) { Ok(TrieBackend::new(db, root)) } else { Err(Box::new(ExecutionError::InvalidProof)) @@ -215,7 +231,7 @@ where { let mut db = MemoryDB::default(); for item in proof { - db.insert(&[], &item); + db.insert(EMPTY_PREFIX, &item); } db } @@ -307,7 +323,7 @@ mod tests { let mut in_memory = in_memory.update(contents); let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>( ::std::iter::empty(), - in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new())) + in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new())) ).0; (0..64).for_each(|i| assert_eq!( in_memory.storage(&[i]).unwrap().unwrap(), diff --git a/substrate/core/state-machine/src/testing.rs b/substrate/core/state-machine/src/testing.rs index dc19dad7b34..73c0b38ef43 100644 --- a/substrate/core/state-machine/src/testing.rs +++ b/substrate/core/state-machine/src/testing.rs @@ -277,7 +277,7 @@ mod tests { ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); - const ROOT: [u8; 32] = hex!("cc65c26c37ebd4abcdeb3f1ecd727527051620779a2f6c809bac0f8a87dbb816"); + const ROOT: [u8; 32] = hex!("2a340d3dfd52f5992c6b117e9e45f479e6da5afffafeb26ab619cf137a95aeb8"); assert_eq!(ext.storage_root(), H256::from(ROOT)); } diff --git a/substrate/core/state-machine/src/trie_backend.rs b/substrate/core/state-machine/src/trie_backend.rs index 0c57cf3682f..18c1b0eeca7 100644 --- a/substrate/core/state-machine/src/trie_backend.rs +++ b/substrate/core/state-machine/src/trie_backend.rs @@ -18,7 +18,8 @@ use log::{warn, debug}; use hash_db::Hasher; -use trie::{TrieDB, TrieError, Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root}; +use trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root}; +use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}; use crate::Backend; @@ -137,7 +138,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where &mut write_overlay, ); - match delta_trie_root::<H, _, _, _, _>(&mut eph, root, delta) { + match delta_trie_root::<Layout<H>, _, _, _, _>(&mut eph, root, delta) { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), } @@ -151,11 +152,11 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>, H::Out: Ord { - let default_root = default_child_trie_root::<H>(storage_key); + let default_root = default_child_trie_root::<Layout<H>>(storage_key); let mut write_overlay = S::Overlay::default(); let mut root = match self.storage(storage_key) { - Ok(value) => value.unwrap_or(default_child_trie_root::<H>(storage_key)), + Ok(value) => value.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key)), Err(e) => { warn!(target: "trie", "Failed to read child storage root: {}", e); default_root.clone() @@ -168,7 +169,12 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where &mut write_overlay, ); - match child_delta_trie_root::<H, _, _, _, _>(storage_key, &mut eph, root.clone(), delta) { + match child_delta_trie_root::<Layout<H>, _, _, _, _>( + storage_key, + &mut eph, + root.clone(), + delta + ) { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), } @@ -189,7 +195,8 @@ pub mod tests { use std::collections::HashSet; use primitives::{Blake2Hasher, H256}; use parity_codec::Encode; - use trie::{TrieMut, TrieDBMut, PrefixedMemoryDB}; + use trie::{TrieMut, PrefixedMemoryDB}; + use trie::trie_types::TrieDBMut; use super::*; fn test_db() -> (PrefixedMemoryDB<Blake2Hasher>, H256) { diff --git a/substrate/core/state-machine/src/trie_backend_essence.rs b/substrate/core/state-machine/src/trie_backend_essence.rs index cad150d1bc1..06ed92cbae3 100644 --- a/substrate/core/state-machine/src/trie_backend_essence.rs +++ b/substrate/core/state-machine/src/trie_backend_essence.rs @@ -20,17 +20,17 @@ use std::ops::Deref; use std::sync::Arc; use log::{debug, warn}; -use hash_db::{self, Hasher}; -use trie::{ - TrieDB, Trie, MemoryDB, PrefixedMemoryDB, DBValue, TrieError, - default_child_trie_root, read_trie_value, read_child_trie_value, for_keys_in_child_trie, -}; +use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix}; +use trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, + default_child_trie_root, read_trie_value, read_child_trie_value, + for_keys_in_child_trie}; +use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::backend::Consolidate; /// Patricia trie-based storage trait. pub trait Storage<H: Hasher>: Send + Sync { /// Get a trie node. - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String>; + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>; } /// Patricia trie-based pairs storage essence. @@ -73,12 +73,13 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> { let map_e = |e| format!("Trie lookup error: {}", e); - read_trie_value(&eph, &self.root, key).map_err(map_e) + read_trie_value::<Layout<H>, _>(&eph, &self.root, key).map_err(map_e) } /// Get the value of child storage at given key. pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, String> { - let root = self.storage(storage_key)?.unwrap_or(default_child_trie_root::<H>(storage_key)); + let root = self.storage(storage_key)? + .unwrap_or(default_child_trie_root::<Layout<H>>(storage_key)); let mut read_overlay = S::Overlay::default(); let eph = Ephemeral { @@ -88,13 +89,13 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> { let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value(storage_key, &eph, &root, key).map_err(map_e) + read_child_trie_value::<Layout<H>, _>(storage_key, &eph, &root, key).map_err(map_e) } /// Retrieve all entries keys of child storage and call `f` for each of those keys. pub fn for_keys_in_child_storage<F: FnMut(&[u8])>(&self, storage_key: &[u8], f: F) { let root = match self.storage(storage_key) { - Ok(v) => v.unwrap_or(default_child_trie_root::<H>(storage_key)), + Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key)), Err(e) => { debug!(target: "trie", "Error while iterating child storage: {}", e); return; @@ -107,7 +108,12 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> { overlay: &mut read_overlay, }; - if let Err(e) = for_keys_in_child_trie::<H, _, Ephemeral<S, H>>(storage_key, &eph, &root, f) { + if let Err(e) = for_keys_in_child_trie::<Layout<H>, _, Ephemeral<S, H>>( + storage_key, + &eph, + &root, + f, + ) { debug!(target: "trie", "Error while iterating child storage: {}", e); } } @@ -186,10 +192,10 @@ impl<'a, for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out) -> Option<DBValue> { - if let Some(val) = hash_db::HashDB::get(self.overlay, key, &[]) { + if let Some(val) = hash_db::HashDB::get(self.overlay, key, EMPTY_PREFIX) { Some(val) } else { - match self.storage.get(&key, &[]) { + match self.storage.get(&key, EMPTY_PREFIX) { Ok(x) => x, Err(e) => { warn!(target: "trie", "Failed to read from DB: {}", e); @@ -200,15 +206,15 @@ impl<'a, } fn contains(&self, key: &H::Out) -> bool { - hash_db::HashDB::get(self, key, &[]).is_some() + hash_db::HashDB::get(self, key, EMPTY_PREFIX).is_some() } fn emplace(&mut self, key: H::Out, value: DBValue) { - hash_db::HashDB::emplace(self.overlay, key, &[], value) + hash_db::HashDB::emplace(self.overlay, key, EMPTY_PREFIX, value) } fn remove(&mut self, key: &H::Out) { - hash_db::HashDB::remove(self.overlay, key, &[]) + hash_db::HashDB::remove(self.overlay, key, EMPTY_PREFIX) } } @@ -228,7 +234,7 @@ impl<'a, > hash_db::HashDB<H, DBValue> for Ephemeral<'a, S, H> { - fn get(&self, key: &H::Out, prefix: &[u8]) -> Option<DBValue> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> { if let Some(val) = hash_db::HashDB::get(self.overlay, key, prefix) { Some(val) } else { @@ -242,19 +248,19 @@ impl<'a, } } - fn contains(&self, key: &H::Out, prefix: &[u8]) -> bool { + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::get(self, key, prefix).is_some() } - fn insert(&mut self, prefix: &[u8], value: &[u8]) -> H::Out { + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { hash_db::HashDB::insert(self.overlay, prefix, value) } - fn emplace(&mut self, key: H::Out, prefix: &[u8], value: DBValue) { + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { hash_db::HashDB::emplace(self.overlay, key, prefix, value) } - fn remove(&mut self, key: &H::Out, prefix: &[u8]) { + fn remove(&mut self, key: &H::Out, prefix: Prefix) { hash_db::HashDB::remove(self.overlay, key, prefix) } } @@ -265,8 +271,8 @@ impl<'a, > hash_db::HashDBRef<H, DBValue> for Ephemeral<'a, S, H> { - fn get(&self, key: &H::Out, prefix: &[u8]) -> Option<DBValue> { hash_db::HashDB::get(self, key, prefix) } - fn contains(&self, key: &H::Out, prefix: &[u8]) -> bool { hash_db::HashDB::contains(self, key, prefix) } + fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> { hash_db::HashDB::get(self, key, prefix) } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::contains(self, key, prefix) } } /// Key-value pairs storage that is used by trie backend essence. @@ -274,14 +280,14 @@ pub trait TrieBackendStorage<H: Hasher>: Send + Sync { /// Type of in-memory overlay. type Overlay: hash_db::HashDB<H, DBValue> + Default + Consolidate; /// Get the value stored at key. - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String>; + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>; } // This implementation is used by normal storage trie clients. impl<H: Hasher> TrieBackendStorage<H> for Arc<dyn Storage<H>> { type Overlay = PrefixedMemoryDB<H>; - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> { Storage::<H>::get(self.deref(), key, prefix) } } @@ -290,7 +296,7 @@ impl<H: Hasher> TrieBackendStorage<H> for Arc<dyn Storage<H>> { impl<H: Hasher> TrieBackendStorage<H> for PrefixedMemoryDB<H> { type Overlay = PrefixedMemoryDB<H>; - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> { Ok(hash_db::HashDB::get(self, key, prefix)) } } @@ -298,7 +304,7 @@ impl<H: Hasher> TrieBackendStorage<H> for PrefixedMemoryDB<H> { impl<H: Hasher> TrieBackendStorage<H> for MemoryDB<H> { type Overlay = MemoryDB<H>; - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> { Ok(hash_db::HashDB::get(self, key, prefix)) } } diff --git a/substrate/core/test-client/Cargo.toml b/substrate/core/test-client/Cargo.toml index abb42daea29..bd3b0db2ce9 100644 --- a/substrate/core/test-client/Cargo.toml +++ b/substrate/core/test-client/Cargo.toml @@ -10,7 +10,7 @@ client-db = { package = "substrate-client-db", path = "../client/db", features = consensus = { package = "substrate-consensus-common", path = "../consensus/common" } executor = { package = "substrate-executor", path = "../executor" } futures-preview = "0.3.0-alpha.17" -hash-db = "0.14.0" +hash-db = "0.15.0" keyring = { package = "substrate-keyring", path = "../keyring" } parity-codec = "4.1.1" primitives = { package = "substrate-primitives", path = "../primitives" } diff --git a/substrate/core/test-runtime/Cargo.toml b/substrate/core/test-runtime/Cargo.toml index 532d5a4b38c..03ba10c5fbb 100644 --- a/substrate/core/test-runtime/Cargo.toml +++ b/substrate/core/test-runtime/Cargo.toml @@ -21,7 +21,8 @@ sr-primitives = { path = "../sr-primitives", default-features = false } runtime_version = { package = "sr-version", path = "../sr-version", default-features = false } runtime_support = { package = "srml-support", path = "../../srml/support", default-features = false } substrate-trie = { path = "../trie", default-features = false } -trie-db = { version = "0.14.0", default-features = false } +trie-db = { version = "0.15.0", default-features = false } +memory-db = { version = "0.15.0", default-features = false } offchain-primitives = { package = "substrate-offchain-primitives", path = "../offchain/primitives", default-features = false} executive = { package = "srml-executive", path = "../../srml/executive", default-features = false } cfg-if = "0.1.6" @@ -60,6 +61,7 @@ std = [ "primitives/std", "substrate-trie/std", "trie-db/std", + "memory-db/std", "offchain-primitives/std", "executive/std", "srml-babe/std", diff --git a/substrate/core/test-runtime/src/lib.rs b/substrate/core/test-runtime/src/lib.rs index e2cdcabd2b2..98d24370596 100644 --- a/substrate/core/test-runtime/src/lib.rs +++ b/substrate/core/test-runtime/src/lib.rs @@ -27,7 +27,8 @@ use parity_codec::{Encode, Decode, Input}; use primitives::Blake2Hasher; use trie_db::{TrieMut, Trie}; -use substrate_trie::{TrieDB, TrieDBMut, PrefixedMemoryDB}; +use substrate_trie::PrefixedMemoryDB; +use substrate_trie::trie_types::{TrieDB, TrieDBMut}; use substrate_client::{ runtime_api as client_api, block_builder::api as block_builder_api, decl_runtime_apis, @@ -394,20 +395,24 @@ fn code_using_trie() -> u64 { for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; - t.insert(key, val).expect("static input"); + if !t.insert(key, val).is_ok() { + return 101; + } } t }; - let trie = TrieDB::<Blake2Hasher>::new(&mdb, &root).expect("on memory with static content"); - - let iter = trie.iter().expect("static input"); - let mut iter_pairs = Vec::new(); - for pair in iter { - let (key, value) = pair.expect("on memory with static content"); - iter_pairs.push((key, value.to_vec())); - } - iter_pairs.len() as u64 + if let Ok(trie) = TrieDB::<Blake2Hasher>::new(&mdb, &root) { + if let Ok(iter) = trie.iter() { + let mut iter_pairs = Vec::new(); + for pair in iter { + if let Ok((key, value)) = pair { + iter_pairs.push((key, value.to_vec())); + } + } + iter_pairs.len() as u64 + } else { 102 } + } else { 103 } } #[cfg(not(feature = "std"))] diff --git a/substrate/core/trie/Cargo.toml b/substrate/core/trie/Cargo.toml index d2ba3522571..8659462bc4a 100644 --- a/substrate/core/trie/Cargo.toml +++ b/substrate/core/trie/Cargo.toml @@ -14,16 +14,16 @@ harness = false [dependencies] codec = { package = "parity-codec", version = "4.1.1", default-features = false } rstd = { package = "sr-std", path = "../sr-std", default-features = false } -hash-db = { version = "0.14.0", default-features = false } -trie-db = { version = "0.14.0", default-features = false } -trie-root = { version = "0.14.0", default-features = false } -memory-db = { version = "0.14.0", default-features = false } +hash-db = { version = "0.15.0", default-features = false } +trie-db = { version = "0.15.0", default-features = false } +trie-root = { version = "0.15.0", default-features = false } +memory-db = { version = "0.15.0", default-features = false } primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } [dev-dependencies] -trie-bench = { version = "0.14.0" } -trie-standardmap = { version = "0.14.0" } -keccak-hasher = { version = "0.14.0" } +trie-bench = { version = "0.16.0" } +trie-standardmap = { version = "0.15.0" } +keccak-hasher = { version = "0.15.0" } criterion = "0.2" hex-literal = "0.2.0" diff --git a/substrate/core/trie/benches/bench.rs b/substrate/core/trie/benches/bench.rs index 176bf5971ef..a8a47322228 100644 --- a/substrate/core/trie/benches/bench.rs +++ b/substrate/core/trie/benches/bench.rs @@ -20,13 +20,11 @@ criterion_main!(benches); fn benchmark(c: &mut Criterion) { trie_bench::standard_benchmark::< - primitives::Blake2Hasher, - substrate_trie::NodeCodec<primitives::Blake2Hasher>, + substrate_trie::Layout<primitives::Blake2Hasher>, substrate_trie::TrieStream, >(c, "substrate-blake2"); trie_bench::standard_benchmark::< - keccak_hasher::KeccakHasher, - substrate_trie::NodeCodec<keccak_hasher::KeccakHasher>, + substrate_trie::Layout<primitives::Blake2Hasher>, substrate_trie::TrieStream, >(c, "substrate-keccak"); } diff --git a/substrate/core/trie/src/lib.rs b/substrate/core/trie/src/lib.rs index 6e8adb4941c..e526a27ebef 100644 --- a/substrate/core/trie/src/lib.rs +++ b/substrate/core/trie/src/lib.rs @@ -33,57 +33,104 @@ pub use trie_stream::TrieStream; /// The Substrate format implementation of `NodeCodec`. pub use node_codec::NodeCodec; /// Various re-exports from the `trie-db` crate. -pub use trie_db::{Trie, TrieMut, DBValue, Recorder, Query}; +pub use trie_db::{Trie, TrieMut, DBValue, Recorder, CError, + Query, TrieLayout, TrieConfiguration, nibble_ops}; /// Various re-exports from the `memory-db` crate. -pub use memory_db::{KeyFunction, prefixed_key}; +pub use memory_db::KeyFunction; +pub use memory_db::prefixed_key; /// Various re-exports from the `hash-db` crate. -pub use hash_db::HashDB as HashDBT; +pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX}; -/// As in `trie_db`, but less generic, error type for the crate. -pub type TrieError<H> = trie_db::TrieError<H, Error>; -/// As in `hash_db`, but less generic, trait exposed. + +#[derive(Default)] +/// substrate trie layout +pub struct Layout<H>(rstd::marker::PhantomData<H>); + +impl<H: Hasher> TrieLayout for Layout<H> { + const USE_EXTENSION: bool = false; + type Hash = H; + type Codec = NodeCodec<Self::Hash>; +} + +impl<H: Hasher> TrieConfiguration for Layout<H> { + fn trie_root<I, A, B>(input: I) -> <Self::Hash as Hasher>::Out where + I: IntoIterator<Item = (A, B)>, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::trie_root_no_extension::<H, TrieStream, _, _, _>(input) + } + + fn trie_root_unhashed<I, A, B>(input: I) -> Vec<u8> where + I: IntoIterator<Item = (A, B)>, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::unhashed_trie_no_extension::<H, TrieStream, _, _, _>(input) + } + + fn encode_index(input: u32) -> Vec<u8> { + codec::Encode::encode(&codec::Compact(input)) + } +} + +/// TrieDB error over `TrieConfiguration` trait. +pub type TrieError<L> = trie_db::TrieError<TrieHash<L>, CError<L>>; +/// Reexport from `hash_db`, with genericity set for `Hasher` trait. pub trait AsHashDB<H: Hasher>: hash_db::AsHashDB<H, trie_db::DBValue> {} impl<H: Hasher, T: hash_db::AsHashDB<H, trie_db::DBValue>> AsHashDB<H> for T {} -/// As in `hash_db`, but less generic, trait exposed. +/// Reexport from `hash_db`, with genericity set for `Hasher` trait. pub type HashDB<'a, H> = dyn hash_db::HashDB<H, trie_db::DBValue> + 'a; -/// As in `hash_db`, but less generic, trait exposed. +/// Reexport from `hash_db`, with genericity set for key only. pub type PlainDB<'a, K> = dyn hash_db::PlainDB<K, trie_db::DBValue> + 'a; -/// As in `memory_db::MemoryDB` that uses prefixed storage key scheme. +/// Reexport from `hash_db`, with genericity set for `Hasher` trait. +/// This uses a `KeyFunction` for prefixing keys internally (avoiding +/// key conflict for non random keys). pub type PrefixedMemoryDB<H> = memory_db::MemoryDB<H, memory_db::PrefixedKey<H>, trie_db::DBValue>; -/// As in `memory_db::MemoryDB` that uses prefixed storage key scheme. +/// Reexport from `hash_db`, with genericity set for `Hasher` trait. +/// This uses the `KeyFunction` for prefixing keys internally (avoiding +/// This uses a noops `KeyFunction` (key addressing must be hashed or using +/// an encoding scheme that avoid key conflict). pub type MemoryDB<H> = memory_db::MemoryDB<H, memory_db::HashKey<H>, trie_db::DBValue>; -/// As in `memory_db`, but less generic, trait exposed. +/// Reexport from `hash_db`, with genericity set for `Hasher` trait. pub type GenericMemoryDB<H, KF> = memory_db::MemoryDB<H, KF, trie_db::DBValue>; /// Persistent trie database read-access interface for the a given hasher. -pub type TrieDB<'a, H> = trie_db::TrieDB<'a, H, NodeCodec<H>>; +pub type TrieDB<'a, L> = trie_db::TrieDB<'a, L>; /// Persistent trie database write-access interface for the a given hasher. -pub type TrieDBMut<'a, H> = trie_db::TrieDBMut<'a, H, NodeCodec<H>>; +pub type TrieDBMut<'a, L> = trie_db::TrieDBMut<'a, L>; /// Querying interface, as in `trie_db` but less generic. -pub type Lookup<'a, H, Q> = trie_db::Lookup<'a, H, NodeCodec<H>, Q>; - -/// Determine a trie root given its ordered contents, closed form. -pub fn trie_root<H: Hasher, I, A, B>(input: I) -> H::Out where - I: IntoIterator<Item = (A, B)>, - A: AsRef<[u8]> + Ord, - B: AsRef<[u8]>, -{ - trie_root::trie_root::<H, TrieStream, _, _, _>(input) +pub type Lookup<'a, L, Q> = trie_db::Lookup<'a, L, Q>; +/// Hash type for a trie layout. +pub type TrieHash<L> = <<L as TrieLayout>::Hash as Hasher>::Out; + +/// This module is for non generic definition of trie type. +/// Only the `Hasher` trait is generic in this case. +pub mod trie_types { + pub type Layout<H> = super::Layout<H>; + /// Persistent trie database read-access interface for the a given hasher. + pub type TrieDB<'a, H> = super::TrieDB<'a, Layout<H>>; + /// Persistent trie database write-access interface for the a given hasher. + pub type TrieDBMut<'a, H> = super::TrieDBMut<'a, Layout<H>>; + /// Querying interface, as in `trie_db` but less generic. + pub type Lookup<'a, H, Q> = trie_db::Lookup<'a, Layout<H>, Q>; + /// As in `trie_db`, but less generic, error type for the crate. + pub type TrieError<H> = trie_db::TrieError<H, super::Error>; } /// Determine a trie root given a hash DB and delta values. -pub fn delta_trie_root<H: Hasher, I, A, B, DB>( +pub fn delta_trie_root<L: TrieConfiguration, I, A, B, DB>( db: &mut DB, - mut root: H::Out, + mut root: TrieHash<L>, delta: I -) -> Result<H::Out, Box<TrieError<H::Out>>> where +) -> Result<TrieHash<L>, Box<TrieError<L>>> where I: IntoIterator<Item = (A, Option<B>)>, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, - DB: hash_db::HashDB<H, trie_db::DBValue>, + DB: hash_db::HashDB<L::Hash, trie_db::DBValue>, { { - let mut trie = TrieDBMut::<H>::from_existing(&mut *db, &mut root)?; + let mut trie = TrieDBMut::<L>::from_existing(&mut *db, &mut root)?; for (key, change) in delta { match change { @@ -97,45 +144,26 @@ pub fn delta_trie_root<H: Hasher, I, A, B, DB>( } /// Read a value from the trie. -pub fn read_trie_value<H: Hasher, DB: hash_db::HashDBRef<H, trie_db::DBValue>>( +pub fn read_trie_value<L: TrieConfiguration, DB: hash_db::HashDBRef<L::Hash, trie_db::DBValue>>( db: &DB, - root: &H::Out, + root: &TrieHash<L>, key: &[u8] -) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> { - Ok(TrieDB::<H>::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) +) -> Result<Option<Vec<u8>>, Box<TrieError<L>>> { + Ok(TrieDB::<L>::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the trie with given Query. -pub fn read_trie_value_with<H: Hasher, Q: Query<H, Item=DBValue>, DB: hash_db::HashDBRef<H, trie_db::DBValue>>( +pub fn read_trie_value_with< + L: TrieConfiguration, + Q: Query<L::Hash, Item=DBValue>, + DB: hash_db::HashDBRef<L::Hash, trie_db::DBValue> +>( db: &DB, - root: &H::Out, + root: &TrieHash<L>, key: &[u8], query: Q -) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> { - Ok(TrieDB::<H>::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) -} - -/// Determine a trie root node's data given its ordered contents, closed form. -pub fn unhashed_trie<H: Hasher, I, A, B>(input: I) -> Vec<u8> where - I: IntoIterator<Item = (A, B)>, - A: AsRef<[u8]> + Ord, - B: AsRef<[u8]>, -{ - trie_root::unhashed_trie::<H, TrieStream, _, _, _>(input) -} - -/// A trie root formed from the items, with keys attached according to their -/// compact-encoded index (using `parity-codec` crate). -pub fn ordered_trie_root<H: Hasher, I, A>(input: I) -> H::Out -where - I: IntoIterator<Item = A>, - A: AsRef<[u8]>, -{ - trie_root::<H, _, _, _>(input - .into_iter() - .enumerate() - .map(|(i, v)| (codec::Encode::encode(&codec::Compact(i as u32)), v)) - ) +) -> Result<Option<Vec<u8>>, Box<TrieError<L>>> { + Ok(TrieDB::<L>::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) } /// Determine whether a child trie key is valid. @@ -143,7 +171,7 @@ where /// For now, the only valid child trie key is `:child_storage:default:`. /// /// `child_trie_root` and `child_delta_trie_root` can panic if invalid value is provided to them. -pub fn is_child_trie_key_valid<H: Hasher>(storage_key: &[u8]) -> bool { +pub fn is_child_trie_key_valid<L: TrieConfiguration>(storage_key: &[u8]) -> bool { use primitives::storage::well_known_keys; let has_right_prefix = storage_key.starts_with(b":child_storage:default:"); if has_right_prefix { @@ -158,37 +186,42 @@ pub fn is_child_trie_key_valid<H: Hasher>(storage_key: &[u8]) -> bool { } /// Determine the default child trie root. -pub fn default_child_trie_root<H: Hasher>(_storage_key: &[u8]) -> Vec<u8> { - trie_root::<H, _, Vec<u8>, Vec<u8>>(core::iter::empty()).as_ref().iter().cloned().collect() +pub fn default_child_trie_root<L: TrieConfiguration>(_storage_key: &[u8]) -> Vec<u8> { + L::trie_root::<_, Vec<u8>, Vec<u8>>(core::iter::empty()).as_ref().iter().cloned().collect() } -/// Determine a child trie root given its ordered contents, closed form. H is the default hasher, but a generic -/// implementation may ignore this type parameter and use other hashers. -pub fn child_trie_root<H: Hasher, I, A, B>(_storage_key: &[u8], input: I) -> Vec<u8> where - I: IntoIterator<Item = (A, B)>, - A: AsRef<[u8]> + Ord, - B: AsRef<[u8]>, +/// Determine a child trie root given its ordered contents, closed form. H is the default hasher, +/// but a generic implementation may ignore this type parameter and use other hashers. +pub fn child_trie_root<L: TrieConfiguration, I, A, B>(_storage_key: &[u8], input: I) -> Vec<u8> + where + I: IntoIterator<Item = (A, B)>, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, { - trie_root::<H, _, _, _>(input).as_ref().iter().cloned().collect() + L::trie_root(input).as_ref().iter().cloned().collect() } -/// Determine a child trie root given a hash DB and delta values. H is the default hasher, but a generic implementation may ignore this type parameter and use other hashers. -pub fn child_delta_trie_root<H: Hasher, I, A, B, DB>( +/// Determine a child trie root given a hash DB and delta values. H is the default hasher, +/// but a generic implementation may ignore this type parameter and use other hashers. +pub fn child_delta_trie_root<L: TrieConfiguration, I, A, B, DB>( _storage_key: &[u8], db: &mut DB, root_vec: Vec<u8>, delta: I -) -> Result<Vec<u8>, Box<TrieError<H::Out>>> where - I: IntoIterator<Item = (A, Option<B>)>, - A: AsRef<[u8]> + Ord, - B: AsRef<[u8]>, - DB: hash_db::HashDB<H, trie_db::DBValue> + hash_db::PlainDB<H::Out, trie_db::DBValue>, +) -> Result<Vec<u8>, Box<TrieError<L>>> + where + I: IntoIterator<Item = (A, Option<B>)>, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + DB: hash_db::HashDB<L::Hash, trie_db::DBValue> + + hash_db::PlainDB<TrieHash<L>, trie_db::DBValue>, { - let mut root = H::Out::default(); - root.as_mut().copy_from_slice(&root_vec); // root is fetched from DB, not writable by runtime, so it's always valid. + let mut root = TrieHash::<L>::default(); + // root is fetched from DB, not writable by runtime, so it's always valid. + root.as_mut().copy_from_slice(&root_vec); { - let mut trie = TrieDBMut::<H>::from_existing(&mut *db, &mut root)?; + let mut trie = TrieDBMut::<L>::from_existing(&mut *db, &mut root)?; for (key, change) in delta { match change { @@ -202,18 +235,21 @@ pub fn child_delta_trie_root<H: Hasher, I, A, B, DB>( } /// Call `f` for all keys in a child trie. -pub fn for_keys_in_child_trie<H: Hasher, F: FnMut(&[u8]), DB>( +pub fn for_keys_in_child_trie<L: TrieConfiguration, F: FnMut(&[u8]), DB>( _storage_key: &[u8], db: &DB, root_slice: &[u8], mut f: F -) -> Result<(), Box<TrieError<H::Out>>> where - DB: hash_db::HashDBRef<H, trie_db::DBValue> + hash_db::PlainDBRef<H::Out, trie_db::DBValue>, +) -> Result<(), Box<TrieError<L>>> + where + DB: hash_db::HashDBRef<L::Hash, trie_db::DBValue> + + hash_db::PlainDBRef<TrieHash<L>, trie_db::DBValue>, { - let mut root = H::Out::default(); - root.as_mut().copy_from_slice(root_slice); // root is fetched from DB, not writable by runtime, so it's always valid. + let mut root = TrieHash::<L>::default(); + // root is fetched from DB, not writable by runtime, so it's always valid. + root.as_mut().copy_from_slice(root_slice); - let trie = TrieDB::<H>::new(&*db, &root)?; + let trie = TrieDB::<L>::new(&*db, &root)?; let iter = trie.iter()?; for x in iter { @@ -225,14 +261,14 @@ pub fn for_keys_in_child_trie<H: Hasher, F: FnMut(&[u8]), DB>( } /// Record all keys for a given root. -pub fn record_all_keys<H: Hasher, DB>( +pub fn record_all_keys<L: TrieConfiguration, DB>( db: &DB, - root: &H::Out, - recorder: &mut Recorder<H::Out> -) -> Result<(), Box<TrieError<H::Out>>> where - DB: hash_db::HashDBRef<H, trie_db::DBValue> + root: &TrieHash<L>, + recorder: &mut Recorder<TrieHash<L>> +) -> Result<(), Box<TrieError<L>>> where + DB: hash_db::HashDBRef<L::Hash, trie_db::DBValue> { - let trie = TrieDB::<H>::new(&*db, root)?; + let trie = TrieDB::<L>::new(&*db, root)?; let iter = trie.iter()?; for x in iter { @@ -248,84 +284,49 @@ pub fn record_all_keys<H: Hasher, DB>( } /// Read a value from the child trie. -pub fn read_child_trie_value<H: Hasher, DB>( +pub fn read_child_trie_value<L: TrieConfiguration, DB>( _storage_key: &[u8], db: &DB, root_slice: &[u8], key: &[u8] -) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> where - DB: hash_db::HashDBRef<H, trie_db::DBValue> + hash_db::PlainDBRef<H::Out, trie_db::DBValue>, +) -> Result<Option<Vec<u8>>, Box<TrieError<L>>> + where + DB: hash_db::HashDBRef<L::Hash, trie_db::DBValue> + + hash_db::PlainDBRef<TrieHash<L>, trie_db::DBValue>, { - let mut root = H::Out::default(); - root.as_mut().copy_from_slice(root_slice); // root is fetched from DB, not writable by runtime, so it's always valid. + let mut root = TrieHash::<L>::default(); + // root is fetched from DB, not writable by runtime, so it's always valid. + root.as_mut().copy_from_slice(root_slice); - Ok(TrieDB::<H>::new(&*db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) + Ok(TrieDB::<L>::new(&*db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the child trie with given query. -pub fn read_child_trie_value_with<H: Hasher, Q: Query<H, Item=DBValue>, DB>( +pub fn read_child_trie_value_with<L: TrieConfiguration, Q: Query<L::Hash, Item=DBValue>, DB>( _storage_key: &[u8], db: &DB, root_slice: &[u8], key: &[u8], query: Q -) -> Result<Option<Vec<u8>>, Box<TrieError<H::Out>>> where - DB: hash_db::HashDBRef<H, trie_db::DBValue> + hash_db::PlainDBRef<H::Out, trie_db::DBValue>, +) -> Result<Option<Vec<u8>>, Box<TrieError<L>>> + where + DB: hash_db::HashDBRef<L::Hash, trie_db::DBValue> + + hash_db::PlainDBRef<TrieHash<L>, trie_db::DBValue>, { - let mut root = H::Out::default(); - root.as_mut().copy_from_slice(root_slice); // root is fetched from DB, not writable by runtime, so it's always valid. + let mut root = TrieHash::<L>::default(); + // root is fetched from DB, not writable by runtime, so it's always valid. + root.as_mut().copy_from_slice(root_slice); - Ok(TrieDB::<H>::new(&*db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) + Ok(TrieDB::<L>::new(&*db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) } -// Utilities (not exported): - -const EMPTY_TRIE: u8 = 0; -const LEAF_NODE_OFFSET: u8 = 1; -const LEAF_NODE_BIG: u8 = 127; -const EXTENSION_NODE_OFFSET: u8 = 128; -const EXTENSION_NODE_BIG: u8 = 253; -const BRANCH_NODE_NO_VALUE: u8 = 254; -const BRANCH_NODE_WITH_VALUE: u8 = 255; -const LEAF_NODE_THRESHOLD: u8 = LEAF_NODE_BIG - LEAF_NODE_OFFSET; -const EXTENSION_NODE_THRESHOLD: u8 = EXTENSION_NODE_BIG - EXTENSION_NODE_OFFSET; //125 -const LEAF_NODE_SMALL_MAX: u8 = LEAF_NODE_BIG - 1; -const EXTENSION_NODE_SMALL_MAX: u8 = EXTENSION_NODE_BIG - 1; - -fn take<'a>(input: &mut &'a[u8], count: usize) -> Option<&'a[u8]> { - if input.len() < count { - return None - } - let r = &(*input)[..count]; - *input = &(*input)[count..]; - Some(r) -} - -fn partial_to_key(partial: &[u8], offset: u8, big: u8) -> Vec<u8> { - let nibble_count = (partial.len() - 1) * 2 + if partial[0] & 16 == 16 { 1 } else { 0 }; - let (first_byte_small, big_threshold) = (offset, (big - offset) as usize); - let mut output = [first_byte_small + nibble_count.min(big_threshold) as u8].to_vec(); - if nibble_count >= big_threshold { output.push((nibble_count - big_threshold) as u8) } - if nibble_count % 2 == 1 { - output.push(partial[0] & 0x0f); - } - output.extend_from_slice(&partial[1..]); - output -} - -fn branch_node(has_value: bool, has_children: impl Iterator<Item = bool>) -> [u8; 3] { - let first = if has_value { - BRANCH_NODE_WITH_VALUE - } else { - BRANCH_NODE_NO_VALUE - }; - let mut bitmap: u16 = 0; - let mut cursor: u16 = 1; - for v in has_children { - if v { bitmap |= cursor } - cursor <<= 1; - } - [first, (bitmap % 256 ) as u8, (bitmap / 256 ) as u8] +/// Constants used into trie simplification codec. +mod trie_constants { + pub const EMPTY_TRIE: u8 = 0; + pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; + pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; + pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; + pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; } #[cfg(test)] @@ -334,19 +335,25 @@ mod tests { use codec::{Encode, Compact}; use primitives::Blake2Hasher; use hash_db::{HashDB, Hasher}; - use trie_db::{DBValue, TrieMut, Trie}; + use trie_db::{DBValue, TrieMut, Trie, NodeCodec as NodeCodecT}; use trie_standardmap::{Alphabet, ValueMode, StandardMap}; use hex_literal::hex; - fn check_equivalent(input: &Vec<(&[u8], &[u8])>) { + type Layout = super::Layout<Blake2Hasher>; + + fn hashed_null_node<T: TrieConfiguration>() -> TrieHash<T> { + <T::Codec as NodeCodecT<_>>::hashed_null_node() + } + + fn check_equivalent<T: TrieConfiguration>(input: &Vec<(&[u8], &[u8])>) { { - let closed_form = trie_root::<Blake2Hasher, _, _, _>(input.clone()); - let d = unhashed_trie::<Blake2Hasher, _, _, _>(input.clone()); + let closed_form = T::trie_root(input.clone()); + let d = T::trie_root_unhashed(input.clone()); println!("Data: {:#x?}, {:#x?}", d, Blake2Hasher::hash(&d[..])); let persistent = { let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut t = TrieDBMut::<Blake2Hasher>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<T>::new(&mut memdb, &mut root); for (x, y) in input.iter().rev() { t.insert(x, y).unwrap(); } @@ -356,20 +363,22 @@ mod tests { } } - fn check_iteration(input: &Vec<(&[u8], &[u8])>) { + fn check_iteration<T: TrieConfiguration>(input: &Vec<(&[u8], &[u8])>) { let mut memdb = MemoryDB::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::<Blake2Hasher>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<T>::new(&mut memdb, &mut root); for (x, y) in input.clone() { t.insert(x, y).unwrap(); } } { - let t = TrieDB::<Blake2Hasher>::new(&mut memdb, &root).unwrap(); + let t = TrieDB::<T>::new(&mut memdb, &root).unwrap(); assert_eq!( input.iter().map(|(i, j)| (i.to_vec(), j.to_vec())).collect::<Vec<_>>(), - t.iter().unwrap().map(|x| x.map(|y| (y.0, y.1.to_vec())).unwrap()).collect::<Vec<_>>() + t.iter().unwrap() + .map(|x| x.map(|y| (y.0, y.1.to_vec())).unwrap()) + .collect::<Vec<_>>() ); } } @@ -377,11 +386,11 @@ mod tests { #[test] fn default_trie_root() { let mut db = MemoryDB::default(); - let mut root = <Blake2Hasher as Hasher>::Out::default(); - let mut empty = TrieDBMut::<Blake2Hasher>::new(&mut db, &mut root); + let mut root = TrieHash::<Layout>::default(); + let mut empty = TrieDBMut::<Layout>::new(&mut db, &mut root); empty.commit(); let root1 = empty.root().as_ref().to_vec(); - let root2: Vec<u8> = trie_root::<Blake2Hasher, _, Vec<u8>, Vec<u8>>( + let root2: Vec<u8> = Layout::trie_root::<_, Vec<u8>, Vec<u8>>( std::iter::empty(), ).as_ref().iter().cloned().collect(); @@ -391,29 +400,35 @@ mod tests { #[test] fn empty_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![]; - check_equivalent(&input); - check_iteration(&input); + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } #[test] fn leaf_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0xbb][..])]; - check_equivalent(&input); - check_iteration(&input); + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } #[test] fn branch_is_equivalent() { - let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0x10][..]), (&[0xba][..], &[0x11][..])]; - check_equivalent(&input); - check_iteration(&input); + let input: Vec<(&[u8], &[u8])> = vec![ + (&[0xaa][..], &[0x10][..]), + (&[0xba][..], &[0x11][..]), + ]; + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } #[test] fn extension_and_branch_is_equivalent() { - let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0x10][..]), (&[0xab][..], &[0x11][..])]; - check_equivalent(&input); - check_iteration(&input); + let input: Vec<(&[u8], &[u8])> = vec![ + (&[0xaa][..], &[0x10][..]), + (&[0xab][..], &[0x11][..]), + ]; + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } #[test] @@ -428,8 +443,8 @@ mod tests { let mut d = st.make(); d.sort_unstable_by(|&(ref a, _), &(ref b, _)| a.cmp(b)); let dr = d.iter().map(|v| (&v.0[..], &v.1[..])).collect(); - check_equivalent(&dr); - check_iteration(&dr); + check_equivalent::<Layout>(&dr); + check_iteration::<Layout>(&dr); } #[test] @@ -439,8 +454,8 @@ mod tests { (&[0xaa, 0xaa][..], &[0xaa][..]), (&[0xaa, 0xbb][..], &[0xab][..]) ]; - check_equivalent(&input); - check_iteration(&input); + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } #[test] @@ -453,8 +468,8 @@ mod tests { (&[0xbb, 0xbb][..], &[0xbb][..]), (&[0xbb, 0xcc][..], &[0xbc][..]), ]; - check_equivalent(&input); - check_iteration(&input); + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } #[test] @@ -463,8 +478,8 @@ mod tests { (&[0xaa][..], &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..]), (&[0xba][..], &[0x11][..]), ]; - check_equivalent(&input); - check_iteration(&input); + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } #[test] @@ -473,16 +488,16 @@ mod tests { (&[0xaa][..], &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..]), (&[0xba][..], &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..]) ]; - check_equivalent(&input); - check_iteration(&input); + check_equivalent::<Layout>(&input); + check_iteration::<Layout>(&input); } - fn populate_trie<'db>( - db: &'db mut dyn HashDB<Blake2Hasher, DBValue>, - root: &'db mut <Blake2Hasher as Hasher>::Out, + fn populate_trie<'db, T: TrieConfiguration>( + db: &'db mut dyn HashDB<T::Hash, DBValue>, + root: &'db mut TrieHash<T>, v: &[(Vec<u8>, Vec<u8>)] - ) -> TrieDBMut<'db, Blake2Hasher> { - let mut t = TrieDBMut::<Blake2Hasher>::new(db, root); + ) -> TrieDBMut<'db, T> { + let mut t = TrieDBMut::<T>::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -491,7 +506,10 @@ mod tests { t } - fn unpopulate_trie<'db>(t: &mut TrieDBMut<'db, Blake2Hasher>, v: &[(Vec<u8>, Vec<u8>)]) { + fn unpopulate_trie<'db, T: TrieConfiguration>( + t: &mut TrieDBMut<'db, T>, + v: &[(Vec<u8>, Vec<u8>)], + ) { for i in v { let key: &[u8]= &i.0; t.remove(key).unwrap(); @@ -513,10 +531,10 @@ mod tests { count: 100, }.make_with(seed.as_fixed_bytes_mut()); - let real = trie_root::<Blake2Hasher,_, _, _>(x.clone()); + let real = Layout::trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + let mut memtrie = populate_trie::<Layout>(&mut memdb, &mut root, &x); memtrie.commit(); if *memtrie.root() != real { @@ -528,17 +546,18 @@ mod tests { } } assert_eq!(*memtrie.root(), real); - unpopulate_trie(&mut memtrie, &x); + unpopulate_trie::<Layout>(&mut memtrie, &x); memtrie.commit(); - if *memtrie.root() != <NodeCodec<Blake2Hasher> as trie_db::NodeCodec<Blake2Hasher>>::hashed_null_node() { + let hashed_null_node = hashed_null_node::<Layout>(); + if *memtrie.root() != hashed_null_node { println!("- TRIE MISMATCH"); println!(""); - println!("{:?} vs {:?}", memtrie.root(), <NodeCodec<Blake2Hasher> as trie_db::NodeCodec<Blake2Hasher>>::hashed_null_node()); + println!("{:?} vs {:?}", memtrie.root(), hashed_null_node); for i in &x { println!("{:#x?} -> {:#x?}", i.0, i.1); } } - assert_eq!(*memtrie.root(), <NodeCodec<Blake2Hasher> as trie_db::NodeCodec<Blake2Hasher>>::hashed_null_node()); + assert_eq!(*memtrie.root(), hashed_null_node); } } @@ -549,7 +568,7 @@ mod tests { #[test] fn codec_trie_empty() { let input: Vec<(&[u8], &[u8])> = vec![]; - let trie = unhashed_trie::<Blake2Hasher, _, _, _>(input); + let trie = Layout::trie_root_unhashed::<_, _, _>(input); println!("trie: {:#x?}", trie); assert_eq!(trie, vec![0x0]); } @@ -559,11 +578,10 @@ mod tests { let input = vec![ (vec![0xaa], vec![0xbb]) ]; - let trie = unhashed_trie::<Blake2Hasher, _, _, _>(input); + let trie = Layout::trie_root_unhashed::<_, _, _>(input); println!("trie: {:#x?}", trie); - assert_eq!(trie, vec![ - 0x03, // leaf (0x01) with (+) key of 2 nibbles (0x02) + 0x42, // leaf 0x40 (2^6) with (+) key of 2 nibbles (0x02) 0xaa, // key data to_compact(1), // length of value in bytes as Compact 0xbb // value data @@ -573,21 +591,20 @@ mod tests { #[test] fn codec_trie_two_tuples_disjoint_keys() { let input = vec![(&[0x48, 0x19], &[0xfe]), (&[0x13, 0x14], &[0xff])]; - let trie = unhashed_trie::<Blake2Hasher, _, _, _>(input); + let trie = Layout::trie_root_unhashed::<_, _, _>(input); println!("trie: {:#x?}", trie); - let mut ex = Vec::<u8>::new(); - ex.push(0xfe); // branch, no value + ex.push(0x80); // branch, no value (0b_10..) no nibble ex.push(0x12); // slots 1 & 4 are taken from 0-7 ex.push(0x00); // no slots from 8-15 ex.push(to_compact(0x05)); // first slot: LEAF, 5 bytes long. - ex.push(0x04); // leaf with 3 nibbles + ex.push(0x43); // leaf 0x40 with 3 nibbles ex.push(0x03); // first nibble ex.push(0x14); // second & third nibble ex.push(to_compact(0x01)); // 1 byte data ex.push(0xff); // value data ex.push(to_compact(0x05)); // second slot: LEAF, 5 bytes long. - ex.push(0x04); // leaf with 3 nibbles + ex.push(0x43); // leaf with 3 nibbles ex.push(0x08); // first nibble ex.push(0x19); // second & third nibble ex.push(to_compact(0x01)); // 1 byte data @@ -605,9 +622,9 @@ mod tests { let mut mdb = MemoryDB::default(); let mut root = Default::default(); - let _ = populate_trie(&mut mdb, &mut root, &pairs); + let _ = populate_trie::<Layout>(&mut mdb, &mut root, &pairs); - let trie = TrieDB::<Blake2Hasher>::new(&mdb, &root).unwrap(); + let trie = TrieDB::<Layout>::new(&mdb, &root).unwrap(); let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); diff --git a/substrate/core/trie/src/node_codec.rs b/substrate/core/trie/src/node_codec.rs index 1b0d2be6524..8c01beda6b5 100644 --- a/substrate/core/trie/src/node_codec.rs +++ b/substrate/core/trie/src/node_codec.rs @@ -18,72 +18,95 @@ use rstd::marker::PhantomData; use rstd::vec::Vec; +use rstd::borrow::Borrow; use codec::{Encode, Decode, Compact}; use hash_db::Hasher; -use trie_db::{self, DBValue, NibbleSlice, node::Node, ChildReference}; +use trie_db::{self, NibbleSlice, node::Node, ChildReference, + nibble_ops, Partial, NodeCodec as NodeCodecT}; use crate::error::Error; -use super::{EMPTY_TRIE, LEAF_NODE_OFFSET, LEAF_NODE_BIG, EXTENSION_NODE_OFFSET, - EXTENSION_NODE_BIG, take, partial_to_key, node_header::NodeHeader, branch_node}; +use crate::trie_constants; +use super::{node_header::{NodeHeader, NodeKind}}; + +fn take<'a>(input: &mut &'a[u8], count: usize) -> Option<&'a[u8]> { + if input.len() < count { + return None + } + let r = &(*input)[..count]; + *input = &(*input)[count..]; + Some(r) +} /// Concrete implementation of a `NodeCodec` with Parity Codec encoding, generic over the `Hasher` #[derive(Default, Clone)] -pub struct NodeCodec<H: Hasher>(PhantomData<H>); +pub struct NodeCodec<H>(PhantomData<H>); -impl<H: Hasher> trie_db::NodeCodec<H> for NodeCodec<H> { +impl<H: Hasher> NodeCodecT<H> for NodeCodec<H> { type Error = Error; - fn hashed_null_node() -> H::Out { - H::hash(&[0u8][..]) + fn hashed_null_node() -> <H as Hasher>::Out { + H::hash(<Self as NodeCodecT<_>>::empty_node()) } - fn decode(data: &[u8]) -> ::rstd::result::Result<Node, Self::Error> { - use Error::BadFormat; + fn decode(data: &[u8]) -> rstd::result::Result<Node, Self::Error> { let input = &mut &*data; - match NodeHeader::decode(input).ok_or(BadFormat)? { + let head = NodeHeader::decode(input).ok_or(Error::BadFormat)?; + match head { NodeHeader::Null => Ok(Node::Empty), - NodeHeader::Branch(has_value) => { - let bitmap = u16::decode(input).ok_or(BadFormat)?; + NodeHeader::Branch(has_value, nibble_count) => { + let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; + // check that the padding is valid (if any) + if padding && nibble_ops::pad_left(input[0]) != 0 { + return Err(Error::BadFormat); + } + let nibble_data = take( + input, + (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / nibble_ops::NIBBLE_PER_BYTE, + ).ok_or(Error::BadFormat)?; + let nibble_slice = NibbleSlice::new_offset( + nibble_data, + nibble_ops::number_padding(nibble_count), + ); + let bitmap_slice = take(input, BITMAP_LENGTH).ok_or(Error::BadFormat)?; + let bitmap = Bitmap::decode(&bitmap_slice[..])?; let value = if has_value { - let count = <Compact<u32>>::decode(input).ok_or(BadFormat)?.0 as usize; - Some(take(input, count).ok_or(BadFormat)?) + let count = <Compact<u32>>::decode(input).ok_or(Error::BadFormat)?.0 as usize; + Some(take(input, count).ok_or(Error::BadFormat)?) } else { None }; let mut children = [None; 16]; - let mut pot_cursor = 1; - for i in 0..16 { - if bitmap & pot_cursor != 0 { - let count = <Compact<u32>>::decode(input).ok_or(BadFormat)?.0 as usize; - children[i] = Some(take(input, count).ok_or(BadFormat)?); + + for i in 0..nibble_ops::NIBBLE_LENGTH { + if bitmap.value_at(i) { + let count = <Compact<u32>>::decode(input).ok_or(Error::BadFormat)?.0 as usize; + children[i] = Some(take(input, count).ok_or(Error::BadFormat)?); } - pot_cursor <<= 1; } - Ok(Node::Branch(children, value)) - } - NodeHeader::Extension(nibble_count) => { - if nibble_count % 2 == 1 && input[0] & 0xf0 != 0x00 { - return Err(BadFormat); - } - let nibble_data = take(input, (nibble_count + 1) / 2).ok_or(BadFormat)?; - let nibble_slice = NibbleSlice::new_offset(nibble_data, nibble_count % 2); - let count = <Compact<u32>>::decode(input).ok_or(BadFormat)?.0 as usize; - Ok(Node::Extension(nibble_slice, take(input, count).ok_or(BadFormat)?)) + Ok(Node::NibbledBranch(nibble_slice, children, value)) } NodeHeader::Leaf(nibble_count) => { - if nibble_count % 2 == 1 && input[0] & 0xf0 != 0x00 { - return Err(BadFormat); + let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; + // check that the padding is valid (if any) + if padding && nibble_ops::pad_left(input[0]) != 0 { + return Err(Error::BadFormat); } - let nibble_data = take(input, (nibble_count + 1) / 2).ok_or(BadFormat)?; - let nibble_slice = NibbleSlice::new_offset(nibble_data, nibble_count % 2); - let count = <Compact<u32>>::decode(input).ok_or(BadFormat)?.0 as usize; - Ok(Node::Leaf(nibble_slice, take(input, count).ok_or(BadFormat)?)) + let nibble_data = take( + input, + (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / nibble_ops::NIBBLE_PER_BYTE, + ).ok_or(Error::BadFormat)?; + let nibble_slice = NibbleSlice::new_offset( + nibble_data, + nibble_ops::number_padding(nibble_count), + ); + let count = <Compact<u32>>::decode(input).ok_or(Error::BadFormat)?.0 as usize; + Ok(Node::Leaf(nibble_slice, take(input, count).ok_or(Error::BadFormat)?)) } } } - fn try_decode_hash(data: &[u8]) -> Option<H::Out> { + fn try_decode_hash(data: &[u8]) -> Option<<H as Hasher>::Out> { if data.len() == H::LENGTH { - let mut r = H::Out::default(); + let mut r = <H as Hasher>::Out::default(); r.as_mut().copy_from_slice(data); Some(r) } else { @@ -92,53 +115,140 @@ impl<H: Hasher> trie_db::NodeCodec<H> for NodeCodec<H> { } fn is_empty_node(data: &[u8]) -> bool { - data == &[EMPTY_TRIE][..] + data == <Self as NodeCodecT<_>>::empty_node() } - fn empty_node() -> Vec<u8> { - [EMPTY_TRIE].to_vec() + + fn empty_node() -> &'static [u8] { + &[trie_constants::EMPTY_TRIE] } - // FIXME: refactor this so that `partial` isn't already encoded with HPE. Should just be an `impl Iterator<Item=u8>`. - fn leaf_node(partial: &[u8], value: &[u8]) -> Vec<u8> { - let mut output = partial_to_key(partial, LEAF_NODE_OFFSET, LEAF_NODE_BIG); + fn leaf_node(partial: Partial, value: &[u8]) -> Vec<u8> { + let mut output = partial_encode(partial, NodeKind::Leaf); value.encode_to(&mut output); output } - // FIXME: refactor this so that `partial` isn't already encoded with HPE. Should just be an `impl Iterator<Item=u8>`. - fn ext_node(partial: &[u8], child: ChildReference<H::Out>) -> Vec<u8> { - let mut output = partial_to_key(partial, EXTENSION_NODE_OFFSET, EXTENSION_NODE_BIG); - match child { - ChildReference::Hash(h) => - h.as_ref().encode_to(&mut output), - ChildReference::Inline(inline_data, len) => - (&AsRef::<[u8]>::as_ref(&inline_data)[..len]).encode_to(&mut output), - }; - output + fn extension_node( + _partial: impl Iterator<Item = u8>, + _nbnibble: usize, + _child: ChildReference<<H as Hasher>::Out>, + ) -> Vec<u8> { + unreachable!() + } + + fn branch_node( + _children: impl Iterator<Item = impl Borrow<Option<ChildReference<<H as Hasher>::Out>>>>, + _maybe_value: Option<&[u8]>, + ) -> Vec<u8> { + unreachable!() } - fn branch_node<I>(children: I, maybe_value: Option<DBValue>) -> Vec<u8> - where I: IntoIterator<Item=Option<ChildReference<H::Out>>> + Iterator<Item=Option<ChildReference<H::Out>>> - { - let mut output = [0, 0, 0].to_vec(); - let have_value = if let Some(value) = maybe_value { - (&*value).encode_to(&mut output); - true + fn branch_node_nibbled( + partial: impl Iterator<Item = u8>, + number_nibble: usize, + children: impl Iterator<Item = impl Borrow<Option<ChildReference<<H as Hasher>::Out>>>>, + maybe_value: Option<&[u8]>, + ) -> Vec<u8> { + let mut output = if maybe_value.is_some() { + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) } else { - false + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }; - let prefix = branch_node(have_value, children.map(|maybe_child| match maybe_child { + let bitmap_index = output.len(); + let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; + (0..BITMAP_LENGTH).for_each(|_|output.push(0)); + if let Some(value) = maybe_value { + value.encode_to(&mut output); + }; + Bitmap::encode(children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { h.as_ref().encode_to(&mut output); true } - Some(ChildReference::Inline(inline_data, len)) => { - (&AsRef::<[u8]>::as_ref(&inline_data)[..len]).encode_to(&mut output); + &Some(ChildReference::Inline(inline_data, len)) => { + inline_data.as_ref()[..len].encode_to(&mut output); true } None => false, - })); - output[0..3].copy_from_slice(&prefix[..]); + }), bitmap.as_mut()); + output[bitmap_index..bitmap_index + BITMAP_LENGTH] + .copy_from_slice(&bitmap.as_ref()[..BITMAP_LENGTH]); output } + +} + +// utils + +/// Encode and allocate node type header (type and size), and partial value. +/// It uses an iterator over encoded partial bytes as input. +fn partial_from_iterator_encode<I: Iterator<Item = u8>>( + partial: I, + nibble_count: usize, + node_kind: NodeKind, +) -> Vec<u8> { + let nibble_count = rstd::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); + + let mut output = Vec::with_capacity(3 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); + match node_kind { + NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), + NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), + NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + }; + output.extend(partial); + output } + +/// Encode and allocate node type header (type and size), and partial value. +/// Same as `partial_from_iterator_encode` but uses non encoded `Partial` as input. +fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec<u8> { + let number_nibble_encoded = (partial.0).0 as usize; + let nibble_count = partial.1.len() * nibble_ops::NIBBLE_PER_BYTE + number_nibble_encoded; + + let nibble_count = rstd::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); + + let mut output = Vec::with_capacity(3 + partial.1.len()); + match node_kind { + NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), + NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), + NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + }; + if number_nibble_encoded > 0 { + output.push(nibble_ops::pad_right((partial.0).1)); + } + output.extend_from_slice(&partial.1[..]); + output +} + +const BITMAP_LENGTH: usize = 2; + +/// Radix 16 trie, bitmap encoding implementation, +/// it contains children mapping information for a branch +/// (children presence only), it encodes into +/// a compact bitmap encoding representation. +pub(crate) struct Bitmap(u16); + +impl Bitmap { + + pub fn decode(data: &[u8]) -> Result<Self, Error> { + u16::decode(&mut &data[..]) + .ok_or(Error::BadFormat) + .map(|v|Bitmap(v)) + } + + pub fn value_at(&self, i: usize) -> bool { + self.0 & (1u16 << i) != 0 + } + + pub fn encode<I: Iterator<Item = bool>>(has_children: I , dest: &mut [u8]) { + let mut bitmap: u16 = 0; + let mut cursor: u16 = 1; + for v in has_children { + if v { bitmap |= cursor } + cursor <<= 1; + } + dest[0] = (bitmap % 256) as u8; + dest[1] = (bitmap / 256) as u8; + } +} + diff --git a/substrate/core/trie/src/node_header.rs b/substrate/core/trie/src/node_header.rs index 2c01189f8a1..e4c38bd0751 100644 --- a/substrate/core/trie/src/node_header.rs +++ b/substrate/core/trie/src/node_header.rs @@ -16,62 +16,105 @@ //! The node header. +use crate::trie_constants; use codec::{Encode, Decode, Input, Output}; -use super::{EMPTY_TRIE, LEAF_NODE_OFFSET, LEAF_NODE_BIG, EXTENSION_NODE_OFFSET, - EXTENSION_NODE_BIG, BRANCH_NODE_NO_VALUE, BRANCH_NODE_WITH_VALUE, LEAF_NODE_THRESHOLD, - EXTENSION_NODE_THRESHOLD, LEAF_NODE_SMALL_MAX, EXTENSION_NODE_SMALL_MAX}; +use rstd::iter::once; -/// A node header. -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum NodeHeader { +/// A node header +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub(crate) enum NodeHeader { Null, - Branch(bool), - Extension(usize), + Branch(bool, usize), Leaf(usize), } +/// NodeHeader without content +pub(crate) enum NodeKind { + Leaf, + BranchNoValue, + BranchWithValue, +} + impl Encode for NodeHeader { fn encode_to<T: Output>(&self, output: &mut T) { match self { - NodeHeader::Null => output.push_byte(EMPTY_TRIE), - - NodeHeader::Branch(true) => output.push_byte(BRANCH_NODE_WITH_VALUE), - NodeHeader::Branch(false) => output.push_byte(BRANCH_NODE_NO_VALUE), - - NodeHeader::Leaf(nibble_count) if *nibble_count < LEAF_NODE_THRESHOLD as usize => - output.push_byte(LEAF_NODE_OFFSET + *nibble_count as u8), - NodeHeader::Leaf(nibble_count) => { - output.push_byte(LEAF_NODE_BIG); - output.push_byte((*nibble_count - LEAF_NODE_THRESHOLD as usize) as u8); - } - - NodeHeader::Extension(nibble_count) if *nibble_count < EXTENSION_NODE_THRESHOLD as usize => - output.push_byte(EXTENSION_NODE_OFFSET + *nibble_count as u8), - NodeHeader::Extension(nibble_count) => { - output.push_byte(EXTENSION_NODE_BIG); - output.push_byte((*nibble_count - EXTENSION_NODE_THRESHOLD as usize) as u8); - } + NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), + NodeHeader::Branch(true, nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, output), + NodeHeader::Branch(false, nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, output), + NodeHeader::Leaf(nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, output), } } } impl Decode for NodeHeader { fn decode<I: Input>(input: &mut I) -> Option<Self> { - Some(match input.read_byte()? { - EMPTY_TRIE => NodeHeader::Null, // 0 + let i = input.read_byte()?; + if i == trie_constants::EMPTY_TRIE { + return Some(NodeHeader::Null); + } + match i & (0b11 << 6) { + trie_constants::LEAF_PREFIX_MASK => Some(NodeHeader::Leaf(decode_size(i, input)?)), + trie_constants::BRANCH_WITHOUT_MASK => Some(NodeHeader::Branch(false, decode_size(i, input)?)), + trie_constants::BRANCH_WITH_MASK => Some(NodeHeader::Branch(true, decode_size(i, input)?)), + // do not allow any special encoding + _ => None, + } + } +} - i @ LEAF_NODE_OFFSET ..= LEAF_NODE_SMALL_MAX => // 1 ... (127 - 1) - NodeHeader::Leaf((i - LEAF_NODE_OFFSET) as usize), - LEAF_NODE_BIG => // 127 - NodeHeader::Leaf(input.read_byte()? as usize + LEAF_NODE_THRESHOLD as usize), +/// Returns an iterator over encoded bytes for node header and size. +/// Size encoding allows unlimited, length unefficient, representation, but +/// is bounded to 16 bit maximum value to avoid possible DOS. +pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8) -> impl Iterator<Item = u8> { + let size = rstd::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - i @ EXTENSION_NODE_OFFSET ..= EXTENSION_NODE_SMALL_MAX =>// 128 ... (253 - 1) - NodeHeader::Extension((i - EXTENSION_NODE_OFFSET) as usize), - EXTENSION_NODE_BIG => // 253 - NodeHeader::Extension(input.read_byte()? as usize + EXTENSION_NODE_THRESHOLD as usize), + let l1 = rstd::cmp::min(62, size); + let (first_byte, mut rem) = if size == l1 { + (once(prefix + l1 as u8), 0) + } else { + (once(prefix + 63), size - l1) + }; + let next_bytes = move || { + if rem > 0 { + if rem < 256 { + let result = rem - 1; + rem = 0; + Some(result as u8) + } else { + rem = rem.saturating_sub(255); + Some(255) + } + } else { + None + } + }; + first_byte.chain(rstd::iter::from_fn(next_bytes)) +} - BRANCH_NODE_NO_VALUE => NodeHeader::Branch(false), // 254 - BRANCH_NODE_WITH_VALUE => NodeHeader::Branch(true), // 255 - }) +/// Encodes size and prefix to a stream output. +fn encode_size_and_prefix(size: usize, prefix: u8, out: &mut impl Output) { + for b in size_and_prefix_iterator(size, prefix) { + out.push_byte(b) + } +} + +/// Decode size only from stream input and header byte. +fn decode_size(first: u8, input: &mut impl Input) -> Option<usize> { + let mut result = (first & 255u8 >> 2) as usize; + if result < 63 { + return Some(result); + } + result -= 1; + while result <= trie_constants::NIBBLE_SIZE_BOUND { + let n = input.read_byte()? as usize; + if n < 255 { + return Some(result + n + 1); + } + result += 255; } + Some(trie_constants::NIBBLE_SIZE_BOUND) } diff --git a/substrate/core/trie/src/trie_stream.rs b/substrate/core/trie/src/trie_stream.rs index 913cff2c5a9..2629cefac8c 100644 --- a/substrate/core/trie/src/trie_stream.rs +++ b/substrate/core/trie/src/trie_stream.rs @@ -16,16 +16,19 @@ //! `TrieStream` implementation for Substrate's trie format. -use rstd::iter::once; use hash_db::Hasher; use trie_root; use codec::Encode; use rstd::vec::Vec; +use crate::trie_constants; +use crate::node_header::{NodeKind, size_and_prefix_iterator}; +use crate::node_codec::Bitmap; -use super::{EMPTY_TRIE, LEAF_NODE_OFFSET, LEAF_NODE_BIG, EXTENSION_NODE_OFFSET, - EXTENSION_NODE_BIG, branch_node}; +const BRANCH_NODE_NO_VALUE: u8 = 254; +const BRANCH_NODE_WITH_VALUE: u8 = 255; -/// Codec-flavored TrieStream +#[derive(Default, Clone)] +/// Codec-flavored TrieStream. pub struct TrieStream { buffer: Vec<u8>, } @@ -35,62 +38,102 @@ impl TrieStream { pub fn as_raw(&self) -> &[u8] { &self.buffer } } -/// Create a leaf/extension node, encoding a number of nibbles. Note that this -/// cannot handle a number of nibbles that is zero or greater than 127 and if -/// you attempt to do so *IT WILL PANIC*. -fn fuse_nibbles_node<'a>(nibbles: &'a [u8], leaf: bool) -> impl Iterator<Item = u8> + 'a { - debug_assert!(nibbles.len() < 255 + 126, "nibbles length too long. what kind of size of key are you trying to include in the trie!?!"); - // We use two ranges of possible values; one for leafs and the other for extensions. - // Each range encodes zero following nibbles up to some maximum. If the maximum is - // reached, then it is considered "big" and a second byte follows it in order to - // encode a further offset to the number of nibbles of up to 255. Beyond that, we - // cannot encode. This shouldn't be a problem though since that allows for keys of - // up to 380 nibbles (190 bytes) and we expect key sizes to be generally 128-bit (16 - // bytes) or, at a push, 384-bit (48 bytes). - - let (first_byte_small, big_threshold) = if leaf { - (LEAF_NODE_OFFSET, (LEAF_NODE_BIG - LEAF_NODE_OFFSET) as usize) - } else { - (EXTENSION_NODE_OFFSET, (EXTENSION_NODE_BIG - EXTENSION_NODE_OFFSET) as usize) +fn branch_node_bit_mask(has_children: impl Iterator<Item = bool>) -> (u8, u8) { + let mut bitmap: u16 = 0; + let mut cursor: u16 = 1; + for v in has_children { + if v { bitmap |= cursor } + cursor <<= 1; + } + ((bitmap % 256 ) as u8, (bitmap / 256 ) as u8) +} + + +/// Create a leaf/branch node, encoding a number of nibbles. +fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator<Item = u8> + 'a { + let size = rstd::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibbles.len()); + + let iter_start = match kind { + NodeKind::Leaf => size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK), + NodeKind::BranchNoValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK), + NodeKind::BranchWithValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK), }; - let first_byte = first_byte_small + nibbles.len().min(big_threshold) as u8; - once(first_byte) - .chain(if nibbles.len() >= big_threshold { Some((nibbles.len() - big_threshold) as u8) } else { None }) + iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) } + impl trie_root::TrieStream for TrieStream { - fn new() -> Self { Self {buffer: Vec::new() } } + + fn new() -> Self { + TrieStream { + buffer: Vec::new() + } + } + fn append_empty_data(&mut self) { - self.buffer.push(EMPTY_TRIE); + self.buffer.push(trie_constants::EMPTY_TRIE); } fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - self.buffer.extend(fuse_nibbles_node(key, true)); + self.buffer.extend(fuse_nibbles_node(key, NodeKind::Leaf)); value.encode_to(&mut self.buffer); } - fn begin_branch(&mut self, maybe_value: Option<&[u8]>, has_children: impl Iterator<Item = bool>) { - self.buffer.extend(&branch_node(maybe_value.is_some(), has_children)); - // Push the value if one exists. + + fn begin_branch( + &mut self, + maybe_partial: Option<&[u8]>, + maybe_value: Option<&[u8]>, + has_children: impl Iterator<Item = bool>, + ) { + if let Some(partial) = maybe_partial { + if maybe_value.is_some() { + self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchWithValue)); + } else { + self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchNoValue)); + } + let bm = branch_node_bit_mask(has_children); + self.buffer.extend([bm.0,bm.1].iter()); + } else { + debug_assert!(false, "trie stream codec only for no extension trie"); + self.buffer.extend(&branch_node(maybe_value.is_some(), has_children)); + } if let Some(value) = maybe_value { value.encode_to(&mut self.buffer); } } - fn append_extension(&mut self, key: &[u8]) { - self.buffer.extend(fuse_nibbles_node(key, false)); + + fn append_extension(&mut self, _key: &[u8]) { + debug_assert!(false, "trie stream codec only for no extension trie"); } + fn append_substream<H: Hasher>(&mut self, other: Self) { let data = other.out(); match data.len() { - 0..=31 => { - data.encode_to(&mut self.buffer) - }, - _ => { - H::hash(&data).as_ref().encode_to(&mut self.buffer) - } + 0..=31 => data.encode_to(&mut self.buffer), + _ => H::hash(&data).as_ref().encode_to(&mut self.buffer), } } fn out(self) -> Vec<u8> { self.buffer } } + +fn branch_node(has_value: bool, has_children: impl Iterator<Item = bool>) -> [u8; 3] { + let mut result = [0, 0, 0]; + branch_node_buffered(has_value, has_children, &mut result[..]); + result +} + +fn branch_node_buffered<I>(has_value: bool, has_children: I, output: &mut[u8]) + where + I: Iterator<Item = bool>, +{ + let first = if has_value { + BRANCH_NODE_WITH_VALUE + } else { + BRANCH_NODE_NO_VALUE + }; + output[0] = first; + Bitmap::encode(has_children, &mut output[1..]); +} diff --git a/substrate/node-template/Cargo.toml b/substrate/node-template/Cargo.toml index 4c85ef38174..ad626beb0bf 100644 --- a/substrate/node-template/Cargo.toml +++ b/substrate/node-template/Cargo.toml @@ -18,7 +18,7 @@ tokio = "0.1" exit-future = "0.1" parking_lot = "0.8.0" parity-codec = "4.1.1" -trie-root = "0.14.0" +trie-root = "0.15.0" sr-io = { path = "../core/sr-io" } substrate-cli = { path = "../core/cli" } primitives = { package = "substrate-primitives", path = "../core/primitives" } diff --git a/substrate/node/executor/Cargo.toml b/substrate/node/executor/Cargo.toml index 1849e1eeed7..44a56ee4a63 100644 --- a/substrate/node/executor/Cargo.toml +++ b/substrate/node/executor/Cargo.toml @@ -6,7 +6,7 @@ description = "Substrate node implementation in Rust." edition = "2018" [dependencies] -trie-root = "0.14.0" +trie-root = "0.15.0" parity-codec = "4.1.1" runtime_io = { package = "sr-io", path = "../../core/sr-io" } state_machine = { package = "substrate-state-machine", path = "../../core/state-machine" } diff --git a/substrate/node/executor/src/lib.rs b/substrate/node/executor/src/lib.rs index e9c7f4e1cb6..572d0ca1499 100644 --- a/substrate/node/executor/src/lib.rs +++ b/substrate/node/executor/src/lib.rs @@ -403,13 +403,13 @@ mod tests { parent_hash: Hash, extrinsics: Vec<CheckedExtrinsic>, ) -> (Vec<u8>, Hash) { - use trie::ordered_trie_root; + use trie::{TrieConfiguration, trie_types::Layout}; // sign extrinsics. let extrinsics = extrinsics.into_iter().map(sign).collect::<Vec<_>>(); // calculate the header fields that we can. - let extrinsics_root = ordered_trie_root::<Blake2Hasher, _, _>( + let extrinsics_root = Layout::<Blake2Hasher>::ordered_trie_root( extrinsics.iter().map(Encode::encode) ).to_fixed_bytes() .into(); diff --git a/substrate/node/runtime/src/lib.rs b/substrate/node/runtime/src/lib.rs index e9ae43be037..c6b74293b8f 100644 --- a/substrate/node/runtime/src/lib.rs +++ b/substrate/node/runtime/src/lib.rs @@ -79,8 +79,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to equal spec_version. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 126, - impl_version: 126, + spec_version: 127, + impl_version: 127, apis: RUNTIME_API_VERSIONS, }; diff --git a/substrate/srml/executive/src/lib.rs b/substrate/srml/executive/src/lib.rs index 835b167e3dd..062badd9320 100644 --- a/substrate/srml/executive/src/lib.rs +++ b/substrate/srml/executive/src/lib.rs @@ -500,7 +500,7 @@ mod tests { header: Header { parent_hash: [69u8; 32].into(), number: 1, - state_root: hex!("ba811447b8ae3bf798a07a18f5355ea59926917c8a9cc7527ede20b261aacfdf").into(), + state_root: hex!("3e51b47b6cc8449eece93eee4b01f03b00a0ca7981c0b6c0447b6e0d50ca886d").into(), extrinsics_root: hex!("03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314").into(), digest: Digest { logs: vec![], }, }, diff --git a/substrate/srml/session/src/historical.rs b/substrate/srml/session/src/historical.rs index 402a73adf36..1e8e7faaaf3 100644 --- a/substrate/srml/session/src/historical.rs +++ b/substrate/srml/session/src/historical.rs @@ -33,8 +33,8 @@ use srml_support::{ StorageValue, StorageMap, decl_module, decl_storage, }; use srml_support::{Parameter, print}; -use substrate_trie::{MemoryDB, Trie, TrieMut, TrieDBMut, TrieDB, Recorder}; - +use substrate_trie::{MemoryDB, Trie, TrieMut, Recorder, EMPTY_PREFIX}; +use substrate_trie::trie_types::{TrieDBMut, TrieDB}; use super::{SessionIndex, Module as SessionModule}; /// Trait necessary for the historical module. @@ -219,7 +219,7 @@ impl<T: Trait> ProvingTrie<T> { let mut memory_db = MemoryDB::default(); for node in nodes { - HashDBT::insert(&mut memory_db, &[], &node[..]); + HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); } ProvingTrie { -- GitLab