blockchain.rs

// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.

// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Parity.  If not, see <http://www.gnu.org/licenses/>.

//! Blockchain database.

use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrder};
use bloomchain as bc;
use util::*;
use header::*;
use super::extras::*;
use transaction::*;
use views::*;
use receipt::Receipt;
use blooms::{Bloom, BloomGroup};
use blockchain::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData};
use blockchain::best_block::BestBlock;
use types::tree_route::TreeRoute;
use blockchain::update::ExtrasUpdate;
use blockchain::{CacheSize, ImportRoute, Config};
use db::{Writable, Readable, CacheUpdatePolicy};
use client::{DB_COL_EXTRA, DB_COL_HEADERS, DB_COL_BODIES};

const LOG_BLOOMS_LEVELS: usize = 3;
const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16;

/// Interface for querying blocks by hash and by number.
pub trait BlockProvider {
	/// Returns true if the given block is known
	/// (though not necessarily a part of the canon chain).
	fn is_known(&self, hash: &H256) -> bool;

	/// Get raw block data
	fn block(&self, hash: &H256) -> Option<Bytes>;

	/// Get the familial details concerning a block.
	fn block_details(&self, hash: &H256) -> Option<BlockDetails>;

	/// Get the hash of given block's number.
	fn block_hash(&self, index: BlockNumber) -> Option<H256>;

	/// Get the address of transaction with given hash.
	fn transaction_address(&self, hash: &H256) -> Option<TransactionAddress>;

	/// Get receipts of block with given hash.
	fn block_receipts(&self, hash: &H256) -> Option<BlockReceipts>;

	/// Get the partial-header of a block.
	fn block_header(&self, hash: &H256) -> Option<Header> {
		self.block_header_data(hash).map(|header| decode(&header))
	}

	/// Get the header RLP of a block.
	fn block_header_data(&self, hash: &H256) -> Option<Bytes>;

	/// Get the block body (uncles and transactions).
	fn block_body(&self, hash: &H256) -> Option<Bytes>;

	/// Get a list of uncles for a given block.
	/// Returns None if block does not exist.
	fn uncles(&self, hash: &H256) -> Option<Vec<Header>> {
		self.block_body(hash).map(|bytes| BodyView::new(&bytes).uncles())
	}

	/// Get a list of uncle hashes for a given block.
	/// Returns None if block does not exist.
	fn uncle_hashes(&self, hash: &H256) -> Option<Vec<H256>> {
		self.block_body(hash).map(|bytes| BodyView::new(&bytes).uncle_hashes())
	}

	/// Get the number of given block's hash.
	fn block_number(&self, hash: &H256) -> Option<BlockNumber> {
		self.block_details(hash).map(|details| details.number)
	}

	/// Get transaction with given transaction hash.
	fn transaction(&self, address: &TransactionAddress) -> Option<LocalizedTransaction> {
		self.block_body(&address.block_hash)
			.and_then(|bytes| self.block_number(&address.block_hash)
			.and_then(|n| BodyView::new(&bytes).localized_transaction_at(&address.block_hash, n, address.index)))
	}

	/// Get transaction receipt.
	fn transaction_receipt(&self, address: &TransactionAddress) -> Option<Receipt> {
		self.block_receipts(&address.block_hash).and_then(|br| br.receipts.into_iter().nth(address.index))
	}

	/// Get a list of transactions for a given block.
	/// Returns None if block does not exist.
	fn transactions(&self, hash: &H256) -> Option<Vec<LocalizedTransaction>> {
		self.block_body(hash)
			.and_then(|bytes| self.block_number(hash)
			.map(|n| BodyView::new(&bytes).localized_transactions(hash, n)))
	}

	/// Returns reference to genesis hash.
	fn genesis_hash(&self) -> H256 {
		self.block_hash(0).expect("Genesis hash should always exist")
	}

	/// Returns the header of the genesis block.
	fn genesis_header(&self) -> Header {
		self.block_header(&self.genesis_hash()).unwrap()
	}

	/// Returns numbers of blocks containing given bloom.
	fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber>;
}

#[derive(Debug, Hash, Eq, PartialEq, Clone)]
enum CacheID {
	BlockHeader(H256),
	BlockBody(H256),
	BlockDetails(H256),
	BlockHashes(BlockNumber),
	TransactionAddresses(H256),
	BlocksBlooms(LogGroupPosition),
	BlockReceipts(H256),
}

struct CacheManager {
	cache_usage: VecDeque<HashSet<CacheID>>,
	in_use: HashSet<CacheID>,
}

impl bc::group::BloomGroupDatabase for BlockChain {
	fn blooms_at(&self, position: &bc::group::GroupPosition) -> Option<bc::group::BloomGroup> {
		let position = LogGroupPosition::from(position.clone());
		self.note_used(CacheID::BlocksBlooms(position.clone()));
		self.db.read_with_cache(DB_COL_EXTRA, &self.blocks_blooms, &position).map(Into::into)
	}
}

/// Structure providing fast access to blockchain data.
///
/// **Does not do input data verification.**
pub struct BlockChain {
	// All locks must be captured in the order declared here.
	pref_cache_size: AtomicUsize,
	max_cache_size: AtomicUsize,
	blooms_config: bc::Config,

	best_block: RwLock<BestBlock>,

	// block cache
	block_headers: RwLock<HashMap<H256, Bytes>>,
	block_bodies: RwLock<HashMap<H256, Bytes>>,

	// extra caches
	block_details: RwLock<HashMap<H256, BlockDetails>>,
	block_hashes: RwLock<HashMap<BlockNumber, H256>>,
	transaction_addresses: RwLock<HashMap<H256, TransactionAddress>>,
	blocks_blooms: RwLock<HashMap<LogGroupPosition, BloomGroup>>,
	block_receipts: RwLock<HashMap<H256, BlockReceipts>>,

	db: Arc<Database>,

	cache_man: RwLock<CacheManager>,
}

impl BlockProvider for BlockChain {
	/// Returns true if the given block is known
	/// (though not necessarily a part of the canon chain).
	fn is_known(&self, hash: &H256) -> bool {
		self.db.exists_with_cache(DB_COL_EXTRA, &self.block_details, hash)
	}

	/// Get raw block data
	fn block(&self, hash: &H256) -> Option<Bytes> {
		match (self.block_header_data(hash), self.block_body(hash)) {
			(Some(header), Some(body)) => {
				let mut block = RlpStream::new_list(3);
				let body_rlp = Rlp::new(&body);
				block.append_raw(&header, 1);
				block.append_raw(body_rlp.at(0).as_raw(), 1);
				block.append_raw(body_rlp.at(1).as_raw(), 1);
				Some(block.out())
			},
			_ => None,
		}
	}

	/// Get block header data
	fn block_header_data(&self, hash: &H256) -> Option<Bytes> {
		// Check cache first
		{
			let read = self.block_headers.read();
			if let Some(v) = read.get(hash) {
				return Some(v.clone());
			}
		}

		// Check if it's the best block
		{
			let best_block = self.best_block.read();
			if &best_block.hash == hash {
				return Some(Rlp::new(&best_block.block).at(0).as_raw().to_vec());
			}
		}

		// Read from DB and populate cache
		let opt = self.db.get(DB_COL_HEADERS, hash)
			.expect("Low level database error. Some issue with disk?");

		self.note_used(CacheID::BlockHeader(hash.clone()));

		match opt {
			Some(b) => {
				let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).to_vec();
				let mut write = self.block_headers.write();
				write.insert(hash.clone(), bytes.clone());
				Some(bytes)
			},
			None => None
		}
	}

	/// Get block body data
	fn block_body(&self, hash: &H256) -> Option<Bytes> {
		// Check cache first
		{
			let read = self.block_bodies.read();
			if let Some(v) = read.get(hash) {
				return Some(v.clone());
			}
		}

		// Check if it's the best block
		{
			let best_block = self.best_block.read();
			if &best_block.hash == hash {
				return Some(Self::block_to_body(&best_block.block));
			}
		}

		// Read from DB and populate cache
		let opt = self.db.get(DB_COL_BODIES, hash)
			.expect("Low level database error. Some issue with disk?");

		self.note_used(CacheID::BlockBody(hash.clone()));

		match opt {
			Some(b) => {
				let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).to_vec();
				let mut write = self.block_bodies.write();
				write.insert(hash.clone(), bytes.clone());
				Some(bytes)
			},
			None => None
		}
	}

	/// Get the familial details concerning a block.
	fn block_details(&self, hash: &H256) -> Option<BlockDetails> {
		self.note_used(CacheID::BlockDetails(hash.clone()));
		self.db.read_with_cache(DB_COL_EXTRA, &self.block_details, hash)
	}

	/// Get the hash of given block's number.
	fn block_hash(&self, index: BlockNumber) -> Option<H256> {
		self.note_used(CacheID::BlockHashes(index));
		self.db.read_with_cache(DB_COL_EXTRA, &self.block_hashes, &index)
	}

	/// Get the address of transaction with given hash.
	fn transaction_address(&self, hash: &H256) -> Option<TransactionAddress> {
		self.note_used(CacheID::TransactionAddresses(hash.clone()));
		self.db.read_with_cache(DB_COL_EXTRA, &self.transaction_addresses, hash)
	}

	/// Get receipts of block with given hash.
	fn block_receipts(&self, hash: &H256) -> Option<BlockReceipts> {
		self.note_used(CacheID::BlockReceipts(hash.clone()));
		self.db.read_with_cache(DB_COL_EXTRA, &self.block_receipts, hash)
	}

	/// Returns numbers of blocks containing given bloom.
	fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber> {
		let range = from_block as bc::Number..to_block as bc::Number;
		let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
		chain.with_bloom(&range, &Bloom::from(bloom.clone()).into())
			.into_iter()
			.map(|b| b as BlockNumber)
			.collect()
	}
}

const COLLECTION_QUEUE_SIZE: usize = 8;

pub struct AncestryIter<'a> {
	current: H256,
	chain: &'a BlockChain,
}

impl<'a> Iterator for AncestryIter<'a> {
	type Item = H256;
	fn next(&mut self) -> Option<H256> {
		if self.current.is_zero() {
			Option::None
		} else {
			let mut n = self.chain.block_details(&self.current).unwrap().parent;
			mem::swap(&mut self.current, &mut n);
			Some(n)
		}
	}
}

impl BlockChain {
	/// Create new instance of blockchain from given Genesis
	pub fn new(config: Config, genesis: &[u8], db: Arc<Database>) -> BlockChain {
		let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()};
		(0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new()));

		let bc = BlockChain {
			pref_cache_size: AtomicUsize::new(config.pref_cache_size),
			max_cache_size: AtomicUsize::new(config.max_cache_size),
			blooms_config: bc::Config {
				levels: LOG_BLOOMS_LEVELS,
				elements_per_index: LOG_BLOOMS_ELEMENTS_PER_INDEX,
			},
			best_block: RwLock::new(BestBlock::default()),
			block_headers: RwLock::new(HashMap::new()),
			block_bodies: RwLock::new(HashMap::new()),
			block_details: RwLock::new(HashMap::new()),
			block_hashes: RwLock::new(HashMap::new()),
			transaction_addresses: RwLock::new(HashMap::new()),
			blocks_blooms: RwLock::new(HashMap::new()),
			block_receipts: RwLock::new(HashMap::new()),
			db: db.clone(),
			cache_man: RwLock::new(cache_man),
		};

		// load best block
		let best_block_hash = match bc.db.get(DB_COL_EXTRA, b"best").unwrap() {
			Some(best) => {
				H256::from_slice(&best)
			}
			None => {
				// best block does not exist
				// we need to insert genesis into the cache
				let block = BlockView::new(genesis);
				let header = block.header_view();
				let hash = block.sha3();

				let details = BlockDetails {
					number: header.number(),
					total_difficulty: header.difficulty(),
					parent: header.parent_hash(),
					children: vec![]
				};

				let batch = DBTransaction::new(&db);
				batch.put(DB_COL_HEADERS, &hash, block.header_rlp().as_raw()).unwrap();
				batch.put(DB_COL_BODIES, &hash, &Self::block_to_body(&genesis)).unwrap();

				batch.write(DB_COL_EXTRA, &hash, &details);
				batch.write(DB_COL_EXTRA, &header.number(), &hash);
				batch.put(DB_COL_EXTRA, b"best", &hash).unwrap();
				bc.db.write(batch).expect("Low level database error. Some issue with disk?");
				hash
			}
		};

		{
			// Fetch best block details
			let best_block_number = bc.block_number(&best_block_hash).unwrap();
			let best_block_total_difficulty = bc.block_details(&best_block_hash).unwrap().total_difficulty;
			let best_block_rlp = bc.block(&best_block_hash).unwrap();

			// and write them
			let mut best_block = bc.best_block.write();
			*best_block = BestBlock {
				number: best_block_number,
				total_difficulty: best_block_total_difficulty,
				hash: best_block_hash,
				block: best_block_rlp,
			};
		}

		bc
	}

	/// Returns true if the given parent block has given child
	/// (though not necessarily a part of the canon chain).
	fn is_known_child(&self, parent: &H256, hash: &H256) -> bool {
		self.db.read_with_cache(DB_COL_EXTRA, &self.block_details, parent).map_or(false, |d| d.children.contains(hash))
	}

	/// Rewind to a previous block
	#[cfg(test)]
	fn rewind(&self) -> Option<H256> {
		use db::Key;
		let batch = self.db.transaction();
		// track back to the best block we have in the blocks database
		if let Some(best_block_hash) = self.db.get(DB_COL_EXTRA, b"best").unwrap() {
			let best_block_hash = H256::from_slice(&best_block_hash);
			if best_block_hash == self.genesis_hash() {
				return None;
			}
			if let Some(extras) = self.db.read(DB_COL_EXTRA, &best_block_hash) as Option<BlockDetails> {
				type DetailsKey = Key<BlockDetails, Target=H264>;
				batch.delete(DB_COL_EXTRA, &(DetailsKey::key(&best_block_hash))).unwrap();
				let hash = extras.parent;
				let range = extras.number as bc::Number .. extras.number as bc::Number;
				let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
				let changes = chain.replace(&range, vec![]);
				for (k, v) in changes.into_iter() {
					batch.write(DB_COL_EXTRA, &LogGroupPosition::from(k), &BloomGroup::from(v));
				}
				batch.put(DB_COL_EXTRA, b"best", &hash).unwrap();

				let best_block_total_difficulty = self.block_details(&hash).unwrap().total_difficulty;
				let best_block_rlp = self.block(&hash).unwrap();

				let mut best_block = self.best_block.write();
				*best_block = BestBlock {
					number: extras.number - 1,
					total_difficulty: best_block_total_difficulty,
					hash: hash,
					block: best_block_rlp,
				};
				// update parent extras
				if let Some(mut details) = self.db.read(DB_COL_EXTRA, &hash) as Option<BlockDetails> {
					details.children.clear();
					batch.write(DB_COL_EXTRA, &hash, &details);
				}
				self.db.write(batch).expect("Writing to db failed");
				self.block_details.write().clear();
				self.block_hashes.write().clear();
				self.block_headers.write().clear();
				self.block_bodies.write().clear();
				self.block_receipts.write().clear();
				return Some(hash);
			}
		}

		None
	}

	/// Set the cache configuration.
	pub fn configure_cache(&self, pref_cache_size: usize, max_cache_size: usize) {
		self.pref_cache_size.store(pref_cache_size, AtomicOrder::Relaxed);
		self.max_cache_size.store(max_cache_size, AtomicOrder::Relaxed);
	}

	/// Returns a tree route between `from` and `to`, which is a tuple of:
	///
	/// - a vector of hashes of all blocks, ordered from `from` to `to`.
	///
	/// - common ancestor of these blocks.
	///
	/// - an index where best common ancestor would be
	///
	/// 1.) from newer to older
	///
	/// - bc: `A1 -> A2 -> A3 -> A4 -> A5`
	/// - from: A5, to: A4
	/// - route:
	///
	///   ```json
	///   { blocks: [A5], ancestor: A4, index: 1 }
	///   ```
	///
	/// 2.) from older to newer
	///
	/// - bc: `A1 -> A2 -> A3 -> A4 -> A5`
	/// - from: A3, to: A4
	/// - route:
	///
	///   ```json
	///   { blocks: [A4], ancestor: A3, index: 0 }
	///   ```
	///
	/// 3.) fork:
	///
	/// - bc:
	///
	///   ```text
	///   A1 -> A2 -> A3 -> A4
	///              -> B3 -> B4
	///   ```
	/// - from: B4, to: A4
	/// - route:
	///
	///   ```json
	///   { blocks: [B4, B3, A3, A4], ancestor: A2, index: 2 }
	///   ```
	pub fn tree_route(&self, from: H256, to: H256) -> TreeRoute {
		let mut from_branch = vec![];
		let mut to_branch = vec![];

		let mut from_details = self.block_details(&from).unwrap_or_else(|| panic!("0. Expected to find details for block {:?}", from));
		let mut to_details = self.block_details(&to).unwrap_or_else(|| panic!("1. Expected to find details for block {:?}", to));
		let mut current_from = from;
		let mut current_to = to;

		// reset from && to to the same level
		while from_details.number > to_details.number {
			from_branch.push(current_from);
			current_from = from_details.parent.clone();
			from_details = self.block_details(&from_details.parent).unwrap_or_else(|| panic!("2. Expected to find details for block {:?}", from_details.parent));
		}

		while to_details.number > from_details.number {
			to_branch.push(current_to);
			current_to = to_details.parent.clone();
			to_details = self.block_details(&to_details.parent).unwrap_or_else(|| panic!("3. Expected to find details for block {:?}", to_details.parent));
		}

		assert_eq!(from_details.number, to_details.number);

		// move to shared parent
		while current_from != current_to {
			from_branch.push(current_from);
			current_from = from_details.parent.clone();
			from_details = self.block_details(&from_details.parent).unwrap_or_else(|| panic!("4. Expected to find details for block {:?}", from_details.parent));

			to_branch.push(current_to);
			current_to = to_details.parent.clone();
			to_details = self.block_details(&to_details.parent).unwrap_or_else(|| panic!("5. Expected to find details for block {:?}", from_details.parent));
		}

		let index = from_branch.len();

		from_branch.extend(to_branch.into_iter().rev());

		TreeRoute {
			blocks: from_branch,
			ancestor: current_from,
			index: index
		}
	}

	#[cfg_attr(feature="dev", allow(similar_names))]
	/// Inserts the block into backing cache database.
	/// Expects the block to be valid and already verified.
	/// If the block is already known, does nothing.
	pub fn insert_block(&self, batch: &DBTransaction, bytes: &[u8], receipts: Vec<Receipt>) -> ImportRoute {
		// create views onto rlp
		let block = BlockView::new(bytes);
		let header = block.header_view();
		let hash = header.sha3();

		if self.is_known_child(&header.parent_hash(), &hash) {
			return ImportRoute::none();
		}

		let block_rlp = UntrustedRlp::new(bytes);
		let compressed_header = block_rlp.at(0).unwrap().compress(RlpType::Blocks);
		let compressed_body = UntrustedRlp::new(&Self::block_to_body(bytes)).compress(RlpType::Blocks);

		// store block in db
		batch.put(DB_COL_HEADERS, &hash, &compressed_header).unwrap();
		batch.put(DB_COL_BODIES, &hash, &compressed_body).unwrap();

		let info = self.block_info(bytes);

		if let BlockLocation::BranchBecomingCanonChain(ref d) = info.location {
			info!(target: "reorg", "Reorg to {} ({} {} {})",
				Colour::Yellow.bold().paint(format!("#{} {}", info.number, info.hash)),
				Colour::Red.paint(d.retracted.iter().fold(String::new(), |acc, h| format!("{} {}", acc, h))),
				Colour::White.paint(format!("#{} {}", d.ancestor, self.block_details(&d.ancestor).expect("`ancestor` is in the route; qed").number)),
				Colour::Green.paint(d.enacted.iter().fold(String::new(), |acc, h| format!("{} {}", acc, h)))
			);
		}

		self.apply_update(batch, ExtrasUpdate {
			block_hashes: self.prepare_block_hashes_update(bytes, &info),
			block_details: self.prepare_block_details_update(bytes, &info),
			block_receipts: self.prepare_block_receipts_update(receipts, &info),
			transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
			blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
			info: info.clone(),
			block: bytes,
		});

		ImportRoute::from(info)
	}

	/// Get inserted block info which is critical to prepare extras updates.
	fn block_info(&self, block_bytes: &[u8]) -> BlockInfo {
		let block = BlockView::new(block_bytes);
		let header = block.header_view();
		let hash = block.sha3();
		let number = header.number();
		let parent_hash = header.parent_hash();
		let parent_details = self.block_details(&parent_hash).unwrap_or_else(|| panic!("Invalid parent hash: {:?}", parent_hash));
		let total_difficulty = parent_details.total_difficulty + header.difficulty();
		let is_new_best = total_difficulty > self.best_block_total_difficulty();

		BlockInfo {
			hash: hash,
			number: number,
			total_difficulty: total_difficulty,
			location: if is_new_best {
				// on new best block we need to make sure that all ancestors
				// are moved to "canon chain"
				// find the route between old best block and the new one
				let best_hash = self.best_block_hash();
				let route = self.tree_route(best_hash, parent_hash);

				assert_eq!(number, parent_details.number + 1);

				match route.blocks.len() {
					0 => BlockLocation::CanonChain,
					_ => {
						let retracted = route.blocks.iter().take(route.index).cloned().collect::<Vec<_>>().into_iter().collect::<Vec<_>>();
						let enacted = route.blocks.into_iter().skip(route.index).collect::<Vec<_>>();
						BlockLocation::BranchBecomingCanonChain(BranchBecomingCanonChainData {
							ancestor: route.ancestor,
							enacted: enacted,
							retracted: retracted,
						})
					}
				}
			} else {
				BlockLocation::Branch
			}
		}
	}

	/// Applies extras update.
	fn apply_update(&self, batch: &DBTransaction, update: ExtrasUpdate) {
		{
			for hash in update.block_details.keys().cloned() {
				self.note_used(CacheID::BlockDetails(hash));
			}

			let mut write_details = self.block_details.write();
			batch.extend_with_cache(DB_COL_EXTRA, &mut *write_details, update.block_details, CacheUpdatePolicy::Overwrite);
		}

		{
			let mut write_receipts = self.block_receipts.write();
			batch.extend_with_cache(DB_COL_EXTRA, &mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove);
		}

		{
			let mut write_blocks_blooms = self.blocks_blooms.write();
			batch.extend_with_cache(DB_COL_EXTRA, &mut *write_blocks_blooms, update.blocks_blooms, CacheUpdatePolicy::Remove);
		}

		// These cached values must be updated last with all three locks taken to avoid
		// cache decoherence
		{
			let mut best_block = self.best_block.write();
			// update best block
			match update.info.location {
				BlockLocation::Branch => (),
				_ => {
					batch.put(DB_COL_EXTRA, b"best", &update.info.hash).unwrap();
					*best_block = BestBlock {
						hash: update.info.hash,
						number: update.info.number,
						total_difficulty: update.info.total_difficulty,
						block: update.block.to_vec(),
					};
				}
			}

			let mut write_hashes = self.block_hashes.write();
			let mut write_txs = self.transaction_addresses.write();

			batch.extend_with_cache(DB_COL_EXTRA, &mut *write_hashes, update.block_hashes, CacheUpdatePolicy::Remove);
			batch.extend_with_cache(DB_COL_EXTRA, &mut *write_txs, update.transactions_addresses, CacheUpdatePolicy::Remove);
		}
	}

	/// Iterator that lists `first` and then all of `first`'s ancestors, by hash.
	pub fn ancestry_iter(&self, first: H256) -> Option<AncestryIter> {
		if self.is_known(&first) {
			Some(AncestryIter {
				current: first,
				chain: self,
			})
		} else {
			None
		}
	}

	/// Given a block's `parent`, find every block header which represents a valid possible uncle.
	pub fn find_uncle_headers(&self, parent: &H256, uncle_generations: usize) -> Option<Vec<Header>> {
		self.find_uncle_hashes(parent, uncle_generations).map(|v| v.into_iter().filter_map(|h| self.block_header(&h)).collect())
	}

	/// Given a block's `parent`, find every block hash which represents a valid possible uncle.
	pub fn find_uncle_hashes(&self, parent: &H256, uncle_generations: usize) -> Option<Vec<H256>> {
		if !self.is_known(parent) { return None; }

		let mut excluded = HashSet::new();
		for a in self.ancestry_iter(parent.clone()).unwrap().take(uncle_generations) {
			excluded.extend(self.uncle_hashes(&a).unwrap().into_iter());
			excluded.insert(a);
		}

		let mut ret = Vec::new();
		for a in self.ancestry_iter(parent.clone()).unwrap().skip(1).take(uncle_generations) {
			ret.extend(self.block_details(&a).unwrap().children.iter()
				.filter(|h| !excluded.contains(h))
			);
		}
		Some(ret)
	}

	/// This function returns modified block hashes.
	fn prepare_block_hashes_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<BlockNumber, H256> {
		let mut block_hashes = HashMap::new();
		let block = BlockView::new(block_bytes);
		let header = block.header_view();
		let number = header.number();

		match info.location {
			BlockLocation::Branch => (),
			BlockLocation::CanonChain => {
				block_hashes.insert(number, info.hash.clone());
			},
			BlockLocation::BranchBecomingCanonChain(ref data) => {
				let ancestor_number = self.block_number(&data.ancestor).expect("Block number of ancestor is always in DB");
				let start_number = ancestor_number + 1;

				for (index, hash) in data.enacted.iter().cloned().enumerate() {
					block_hashes.insert(start_number + index as BlockNumber, hash);
				}

				block_hashes.insert(number, info.hash.clone());
			}
		}

		block_hashes
	}

	/// This function returns modified block details.
	fn prepare_block_details_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<H256, BlockDetails> {
		let block = BlockView::new(block_bytes);
		let header = block.header_view();
		let parent_hash = header.parent_hash();

		// update parent
		let mut parent_details = self.block_details(&parent_hash).unwrap_or_else(|| panic!("Invalid parent hash: {:?}", parent_hash));
		parent_details.children.push(info.hash.clone());

		// create current block details
		let details = BlockDetails {
			number: header.number(),
			total_difficulty: info.total_difficulty,
			parent: parent_hash.clone(),
			children: vec![]
		};

		// write to batch
		let mut block_details = HashMap::new();
		block_details.insert(parent_hash, parent_details);
		block_details.insert(info.hash.clone(), details);
		block_details
	}

	/// This function returns modified block receipts.
	fn prepare_block_receipts_update(&self, receipts: Vec<Receipt>, info: &BlockInfo) -> HashMap<H256, BlockReceipts> {
		let mut block_receipts = HashMap::new();
		block_receipts.insert(info.hash.clone(), BlockReceipts::new(receipts));
		block_receipts
	}

	/// This function returns modified transaction addresses.
	fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<H256, TransactionAddress> {
		let block = BlockView::new(block_bytes);
		let transaction_hashes = block.transaction_hashes();

		transaction_hashes.into_iter()
			.enumerate()
			.fold(HashMap::new(), |mut acc, (i ,tx_hash)| {
				acc.insert(tx_hash, TransactionAddress {
					block_hash: info.hash.clone(),
					index: i
				});
				acc
			})
	}

	/// This functions returns modified blocks blooms.
	///
	/// To accelerate blooms lookups, blomms are stored in multiple
	/// layers (BLOOM_LEVELS, currently 3).
	/// ChainFilter is responsible for building and rebuilding these layers.
	/// It returns them in HashMap, where values are Blooms and
	/// keys are BloomIndexes. BloomIndex represents bloom location on one
	/// of these layers.
	///
	/// To reduce number of queries to databse, block blooms are stored
	/// in BlocksBlooms structure which contains info about several
	/// (BLOOM_INDEX_SIZE, currently 16) consecutive blocks blooms.
	///
	/// Later, BloomIndexer is used to map bloom location on filter layer (BloomIndex)
	/// to bloom location in database (BlocksBloomLocation).
	///
	fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<LogGroupPosition, BloomGroup> {
		let block = BlockView::new(block_bytes);
		let header = block.header_view();

		let log_blooms = match info.location {
			BlockLocation::Branch => HashMap::new(),
			BlockLocation::CanonChain => {
				let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
				chain.insert(info.number as bc::Number, Bloom::from(header.log_bloom()).into())
			},
			BlockLocation::BranchBecomingCanonChain(ref data) => {
				let ancestor_number = self.block_number(&data.ancestor).unwrap();
				let start_number = ancestor_number + 1;
				let range = start_number as bc::Number..self.best_block_number() as bc::Number;

				let mut blooms: Vec<bc::Bloom> = data.enacted.iter()
					.map(|hash| self.block_header_data(hash).unwrap())
					.map(|bytes| HeaderView::new(&bytes).log_bloom())
					.map(Bloom::from)
					.map(Into::into)
					.collect();

				blooms.push(Bloom::from(header.log_bloom()).into());

				let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
				chain.replace(&range, blooms)
			}
		};

		log_blooms.into_iter()
			.map(|p| (From::from(p.0), From::from(p.1)))
			.collect()
	}

	/// Get best block hash.
	pub fn best_block_hash(&self) -> H256 {
		self.best_block.read().hash.clone()
	}

	/// Get best block number.
	pub fn best_block_number(&self) -> BlockNumber {
		self.best_block.read().number
	}

	/// Get best block total difficulty.
	pub fn best_block_total_difficulty(&self) -> U256 {
		self.best_block.read().total_difficulty
	}

	/// Get best block header
	pub fn best_block_header(&self) -> Bytes {
		let block = self.best_block.read();
		BlockView::new(&block.block).header_view().rlp().as_raw().to_vec()
	}

	/// Get current cache size.
	pub fn cache_size(&self) -> CacheSize {
		CacheSize {
			blocks: self.block_headers.read().heap_size_of_children() + self.block_bodies.read().heap_size_of_children(),
			block_details: self.block_details.read().heap_size_of_children(),
			transaction_addresses: self.transaction_addresses.read().heap_size_of_children(),
			blocks_blooms: self.blocks_blooms.read().heap_size_of_children(),
			block_receipts: self.block_receipts.read().heap_size_of_children(),
		}
	}

	/// Let the cache system know that a cacheable item has been used.
	fn note_used(&self, id: CacheID) {
		let mut cache_man = self.cache_man.write();
		if !cache_man.cache_usage[0].contains(&id) {
			cache_man.cache_usage[0].insert(id.clone());
			if cache_man.in_use.contains(&id) {
				if let Some(c) = cache_man.cache_usage.iter_mut().skip(1).find(|e|e.contains(&id)) {
					c.remove(&id);
				}
			} else {
				cache_man.in_use.insert(id);
			}
		}
	}

	/// Ticks our cache system and throws out any old data.
	pub fn collect_garbage(&self) {
		if self.cache_size().total() < self.pref_cache_size.load(AtomicOrder::Relaxed) {
			// rotate cache
			let mut cache_man = self.cache_man.write();
			const AVERAGE_BYTES_PER_CACHE_ENTRY: usize = 400; //estimated
			if cache_man.cache_usage[0].len() > self.pref_cache_size.load(AtomicOrder::Relaxed) / COLLECTION_QUEUE_SIZE / AVERAGE_BYTES_PER_CACHE_ENTRY {
				trace!("Cache rotation, cache_size = {}", self.cache_size().total());
				let cache = cache_man.cache_usage.pop_back().unwrap();
				cache_man.cache_usage.push_front(cache);
			}
			return;
		}

		for i in 0..COLLECTION_QUEUE_SIZE {
			{
				trace!("Cache cleanup round started {}, cache_size = {}", i, self.cache_size().total());
				let mut block_headers = self.block_headers.write();
				let mut block_bodies = self.block_bodies.write();
				let mut block_details = self.block_details.write();
				let mut block_hashes = self.block_hashes.write();
				let mut transaction_addresses = self.transaction_addresses.write();
				let mut blocks_blooms = self.blocks_blooms.write();
				let mut block_receipts = self.block_receipts.write();
				let mut cache_man = self.cache_man.write();

				for id in cache_man.cache_usage.pop_back().unwrap().into_iter() {
					cache_man.in_use.remove(&id);
					match id {
						CacheID::BlockHeader(h) => { block_headers.remove(&h); },
						CacheID::BlockBody(h) => { block_bodies.remove(&h); },
						CacheID::BlockDetails(h) => { block_details.remove(&h); }
						CacheID::BlockHashes(h) => { block_hashes.remove(&h); }
						CacheID::TransactionAddresses(h) => { transaction_addresses.remove(&h); }
						CacheID::BlocksBlooms(h) => { blocks_blooms.remove(&h); }
						CacheID::BlockReceipts(h) => { block_receipts.remove(&h); }
					}
				}
				cache_man.cache_usage.push_front(HashSet::new());

				// TODO: handle block_hashes properly.
				block_hashes.clear();

				block_headers.shrink_to_fit();
				block_bodies.shrink_to_fit();
				block_details.shrink_to_fit();
 				block_hashes.shrink_to_fit();
 				transaction_addresses.shrink_to_fit();
 				blocks_blooms.shrink_to_fit();
 				block_receipts.shrink_to_fit();
			}
			trace!("Cache cleanup round complete {}, cache_size = {}", i, self.cache_size().total());
			if self.cache_size().total() < self.max_cache_size.load(AtomicOrder::Relaxed) { break; }
		}

		// TODO: m_lastCollection = chrono::system_clock::now();
	}

	/// Create a block body from a block.
	pub fn block_to_body(block: &[u8]) -> Bytes {
		let mut body = RlpStream::new_list(2);
		let block_rlp = Rlp::new(block);
		body.append_raw(block_rlp.at(1).as_raw(), 1);
		body.append_raw(block_rlp.at(2).as_raw(), 1);
		body.out()
	}
}

#[cfg(test)]
mod tests {
	#![cfg_attr(feature="dev", allow(similar_names))]
	use std::str::FromStr;
	use std::sync::Arc;
	use rustc_serialize::hex::FromHex;
	use util::{Database, DatabaseConfig};
	use util::hash::*;
	use util::sha3::Hashable;
	use receipt::Receipt;
	use blockchain::{BlockProvider, BlockChain, Config, ImportRoute};
	use tests::helpers::*;
	use devtools::*;
	use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer};
	use views::BlockView;
	use client;

	fn new_db(path: &str) -> Arc<Database> {
		Arc::new(Database::open(&DatabaseConfig::with_columns(client::DB_NO_OF_COLUMNS), path).unwrap())
	}

	#[test]
	fn should_cache_best_block() {
		// given
		let mut canon_chain = ChainGenerator::default();
		let mut finalizer = BlockFinalizer::default();
		let genesis = canon_chain.generate(&mut finalizer).unwrap();
		let first = canon_chain.generate(&mut finalizer).unwrap();

		let temp = RandomTempPath::new();
		let db = new_db(temp.as_str());
		let bc = BlockChain::new(Config::default(), &genesis, db.clone());
		assert_eq!(bc.best_block_number(), 0);

		// when
		let batch = db.transaction();
		bc.insert_block(&batch, &first, vec![]);
		// NOTE no db.write here (we want to check if best block is cached)

		// then
		assert_eq!(bc.best_block_number(), 1);
		assert!(bc.block(&bc.best_block_hash()).is_some(), "Best block should be queryable even without DB write.");
	}