Run 'cargo fmt'

This adds the following new endpoints:

- GET /scripthash/:scripthash
- GET /scripthash/:scripthash/txs
- GET /scripthash/:scripthash/utxo

(cherry picked from commit 16891ae00e)

.gitignore

@@ -4,3 +4,5 @@ target
 *.sublime*
 *~
 *.pyc
+.idea/
+start.sh

.hooks/install.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd `dirname $0`/../.git/hooks/
+ln -s ../../.hooks/pre-commit

.hooks/pre-commit

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+CARGO_FMT="cargo +stable fmt --all"
+
+$CARGO_FMT --version &>/dev/null
+if [ $? != 0 ]; then
+    printf "[pre_commit] \033[0;31merror\033[0m: \"$CARGO_FMT\" not available?\n"
+    exit 1
+fi
+
+$CARGO_FMT -- --check
+result=$?
+
+printf "[pre_commit] $CARGO_FMT → "
+if [ $result != 0 ]; then
+    printf "\033[0;31merror\033[0m \n"
+else
+    printf "\033[0;32mOK\033[0m \n"
+fi
+
+exit $result

Cargo.toml

@@ -14,7 +14,7 @@ readme = "README.md"
 arrayref = "0.3"
 base64 = "0.9"
 bincode = "1.0"
-bitcoin = "0.14"
+bitcoin-bech32 = "0.8.0"
 chan = "0.1"
 chan-signal = "0.3"
 clap = "2.31"
@@ -30,6 +30,7 @@ page_size = "0.4"
 prometheus = "0.4"
 rocksdb = "0.10.1"
 rust-crypto = "0.2"
+secp256k1 = "0.11"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
@@ -37,3 +38,15 @@ stderrlog = "0.4.1"
 sysconf = ">=0.3.4"
 time = "0.1"
 tiny_http = "0.6"
+hyper = "0.12"
+url = "1.0"
+lru-cache = "0.1.1"
+
+[dependencies.elements]
+git = "https://github.com/ElementsProject/rust-elements/"
+rev = "78c1e29809b7420041b1259711f0a2934211a0d3" # from master branch
+features = ["serde-feature"] # Doesn't look to work by now
+
+[dependencies.bitcoin]
+version = "0.15.1"
+

README.md

@@ -1,34 +1,8 @@
-# Electrum Server in Rust
+# Esplora - Electrs backend API
 
-[![Build Status](https://travis-ci.com/romanz/electrs.svg?branch=master)](https://travis-ci.com/romanz/electrs)
-[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
-[![crates.io](http://meritbadge.herokuapp.com/electrs)](https://crates.io/crates/electrs)
-[![gitter.im](https://badges.gitter.im/romanz/electrs.svg)](https://gitter.im/romanz/electrs)
+A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs).
 
-An efficient re-implementation of Electrum Server, inspired by [ElectrumX](https://github.com/kyuupichan/electrumx), [Electrum Personal Server](https://github.com/chris-belcher/electrum-personal-server) and [bitcoincore-indexd](https://github.com/jonasschnelli/bitcoincore-indexd).
+The `liquid_e` branch supports Liquid and other Elements-based chains,
+including confidential transactions, peg-in/out and multi-asset.
 
-The motivation behind this project is to enable a user to run his own Electrum server,
-with required hardware resources not much beyond those of a [full node](https://en.bitcoin.it/wiki/Full_node#Why_should_you_use_a_full_node_wallet).
-The server indexes the entire Bitcoin blockchain, and the resulting index enables fast queries for any given user wallet,
-allowing the user to keep real-time track of his balances and his transaction history using the [Electrum wallet](https://electrum.org/).
-Since it runs on the user's own machine, there is no need for the wallet to communicate with external Electrum servers,
-thus preserving the privacy of the user's addresses and balances.
-
-## Features
-
- * Supports Electrum protocol [v1.2](https://electrumx.readthedocs.io/en/latest/protocol.html)
- * Maintains an index over transaction inputs and outputs, allowing fast balance queries
- * Fast synchronization of the Bitcoin blockchain (~2 hours for ~187GB @ July 2018) on [modest hardware](https://gist.github.com/romanz/cd9324474de0c2f121198afe3d063548)
- * Low index storage overhead (~20%), relying on a local full node for transaction retrieval
- * Efficient mempool tracker (allowing better fee [estimation](https://github.com/spesmilo/electrum/blob/59c1d03f018026ac301c4e74facfc64da8ae4708/RELEASE-NOTES#L34-L46))
- * Low CPU & memory usage (after initial indexing)
- * [`txindex`](https://github.com/bitcoinbook/bitcoinbook/blob/develop/ch03.asciidoc#txindex) is not required for the Bitcoin node
- * Uses a single [RocksDB](https://github.com/spacejam/rust-rocksdb) database, for better consistency and crash recovery
-
-## Usage
-
-See [here](doc/usage.md) for installation, build and usage instructions.
-
-## Index database
-
-The database schema is described [here](doc/schema.md).
+See [the main README in `bitcoin_e`](https://github.com/Blockstream/electrs) for more details.

examples/compact.rs

@@ -1,29 +0,0 @@
-/// Benchmark full compaction.
-extern crate electrs;
-
-#[macro_use]
-extern crate log;
-
-extern crate error_chain;
-
-use electrs::{config::Config, errors::*, store::DBStore};
-
-use error_chain::ChainedError;
-
-fn run(config: Config) -> Result<()> {
-    if !config.db_path.exists() {
-        panic!(
-            "DB {:?} must exist when running this benchmark!",
-            config.db_path
-        );
-    }
-    let store = DBStore::open(&config.db_path, /*low_memory=*/ true);
-    store.compact();
-    Ok(())
-}
-
-fn main() {
-    if let Err(e) = run(Config::from_args()) {
-        error!("{}", e.display_chain());
-    }
-}

examples/index.rs

@@ -1,38 +0,0 @@
-/// Benchmark regular indexing flow (using JSONRPC), don't persist the resulting index.
-extern crate electrs;
-extern crate error_chain;
-
-#[macro_use]
-extern crate log;
-
-use electrs::{
-    config::Config, daemon::Daemon, errors::*, fake::FakeStore, index::Index, metrics::Metrics,
-    signal::Waiter,
-};
-use error_chain::ChainedError;
-
-fn run() -> Result<()> {
-    let signal = Waiter::new();
-    let config = Config::from_args();
-    let metrics = Metrics::new(config.monitoring_addr);
-    metrics.start();
-
-    let daemon = Daemon::new(
-        &config.daemon_dir,
-        config.daemon_rpc_addr,
-        config.cookie_getter(),
-        config.network_type,
-        signal.clone(),
-        &metrics,
-    )?;
-    let fake_store = FakeStore {};
-    let index = Index::load(&fake_store, &daemon, &metrics, config.index_batch_size)?;
-    index.update(&fake_store, &signal)?;
-    Ok(())
-}
-
-fn main() {
-    if let Err(e) = run() {
-        error!("{}", e.display_chain());
-    }
-}

examples/listener.rs

@@ -1,15 +0,0 @@
-extern crate electrs;
-
-#[macro_use]
-extern crate log;
-
-use electrs::config::Config;
-use electrs::notify;
-
-fn main() {
-    let _ = Config::from_args();
-    let rx = notify::run().into_receiver();
-    for blockhash in rx.iter() {
-        info!("block {}", blockhash.be_hex_string())
-    }
-}

examples/txid_collisions.rs

@@ -1,49 +0,0 @@
-extern crate electrs;
-
-extern crate hex;
-extern crate log;
-
-use electrs::{config::Config, store::DBStore};
-
-fn max_collision(store: DBStore, prefix: &[u8]) {
-    let prefix_len = prefix.len();
-    let mut prev: Option<Vec<u8>> = None;
-    let mut collision_max = 0;
-
-    for row in store.iter_scan(prefix) {
-        assert!(row.key.starts_with(prefix));
-        if let Some(prev) = prev {
-            let collision_len = prev
-                .iter()
-                .zip(row.key.iter())
-                .take_while(|(a, b)| a == b)
-                .count();
-            if collision_len > collision_max {
-                eprintln!(
-                    "{} bytes collision found:\n{:?}\n{:?}\n",
-                    collision_len - prefix_len,
-                    revhex(&prev[prefix_len..]),
-                    revhex(&row.key[prefix_len..]),
-                );
-                collision_max = collision_len;
-            }
-        }
-        prev = Some(row.key.to_vec());
-    }
-}
-
-fn revhex(value: &[u8]) -> String {
-    hex::encode(&value.iter().cloned().rev().collect::<Vec<u8>>())
-}
-
-fn run(config: Config) {
-    if !config.db_path.exists() {
-        panic!("DB {:?} must exist when running this tool!", config.db_path);
-    }
-    let store = DBStore::open(&config.db_path, /*low_memory=*/ false);
-    max_collision(store, b"T");
-}
-
-fn main() {
-    run(Config::from_args());
-}

query/block.sh

@@ -0,0 +1,2 @@
+echo '{ "id": 5, "method":"blockchain.block.get", "params": ["997dd1addb13aac407fb7b996ca2f9cb4a9a71338d3ad9432bd62d2302939ac2"] }' | nc 127.0.0.1 51401
+

query/block_from_height.sh

@@ -0,0 +1,2 @@
+echo '{ "id": 5, "method":"blockchain.block.get_from_height", "params": [0] }' | nc 127.0.0.1 51401
+

query/hash_from_height.sh

@@ -0,0 +1,4 @@
+echo '{ "id": 5, "method":"blockchain.block.get_header", "params": [0] }' | nc 127.0.0.1 51401
+echo '{ "id": 5, "method":"blockchain.block.get_header", "params": [1] }' | nc 127.0.0.1 51401
+echo '{ "id": 5, "method":"blockchain.block.get_header", "params": [2] }' | nc 127.0.0.1 51401
+

query/height.sh

@@ -0,0 +1,2 @@
+echo '{ "id": 5, "method":"blockchain.headers.subscribe", "params": [] }' | nc 127.0.0.1 51401
+

query/version.sh

@@ -0,0 +1,2 @@
+echo "{ \"id\": 0, \"method\":\"server.version\", \"params\": [] }" | nc 127.0.0.1 51401
+

src/bin/electrs.rs

@@ -4,6 +4,7 @@ extern crate error_chain;
 #[macro_use]
 extern crate log;
 
+use electrs::rest;
 use error_chain::ChainedError;
 use std::process;
 use std::time::Duration;
@@ -16,13 +17,12 @@ use electrs::{
     errors::*,
     index::Index,
     metrics::Metrics,
-    query::{Query, TransactionCache},
-    rpc::RPC,
+    query::Query,
     signal::Waiter,
-    store::{full_compaction, is_fully_compacted, DBStore},
+    store::{full_compaction, is_fully_compacted, verify_index_compatibility, DBStore},
 };
 
-fn run_server(config: &Config) -> Result<()> {
+fn run_server(config: Config) -> Result<()> {
     let signal = Waiter::new();
     let metrics = Metrics::new(config.monitoring_addr);
     metrics.start();
@@ -37,7 +37,10 @@ fn run_server(config: &Config) -> Result<()> {
     )?;
     // Perform initial indexing from local blk*.dat block files.
     let store = DBStore::open(&config.db_path, /*low_memory=*/ config.jsonrpc_import);
-    let index = Index::load(&store, &daemon, &metrics, config.index_batch_size)?;
+    let index = Index::load(&store, &daemon, &metrics, &config)?;
+
+    verify_index_compatibility(&store, &config);
+
     let store = if is_fully_compacted(&store) {
         store // initial import and full compaction are over
     } else {
@@ -46,24 +49,30 @@ fn run_server(config: &Config) -> Result<()> {
             full_compaction(store)
         } else {
             // faster, but uses more memory
-            let store = bulk::index_blk_files(&daemon, config.bulk_index_threads, &metrics, store)?;
+            let store = bulk::index_blk_files(&daemon, &config, &metrics, store)?;
             let store = full_compaction(store);
             index.reload(&store); // make sure the block header index is up-to-date
             store
         }
-    }.enable_compaction(); // enable auto compactions before starting incremental index updates.
+    }
+    .enable_compaction(); // enable auto compactions before starting incremental index updates.
 
     let app = App::new(store, index, daemon)?;
-    let tx_cache = TransactionCache::new(config.tx_cache_size);
-    let query = Query::new(app.clone(), &metrics, tx_cache);
+    let query = Query::new(app.clone(), config.extended_db_enabled, &metrics);
 
-    let mut server = None; // Electrum RPC server
+    let mut server = None; // HTTP REST server
     loop {
         app.update(&signal)?;
         query.update_mempool()?;
-        server
-            .get_or_insert_with(|| RPC::start(config.electrum_rpc_addr, query.clone(), &metrics))
-            .notify(); // update subscribed clients
+
+        if server.is_none() {
+            if app.daemon().getblockchaininfo()?.verificationprogress > 0.9999 {
+                server = Some(rest::run_server(&config, query.clone()));
+            } else {
+                warn!("bitcoind not fully synced waiting");
+            }
+        }
+
         if let Err(err) = signal.wait(Duration::from_secs(5)) {
             info!("stopping server: {}", err);
             break;
@@ -74,7 +83,7 @@ fn run_server(config: &Config) -> Result<()> {
 
 fn main() {
     let config = Config::from_args();
-    if let Err(e) = run_server(&config) {
+    if let Err(e) = run_server(config) {
         error!("server failed: {}", e.display_chain());
         process::exit(1);
     }

src/bulk.rs

@@ -1,8 +1,6 @@
-use bitcoin::blockdata::block::Block;
-use bitcoin::network::serialize::BitcoinHash;
-use bitcoin::network::serialize::SimpleDecoder;
-use bitcoin::network::serialize::{deserialize, RawDecoder};
-use bitcoin::util::hash::Sha256dHash;
+use bitcoin::consensus::encode::{deserialize, Decodable};
+use bitcoin::util::hash::{BitcoinHash, Sha256dHash};
+use elements::Block;
 use libc;
 use std::collections::HashSet;
 use std::fs;
@@ -14,6 +12,7 @@ use std::sync::{
 };
 use std::thread;
 
+use config::Config;
 use daemon::Daemon;
 use index::{index_block, last_indexed_block, read_indexed_blockhashes};
 use metrics::{CounterVec, Histogram, HistogramOpts, HistogramVec, MetricOpts, Metrics};
@@ -30,6 +29,7 @@ struct Parser {
     duration: HistogramVec,
     block_count: CounterVec,
     bytes_read: Histogram,
+    extended_db_enabled: bool,
 }
 
 impl Parser {
@@ -37,11 +37,13 @@ impl Parser {
         daemon: &Daemon,
         metrics: &Metrics,
         indexed_blockhashes: HashSet<Sha256dHash>,
+        extended_db_enabled: bool,
     ) -> Result<Arc<Parser>> {
         Ok(Arc::new(Parser {
             magic: daemon.magic(),
             current_headers: load_headers(daemon)?,
             indexed_blockhashes: Mutex::new(indexed_blockhashes),
+            extended_db_enabled,
             duration: metrics.histogram_vec(
                 HistogramOpts::new("parse_duration", "blk*.dat parsing duration (in seconds)"),
                 &["step"],
@@ -94,7 +96,11 @@ impl Parser {
                     .expect("indexed_blockhashes")
                     .insert(blockhash.clone())
                 {
-                    rows.extend(index_block(&block, header.height()));
+                    rows.extend(index_block(
+                        &block,
+                        header.height() as u32,
+                        self.extended_db_enabled,
+                    ));
                     self.block_count.with_label_values(&["indexed"]).inc();
                 } else {
                     self.block_count.with_label_values(&["duplicate"]).inc();
@@ -118,11 +124,9 @@ fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<Block>> {
     let mut blocks = vec![];
     let max_pos = blob.len() as u64;
     while cursor.position() < max_pos {
-        let mut decoder = RawDecoder::new(cursor);
-        match decoder.read_u32() {
+        match u32::consensus_decode(&mut cursor) {
             Ok(value) => {
                 if magic != value {
-                    cursor = decoder.into_inner();
                     cursor
                         .seek(SeekFrom::Current(-3))
                         .expect("failed to seek back");
@@ -131,9 +135,7 @@ fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<Block>> {
             }
             Err(_) => break, // EOF
         };
-        let block_size = decoder.read_u32().chain_err(|| "no block size")?;
-        cursor = decoder.into_inner();
-
+        let block_size = u32::consensus_decode(&mut cursor).chain_err(|| "no block size")?;
         let start = cursor.position() as usize;
         cursor
             .seek(SeekFrom::Current(block_size as i64))
@@ -215,7 +217,7 @@ fn start_indexer(
 
 pub fn index_blk_files(
     daemon: &Daemon,
-    index_threads: usize,
+    config: &Config,
     metrics: &Metrics,
     store: DBStore,
 ) -> Result<DBStore> {
@@ -224,10 +226,15 @@ pub fn index_blk_files(
     info!("indexing {} blk*.dat files", blk_files.len());
     let indexed_blockhashes = read_indexed_blockhashes(&store);
     debug!("found {} indexed blocks", indexed_blockhashes.len());
-    let parser = Parser::new(daemon, metrics, indexed_blockhashes)?;
+    let parser = Parser::new(
+        daemon,
+        metrics,
+        indexed_blockhashes,
+        config.extended_db_enabled,
+    )?;
     let (blobs, reader) = start_reader(blk_files, parser.clone());
     let rows_chan = SyncChannel::new(0);
-    let indexers: Vec<JoinHandle> = (0..index_threads)
+    let indexers: Vec<JoinHandle> = (0..config.bulk_index_threads)
         .map(|_| start_indexer(blobs.clone(), parser.clone(), rows_chan.sender()))
         .collect();
     Ok(spawn_thread("bulk_writer", move || -> DBStore {
@@ -247,6 +254,7 @@ pub fn index_blk_files(
         });
         store.write(vec![parser.last_indexed_row()]);
         store
-    }).join()
+    })
+    .join()
     .expect("writer panicked"))
 }

src/config.rs

@@ -1,4 +1,3 @@
-use bitcoin::network::constants::Network;
 use clap::{App, Arg};
 use dirs::home_dir;
 use num_cpus;
@@ -10,23 +9,29 @@ use stderrlog;
 
 use daemon::CookieGetter;
 
+use daemon::Network;
 use errors::*;
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Config {
     // See below for the documentation of each field:
     pub log: stderrlog::StdErrLog,
     pub network_type: Network,
+    pub parent_network: Network,
+    pub parent_genesis_hash: String,
     pub db_path: PathBuf,
     pub daemon_dir: PathBuf,
     pub daemon_rpc_addr: SocketAddr,
     pub cookie: Option<String>,
     pub electrum_rpc_addr: SocketAddr,
+    pub http_addr: SocketAddr,
     pub monitoring_addr: SocketAddr,
     pub jsonrpc_import: bool,
     pub index_batch_size: usize,
     pub bulk_index_threads: usize,
     pub tx_cache_size: usize,
+    pub extended_db_enabled: bool,
+    pub prevout_enabled: bool,
 }
 
 impl Config {
@@ -65,7 +70,13 @@ impl Config {
             .arg(
                 Arg::with_name("network")
                     .long("network")
-                    .help("Select Bitcoin network type ('mainnet', 'testnet' or 'regtest')")
+                    .help("Select Bitcoin network type ('mainnet', 'testnet', 'regtest', 'liquid', 'liquidregtest')")
+                    .takes_value(true),
+            )
+            .arg(
+                Arg::with_name("parent_network")
+                    .long("parent-network")
+                    .help("Select parent network type ('mainnet', 'testnet', 'regtest', 'liquid', 'liquidregtest')")
                     .takes_value(true),
             )
             .arg(
@@ -74,6 +85,12 @@ impl Config {
                     .help("Electrum server JSONRPC 'addr:port' to listen on (default: '127.0.0.1:50001' for mainnet, '127.0.0.1:60001' for testnet and '127.0.0.1:60401' for regtest)")
                     .takes_value(true),
             )
+            .arg(
+                Arg::with_name("http_addr")
+                    .long("http-addr")
+                    .help("HTTP server 'addr:port' to listen on (default: '127.0.0.1:3000' for mainnet/liquid, '127.0.0.1:3001' for testnet and '127.0.0.1:3002' for regtest/liquid-regtest)")
+                    .takes_value(true),
+            )
             .arg(
                 Arg::with_name("daemon_rpc_addr")
                     .long("daemon-rpc-addr")
@@ -109,32 +126,54 @@ impl Config {
                     .help("Number of transactions to keep in for query LRU cache")
                     .default_value("10000")  // should be enough for a small wallet.
             )
+            .arg(
+                Arg::with_name("light")
+                    .long("light")
+                    .help("Enable light operation mode")
+            )
+            .arg(
+                Arg::with_name("disable_prevout")
+                    .long("disable-prevout")
+                    .help("Don't attach previous output details to inputs")
+            )
             .get_matches();
 
         let network_name = m.value_of("network").unwrap_or("mainnet");
-        let network_type = match network_name {
-            "mainnet" => Network::Bitcoin,
-            "testnet" => Network::Testnet,
-            "regtest" => Network::Regtest,
-            _ => panic!("unsupported Bitcoin network: {:?}", network_name),
-        };
+        let network_type = Network::from(network_name);
         let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db"));
         let db_path = db_dir.join(network_name);
 
+        let parent_network = Network::from(m.value_of("parent_network").unwrap_or("mainnet"));
+        let parent_genesis_hash = parent_network.genesis_hash().le_hex_string();
+
         let default_daemon_port = match network_type {
             Network::Bitcoin => 8332,
             Network::Testnet => 18332,
             Network::Regtest => 18443,
+            Network::Liquid => 10099,
+            Network::LiquidV1 => 7041,
+            Network::LiquidRegtest => 7041,
         };
         let default_electrum_port = match network_type {
             Network::Bitcoin => 50001,
             Network::Testnet => 60001,
             Network::Regtest => 60401,
+            Network::Liquid => 51000,
+            Network::LiquidV1 => 51200,
+            Network::LiquidRegtest => 51401,
+        };
+        let default_http_port = match network_type {
+            Network::Bitcoin | Network::Liquid | Network::LiquidV1 => 3000,
+            Network::Testnet => 3001,
+            Network::Regtest | Network::LiquidRegtest => 3002,
         };
         let default_monitoring_port = match network_type {
             Network::Bitcoin => 4224,
             Network::Testnet => 14224,
             Network::Regtest => 24224,
+            Network::Liquid => 34224,
+            Network::LiquidV1 => 31240,
+            Network::LiquidRegtest => 44224,
         };
 
         let daemon_rpc_addr: SocketAddr = m
@@ -147,6 +186,11 @@ impl Config {
             .unwrap_or(&format!("127.0.0.1:{}", default_electrum_port))
             .parse()
             .expect("invalid Electrum RPC address");
+        let http_addr: SocketAddr = m
+            .value_of("http_addr")
+            .unwrap_or(&format!("127.0.0.1:{}", default_http_port))
+            .parse()
+            .expect("invalid HTTP server address");
         let monitoring_addr: SocketAddr = m
             .value_of("monitoring_addr")
             .unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port))
@@ -165,6 +209,9 @@ impl Config {
             Network::Bitcoin => (),
             Network::Testnet => daemon_dir.push("testnet3"),
             Network::Regtest => daemon_dir.push("regtest"),
+            Network::Liquid => daemon_dir.push("liquid"),
+            Network::LiquidV1 => daemon_dir.push("liquidv1"),
+            Network::LiquidRegtest => daemon_dir.push("liquidregtest"),
         }
         let cookie = m.value_of("cookie").map(|s| s.to_owned());
 
@@ -183,16 +230,21 @@ impl Config {
         let config = Config {
             log,
             network_type,
+            parent_network,
+            parent_genesis_hash,
             db_path,
             daemon_dir,
             daemon_rpc_addr,
             cookie,
             electrum_rpc_addr,
+            http_addr,
             monitoring_addr,
             jsonrpc_import: m.is_present("jsonrpc_import"),
             index_batch_size: value_t_or_exit!(m, "index_batch_size", usize),
             bulk_index_threads,
             tx_cache_size: value_t_or_exit!(m, "tx_cache_size", usize),
+            extended_db_enabled: !m.is_present("light"),
+            prevout_enabled: !m.is_present("disable_prevout"),
         };
         eprintln!("{:?}", config);
         config

src/daemon.rs

@@ -1,10 +1,11 @@
 use base64;
-use bitcoin::blockdata::block::{Block, BlockHeader};
-use bitcoin::blockdata::transaction::Transaction;
-use bitcoin::network::constants::Network;
-use bitcoin::network::serialize::BitcoinHash;
-use bitcoin::network::serialize::{deserialize, serialize};
+use bitcoin::blockdata::constants::genesis_block;
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::network::constants::Network as BNetwork;
+use bitcoin::util::hash::BitcoinHash;
 use bitcoin::util::hash::Sha256dHash;
+use bitcoin_bech32::constants::Network as B32Network;
+use elements::{Block, BlockHeader, Transaction};
 use glob;
 use hex;
 use serde_json::{from_str, from_value, Value};
@@ -21,12 +22,80 @@ use util::HeaderList;
 
 use errors::*;
 
+#[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize)]
+pub enum Network {
+    Bitcoin,
+    Testnet,
+    Regtest,
+    Liquid,
+    LiquidV1,
+    LiquidRegtest,
+}
+
+impl Network {
+    pub fn genesis_hash(&self) -> Sha256dHash {
+        let block = genesis_block(BNetwork::from(self));
+        block.bitcoin_hash()
+    }
+}
+
+impl<'a> From<&'a str> for Network {
+    fn from(network_name: &'a str) -> Self {
+        match network_name {
+            "mainnet" => Network::Bitcoin,
+            "testnet" => Network::Testnet,
+            "regtest" => Network::Regtest,
+            "liquid" => Network::Liquid,
+            "liquidv1" => Network::LiquidV1,
+            "liquidregtest" => Network::LiquidRegtest,
+            _ => panic!("unsupported Bitcoin network: {:?}", network_name),
+        }
+    }
+}
+
+impl<'a> From<&'a Network> for BNetwork {
+    fn from(network: &'a Network) -> Self {
+        match network {
+            Network::Bitcoin => BNetwork::Bitcoin,
+            Network::Testnet => BNetwork::Testnet,
+            Network::Regtest => BNetwork::Regtest,
+            Network::Liquid => BNetwork::Bitcoin,   // @FIXME
+            Network::LiquidV1 => BNetwork::Bitcoin, // @FIXME
+            Network::LiquidRegtest => BNetwork::Regtest, // @FIXME
+        }
+    }
+}
+
+impl<'a> From<&'a Network> for B32Network {
+    fn from(network: &'a Network) -> Self {
+        match network {
+            Network::Bitcoin => B32Network::Bitcoin,
+            Network::Testnet => B32Network::Testnet,
+            Network::Regtest => B32Network::Regtest,
+            Network::Liquid => B32Network::Bitcoin, // @FIXME
+            Network::LiquidV1 => B32Network::Bitcoin, // @FIXME
+            Network::LiquidRegtest => B32Network::Regtest, // @FIXME
+        }
+    }
+}
+
+impl<'a> From<&'a BNetwork> for Network {
+    fn from(network: &'a BNetwork) -> Self {
+        match network {
+            BNetwork::Bitcoin => Network::Liquid,        // @FIXME
+            BNetwork::Regtest => Network::LiquidRegtest, // @FIXME
+            BNetwork::Testnet => Network::Testnet,       // @FIXME
+        }
+    }
+}
+
 fn parse_hash(value: &Value) -> Result<Sha256dHash> {
     Ok(Sha256dHash::from_hex(
         value
             .as_str()
             .chain_err(|| format!("non-string value: {}", value))?,
-    ).chain_err(|| format!("non-hex value: {}", value))?)
+    )
+    .chain_err(|| format!("non-hex value: {}", value))?)
 }
 
 fn header_from_value(value: Value) -> Result<BlockHeader> {
@@ -92,13 +161,13 @@ fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Resu
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-struct BlockchainInfo {
-    chain: String,
-    blocks: u32,
-    headers: u32,
-    bestblockhash: String,
-    pruned: bool,
-    initialblockdownload: bool,
+pub struct BlockchainInfo {
+    pub chain: String,
+    pub blocks: u32,
+    pub headers: u32,
+    pub bestblockhash: String,
+    pub pruned: bool,
+    pub verificationprogress: f32,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -206,7 +275,8 @@ impl Connection {
             .next()
             .chain_err(|| {
                 ErrorKind::Connection("disconnected from daemon while receiving".to_owned())
-            })?.chain_err(|| "failed to read status")?;
+            })?
+            .chain_err(|| "failed to read status")?;
         let mut headers = HashMap::new();
         for line in iter {
             let line = line.chain_err(|| ErrorKind::Connection("failed to read".to_owned()))?;
@@ -330,13 +400,7 @@ impl Daemon {
         if blockchain_info.pruned == true {
             bail!("pruned node is not supported (use '-prune=0' bitcoind flag)".to_owned())
         }
-        loop {
-            if daemon.getblockchaininfo()?.initialblockdownload == false {
-                break;
-            }
-            warn!("wait until bitcoind is synced (i.e. initialblockdownload = false)");
-            signal.wait(Duration::from_secs(3))?;
-        }
+
         Ok(daemon)
     }
 
@@ -366,7 +430,14 @@ impl Daemon {
     }
 
     pub fn magic(&self) -> u32 {
-        self.network.magic()
+        match self.network {
+            Network::Bitcoin => 0xD9B4BEF9,
+            Network::Testnet => 0x0709110B,
+            Network::Regtest => 0xDAB5BFFA,
+            Network::Liquid => 0xDAB5BFFA,
+            Network::LiquidV1 => 0xDAB5BFFA,
+            Network::LiquidRegtest => 0xDAB5BFFA,
+        }
     }
 
     fn call_jsonrpc(&self, method: &str, request: &Value) -> Result<Value> {
@@ -430,7 +501,7 @@ impl Daemon {
 
     // bitcoind JSONRPC API:
 
-    fn getblockchaininfo(&self) -> Result<BlockchainInfo> {
+    pub fn getblockchaininfo(&self) -> Result<BlockchainInfo> {
         let info: Value = self.request("getblockchaininfo", json!([]))?;
         Ok(from_value(info).chain_err(|| "invalid blockchain info")?)
     }
@@ -474,6 +545,10 @@ impl Daemon {
         Ok(block)
     }
 
+    pub fn getblock_raw(&self, blockhash: &Sha256dHash, verbose: u32) -> Result<Value> {
+        self.request("getblock", json!([blockhash.be_hex_string(), verbose]))
+    }
+
     pub fn getblocks(&self, blockhashes: &[Sha256dHash]) -> Result<Vec<Block>> {
         let params_list: Vec<Value> = blockhashes
             .iter()
@@ -487,32 +562,14 @@ impl Daemon {
         Ok(blocks)
     }
 
-    pub fn gettransaction(
-        &self,
-        txhash: &Sha256dHash,
-        blockhash: Option<Sha256dHash>,
-    ) -> Result<Transaction> {
-        let mut args = json!([txhash.be_hex_string(), /*verbose=*/ false]);
-        if let Some(blockhash) = blockhash {
-            args.as_array_mut()
-                .unwrap()
-                .push(json!(blockhash.be_hex_string()));
-        }
+    pub fn gettransaction(&self, txhash: &Sha256dHash) -> Result<Transaction> {
+        let args = json!([txhash.be_hex_string(), /*verbose=*/ false]);
         tx_from_value(self.request("getrawtransaction", args)?)
     }
 
-    pub fn gettransaction_raw(
-        &self,
-        txhash: &Sha256dHash,
-        blockhash: Option<Sha256dHash>,
-        verbose: bool,
-    ) -> Result<Value> {
-        let mut args = json!([txhash.be_hex_string(), verbose]);
-        if let Some(blockhash) = blockhash {
-            args.as_array_mut()
-                .unwrap()
-                .push(json!(blockhash.be_hex_string()));
-        }
+    pub fn gettransaction_raw(&self, txhash: &Sha256dHash, verbose: bool) -> Result<Value> {
+        let args = json!([txhash.be_hex_string(), verbose]);
+        debug!("gettransaction_raw args {:?}", args);
         Ok(self.request("getrawtransaction", args)?)
     }
 
@@ -557,7 +614,7 @@ impl Daemon {
     }
 
     pub fn broadcast(&self, tx: &Transaction) -> Result<Sha256dHash> {
-        let tx = hex::encode(serialize(tx).unwrap());
+        let tx = hex::encode(serialize(tx));
         let txid = self.request("sendrawtransaction", json!([tx]))?;
         Ok(
             Sha256dHash::from_hex(txid.as_str().chain_err(|| "non-string txid")?)
@@ -617,7 +674,7 @@ impl Daemon {
             let header = self
                 .getblockheader(&blockhash)
                 .chain_err(|| format!("failed to get {} header", blockhash))?;
-            new_headers.push(header);
+            new_headers.push(header.clone());
             blockhash = header.prev_blockhash;
         }
         trace!("downloaded {} block headers", new_headers.len());

src/errors.rs

@@ -1,6 +1,6 @@
 use chan_signal::Signal;
 
-error_chain!{
+error_chain! {
     types {
         Error, ErrorKind, ResultExt, Result;
     }

src/index.rs

@@ -1,11 +1,10 @@
 use bincode;
-use bitcoin::blockdata::block::{Block, BlockHeader};
-use bitcoin::blockdata::transaction::{Transaction, TxIn, TxOut};
-use bitcoin::network::serialize::BitcoinHash;
-use bitcoin::network::serialize::{deserialize, serialize};
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::util::hash::BitcoinHash;
 use bitcoin::util::hash::Sha256dHash;
 use crypto::digest::Digest;
 use crypto::sha2::Sha256;
+use elements::{Block, BlockHeader, Transaction, TxIn, TxOut};
 use std::collections::{HashMap, HashSet};
 use std::iter::FromIterator;
 use std::sync::RwLock;
@@ -15,10 +14,12 @@ use metrics::{Counter, Gauge, HistogramOpts, HistogramTimer, HistogramVec, Metri
 use signal::Waiter;
 use store::{ReadStore, Row, WriteStore};
 use util::{
-    full_hash, hash_prefix, spawn_thread, Bytes, FullHash, HashPrefix, HeaderEntry, HeaderList,
-    HeaderMap, SyncChannel, HASH_PREFIX_LEN,
+    full_hash, hash_prefix, spawn_thread, BlockMeta, Bytes, FullHash, HashPrefix, HeaderEntry,
+    HeaderList, HeaderMap, SyncChannel, HASH_PREFIX_LEN,
 };
 
+use config::Config;
+
 use errors::*;
 
 #[derive(Serialize, Deserialize)]
@@ -51,7 +52,8 @@ impl TxInRow {
             code: b'I',
             prev_hash_prefix: hash_prefix(&txid[..]),
             prev_index: output_index as u16,
-        }).unwrap()
+        })
+        .unwrap()
     }
 
     pub fn to_row(&self) -> Row {
@@ -93,7 +95,8 @@ impl TxOutRow {
         bincode::serialize(&TxOutKey {
             code: b'O',
             script_hash_prefix: hash_prefix(&script_hash[..HASH_PREFIX_LEN]),
-        }).unwrap()
+        })
+        .unwrap()
     }
 
     pub fn to_row(&self) -> Row {
@@ -108,25 +111,28 @@ impl TxOutRow {
     }
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct TxKey {
     code: u8,
     pub txid: FullHash,
 }
 
+#[derive(Debug)]
 pub struct TxRow {
     pub key: TxKey,
     pub height: u32, // value
+    pub blockhash: Sha256dHash,
 }
 
 impl TxRow {
-    pub fn new(txid: &Sha256dHash, height: u32) -> TxRow {
+    pub fn new(txid: &Sha256dHash, height: u32, blockhash: &Sha256dHash) -> TxRow {
         TxRow {
             key: TxKey {
                 code: b'T',
                 txid: full_hash(&txid[..]),
             },
             height: height,
+            blockhash: blockhash.clone(),
         }
     }
 
@@ -141,14 +147,56 @@ impl TxRow {
     pub fn to_row(&self) -> Row {
         Row {
             key: bincode::serialize(&self.key).unwrap(),
-            value: bincode::serialize(&self.height).unwrap(),
+            value: bincode::serialize(&(&self.height, &self.blockhash)).unwrap(),
         }
     }
 
     pub fn from_row(row: &Row) -> TxRow {
+        let (height, blockhash): (u32, Sha256dHash) =
+            bincode::deserialize(&row.value).expect("failed to parse tx row");
         TxRow {
             key: bincode::deserialize(&row.key).expect("failed to parse TxKey"),
-            height: bincode::deserialize(&row.value).expect("failed to parse height"),
+            height: height,
+            blockhash: blockhash,
+        }
+    }
+}
+
+pub struct RawTxRow {
+    pub key: TxKey,
+    pub rawtx: Bytes,
+}
+
+impl RawTxRow {
+    pub fn new(txid: &Sha256dHash, rawtx: Bytes) -> RawTxRow {
+        RawTxRow {
+            key: TxKey {
+                code: b't',
+                txid: full_hash(&txid[..]),
+            },
+            rawtx: rawtx,
+        }
+    }
+
+    pub fn filter_prefix(txid_prefix: &HashPrefix) -> Bytes {
+        [b"t", &txid_prefix[..]].concat()
+    }
+
+    pub fn filter_full(txid: &Sha256dHash) -> Bytes {
+        [b"t", &txid[..]].concat()
+    }
+
+    pub fn to_row(&self) -> Row {
+        Row {
+            key: bincode::serialize(&self.key).unwrap(),
+            value: bincode::serialize(&self.rawtx).unwrap(),
+        }
+    }
+
+    pub fn from_row(row: &Row) -> RawTxRow {
+        RawTxRow {
+            key: bincode::deserialize(&row.key).expect("failed to parse TxKey for RawTx"),
+            rawtx: bincode::deserialize(&row.value).expect("failed to parse rawtx"),
         }
     }
 }
@@ -167,7 +215,12 @@ pub fn compute_script_hash(data: &[u8]) -> FullHash {
     hash
 }
 
-pub fn index_transaction(txn: &Transaction, height: usize, rows: &mut Vec<Row>) {
+pub fn index_transaction(
+    txn: &Transaction,
+    height: u32,
+    blockhash: &Sha256dHash,
+    rows: &mut Vec<Row>,
+) {
     let null_hash = Sha256dHash::default();
     let txid: Sha256dHash = txn.txid();
     for input in &txn.input {
@@ -179,14 +232,20 @@ pub fn index_transaction(txn: &Transaction, height: usize, rows: &mut Vec<Row>)
     for output in &txn.output {
         rows.push(TxOutRow::new(&txid, &output).to_row());
     }
-    // Persist transaction ID and confirmed height
-    rows.push(TxRow::new(&txid, height as u32).to_row());
+    // Persist transaction ID and confirmed height/hash
+    rows.push(TxRow::new(&txid, height, blockhash).to_row());
 }
 
-pub fn index_block(block: &Block, height: usize) -> Vec<Row> {
+pub fn index_block(block: &Block, height: u32, extended_db_enabled: bool) -> Vec<Row> {
+    let blockhash = block.bitcoin_hash();
     let mut rows = vec![];
     for txn in &block.txdata {
-        index_transaction(&txn, height, &mut rows);
+        index_transaction(&txn, height, &blockhash, &mut rows);
+
+        // Persist raw transaction to txstore
+        if extended_db_enabled {
+            rows.push(RawTxRow::new(&txn.txid(), serialize(txn)).to_row()); // @TODO avoid re-serialization
+        }
     }
     let blockhash = block.bitcoin_hash();
     // Persist block hash and header
@@ -194,9 +253,37 @@ pub fn index_block(block: &Block, height: usize) -> Vec<Row> {
         key: bincode::serialize(&BlockKey {
             code: b'B',
             hash: full_hash(&blockhash[..]),
-        }).unwrap(),
-        value: serialize(&block.header).unwrap(),
+        })
+        .unwrap(),
+        value: serialize(&block.header),
     });
+
+    // Persist block metadata (size, number of txs and sum of txs weight)
+    if extended_db_enabled {
+        let blockmeta = BlockMeta::from(block);
+        rows.push(Row {
+            key: bincode::serialize(&BlockKey {
+                code: b'M',
+                hash: full_hash(&blockhash[..]),
+            })
+            .unwrap(),
+            value: bincode::serialize(&blockmeta).unwrap(),
+        });
+    }
+
+    // Persist list of txids in block
+    if extended_db_enabled {
+        let txids: Vec<Sha256dHash> = block.txdata.iter().map(|tx| tx.txid()).collect();
+        rows.push(Row {
+            key: bincode::serialize(&BlockKey {
+                code: b'X',
+                hash: full_hash(&blockhash[..]),
+            })
+            .unwrap(),
+            value: bincode::serialize(&txids).unwrap(),
+        });
+    }
+
     rows
 }
 
@@ -204,7 +291,7 @@ pub fn last_indexed_block(blockhash: &Sha256dHash) -> Row {
     // Store last indexed block (i.e. all previous blocks were indexed)
     Row {
         key: b"L".to_vec(),
-        value: serialize(blockhash).unwrap(),
+        value: serialize(blockhash),
     }
 }
 
@@ -305,6 +392,7 @@ pub struct Index {
     daemon: Daemon,
     stats: Stats,
     batch_size: usize,
+    extended_db_enabled: bool,
 }
 
 impl Index {
@@ -312,7 +400,7 @@ impl Index {
         store: &ReadStore,
         daemon: &Daemon,
         metrics: &Metrics,
-        batch_size: usize,
+        config: &Config,
     ) -> Result<Index> {
         let stats = Stats::new(metrics);
         let headers = read_indexed_headers(store);
@@ -321,7 +409,8 @@ impl Index {
             headers: RwLock::new(headers),
             daemon: daemon.reconnect()?,
             stats,
-            batch_size,
+            batch_size: config.index_batch_size,
+            extended_db_enabled: config.extended_db_enabled,
         })
     }
 
@@ -330,11 +419,19 @@ impl Index {
         *headers = read_indexed_headers(store);
     }
 
+    pub fn best_height(&self) -> usize {
+        self.headers.read().unwrap().len() - 1
+    }
+
     pub fn best_header(&self) -> Option<HeaderEntry> {
         let headers = self.headers.read().unwrap();
         headers.header_by_blockhash(headers.tip()).cloned()
     }
 
+    pub fn best_header_hash(&self) -> Sha256dHash {
+        self.headers.read().unwrap().tip().clone()
+    }
+
     pub fn get_header(&self, height: usize) -> Option<HeaderEntry> {
         self.headers
             .read()
@@ -343,6 +440,14 @@ impl Index {
             .cloned()
     }
 
+    pub fn get_header_by_hash(&self, hash: &Sha256dHash) -> Option<HeaderEntry> {
+        self.headers
+            .read()
+            .unwrap()
+            .header_by_blockhash(hash)
+            .cloned()
+    }
+
     pub fn update(&self, store: &WriteStore, waiter: &Waiter) -> Result<Sha256dHash> {
         let daemon = self.daemon.reconnect()?;
         let tip = daemon.getbestblockhash()?;
@@ -391,7 +496,7 @@ impl Index {
                     .expect(&format!("missing header for block {}", blockhash));
 
                 let timer = self.stats.start_timer("index");
-                let mut block_rows = index_block(block, height);
+                let mut block_rows = index_block(block, height as u32, self.extended_db_enabled);
                 block_rows.push(last_indexed_block(&blockhash));
                 rows.extend(block_rows);
                 timer.observe_duration();

src/lib.rs

@@ -3,22 +3,28 @@
 extern crate base64;
 extern crate bincode;
 extern crate bitcoin;
+extern crate bitcoin_bech32;
 extern crate chan_signal;
 extern crate crypto;
 extern crate dirs;
+extern crate elements;
 extern crate glob;
 extern crate hex;
+extern crate hyper;
 extern crate libc;
 extern crate lru;
+extern crate lru_cache;
 extern crate num_cpus;
 extern crate page_size;
 extern crate prometheus;
 extern crate rocksdb;
+extern crate secp256k1;
 extern crate serde;
 extern crate stderrlog;
 extern crate sysconf;
 extern crate time;
 extern crate tiny_http;
+extern crate url;
 
 #[macro_use]
 extern crate chan;
@@ -44,9 +50,9 @@ pub mod fake;
 pub mod index;
 pub mod mempool;
 pub mod metrics;
-pub mod notify;
 pub mod query;
-pub mod rpc;
+pub mod rest;
 pub mod signal;
 pub mod store;
 pub mod util;
+pub mod utils;

src/mempool.rs

@@ -1,5 +1,5 @@
-use bitcoin::blockdata::transaction::Transaction;
 use bitcoin::util::hash::Sha256dHash;
+use elements::Transaction;
 use hex;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::iter::FromIterator;
@@ -15,6 +15,7 @@ use util::Bytes;
 use errors::*;
 
 const VSIZE_BIN_WIDTH: u32 = 100_000; // in vbytes
+pub const MEMPOOL_HEIGHT: u32 = u32::max_value(); // special "marker" for mempool transactions
 
 struct MempoolStore {
     map: BTreeMap<Bytes, Vec<Bytes>>,
@@ -29,7 +30,7 @@ impl MempoolStore {
 
     fn add(&mut self, tx: &Transaction) {
         let mut rows = vec![];
-        index_transaction(tx, 0, &mut rows);
+        index_transaction(tx, MEMPOOL_HEIGHT, &Sha256dHash::default(), &mut rows);
         for row in rows {
             let (key, value) = row.into_pair();
             self.map.entry(key).or_insert(vec![]).push(value);
@@ -38,7 +39,7 @@ impl MempoolStore {
 
     fn remove(&mut self, tx: &Transaction) {
         let mut rows = vec![];
-        index_transaction(tx, 0, &mut rows);
+        index_transaction(tx, MEMPOOL_HEIGHT, &Sha256dHash::default(), &mut rows);
         for row in rows {
             let (key, value) = row.into_pair();
             let no_values_left = {
@@ -49,7 +50,7 @@ impl MempoolStore {
                 let last_value = values
                     .pop()
                     .expect(&format!("no values found for key {}", hex::encode(&key)));
-                // TxInRow and TxOutRow have an empty value, TxRow has height=0 as value.
+                // TxInRow and TxOutRow have an empty value, TxRow has MEMPOOL_HEIGHT as value.
                 assert_eq!(
                     value,
                     last_value,
@@ -205,7 +206,8 @@ impl Tracker {
                         None // ignore this transaction for now
                     }
                 }
-            }).collect();
+            })
+            .collect();
         if entries.is_empty() {
             return Ok(());
         }

src/metrics.rs

@@ -101,6 +101,13 @@ struct Stats {
 }
 
 fn parse_stats() -> Result<Stats> {
+    if cfg!(target_os = "macos") {
+        return Ok(Stats {
+            utime: 0f64,
+            rss: 0u64,
+            fds: 0usize,
+        });
+    }
     let value = fs::read_to_string("/proc/self/stat").chain_err(|| "failed to read stats")?;
     let parts: Vec<&str> = value.split_whitespace().collect();
     let page_size = page_size::get() as u64;

src/notify.rs

@@ -1,3 +1,5 @@
+// TODO: network::socket::Socket needs to be reimplemented.
+
 use bitcoin::network::constants::Network;
 use bitcoin::network::message::NetworkMessage;
 use bitcoin::network::message_blockdata::InvType;

src/query.rs

@@ -1,37 +1,55 @@
-use bitcoin::blockdata::block::Block;
-use bitcoin::blockdata::transaction::Transaction;
-use bitcoin::network::serialize::deserialize;
+use bincode;
+use bitcoin::consensus::encode::{deserialize, serialize};
 use bitcoin::util::hash::Sha256dHash;
-use crypto::digest::Digest;
-use crypto::sha2::Sha256;
-use lru::LruCache;
-use std::collections::HashMap;
-use std::sync::{Arc, Mutex, RwLock};
+use elements::{confidential, Block, Transaction};
+use std::collections::{BTreeMap, HashMap};
+use std::sync::{Arc, RwLock};
 
 use app::App;
-use index::{compute_script_hash, TxInRow, TxOutRow, TxRow};
+use index::{compute_script_hash, RawTxRow, TxInRow, TxOutRow, TxRow};
 use mempool::Tracker;
-use metrics::Metrics;
+use metrics::{HistogramOpts, HistogramVec, Metrics};
 use serde_json::Value;
 use store::{ReadStore, Row};
-use util::{FullHash, HashPrefix, HeaderEntry};
+use util::{
+    BlockHeaderMeta, BlockMeta, BlockStatus, Bytes, HashPrefix, HeaderEntry, TransactionStatus,
+};
 
 use errors::*;
 
+const FUNDING_TXN_LIMIT: usize = 100;
+
+#[derive(Clone)]
 pub struct FundingOutput {
+    pub txn: Option<TxnHeight>,
     pub txn_id: Sha256dHash,
     pub height: u32,
     pub output_index: usize,
     pub value: u64,
+    pub asset: Option<Sha256dHash>,
 }
 
+impl From<OutPoint> for FundingOutput {
+    fn from(out: OutPoint) -> Self {
+        FundingOutput {
+            txn_id: out.0,
+            output_index: out.1,
+            txn: None,
+            height: 0,
+            value: 0,
+            asset: None,
+        }
+    }
+}
 type OutPoint = (Sha256dHash, usize); // (txid, output_index)
 
-struct SpendingInput {
-    txn_id: Sha256dHash,
-    height: u32,
-    funding_output: OutPoint,
-    value: u64,
+pub struct SpendingInput {
+    pub txn: Option<TxnHeight>,
+    pub txn_id: Sha256dHash,
+    pub height: u32,
+    pub input_index: usize,
+    pub funding_output: OutPoint,
+    pub value: u64,
 }
 
 pub struct Status {
@@ -62,20 +80,34 @@ impl Status {
         calc_balance(&self.mempool)
     }
 
-    pub fn history(&self) -> Vec<(i32, Sha256dHash)> {
-        let mut txns_map = HashMap::<Sha256dHash, i32>::new();
+    pub fn history(&self) -> Vec<(u32, Sha256dHash)> {
+        let mut txns_map = HashMap::<Sha256dHash, u32>::new();
         for f in self.funding() {
-            txns_map.insert(f.txn_id, f.height as i32);
+            txns_map.insert(f.txn_id, f.height);
         }
         for s in self.spending() {
-            txns_map.insert(s.txn_id, s.height as i32);
+            txns_map.insert(s.txn_id, s.height);
         }
-        let mut txns: Vec<(i32, Sha256dHash)> =
+        let mut txns: Vec<(u32, Sha256dHash)> =
             txns_map.into_iter().map(|item| (item.1, item.0)).collect();
         txns.sort_unstable();
         txns
     }
 
+    pub fn history_txs(&self) -> Vec<&TxnHeight> {
+        let mut txns_map = BTreeMap::<Sha256dHash, &TxnHeight>::new();
+        for f in self.funding() {
+            txns_map.insert(f.txn_id, &f.txn.as_ref().unwrap());
+        }
+        for s in self.spending() {
+            txns_map.insert(s.txn_id, &s.txn.as_ref().unwrap());
+        }
+        let mut txns: Vec<&TxnHeight> = txns_map.into_iter().map(|item| item.1).collect();
+        // Sort in reverse confirmation height order (unconfirmed txns use u32::max_value as their height):
+        txns.sort_by(|a, b| b.height.cmp(&a.height));
+        txns
+    }
+
     pub fn unspent(&self) -> Vec<&FundingOutput> {
         let mut outputs_map = HashMap::<OutPoint, &FundingOutput>::new();
         for f in self.funding() {
@@ -93,27 +125,13 @@ impl Status {
         outputs.sort_unstable_by_key(|out| out.height);
         outputs
     }
-
-    pub fn hash(&self) -> Option<FullHash> {
-        let txns = self.history();
-        if txns.is_empty() {
-            None
-        } else {
-            let mut hash = FullHash::default();
-            let mut sha2 = Sha256::new();
-            for (height, txn_id) in txns {
-                let part = format!("{}:{}:", txn_id.be_hex_string(), height);
-                sha2.input(part.as_bytes());
-            }
-            sha2.result(&mut hash);
-            Some(hash)
-        }
-    }
 }
 
-struct TxnHeight {
-    txn: Transaction,
-    height: u32,
+#[derive(Clone)]
+pub struct TxnHeight {
+    pub txn: Transaction,
+    pub height: u32,
+    pub blockhash: Sha256dHash,
 }
 
 fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash {
@@ -121,13 +139,18 @@ fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash {
     Sha256dHash::from_data(&data)
 }
 
-// TODO: the functions below can be part of ReadStore.
 fn txrow_by_txid(store: &ReadStore, txid: &Sha256dHash) -> Option<TxRow> {
     let key = TxRow::filter_full(&txid);
     let value = store.get(&key)?;
     Some(TxRow::from_row(&Row { key, value }))
 }
 
+fn rawtxrow_by_txid(store: &ReadStore, txid: &Sha256dHash) -> Option<RawTxRow> {
+    let key = RawTxRow::filter_full(&txid);
+    let value = store.get(&key)?;
+    Some(RawTxRow::from_row(&Row { key, value }))
+}
+
 fn txrows_by_prefix(store: &ReadStore, txid_prefix: &HashPrefix) -> Vec<TxRow> {
     store
         .scan(&TxRow::filter_prefix(&txid_prefix))
@@ -140,6 +163,7 @@ fn txids_by_script_hash(store: &ReadStore, script_hash: &[u8]) -> Vec<HashPrefix
     store
         .scan(&TxOutRow::filter(script_hash))
         .iter()
+        .take(FUNDING_TXN_LIMIT + 1)
         .map(|row| TxOutRow::from_row(row).txid_prefix)
         .collect()
 }
@@ -156,42 +180,44 @@ fn txids_by_funding_output(
         .collect()
 }
 
-pub struct TransactionCache {
-    map: Mutex<LruCache<Sha256dHash, Transaction>>,
+pub fn get_block_meta(store: &ReadStore, blockhash: &Sha256dHash) -> Option<BlockMeta> {
+    let key = [b"M", &blockhash[..]].concat();
+    let value = store.get(&key)?;
+    let meta: BlockMeta = bincode::deserialize(&value).unwrap();
+    Some(meta)
 }
 
-impl TransactionCache {
-    pub fn new(capacity: usize) -> TransactionCache {
-        TransactionCache {
-            map: Mutex::new(LruCache::new(capacity)),
-        }
-    }
-
-    fn get_or_else<F>(&self, txid: &Sha256dHash, load_txn_func: F) -> Result<Transaction>
-    where
-        F: FnOnce() -> Result<Transaction>,
-    {
-        if let Some(txn) = self.map.lock().unwrap().get(txid) {
-            return Ok(txn.clone());
-        }
-        let txn = load_txn_func()?;
-        self.map.lock().unwrap().put(*txid, txn.clone());
-        Ok(txn)
-    }
+pub fn get_block_txids(store: &ReadStore, blockhash: &Sha256dHash) -> Option<Vec<Sha256dHash>> {
+    let key = [b"X", &blockhash[..]].concat();
+    let value = store.get(&key)?;
+    let txids: Vec<Sha256dHash> = bincode::deserialize(&value).unwrap();
+    Some(txids)
 }
 
 pub struct Query {
     app: Arc<App>,
     tracker: RwLock<Tracker>,
-    tx_cache: TransactionCache,
+    extended_db_enabled: bool,
+
+    // monitoring
+    latency: HistogramVec,
 }
 
 impl Query {
-    pub fn new(app: Arc<App>, metrics: &Metrics, tx_cache: TransactionCache) -> Arc<Query> {
+    pub fn new(app: Arc<App>, extended_db_enabled: bool, metrics: &Metrics) -> Arc<Query> {
+        let latency_buckets = vec![
+            1e-4, 2e-4, 5e-4, 1e-3, 2e-3, 5e-3, 1e-2, 2e-2, 5e-2, 0.1, 0.2, 0.5, 1., 2., 5., 10.,
+            20., 50., 100.,
+        ];
         Arc::new(Query {
             app,
+            extended_db_enabled,
             tracker: RwLock::new(Tracker::new(metrics)),
-            tx_cache,
+            latency: metrics.histogram_vec(
+                HistogramOpts::new("query_latency", "Query latency (in seconds)")
+                    .buckets(latency_buckets),
+                &["type"],
+            ),
         })
     }
 
@@ -200,16 +226,23 @@ impl Query {
         store: &ReadStore,
         prefixes: Vec<HashPrefix>,
     ) -> Result<Vec<TxnHeight>> {
+        if prefixes.len() > FUNDING_TXN_LIMIT {
+            bail!("Too many txs");
+        }
+
         let mut txns = vec![];
+        let _timer = self
+            .latency
+            .with_label_values(&["load_txns_by_prefix"])
+            .start_timer();
         for txid_prefix in prefixes {
             for tx_row in txrows_by_prefix(store, &txid_prefix) {
                 let txid: Sha256dHash = deserialize(&tx_row.key.txid).unwrap();
-                let txn = self
-                    .tx_cache
-                    .get_or_else(&txid, || self.load_txn(&txid, tx_row.height))?;
+                let txn = self.load_txn(&txid).chain_err(|| "cannot locate tx")?;
                 txns.push(TxnHeight {
                     txn,
                     height: tx_row.height,
+                    blockhash: tx_row.blockhash,
                 })
             }
         }
@@ -221,19 +254,25 @@ impl Query {
         store: &ReadStore,
         funding: &FundingOutput,
     ) -> Result<Option<SpendingInput>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_spending_input"])
+            .start_timer();
         let spending_txns: Vec<TxnHeight> = self.load_txns_by_prefix(
             store,
             txids_by_funding_output(store, &funding.txn_id, funding.output_index),
         )?;
         let mut spending_inputs = vec![];
         for t in &spending_txns {
-            for input in t.txn.input.iter() {
+            for (input_index, input) in t.txn.input.iter().enumerate() {
                 if input.previous_output.txid == funding.txn_id
                     && input.previous_output.vout == funding.output_index as u32
                 {
                     spending_inputs.push(SpendingInput {
+                        txn: Some(t.clone()),
                         txn_id: t.txn.txid(),
                         height: t.height,
+                        input_index: input_index,
                         funding_output: (funding.txn_id, funding.output_index),
                         value: funding.value,
                     })
@@ -249,15 +288,30 @@ impl Query {
     }
 
     fn find_funding_outputs(&self, t: &TxnHeight, script_hash: &[u8]) -> Vec<FundingOutput> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_funding_outputs"])
+            .start_timer();
         let mut result = vec![];
         let txn_id = t.txn.txid();
         for (index, output) in t.txn.output.iter().enumerate() {
             if compute_script_hash(&output.script_pubkey[..]) == script_hash {
+                let value = match output.value {
+                    confidential::Value::Explicit(val) => val,
+                    _ => 0,
+                };
+                let asset = match output.asset {
+                    confidential::Asset::Explicit(val) => Some(val),
+                    _ => None,
+                };
+
                 result.push(FundingOutput {
+                    txn: Some(t.clone()),
                     txn_id: txn_id,
                     height: t.height,
                     output_index: index,
-                    value: output.value,
+                    value: value,
+                    asset: asset,
                 })
             }
         }
@@ -268,6 +322,10 @@ impl Query {
         &self,
         script_hash: &[u8],
     ) -> Result<(Vec<FundingOutput>, Vec<SpendingInput>)> {
+        let _timer = self
+            .latency
+            .with_label_values(&["confirmed_status"])
+            .start_timer();
         let mut funding = vec![];
         let mut spending = vec![];
         let read_store = self.app.read_store();
@@ -288,6 +346,10 @@ impl Query {
         script_hash: &[u8],
         confirmed_funding: &[FundingOutput],
     ) -> Result<(Vec<FundingOutput>, Vec<SpendingInput>)> {
+        let _timer = self
+            .latency
+            .with_label_values(&["mempool_status"])
+            .start_timer();
         let mut funding = vec![];
         let mut spending = vec![];
         let tracker = self.tracker.read().unwrap();
@@ -305,54 +367,168 @@ impl Query {
     }
 
     pub fn status(&self, script_hash: &[u8]) -> Result<Status> {
-        let confirmed = self
-            .confirmed_status(script_hash)
-            .chain_err(|| "failed to get confirmed status")?;
-        let mempool = self
-            .mempool_status(script_hash, &confirmed.0)
-            .chain_err(|| "failed to get mempool status")?;
+        let _timer = self.latency.with_label_values(&["status"]).start_timer();
+        let confirmed = self.confirmed_status(script_hash)?;
+        //.chain_err(|| "failed to get confirmed status")?;
+        let mempool = self.mempool_status(script_hash, &confirmed.0)?;
+        //.chain_err(|| "failed to get mempool status")?;
         Ok(Status { confirmed, mempool })
     }
 
-    fn lookup_confirmed_blockhash(
-        &self,
-        tx_hash: &Sha256dHash,
-        block_height: Option<u32>,
-    ) -> Result<Option<Sha256dHash>> {
-        let blockhash = if self.tracker.read().unwrap().get_txn(&tx_hash).is_some() {
-            None // found in mempool (as unconfirmed transaction)
-        } else {
-            // Lookup in confirmed transactions' index
-            let height = match block_height {
-                Some(height) => height,
-                None => {
-                    txrow_by_txid(self.app.read_store(), &tx_hash)
-                        .chain_err(|| format!("not indexed tx {}", tx_hash))?
-                        .height
-                }
-            };
-            let header = self
-                .app
-                .index()
-                .get_header(height as usize)
-                .chain_err(|| format!("missing header at height {}", height))?;
-            Some(*header.hash())
-        };
-        Ok(blockhash)
+    pub fn find_spending_by_outpoint(&self, outpoint: OutPoint) -> Result<Option<SpendingInput>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_spending_by_outpoint"])
+            .start_timer();
+        let funding_output = FundingOutput::from(outpoint);
+        let read_store = self.app.read_store();
+        let tracker = self.tracker.read().unwrap();
+        Ok(
+            if let Some(spent) = self.find_spending_input(read_store, &funding_output)? {
+                Some(spent)
+            } else if let Some(spent) =
+                self.find_spending_input(tracker.index(), &funding_output)?
+            {
+                Some(spent)
+            } else {
+                None
+            },
+        )
     }
 
-    // Internal API for transaction retrieval
-    fn load_txn(&self, tx_hash: &Sha256dHash, block_height: u32) -> Result<Transaction> {
-        let blockhash = self.lookup_confirmed_blockhash(tx_hash, Some(block_height))?;
-        self.app.daemon().gettransaction(tx_hash, blockhash)
+    pub fn find_spending_for_funding_tx(
+        &self,
+        tx: Transaction,
+    ) -> Result<Vec<Option<SpendingInput>>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_spending_for_funding_tx"])
+            .start_timer();
+        let txid = tx.txid();
+        let mut spends = vec![];
+        for (output_index, output) in tx.output.iter().enumerate() {
+            let spend = if !output.is_fee() && !output.script_pubkey.is_provably_unspendable() {
+                self.find_spending_by_outpoint((txid, output_index))?
+            } else {
+                None
+            };
+            spends.push(spend)
+        }
+        Ok(spends)
+    }
+
+    // Load transaction by txid
+    pub fn load_txn(&self, txid: &Sha256dHash) -> Result<Transaction> {
+        let _timer = self.latency.with_label_values(&["load_txn"]).start_timer();
+        if self.extended_db_enabled {
+            // fetch from our txstore or mempool tracker
+            rawtxrow_by_txid(self.app.read_store(), txid)
+                .map(|row| deserialize(&row.rawtx).expect("cannot parse tx from txstore"))
+                .or_else(|| self.tracker.read().unwrap().get_txn(&txid))
+                .chain_err(|| format!("cannot find tx {}", txid))
+        } else {
+            // fetch from bitcoind
+            self.app.daemon().gettransaction(txid)
+        }
+    }
+
+    // Load raw transaction by txid
+    pub fn load_raw_txn(&self, txid: &Sha256dHash) -> Result<Bytes> {
+        let _timer = self
+            .latency
+            .with_label_values(&["load_raw_txn"])
+            .start_timer();
+        if self.extended_db_enabled {
+            // fetch from our txstore or mempool tracker
+            Ok(rawtxrow_by_txid(self.app.read_store(), txid)
+                .map(|row| row.rawtx)
+                .or_else(|| {
+                    self.tracker
+                        .read()
+                        .unwrap()
+                        .get_txn(&txid)
+                        .map(|tx| serialize(&tx))
+                })
+                .chain_err(|| format!("cannot find tx {}", txid))?)
+        } else {
+            // fetch from bitcoind
+            let tx_val = self.app.daemon().gettransaction_raw(txid, false)?;
+            Ok(
+                ::hex::decode(tx_val.as_str().chain_err(|| "non-string tx hex")?)
+                    .chain_err(|| "invalid hex")?,
+            )
+        }
     }
 
     // Public API for transaction retrieval (for Electrum RPC)
+    // Fetched from bitcoind, includes tx confirmation information (number of confirmations and block hash)
     pub fn get_transaction(&self, tx_hash: &Sha256dHash, verbose: bool) -> Result<Value> {
-        let blockhash = self.lookup_confirmed_blockhash(tx_hash, /*block_height*/ None)?;
-        self.app
-            .daemon()
-            .gettransaction_raw(tx_hash, blockhash, verbose)
+        let _timer = self
+            .latency
+            .with_label_values(&["get_transaction"])
+            .start_timer();
+        self.app.daemon().gettransaction_raw(tx_hash, verbose)
+    }
+
+    pub fn get_block(&self, blockhash: &Sha256dHash) -> Result<Block> {
+        let _timer = self.latency.with_label_values(&["get_block"]).start_timer();
+        self.app.daemon().getblock(blockhash)
+    }
+
+    pub fn get_block_header_with_meta(&self, blockhash: &Sha256dHash) -> Result<BlockHeaderMeta> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_header_with_meta"])
+            .start_timer();
+        Ok(BlockHeaderMeta {
+            header_entry: self.get_header_by_hash(blockhash)?,
+            meta: self.get_block_meta(blockhash)?,
+        })
+    }
+
+    pub fn get_block_txids(&self, blockhash: &Sha256dHash) -> Result<Vec<Sha256dHash>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_txids"])
+            .start_timer();
+        if self.extended_db_enabled {
+            // fetch from our blockhash=>txids index
+            get_block_txids(self.app.read_store(), blockhash)
+                .chain_err(|| "cannot load block txids")
+        } else {
+            // fetch from bitcoind
+            let block = self
+                .app
+                .daemon()
+                .getblock_raw(blockhash, 1)
+                .chain_err(|| "cannot load block")?;
+            let txids = block
+                .get("tx")
+                .chain_err(|| "block missing txids")?
+                .as_array()
+                .chain_err(|| "invalid block txids")?;
+            Ok(txids
+                .iter()
+                .map(|txid| {
+                    Sha256dHash::from_hex(txid.as_str().chain_err(|| "txid not string")?)
+                        .chain_err(|| "invalid hex")
+                })
+                .collect::<Result<Vec<Sha256dHash>>>()?)
+        }
+    }
+
+    pub fn get_block_meta(&self, blockhash: &Sha256dHash) -> Result<BlockMeta> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_meta"])
+            .start_timer();
+        if self.extended_db_enabled {
+            // fetch from our blockhash=>txids index
+            get_block_meta(self.app.read_store(), blockhash).chain_err(|| "cannot load block meta")
+        } else {
+            // fetch from bitcoind
+            BlockMeta::parse_getblock(self.app.daemon().getblock_raw(blockhash, 1)?)
+        }
     }
 
     pub fn get_headers(&self, heights: &[usize]) -> Vec<HeaderEntry> {
@@ -363,23 +539,87 @@ impl Query {
             .collect()
     }
 
+    pub fn get_header_by_hash(&self, hash: &Sha256dHash) -> Result<HeaderEntry> {
+        let header = self.app.index().get_header_by_hash(hash);
+        Ok(header.chain_err(|| "no header found")?.clone())
+    }
+
     pub fn get_best_header(&self) -> Result<HeaderEntry> {
         let last_header = self.app.index().best_header();
         Ok(last_header.chain_err(|| "no headers indexed")?.clone())
     }
 
+    pub fn get_best_header_hash(&self) -> Sha256dHash {
+        self.app.index().best_header_hash()
+    }
+
+    pub fn get_best_height(&self) -> usize {
+        self.app.index().best_height()
+    }
+
+    pub fn get_block_status(&self, hash: &Sha256dHash) -> BlockStatus {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_status"])
+            .start_timer();
+        // get_header_by_hash looks up the height first, then fetches the header by that.
+        // if the block is no longer the best block at this height, it'll return None.
+        match self.app.index().get_header_by_hash(hash) {
+            Some(header) => BlockStatus {
+                in_best_chain: true,
+                height: Some(header.height()),
+                next_best: self
+                    .app
+                    .index()
+                    .get_header(header.height() + 1)
+                    .map(|h| h.hash().clone()),
+            },
+            None => BlockStatus {
+                in_best_chain: false,
+                height: None,
+                next_best: None,
+            },
+        }
+    }
+
+    pub fn get_tx_status(&self, tx_hash: &Sha256dHash) -> Result<TransactionStatus> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_tx_status"])
+            .start_timer();
+        // try fetching the height/hash of the block seen to confirm the tx
+        let (height, blockhash) = match txrow_by_txid(self.app.read_store(), &tx_hash) {
+            None => return Ok(TransactionStatus::unconfirmed()),
+            Some(txrow) => (txrow.height, txrow.blockhash),
+        };
+
+        // fetch the block header at the recorded confirmation height
+        let header = self
+            .app
+            .index()
+            .get_header(height as usize)
+            .chain_err(|| "invalid block height for tx")?;
+
+        // the block at confirmation height is not the one containing the tx, must've reorged!
+        if header.hash() != &blockhash {
+            Ok(TransactionStatus::unconfirmed())
+        } else {
+            Ok(TransactionStatus::confirmed(&header))
+        }
+    }
+
     pub fn get_merkle_proof(
         &self,
         tx_hash: &Sha256dHash,
-        height: usize,
+        block_hash: &Sha256dHash,
     ) -> Result<(Vec<Sha256dHash>, usize)> {
-        let header_entry = self
-            .app
-            .index()
-            .get_header(height)
-            .chain_err(|| format!("missing block #{}", height))?;
-        let block: Block = self.app.daemon().getblock(&header_entry.hash())?;
-        let mut txids: Vec<Sha256dHash> = block.txdata.iter().map(|tx| tx.txid()).collect();
+        let _timer = self
+            .latency
+            .with_label_values(&["get_merkle_proof"])
+            .start_timer();
+        let mut txids = self
+            .get_block_txids(&block_hash)
+            .chain_err(|| format!("missing txids for block #{}", block_hash))?;
         let pos = txids
             .iter()
             .position(|txid| txid == tx_hash)

src/rest.rs

@@ -0,0 +1,908 @@
+use bitcoin::consensus::{self, encode::serialize};
+use bitcoin::util::hash::{HexError, Sha256dHash};
+use bitcoin::{BitcoinHash, Script};
+use config::Config;
+use daemon::Network;
+use elements::confidential::{Asset, Value};
+use elements::{Proof, Transaction, TxIn, TxOut};
+use errors;
+use hex::{self, FromHexError};
+use hyper::rt::{self, Future};
+use hyper::service::service_fn_ok;
+use hyper::{Body, Method, Request, Response, Server, StatusCode};
+use index::compute_script_hash;
+use mempool::MEMPOOL_HEIGHT;
+use query::{FundingOutput, Query, SpendingInput, TxnHeight};
+use serde::Serialize;
+use serde_json;
+use std::collections::BTreeMap;
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::sync::Arc;
+use std::thread;
+use util::{
+    full_hash, get_script_asm, script_to_address, BlockHeaderMeta, FullHash, PegOutRequest,
+    TransactionStatus,
+};
+use utils::address::Address;
+
+const TX_LIMIT: usize = 25;
+const BLOCK_LIMIT: usize = 10;
+
+const TTL_LONG: u32 = 157784630; // ttl for static resources (5 years)
+const TTL_SHORT: u32 = 10; // ttl for volatie resources
+const CONF_FINAL: usize = 10; // reorgs deeper than this are considered unlikely
+
+#[derive(Serialize, Deserialize)]
+struct BlockValue {
+    id: String,
+    height: u32,
+    version: u32,
+    timestamp: u32,
+    tx_count: u32,
+    size: u32,
+    weight: u32,
+    merkle_root: String,
+    previousblockhash: Option<String>,
+    proof: Option<BlockProofValue>,
+}
+
+impl From<BlockHeaderMeta> for BlockValue {
+    fn from(blockhm: BlockHeaderMeta) -> Self {
+        let header = blockhm.header_entry.header();
+        BlockValue {
+            id: header.bitcoin_hash().be_hex_string(),
+            height: blockhm.header_entry.height() as u32,
+            proof: Some(BlockProofValue::from(header.proof.clone())),
+            version: header.version,
+            timestamp: header.time,
+            tx_count: blockhm.meta.tx_count,
+            size: blockhm.meta.size,
+            weight: blockhm.meta.weight,
+            merkle_root: header.merkle_root.be_hex_string(),
+            previousblockhash: if &header.prev_blockhash != &Sha256dHash::default() {
+                Some(header.prev_blockhash.be_hex_string())
+            } else {
+                None
+            },
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct BlockProofValue {
+    challenge: Script,
+    challenge_asm: String,
+    solution: Script,
+    solution_asm: String,
+}
+impl From<Proof> for BlockProofValue {
+    fn from(proof: Proof) -> Self {
+        BlockProofValue {
+            challenge_asm: get_script_asm(&proof.challenge),
+            challenge: proof.challenge,
+            solution_asm: get_script_asm(&proof.solution),
+            solution: proof.solution,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct TransactionValue {
+    txid: Sha256dHash,
+    version: u32,
+    locktime: u32,
+    vin: Vec<TxInValue>,
+    vout: Vec<TxOutValue>,
+    size: u32,
+    weight: u32,
+    fee: u64,
+    status: Option<TransactionStatus>,
+}
+
+impl From<Transaction> for TransactionValue {
+    fn from(tx: Transaction) -> Self {
+        let vin = tx
+            .input
+            .iter()
+            .map(|el| TxInValue::from(el.clone()))
+            .collect();
+        let vout: Vec<TxOutValue> = tx
+            .output
+            .iter()
+            .map(|el| TxOutValue::from(el.clone()))
+            .collect();
+        let bytes = serialize(&tx);
+        let fee = vout
+            .iter()
+            .find(|vout| vout.scriptpubkey_type == "fee")
+            .map_or(0, |vout| vout.value.unwrap());
+
+        TransactionValue {
+            txid: tx.txid(),
+            version: tx.version,
+            locktime: tx.lock_time,
+            vin,
+            vout,
+            size: bytes.len() as u32,
+            weight: tx.get_weight() as u32,
+            fee,
+            status: None,
+        }
+    }
+}
+
+impl From<TxnHeight> for TransactionValue {
+    fn from(t: TxnHeight) -> Self {
+        let TxnHeight {
+            txn,
+            height,
+            blockhash,
+        } = t;
+        let mut value = TransactionValue::from(txn);
+        value.status = Some(if height != MEMPOOL_HEIGHT {
+            TransactionStatus {
+                confirmed: true,
+                block_height: Some(height as usize),
+                block_hash: Some(blockhash),
+            }
+        } else {
+            TransactionStatus::unconfirmed()
+        });
+        value
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+struct TxInValue {
+    txid: Sha256dHash,
+    vout: u32,
+    prevout: Option<TxOutValue>,
+    scriptsig: Script,
+    scriptsig_asm: String,
+    is_coinbase: bool,
+    is_pegin: bool,
+    sequence: u32,
+    issuance: Option<IssuanceValue>,
+}
+
+impl From<TxIn> for TxInValue {
+    fn from(txin: TxIn) -> Self {
+        let is_coinbase = txin.is_coinbase();
+
+        let zero = [0u8; 32];
+        let issuance = txin.asset_issuance;
+        let is_reissuance = issuance.asset_blinding_nonce != zero;
+
+        let issuance_val = if txin.has_issuance() {
+            Some(IssuanceValue {
+                is_reissuance: is_reissuance,
+                asset_blinding_nonce: if is_reissuance {
+                    Some(hex::encode(issuance.asset_blinding_nonce))
+                } else {
+                    None
+                },
+                asset_entropy: if issuance.asset_entropy != zero {
+                    Some(hex::encode(issuance.asset_entropy))
+                } else {
+                    None
+                },
+                assetamount: match issuance.amount {
+                    Value::Explicit(value) => Some(value),
+                    _ => None,
+                },
+                assetamountcommitment: match issuance.amount {
+                    Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
+                    _ => None,
+                },
+                tokenamount: match issuance.inflation_keys {
+                    Value::Explicit(value) => Some(value / 100000000), // https://github.com/ElementsProject/rust-elements/issues/7
+                    _ => None,
+                },
+                tokenamountcommitment: match issuance.inflation_keys {
+                    Value::Confidential(..) => {
+                        Some(hex::encode(serialize(&issuance.inflation_keys)))
+                    }
+                    _ => None,
+                },
+            })
+        } else {
+            None
+        };
+
+        let script = txin.script_sig;
+
+        TxInValue {
+            txid: txin.previous_output.txid,
+            vout: txin.previous_output.vout,
+            is_pegin: txin.is_pegin,
+            prevout: None, // added later
+            scriptsig_asm: get_script_asm(&script),
+            scriptsig: script,
+            is_coinbase,
+            sequence: txin.sequence,
+            //issuance: if txin.has_issuance() { Some(IssuanceValue::from(txin.asset_issuance)) } else { None },
+            issuance: issuance_val,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+struct IssuanceValue {
+    pub is_reissuance: bool,
+    pub asset_blinding_nonce: Option<String>,
+    pub asset_entropy: Option<String>,
+    pub assetamount: Option<u64>,
+    pub assetamountcommitment: Option<String>,
+    pub tokenamount: Option<u64>,
+    pub tokenamountcommitment: Option<String>,
+}
+
+/*
+// pending https://github.com/ElementsProject/rust-elements/pull/6
+impl From<AssetIssuance> for IssuanceValue {
+    fn from(issuance: AssetIssuance) -> Self {
+        let zero = [0u8;32];
+        let is_reissuance = issuance.asset_blinding_nonce != zero;
+
+        IssuanceValue {
+            is_reissuance: is_reissuance,
+            asset_blinding_nonce: if is_reissuance { Some(hex::encode(issuance.asset_blinding_nonce)) } else { None },
+            asset_entropy: hex::encode(issuance.asset_entropy),
+            assetamount: match issuance.amount {
+                Asset::Explicit(value) => Some(value.be_hex_string()),
+                _ => None
+            },
+            amountcommitment: match issuance.amount {
+                Asset::Confidential(..) => Some(hex::encode(serialize(&issuance.amount).unwrap())),
+                _ => None
+            },
+            tokenamount: match issuance.inflation_keys {
+                Value::Explicit(value) => Some(value),
+                _ => None,
+            },
+            tokenamountcommitment: match issuance.inflation_keys {
+                Asset::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys).unwrap())),
+                _ => None
+            },
+        }
+    }
+}
+*/
+
+#[derive(Serialize, Deserialize, Clone)]
+struct TxOutValue {
+    scriptpubkey: Script,
+    scriptpubkey_asm: String,
+    scriptpubkey_address: Option<String>,
+    asset: Option<String>,
+    assetcommitment: Option<String>,
+    value: Option<u64>,
+    valuecommitment: Option<String>,
+    scriptpubkey_type: String,
+    pegout: Option<PegOutRequest>,
+}
+
+impl From<TxOut> for TxOutValue {
+    fn from(txout: TxOut) -> Self {
+        let asset = match txout.asset {
+            Asset::Explicit(value) => Some(value.be_hex_string()),
+            _ => None,
+        };
+        let assetcommitment = match txout.asset {
+            Asset::Confidential(..) => Some(hex::encode(serialize(&txout.asset))),
+            _ => None,
+        };
+        let value = match txout.value {
+            Value::Explicit(value) => Some(value),
+            _ => None,
+        };
+        let valuecommitment = match txout.value {
+            Value::Confidential(..) => Some(hex::encode(serialize(&txout.value))),
+            _ => None,
+        };
+        let is_fee = txout.is_fee();
+        let script = txout.script_pubkey;
+        let script_asm = get_script_asm(&script);
+
+        // TODO should the following something to put inside rust-elements lib?
+        let script_type = if is_fee {
+            "fee"
+        } else if script.is_op_return() {
+            "op_return"
+        } else if script.is_p2pk() {
+            "p2pk"
+        } else if script.is_p2pkh() {
+            "p2pkh"
+        } else if script.is_p2sh() {
+            "p2sh"
+        } else if script.is_v0_p2wpkh() {
+            "v0_p2wpkh"
+        } else if script.is_v0_p2wsh() {
+            "v0_p2wsh"
+        } else if script.is_provably_unspendable() {
+            "provably_unspendable"
+        } else {
+            "unknown"
+        };
+
+        TxOutValue {
+            scriptpubkey: script,
+            scriptpubkey_asm: script_asm,
+            scriptpubkey_address: None, // added later
+            asset,
+            assetcommitment,
+            value,
+            valuecommitment,
+            scriptpubkey_type: script_type.to_string(),
+            pegout: None, // added later
+        }
+    }
+}
+
+#[derive(Serialize)]
+struct UtxoValue {
+    txid: Sha256dHash,
+    vout: u32,
+    value: Option<u64>,
+    asset: Option<String>,
+    status: TransactionStatus,
+}
+impl From<FundingOutput> for UtxoValue {
+    fn from(out: FundingOutput) -> Self {
+        let FundingOutput {
+            txn,
+            txn_id,
+            output_index,
+            value,
+            asset,
+            ..
+        } = out;
+        let TxnHeight {
+            height, blockhash, ..
+        } = txn.unwrap(); // we should never get a FundingOutput without a txn here
+
+        UtxoValue {
+            txid: txn_id,
+            vout: output_index as u32,
+            value: if value != 0 { Some(value) } else { None },
+            asset: asset.map(|val| val.be_hex_string()),
+            status: if height != MEMPOOL_HEIGHT {
+                TransactionStatus {
+                    confirmed: true,
+                    block_height: Some(height as usize),
+                    block_hash: Some(blockhash),
+                }
+            } else {
+                TransactionStatus::unconfirmed()
+            },
+        }
+    }
+}
+
+#[derive(Serialize)]
+struct SpendingValue {
+    spent: bool,
+    txid: Option<Sha256dHash>,
+    vin: Option<u32>,
+    status: Option<TransactionStatus>,
+}
+impl From<SpendingInput> for SpendingValue {
+    fn from(spend: SpendingInput) -> Self {
+        let SpendingInput {
+            txn,
+            txn_id,
+            input_index,
+            ..
+        } = spend;
+        let TxnHeight {
+            height, blockhash, ..
+        } = txn.unwrap(); // we should never get a SpendingInput without a txn here
+
+        SpendingValue {
+            spent: true,
+            txid: Some(txn_id),
+            vin: Some(input_index as u32),
+            status: Some(if height != MEMPOOL_HEIGHT {
+                TransactionStatus {
+                    confirmed: true,
+                    block_height: Some(height as usize),
+                    block_hash: Some(blockhash),
+                }
+            } else {
+                TransactionStatus::unconfirmed()
+            }),
+        }
+    }
+}
+impl Default for SpendingValue {
+    fn default() -> Self {
+        SpendingValue {
+            spent: false,
+            txid: None,
+            vin: None,
+            status: None,
+        }
+    }
+}
+
+fn ttl_by_depth(height: Option<usize>, query: &Query) -> u32 {
+    height.map_or(TTL_SHORT, |height| {
+        if query.get_best_height() - height >= CONF_FINAL {
+            TTL_LONG
+        } else {
+            TTL_SHORT
+        }
+    })
+}
+
+fn attach_tx_data(tx: TransactionValue, config: &Config, query: &Arc<Query>) -> TransactionValue {
+    let mut txs = vec![tx];
+    attach_txs_data(&mut txs, config, query);
+    txs.remove(0)
+}
+
+fn attach_txs_data(txs: &mut Vec<TransactionValue>, config: &Config, query: &Arc<Query>) {
+    // a map of prev txids/vouts to lookup, with a reference to the "next in" that spends them
+    let mut lookups: BTreeMap<Sha256dHash, Vec<(u32, &mut TxInValue)>> = BTreeMap::new();
+    // using BTreeMap ensures the txid keys are in order. querying the db with keys in order leverage memory
+    // locality from empirical test up to 2 or 3 times faster
+
+    for mut tx in txs.iter_mut() {
+        // collect lookups
+        if config.prevout_enabled {
+            for mut vin in tx.vin.iter_mut() {
+                if !vin.is_coinbase && !vin.is_pegin {
+                    lookups
+                        .entry(vin.txid)
+                        .or_insert(vec![])
+                        .push((vin.vout, vin));
+                }
+            }
+        }
+        // attach encoded address and pegout info (should ideally happen in TxOutValue::from(),
+        // but it cannot easily access the network)
+        for mut vout in tx.vout.iter_mut() {
+            vout.scriptpubkey_address = script_to_address(&vout.scriptpubkey, &config.network_type);
+            vout.pegout = PegOutRequest::parse(
+                &vout.scriptpubkey,
+                &config.parent_network,
+                &config.parent_genesis_hash,
+            );
+        }
+    }
+
+    // fetch prevtxs and attach prevouts to nextins
+    if config.prevout_enabled {
+        for (prev_txid, prev_vouts) in lookups {
+            let prevtx = query.load_txn(&prev_txid).unwrap();
+            for (prev_out_idx, ref mut nextin) in prev_vouts {
+                let mut prevout = TxOutValue::from(prevtx.output[prev_out_idx as usize].clone());
+                prevout.scriptpubkey_address =
+                    script_to_address(&prevout.scriptpubkey, &config.network_type);
+                nextin.prevout = Some(prevout);
+            }
+        }
+    }
+}
+
+pub fn run_server(config: &Config, query: Arc<Query>) {
+    let addr = &config.http_addr;
+    info!("REST server running on {}", addr);
+    let config = Arc::new(config.clone());
+
+    let new_service = move || {
+        let query = query.clone();
+        let config = config.clone();
+
+        service_fn_ok(
+            move |req: Request<Body>| match handle_request(req, &query, &config) {
+                Ok(response) => response,
+                Err(e) => {
+                    warn!("{:?}", e);
+                    Response::builder()
+                        .status(e.0)
+                        .header("Content-Type", "text/plain")
+                        .body(Body::from(e.1))
+                        .unwrap()
+                }
+            },
+        )
+    };
+
+    let server = Server::bind(&addr)
+        .serve(new_service)
+        .map_err(|e| eprintln!("server error: {}", e));
+
+    thread::spawn(move || {
+        rt::run(server);
+    });
+}
+
+fn handle_request(
+    req: Request<Body>,
+    query: &Arc<Query>,
+    config: &Config,
+) -> Result<Response<Body>, HttpError> {
+    // TODO it looks hyper does not have routing and query parsing :(
+    let uri = req.uri();
+    let path: Vec<&str> = uri.path().split('/').skip(1).collect();
+    info!("path {:?}", path);
+    match (
+        req.method(),
+        path.get(0),
+        path.get(1),
+        path.get(2),
+        path.get(3),
+    ) {
+        (&Method::GET, Some(&"blocks"), Some(&"tip"), Some(&"hash"), None) => http_message(
+            StatusCode::OK,
+            query.get_best_header_hash().be_hex_string(),
+            TTL_SHORT,
+        ),
+
+        (&Method::GET, Some(&"blocks"), Some(&"tip"), Some(&"height"), None) => http_message(
+            StatusCode::OK,
+            query.get_best_height().to_string(),
+            TTL_SHORT,
+        ),
+
+        (&Method::GET, Some(&"blocks"), start_height, None, None) => {
+            let start_height = start_height.and_then(|height| height.parse::<usize>().ok());
+            blocks(&query, start_height)
+        }
+        (&Method::GET, Some(&"block-height"), Some(height), None, None) => {
+            let height = height.parse::<usize>()?;
+            let headers = query.get_headers(&[height]);
+            let header = headers
+                .get(0)
+                .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?;
+            let ttl = ttl_by_depth(Some(height), query);
+            http_message(StatusCode::OK, header.hash().be_hex_string(), ttl)
+        }
+        (&Method::GET, Some(&"block"), Some(hash), None, None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let blockhm = query.get_block_header_with_meta(&hash)?;
+            let block_value = BlockValue::from(blockhm);
+            json_response(block_value, TTL_LONG)
+        }
+        (&Method::GET, Some(&"block"), Some(hash), Some(&"status"), None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let status = query.get_block_status(&hash);
+            let ttl = ttl_by_depth(status.height, query);
+            json_response(status, ttl)
+        }
+        (&Method::GET, Some(&"block"), Some(hash), Some(&"txids"), None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let txids = query
+                .get_block_txids(&hash)
+                .map_err(|_| HttpError::not_found("Block not found".to_string()))?;
+            json_response(txids, TTL_LONG)
+        }
+        (&Method::GET, Some(&"block"), Some(hash), Some(&"txs"), start_index) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let txids = query
+                .get_block_txids(&hash)
+                .map_err(|_| HttpError::not_found("Block not found".to_string()))?;
+
+            let start_index = start_index
+                .map_or(0u32, |el| el.parse().unwrap_or(0))
+                .max(0u32) as usize;
+            if start_index >= txids.len() {
+                bail!(HttpError::not_found("start index out of range".to_string()));
+            } else if start_index % TX_LIMIT != 0 {
+                bail!(HttpError::from(format!(
+                    "start index must be a multipication of {}",
+                    TX_LIMIT
+                )));
+            }
+
+            let mut txs = txids
+                .iter()
+                .skip(start_index)
+                .take(TX_LIMIT)
+                .map(|txid| query.load_txn(&txid).map(TransactionValue::from))
+                .collect::<Result<Vec<TransactionValue>, _>>()?;
+            attach_txs_data(&mut txs, config, query);
+            json_response(txs, TTL_LONG)
+        }
+        (&Method::GET, Some(script_type @ &"address"), Some(script_str), None, None)
+        | (&Method::GET, Some(script_type @ &"scripthash"), Some(script_str), None, None) => {
+            // @TODO create new AddressStatsValue struct?
+            let script_hash = to_scripthash(script_type, script_str, &config.network_type)?;
+            match query.status(&script_hash[..]) {
+                Ok(status) => json_response(
+                    json!({
+                        *script_type: script_str,
+                        "tx_count": status.history().len(),
+                    }),
+                    TTL_SHORT,
+                ),
+
+                // if the address has too many txs, just return the address with no additional info (but no error)
+                Err(errors::Error(errors::ErrorKind::Msg(ref msg), _))
+                    if *msg == "Too many txs".to_string() =>
+                {
+                    json_response(json!({ *script_type: script_str }), TTL_SHORT)
+                }
+
+                Err(err) => bail!(err),
+            }
+        }
+        (
+            &Method::GET,
+            Some(script_type @ &"address"),
+            Some(script_str),
+            Some(&"txs"),
+            start_index,
+        )
+        | (
+            &Method::GET,
+            Some(script_type @ &"scripthash"),
+            Some(script_str),
+            Some(&"txs"),
+            start_index,
+        ) => {
+            let start_index = start_index
+                .map_or(0u32, |el| el.parse().unwrap_or(0))
+                .max(0u32) as usize;
+
+            let script_hash = to_scripthash(script_type, script_str, &config.network_type)?;
+            let status = query.status(&script_hash[..])?;
+            let txs = status.history_txs();
+
+            if txs.len() == 0 {
+                return json_response(json!([]), TTL_SHORT);
+            } else if start_index >= txs.len() {
+                bail!(HttpError::not_found("start index out of range".to_string()));
+            } else if start_index % TX_LIMIT != 0 {
+                bail!(HttpError::from(format!(
+                    "start index must be a multipication of {}",
+                    TX_LIMIT
+                )));
+            }
+
+            let mut txs = txs
+                .iter()
+                .skip(start_index)
+                .take(TX_LIMIT)
+                .map(|t| TransactionValue::from((*t).clone()))
+                .collect();
+            attach_txs_data(&mut txs, config, query);
+
+            json_response(txs, TTL_SHORT)
+        }
+        (&Method::GET, Some(script_type @ &"address"), Some(script_str), Some(&"utxo"), None)
+        | (
+            &Method::GET,
+            Some(script_type @ &"scripthash"),
+            Some(script_str),
+            Some(&"utxo"),
+            None,
+        ) => {
+            let script_hash = to_scripthash(script_type, script_str, &config.network_type)?;
+            let status = query.status(&script_hash[..])?;
+            let utxos: Vec<UtxoValue> = status
+                .unspent()
+                .into_iter()
+                .map(|o| UtxoValue::from(o.clone()))
+                .collect();
+            // @XXX no paging, but query.status() is limited to 30 funding txs
+            json_response(utxos, TTL_SHORT)
+        }
+        (&Method::GET, Some(&"tx"), Some(hash), None, None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let transaction = query
+                .load_txn(&hash)
+                .map_err(|_| HttpError::not_found("Transaction not found".to_string()))?;
+            let status = query.get_tx_status(&hash)?;
+            let ttl = ttl_by_depth(status.block_height, query);
+
+            let mut value = TransactionValue::from(transaction);
+            value.status = Some(status);
+            let value = attach_tx_data(value, config, query);
+            json_response(value, ttl)
+        }
+        (&Method::GET, Some(&"tx"), Some(hash), Some(&"hex"), None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let rawtx = query
+                .load_raw_txn(&hash)
+                .map_err(|_| HttpError::not_found("Transaction not found".to_string()))?;
+            let ttl = ttl_by_depth(query.get_tx_status(&hash)?.block_height, query);
+            http_message(StatusCode::OK, hex::encode(rawtx), ttl)
+        }
+        (&Method::GET, Some(&"tx"), Some(hash), Some(&"status"), None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let status = query.get_tx_status(&hash)?;
+            let ttl = ttl_by_depth(status.block_height, query);
+            json_response(status, ttl)
+        }
+        (&Method::GET, Some(&"tx"), Some(hash), Some(&"merkle-proof"), None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let status = query.get_tx_status(&hash)?;
+            if !status.confirmed {
+                bail!("Transaction is unconfirmed".to_string())
+            };
+            let proof = query.get_merkle_proof(&hash, &status.block_hash.unwrap())?;
+            let ttl = ttl_by_depth(status.block_height, query);
+            json_response(
+                json!({ "block_height": status.block_height, "merkle": proof.0, "pos": proof.1 }),
+                ttl,
+            )
+        }
+        (&Method::GET, Some(&"tx"), Some(hash), Some(&"outspend"), Some(index)) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let outpoint = (hash, index.parse::<usize>()?);
+            let spend = query.find_spending_by_outpoint(outpoint)?.map_or_else(
+                || SpendingValue::default(),
+                |spend| SpendingValue::from(spend),
+            );
+            let ttl = ttl_by_depth(
+                spend
+                    .status
+                    .as_ref()
+                    .and_then(|ref status| status.block_height),
+                query,
+            );
+            json_response(spend, ttl)
+        }
+        (&Method::GET, Some(&"tx"), Some(hash), Some(&"outspends"), None) => {
+            let hash = Sha256dHash::from_hex(hash)?;
+            let tx = query
+                .load_txn(&hash)
+                .map_err(|_| HttpError::not_found("Transaction not found".to_string()))?;
+            let spends: Vec<SpendingValue> = query
+                .find_spending_for_funding_tx(tx)?
+                .into_iter()
+                .map(|spend| {
+                    spend.map_or_else(
+                        || SpendingValue::default(),
+                        |spend| SpendingValue::from(spend),
+                    )
+                })
+                .collect();
+            // @TODO long ttl if all outputs are either spent long ago or unspendable
+            json_response(spends, TTL_SHORT)
+        }
+        _ => Err(HttpError::not_found(format!(
+            "endpoint does not exist {:?}",
+            uri.path()
+        ))),
+    }
+}
+
+fn http_message(
+    status: StatusCode,
+    message: String,
+    ttl: u32,
+) -> Result<Response<Body>, HttpError> {
+    Ok(Response::builder()
+        .status(status)
+        .header("Content-Type", "text/plain")
+        .header("Cache-Control", format!("public, max-age={:}", ttl))
+        .body(Body::from(message))
+        .unwrap())
+}
+
+fn json_response<T: Serialize>(value: T, ttl: u32) -> Result<Response<Body>, HttpError> {
+    let value = serde_json::to_string(&value)?;
+    Ok(Response::builder()
+        .header("Content-Type", "application/json")
+        .header("Cache-Control", format!("public, max-age={:}", ttl))
+        .body(Body::from(value))
+        .unwrap())
+}
+
+fn blocks(query: &Arc<Query>, start_height: Option<usize>) -> Result<Response<Body>, HttpError> {
+    let mut values = Vec::new();
+    let mut current_hash = match start_height {
+        Some(height) => query
+            .get_headers(&[height])
+            .get(0)
+            .ok_or(HttpError::not_found("Block not found".to_string()))?
+            .hash()
+            .clone(),
+        None => query.get_best_header()?.hash().clone(),
+    };
+
+    let zero = [0u8; 32];
+    for _ in 0..BLOCK_LIMIT {
+        let blockhm = query.get_block_header_with_meta(&current_hash)?;
+        current_hash = blockhm.header_entry.header().prev_blockhash.clone();
+        let mut value = BlockValue::from(blockhm);
+        value.proof = None;
+        values.push(value);
+
+        if &current_hash[..] == &zero[..] {
+            break;
+        }
+    }
+    json_response(values, TTL_SHORT)
+}
+
+fn to_scripthash(
+    script_type: &str,
+    script_str: &str,
+    network: &Network,
+) -> Result<FullHash, HttpError> {
+    match script_type {
+        "address" => address_to_scripthash(script_str, network),
+        "scripthash" => Ok(full_hash(&hex::decode(script_str)?)),
+        _ => bail!("Invalid script type".to_string()),
+    }
+}
+
+fn address_to_scripthash(addr: &str, network: &Network) -> Result<FullHash, HttpError> {
+    let addr = Address::from_str(addr)?;
+    let addr_network = addr.network;
+    if addr_network != *network
+        && !(addr_network == Network::Testnet && *network == Network::LiquidRegtest)
+    {
+        bail!(HttpError::from("Address on invalid network".to_string()))
+    }
+    Ok(compute_script_hash(&addr.script_pubkey().into_bytes()))
+}
+
+#[derive(Debug)]
+struct HttpError(StatusCode, String);
+
+impl HttpError {
+    fn not_found(msg: String) -> Self {
+        HttpError(StatusCode::NOT_FOUND, msg)
+    }
+    fn generic() -> Self {
+        HttpError::from("We encountered an error. Please try again later.".to_string())
+    }
+}
+
+impl From<String> for HttpError {
+    fn from(msg: String) -> Self {
+        HttpError(StatusCode::BAD_REQUEST, msg)
+    }
+}
+impl From<ParseIntError> for HttpError {
+    fn from(_e: ParseIntError) -> Self {
+        //HttpError::from(e.description().to_string())
+        HttpError::from("Invalid number".to_string())
+    }
+}
+impl From<HexError> for HttpError {
+    fn from(_e: HexError) -> Self {
+        //HttpError::from(e.description().to_string())
+        HttpError::from("Invalid hex string".to_string())
+    }
+}
+impl From<FromHexError> for HttpError {
+    fn from(_e: FromHexError) -> Self {
+        //HttpError::from(e.description().to_string())
+        HttpError::from("Invalid hex string".to_string())
+    }
+}
+impl From<errors::Error> for HttpError {
+    fn from(e: errors::Error) -> Self {
+        warn!("errors::Error: {:?}", e);
+        match e.description().to_string().as_ref() {
+            "getblock RPC error: {\"code\":-5,\"message\":\"Block not found\"}" => {
+                HttpError::not_found("Block not found".to_string())
+            }
+            "Too many txs" => HttpError(
+                StatusCode::TOO_MANY_REQUESTS,
+                "Sorry! Addresses with a large number of transactions aren\'t currently supported."
+                    .to_string(),
+            ),
+            _ => HttpError::generic(),
+        }
+    }
+}
+impl From<serde_json::Error> for HttpError {
+    fn from(_e: serde_json::Error) -> Self {
+        //HttpError::from(e.description().to_string())
+        HttpError::generic()
+    }
+}
+impl From<consensus::encode::Error> for HttpError {
+    fn from(_e: consensus::encode::Error) -> Self {
+        //HttpError::from(e.description().to_string())
+        HttpError::generic()
+    }
+}

src/rpc.rs

@@ -1,561 +0,0 @@
-use bitcoin::blockdata::transaction::Transaction;
-use bitcoin::network::serialize::{deserialize, serialize};
-use bitcoin::util::address::Address;
-use bitcoin::util::hash::Sha256dHash;
-use error_chain::ChainedError;
-use hex;
-use serde_json::{from_str, Number, Value};
-use std::collections::HashMap;
-use std::io::{BufRead, BufReader, Write};
-use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
-use std::str::FromStr;
-use std::sync::mpsc::{Sender, SyncSender, TrySendError};
-use std::sync::{Arc, Mutex};
-use std::thread;
-
-use index::compute_script_hash;
-use metrics::{Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics};
-use query::{Query, Status};
-use util::{spawn_thread, Channel, HeaderEntry, SyncChannel};
-
-use errors::*;
-
-// TODO: Sha256dHash should be a generic hash-container (since script hash is single SHA256)
-fn hash_from_value(val: Option<&Value>) -> Result<Sha256dHash> {
-    let script_hash = val.chain_err(|| "missing hash")?;
-    let script_hash = script_hash.as_str().chain_err(|| "non-string hash")?;
-    let script_hash = Sha256dHash::from_hex(script_hash).chain_err(|| "non-hex hash")?;
-    Ok(script_hash)
-}
-
-fn usize_from_value(val: Option<&Value>, name: &str) -> Result<usize> {
-    let val = val.chain_err(|| format!("missing {}", name))?;
-    let val = val.as_u64().chain_err(|| format!("non-integer {}", name))?;
-    Ok(val as usize)
-}
-
-fn unspent_from_status(status: &Status) -> Value {
-    json!(Value::Array(
-        status
-            .unspent()
-            .into_iter()
-            .map(|out| json!({
-                "height": out.height,
-                "tx_pos": out.output_index,
-                "tx_hash": out.txn_id.be_hex_string(),
-                "value": out.value,
-            })).collect()
-    ))
-}
-
-fn address_from_value(val: Option<&Value>) -> Result<Address> {
-    let addr = val
-        .chain_err(|| "no address")?
-        .as_str()
-        .chain_err(|| "non-string address")?;
-    Address::from_str(addr).chain_err(|| format!("invalid address {}", addr))
-}
-
-fn jsonify_header(entry: &HeaderEntry) -> Value {
-    let header = entry.header();
-    json!({
-        "block_height": entry.height(),
-        "version": header.version,
-        "prev_block_hash": header.prev_blockhash.be_hex_string(),
-        "merkle_root": header.merkle_root.be_hex_string(),
-        "timestamp": header.time,
-        "bits": header.bits,
-        "nonce": header.nonce
-    })
-}
-
-struct Connection {
-    query: Arc<Query>,
-    last_header_entry: Option<HeaderEntry>,
-    status_hashes: HashMap<Sha256dHash, Value>, // ScriptHash -> StatusHash
-    stream: TcpStream,
-    addr: SocketAddr,
-    chan: SyncChannel<Message>,
-    stats: Arc<Stats>,
-}
-
-impl Connection {
-    pub fn new(
-        query: Arc<Query>,
-        stream: TcpStream,
-        addr: SocketAddr,
-        stats: Arc<Stats>,
-    ) -> Connection {
-        Connection {
-            query,
-            last_header_entry: None, // disable header subscription for now
-            status_hashes: HashMap::new(),
-            stream,
-            addr,
-            chan: SyncChannel::new(10),
-            stats,
-        }
-    }
-
-    fn blockchain_headers_subscribe(&mut self) -> Result<Value> {
-        let entry = self.query.get_best_header()?;
-        let hex_header = hex::encode(serialize(entry.header()).unwrap());
-        let result = json!({"hex": hex_header, "height": entry.height()});
-        self.last_header_entry = Some(entry);
-        Ok(result)
-    }
-
-    fn server_version(&self) -> Result<Value> {
-        Ok(json!(["RustElectrum 0.1.0", "1.2"]))
-    }
-
-    fn server_banner(&self) -> Result<Value> {
-        Ok(json!("Welcome to RustElectrum Server!\n"))
-    }
-
-    fn server_donation_address(&self) -> Result<Value> {
-        Ok(Value::Null)
-    }
-
-    fn server_peers_subscribe(&self) -> Result<Value> {
-        Ok(json!([]))
-    }
-
-    fn mempool_get_fee_histogram(&self) -> Result<Value> {
-        Ok(json!(self.query.get_fee_histogram()))
-    }
-
-    fn blockchain_block_headers(&self, params: &[Value]) -> Result<Value> {
-        let start_height = usize_from_value(params.get(0), "start_height")?;
-        let count = usize_from_value(params.get(1), "count")?;
-        let heights: Vec<usize> = (start_height..(start_height + count)).collect();
-        let headers: Vec<String> = self
-            .query
-            .get_headers(&heights)
-            .into_iter()
-            .map(|entry| hex::encode(&serialize(entry.header()).unwrap()))
-            .collect();
-        Ok(json!({
-            "count": headers.len(),
-            "hex": headers.join(""),
-            "max": 2016,
-        }))
-    }
-
-    fn blockchain_block_get_header(&self, params: &[Value]) -> Result<Value> {
-        let height = usize_from_value(params.get(0), "missing height")?;
-        let mut entries = self.query.get_headers(&[height]);
-        let entry = entries
-            .pop()
-            .chain_err(|| format!("missing header #{}", height))?;
-        assert_eq!(entries.len(), 0);
-        Ok(json!(jsonify_header(&entry)))
-    }
-
-    fn blockchain_estimatefee(&self, params: &[Value]) -> Result<Value> {
-        let blocks_count = usize_from_value(params.get(0), "blocks_count")?;
-        let fee_rate = self.query.estimate_fee(blocks_count); // in BTC/kB
-        Ok(json!(fee_rate))
-    }
-
-    fn blockchain_relayfee(&self) -> Result<Value> {
-        Ok(json!(0.0)) // allow sending transactions with any fee.
-    }
-
-    fn blockchain_scripthash_subscribe(&mut self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-        let status = self.query.status(&script_hash[..])?;
-        let result = status.hash().map_or(Value::Null, |h| json!(hex::encode(h)));
-        self.status_hashes.insert(script_hash, result.clone());
-        Ok(result)
-    }
-
-    fn blockchain_address_subscribe(&mut self, params: &[Value]) -> Result<Value> {
-        let addr = address_from_value(params.get(0)).chain_err(|| "bad address")?;
-        let script_hash = compute_script_hash(&addr.script_pubkey().into_bytes());
-        let status = self.query.status(&script_hash[..])?;
-        let result = status.hash().map_or(Value::Null, |h| json!(hex::encode(h)));
-        let script_hash: Sha256dHash = deserialize(&script_hash).unwrap();
-        self.status_hashes.insert(script_hash, result.clone());
-        Ok(result)
-    }
-
-    fn blockchain_scripthash_get_balance(&self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-        let status = self.query.status(&script_hash[..])?;
-        Ok(
-            json!({ "confirmed": status.confirmed_balance(), "unconfirmed": status.mempool_balance() }),
-        )
-    }
-
-    fn blockchain_address_get_balance(&self, params: &[Value]) -> Result<Value> {
-        let addr = address_from_value(params.get(0)).chain_err(|| "bad address")?;
-        let script_hash = compute_script_hash(&addr.script_pubkey().into_bytes());
-        let status = self.query.status(&script_hash[..])?;
-        Ok(
-            json!({ "confirmed": status.confirmed_balance(), "unconfirmed": status.mempool_balance() }),
-        )
-    }
-
-    fn blockchain_scripthash_get_history(&self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-        let status = self.query.status(&script_hash[..])?;
-        Ok(json!(Value::Array(
-            status
-                .history()
-                .into_iter()
-                .map(|item| json!({"height": item.0, "tx_hash": item.1.be_hex_string()}))
-                .collect()
-        )))
-    }
-
-    fn blockchain_address_get_history(&self, params: &[Value]) -> Result<Value> {
-        let addr = address_from_value(params.get(0)).chain_err(|| "bad address")?;
-        let script_hash = compute_script_hash(&addr.script_pubkey().into_bytes());
-        let status = self.query.status(&script_hash[..])?;
-        Ok(json!(Value::Array(
-            status
-                .history()
-                .into_iter()
-                .map(|item| json!({"height": item.0, "tx_hash": item.1.be_hex_string()}))
-                .collect()
-        )))
-    }
-
-    fn blockchain_scripthash_listunspent(&self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-        Ok(unspent_from_status(&self.query.status(&script_hash[..])?))
-    }
-
-    fn blockchain_address_listunspent(&self, params: &[Value]) -> Result<Value> {
-        let addr = address_from_value(params.get(0)).chain_err(|| "bad address")?;
-        let script_hash = compute_script_hash(&addr.script_pubkey().into_bytes());
-        Ok(unspent_from_status(&self.query.status(&script_hash[..])?))
-    }
-
-    fn blockchain_transaction_broadcast(&self, params: &[Value]) -> Result<Value> {
-        let tx = params.get(0).chain_err(|| "missing tx")?;
-        let tx = tx.as_str().chain_err(|| "non-string tx")?;
-        let tx = hex::decode(&tx).chain_err(|| "non-hex tx")?;
-        let tx: Transaction = deserialize(&tx).chain_err(|| "failed to parse tx")?;
-        let txid = self.query.broadcast(&tx)?;
-        self.query.update_mempool()?;
-        if let Err(e) = self.chan.sender().try_send(Message::PeriodicUpdate) {
-            warn!("failed to issue PeriodicUpdate after broadcast: {}", e);
-        }
-        Ok(json!(txid.be_hex_string()))
-    }
-
-    fn blockchain_transaction_get(&self, params: &[Value]) -> Result<Value> {
-        let tx_hash = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
-        let verbose = match params.get(1) {
-            Some(value) => value.as_bool().chain_err(|| "non-bool verbose value")?,
-            None => false,
-        };
-        Ok(self.query.get_transaction(&tx_hash, verbose)?)
-    }
-
-    fn blockchain_transaction_get_merkle(&self, params: &[Value]) -> Result<Value> {
-        let tx_hash = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
-        let height = usize_from_value(params.get(1), "height")?;
-        let (merkle, pos) = self
-            .query
-            .get_merkle_proof(&tx_hash, height)
-            .chain_err(|| "cannot create merkle proof")?;
-        let merkle: Vec<String> = merkle
-            .into_iter()
-            .map(|txid| txid.be_hex_string())
-            .collect();
-        Ok(json!({
-                "block_height": height,
-                "merkle": merkle,
-                "pos": pos}))
-    }
-
-    fn handle_command(&mut self, method: &str, params: &[Value], id: &Number) -> Result<Value> {
-        let timer = self
-            .stats
-            .latency
-            .with_label_values(&[method])
-            .start_timer();
-        let result = match method {
-            "blockchain.headers.subscribe" => self.blockchain_headers_subscribe(),
-            "server.version" => self.server_version(),
-            "server.banner" => self.server_banner(),
-            "server.donation_address" => self.server_donation_address(),
-            "server.peers.subscribe" => self.server_peers_subscribe(),
-            "mempool.get_fee_histogram" => self.mempool_get_fee_histogram(),
-            "blockchain.block.headers" => self.blockchain_block_headers(&params),
-            "blockchain.block.get_header" => self.blockchain_block_get_header(&params),
-            "blockchain.estimatefee" => self.blockchain_estimatefee(&params),
-            "blockchain.relayfee" => self.blockchain_relayfee(),
-            "blockchain.address.subscribe" => self.blockchain_address_subscribe(&params),
-            "blockchain.address.get_balance" => self.blockchain_address_get_balance(&params),
-            "blockchain.address.get_history" => self.blockchain_address_get_history(&params),
-            "blockchain.address.listunspent" => self.blockchain_address_listunspent(&params),
-            "blockchain.scripthash.subscribe" => self.blockchain_scripthash_subscribe(&params),
-            "blockchain.scripthash.get_balance" => self.blockchain_scripthash_get_balance(&params),
-            "blockchain.scripthash.get_history" => self.blockchain_scripthash_get_history(&params),
-            "blockchain.scripthash.listunspent" => self.blockchain_scripthash_listunspent(&params),
-            "blockchain.transaction.broadcast" => self.blockchain_transaction_broadcast(&params),
-            "blockchain.transaction.get" => self.blockchain_transaction_get(&params),
-            "blockchain.transaction.get_merkle" => self.blockchain_transaction_get_merkle(&params),
-            &_ => bail!("unknown method {} {:?}", method, params),
-        };
-        timer.observe_duration();
-        // TODO: return application errors should be sent to the client
-        Ok(match result {
-            Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}),
-            Err(e) => {
-                warn!(
-                    "rpc #{} {} {:?} failed: {}",
-                    id,
-                    method,
-                    params,
-                    e.display_chain()
-                );
-                json!({"jsonrpc": "2.0", "id": id, "error": format!("{}", e)})
-            }
-        })
-    }
-
-    fn update_subscriptions(&mut self) -> Result<Vec<Value>> {
-        let timer = self
-            .stats
-            .latency
-            .with_label_values(&["periodic_update"])
-            .start_timer();
-        let mut result = vec![];
-        if let Some(ref mut last_entry) = self.last_header_entry {
-            let entry = self.query.get_best_header()?;
-            if *last_entry != entry {
-                *last_entry = entry;
-                let hex_header = hex::encode(serialize(last_entry.header()).unwrap());
-                let header = json!({"hex": hex_header, "height": last_entry.height()});
-                result.push(json!({
-                    "jsonrpc": "2.0",
-                    "method": "blockchain.headers.subscribe",
-                    "params": [header]}));
-            }
-        }
-        for (script_hash, status_hash) in self.status_hashes.iter_mut() {
-            let status = self.query.status(&script_hash[..])?;
-            let new_status_hash = status.hash().map_or(Value::Null, |h| json!(hex::encode(h)));
-            if new_status_hash == *status_hash {
-                continue;
-            }
-            result.push(json!({
-                "jsonrpc": "2.0",
-                "method": "blockchain.scripthash.subscribe",
-                "params": [script_hash.be_hex_string(), new_status_hash]}));
-            *status_hash = new_status_hash;
-        }
-        timer.observe_duration();
-        self.stats
-            .subscriptions
-            .set(self.status_hashes.len() as i64);
-        Ok(result)
-    }
-
-    fn send_values(&mut self, values: &[Value]) -> Result<()> {
-        for value in values {
-            let line = value.to_string() + "\n";
-            self.stream
-                .write_all(line.as_bytes())
-                .chain_err(|| format!("failed to send {}", value))?;
-        }
-        Ok(())
-    }
-
-    fn handle_replies(&mut self) -> Result<()> {
-        loop {
-            let msg = self.chan.receiver().recv().chain_err(|| "channel closed")?;
-            trace!("RPC {:?}", msg);
-            match msg {
-                Message::Request(line) => {
-                    let cmd: Value = from_str(&line).chain_err(|| "invalid JSON format")?;
-                    let reply = match (cmd.get("method"), cmd.get("params"), cmd.get("id")) {
-                        (
-                            Some(&Value::String(ref method)),
-                            Some(&Value::Array(ref params)),
-                            Some(&Value::Number(ref id)),
-                        ) => self.handle_command(method, params, id)?,
-                        _ => bail!("invalid command: {}", cmd),
-                    };
-                    self.send_values(&[reply])?
-                }
-                Message::PeriodicUpdate => {
-                    let values = self
-                        .update_subscriptions()
-                        .chain_err(|| "failed to update subscriptions")?;
-                    self.send_values(&values)?
-                }
-                Message::Done => return Ok(()),
-            }
-        }
-    }
-
-    fn handle_requests(mut reader: BufReader<TcpStream>, tx: SyncSender<Message>) -> Result<()> {
-        loop {
-            let mut line = Vec::<u8>::new();
-            reader
-                .read_until(b'\n', &mut line)
-                .chain_err(|| "failed to read a request")?;
-            if line.is_empty() {
-                tx.send(Message::Done).chain_err(|| "channel closed")?;
-                return Ok(());
-            } else {
-                if line.starts_with(&[22, 3, 1]) {
-                    // (very) naive SSL handshake detection
-                    let _ = tx.send(Message::Done);
-                    bail!("invalid request - maybe SSL-encrypted data?: {:?}", line)
-                }
-                match String::from_utf8(line) {
-                    Ok(req) => tx
-                        .send(Message::Request(req))
-                        .chain_err(|| "channel closed")?,
-                    Err(err) => {
-                        let _ = tx.send(Message::Done);
-                        bail!("invalid UTF8: {}", err)
-                    }
-                }
-            }
-        }
-    }
-
-    pub fn run(mut self) {
-        let reader = BufReader::new(self.stream.try_clone().expect("failed to clone TcpStream"));
-        let tx = self.chan.sender();
-        let child = spawn_thread("reader", || Connection::handle_requests(reader, tx));
-        if let Err(e) = self.handle_replies() {
-            error!(
-                "[{}] connection handling failed: {}",
-                self.addr,
-                e.display_chain().to_string()
-            );
-        }
-        debug!("[{}] shutting down connection", self.addr);
-        let _ = self.stream.shutdown(Shutdown::Both);
-        if let Err(err) = child.join().expect("receiver panicked") {
-            error!("[{}] receiver failed: {}", self.addr, err);
-        }
-    }
-}
-
-#[derive(Debug)]
-pub enum Message {
-    Request(String),
-    PeriodicUpdate,
-    Done,
-}
-
-pub enum Notification {
-    Periodic,
-    Exit,
-}
-
-pub struct RPC {
-    notification: Sender<Notification>,
-    server: Option<thread::JoinHandle<()>>, // so we can join the server while dropping this ojbect
-}
-
-struct Stats {
-    latency: HistogramVec,
-    subscriptions: Gauge,
-}
-
-impl RPC {
-    fn start_notifier(
-        notification: Channel<Notification>,
-        senders: Arc<Mutex<Vec<SyncSender<Message>>>>,
-        acceptor: Sender<Option<(TcpStream, SocketAddr)>>,
-    ) {
-        spawn_thread("notification", move || {
-            for msg in notification.receiver().iter() {
-                let mut senders = senders.lock().unwrap();
-                match msg {
-                    Notification::Periodic => for sender in senders.split_off(0) {
-                        if let Err(TrySendError::Disconnected(_)) =
-                            sender.try_send(Message::PeriodicUpdate)
-                        {
-                            continue;
-                        }
-                        senders.push(sender);
-                    },
-                    Notification::Exit => acceptor.send(None).unwrap(),
-                }
-            }
-        });
-    }
-
-    fn start_acceptor(addr: SocketAddr) -> Channel<Option<(TcpStream, SocketAddr)>> {
-        let chan = Channel::new();
-        let acceptor = chan.sender();
-        spawn_thread("acceptor", move || {
-            let listener = TcpListener::bind(addr).expect(&format!("bind({}) failed", addr));
-            info!("RPC server running on {}", addr);
-            loop {
-                let (stream, addr) = listener.accept().expect("accept failed");
-                acceptor.send(Some((stream, addr))).expect("send failed");
-            }
-        });
-        chan
-    }
-
-    pub fn start(addr: SocketAddr, query: Arc<Query>, metrics: &Metrics) -> RPC {
-        let stats = Arc::new(Stats {
-            latency: metrics.histogram_vec(
-                HistogramOpts::new("electrum_rpc", "Electrum RPC latency (seconds)"),
-                &["method"],
-            ),
-            subscriptions: metrics.gauge(MetricOpts::new(
-                "electrum_subscriptions",
-                "# of Electrum subscriptions",
-            )),
-        });
-        let notification = Channel::new();
-        let handle = RPC {
-            notification: notification.sender(),
-            server: Some(spawn_thread("rpc", move || {
-                let senders = Arc::new(Mutex::new(Vec::<SyncSender<Message>>::new()));
-                let acceptor = RPC::start_acceptor(addr);
-                RPC::start_notifier(notification, senders.clone(), acceptor.sender());
-                let mut children = vec![];
-                while let Some((stream, addr)) = acceptor.receiver().recv().unwrap() {
-                    let query = query.clone();
-                    let senders = senders.clone();
-                    let stats = stats.clone();
-                    children.push(spawn_thread("peer", move || {
-                        info!("[{}] connected peer", addr);
-                        let conn = Connection::new(query, stream, addr, stats);
-                        senders.lock().unwrap().push(conn.chan.sender());
-                        conn.run();
-                        info!("[{}] disconnected peer", addr);
-                    }));
-                }
-                trace!("closing RPC connections");
-                for sender in senders.lock().unwrap().iter() {
-                    let _ = sender.send(Message::Done);
-                }
-                for child in children {
-                    let _ = child.join();
-                }
-                trace!("RPC connections are closed");
-            })),
-        };
-        handle
-    }
-
-    pub fn notify(&self) {
-        self.notification.send(Notification::Periodic).unwrap();
-    }
-}
-
-impl Drop for RPC {
-    fn drop(&mut self) {
-        trace!("stop accepting new RPCs");
-        self.notification.send(Notification::Exit).unwrap();
-        self.server.take().map(|t| t.join().unwrap());
-        trace!("RPC server is stopped");
-    }
-}

src/store.rs

@@ -1,9 +1,13 @@
+use bincode;
 use rocksdb;
 
 use std::path::{Path, PathBuf};
 
+use config::Config;
 use util::Bytes;
 
+const DB_VERSION: u32 = 1;
+
 #[derive(Clone)]
 pub struct Row {
     pub key: Bytes,
@@ -198,3 +202,20 @@ pub fn is_fully_compacted(store: &ReadStore) -> bool {
     let marker = store.get(&full_compaction_marker().key);
     marker.is_some()
 }
+
+pub fn verify_index_compatibility(store: &DBStore, config: &Config) {
+    let compatibility_bytes =
+        bincode::serialize(&(config.network_type, DB_VERSION, config.extended_db_enabled)).unwrap();
+
+    match store.get(b"C") {
+        None => store.write(vec![Row {
+            key: b"C".to_vec(),
+            value: compatibility_bytes,
+        }]),
+        Some(x) => {
+            if x != compatibility_bytes {
+                panic!("Incompatible database found. Changing --light mode requires a reindex.");
+            }
+        }
+    }
+}

src/util.rs

@@ -1,6 +1,7 @@
-use bitcoin::blockdata::block::BlockHeader;
-use bitcoin::network::serialize::BitcoinHash;
-use bitcoin::util::hash::Sha256dHash;
+use bitcoin::consensus::encode::serialize;
+use bitcoin::util::hash::{BitcoinHash, Sha256dHash};
+use elements::{Block, BlockHeader};
+use errors::*;
 use std::collections::HashMap;
 use std::fmt;
 use std::iter::FromIterator;
@@ -27,6 +28,81 @@ pub fn full_hash(hash: &[u8]) -> FullHash {
     array_ref![hash, 0, HASH_LEN].clone()
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct TransactionStatus {
+    pub confirmed: bool,
+    pub block_height: Option<usize>,
+    pub block_hash: Option<Sha256dHash>,
+}
+
+impl TransactionStatus {
+    pub fn unconfirmed() -> Self {
+        TransactionStatus {
+            confirmed: false,
+            block_height: None,
+            block_hash: None,
+        }
+    }
+    pub fn confirmed(header: &HeaderEntry) -> Self {
+        TransactionStatus {
+            confirmed: true,
+            block_height: Some(header.height()),
+            block_hash: Some(header.hash().clone()),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BlockStatus {
+    pub in_best_chain: bool,
+    pub height: Option<usize>,
+    pub next_best: Option<Sha256dHash>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BlockMeta {
+    pub tx_count: u32,
+    pub size: u32,
+    pub weight: u32,
+}
+
+pub struct BlockHeaderMeta {
+    pub header_entry: HeaderEntry,
+    pub meta: BlockMeta,
+}
+
+impl<'a> From<&'a Block> for BlockMeta {
+    fn from(block: &'a Block) -> BlockMeta {
+        BlockMeta {
+            tx_count: block.txdata.len() as u32,
+            size: serialize(block).len() as u32,
+            weight: block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
+        }
+    }
+}
+
+impl BlockMeta {
+    pub fn parse_getblock(val: ::serde_json::Value) -> Result<BlockMeta> {
+        Ok(BlockMeta {
+            tx_count: val
+                .get("nTx")
+                .chain_err(|| "missing nTx")?
+                .as_f64()
+                .chain_err(|| "nTx not a number")? as u32,
+            size: val
+                .get("size")
+                .chain_err(|| "missing size")?
+                .as_f64()
+                .chain_err(|| "size not a number")? as u32,
+            weight: val
+                .get("weight")
+                .chain_err(|| "missing weight")?
+                .as_f64()
+                .chain_err(|| "weight not a number")? as u32,
+        })
+    }
+}
+
 #[derive(Eq, PartialEq, Clone)]
 pub struct HeaderEntry {
     height: usize,
@@ -114,7 +190,8 @@ impl HeaderList {
                 height: height,
                 hash: hashed_header.blockhash,
                 header: hashed_header.header,
-            }).collect()
+            })
+            .collect()
     }
 
     pub fn apply(&mut self, new_headers: Vec<HeaderEntry>) {
@@ -254,3 +331,108 @@ where
         .spawn(f)
         .unwrap()
 }
+
+use bitcoin::util::hash::Hash160;
+use bitcoin::Script;
+use bitcoin_bech32::constants::Network as B32Network;
+use bitcoin_bech32::{u5, WitnessProgram};
+use daemon::Network;
+use utils::address::{Address, Payload};
+
+// @XXX we can't use any of the Address:p2{...}h utility methods, since they expect the pre-image data, which we don't have.
+// we must instead create the Payload manually, which results in code duplication with the p2{...}h methods, especially for witness programs.
+// ideally, this should be implemented as part of the rust-bitcoin lib.
+pub fn script_to_address(script: &Script, network: &Network) -> Option<String> {
+    let payload = if script.is_p2pkh() {
+        Some(Payload::PubkeyHash(Hash160::from(&script[3..23])))
+    } else if script.is_p2sh() {
+        Some(Payload::ScriptHash(Hash160::from(&script[2..22])))
+    } else if script.is_v0_p2wpkh() {
+        Some(Payload::WitnessProgram(
+            WitnessProgram::new(
+                u5::try_from_u8(0).expect("0<32"),
+                script[2..22].to_vec(),
+                B32Network::from(network),
+            )
+            .unwrap(),
+        ))
+    } else if script.is_v0_p2wsh() {
+        Some(Payload::WitnessProgram(
+            WitnessProgram::new(
+                u5::try_from_u8(0).expect("0<32"),
+                script[2..34].to_vec(),
+                B32Network::from(network),
+            )
+            .unwrap(),
+        ))
+    } else {
+        None
+    };
+
+    Some(
+        Address {
+            payload: payload?,
+            network: *network,
+        }
+        .to_string(),
+    )
+}
+
+use bitcoin::blockdata::script::Instruction::PushBytes;
+use hex;
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct PegOutRequest {
+    pub genesis_hash: String,
+    pub scriptpubkey: Script,
+    pub scriptpubkey_asm: String,
+    pub scriptpubkey_address: Option<String>,
+}
+
+impl PegOutRequest {
+    pub fn parse(
+        script: &Script,
+        parent_network: &Network,
+        parent_genesis_hash: &str,
+    ) -> Option<PegOutRequest> {
+        if !script.is_op_return() {
+            return None;
+        }
+
+        let nulldata: Vec<_> = script.iter(true).skip(1).collect();
+        if nulldata.len() < 2 {
+            return None;
+        }
+
+        let genesis_hash = if let PushBytes(data) = nulldata[0] {
+            hex::encode(data.to_vec())
+        } else {
+            return None;
+        };
+
+        let scriptpubkey = if let PushBytes(data) = nulldata[1] {
+            Script::from(data.to_vec())
+        } else {
+            return None;
+        };
+
+        if genesis_hash != parent_genesis_hash {
+            return None;
+        }
+
+        let scriptpubkey_asm = get_script_asm(&scriptpubkey);
+        let scriptpubkey_address = script_to_address(&scriptpubkey, parent_network);
+
+        Some(PegOutRequest {
+            genesis_hash,
+            scriptpubkey,
+            scriptpubkey_asm,
+            scriptpubkey_address,
+        })
+    }
+}
+
+pub fn get_script_asm(script: &Script) -> String {
+    let asm = format!("{:?}", script);
+    (&asm[7..asm.len() - 1]).to_string()
+}

src/utils/address.rs

@@ -0,0 +1,349 @@
+// Rust Bitcoin Library
+// Written in 2014 by
+//     Andrew Poelstra <apoelstra@wpsoftware.net>
+// To the extent possible under law, the author(s) have dedicated all
+// copyright and related and neighboring rights to this software to
+// the public domain worldwide. This software is distributed without
+// any warranty.
+//
+// You should have received a copy of the CC0 Public Domain Dedication
+// along with this software.
+// If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+//
+
+//! Addresses
+//!
+//! Support for ordinary base58 Bitcoin addresses and private keys
+//!
+
+use std::fmt::{self, Display, Formatter};
+use std::str::FromStr;
+
+use bitcoin_bech32::{self, u5, WitnessProgram};
+use secp256k1::key::PublicKey;
+
+use bitcoin::blockdata::opcodes;
+use bitcoin::blockdata::script;
+use bitcoin::consensus::encode;
+use bitcoin::util::base58;
+use bitcoin::util::hash::Hash160;
+use daemon::Network;
+
+/// The method used to produce an address
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Payload {
+    /// pay-to-pubkey
+    Pubkey(PublicKey),
+    /// pay-to-pkhash address
+    PubkeyHash(Hash160),
+    /// P2SH address
+    ScriptHash(Hash160),
+    /// Segwit address
+    WitnessProgram(WitnessProgram),
+}
+
+// Originally was: #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, PartialEq, Hash)]
+/// A Bitcoin address
+pub struct Address {
+    /// The type of the address
+    pub payload: Payload,
+    /// The network on which this address is usable
+    pub network: Network,
+}
+
+impl Address {
+    /// Creates a pay to (compressed) public key hash address from a public key
+    /// This is the preferred non-witness type address
+    #[inline]
+    pub fn p2pkh(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::PubkeyHash(Hash160::from_data(&pk.serialize()[..])),
+        }
+    }
+
+    /// Creates a pay to uncompressed public key hash address from a public key
+    /// This address type is discouraged as it uses more space but otherwise equivalent to p2pkh
+    /// therefore only adds ambiguity
+    #[inline]
+    pub fn p2upkh(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::PubkeyHash(Hash160::from_data(&pk.serialize_uncompressed()[..])),
+        }
+    }
+
+    /// Creates a pay to public key address from a public key
+    /// This address type was used in the early history of Bitcoin.
+    /// Satoshi's coins are still on addresses of this type.
+    #[inline]
+    pub fn p2pk(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::Pubkey(*pk),
+        }
+    }
+
+    /// Creates a pay to script hash P2SH address from a script
+    /// This address type was introduced with BIP16 and is the popular ty implement multi-sig these days.
+    #[inline]
+    pub fn p2sh(script: &script::Script, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::ScriptHash(Hash160::from_data(&script[..])),
+        }
+    }
+
+    /// Create a witness pay to public key address from a public key
+    /// This is the native segwit address type for an output redemable with a single signature
+    pub fn p2wpkh(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::WitnessProgram(
+                // unwrap is safe as witness program is known to be correct as above
+                WitnessProgram::new(
+                    u5::try_from_u8(0).expect("0<32"),
+                    Hash160::from_data(&pk.serialize()[..])[..].to_vec(),
+                    Address::bech_network(network),
+                )
+                .unwrap(),
+            ),
+        }
+    }
+
+    /// Create a pay to script address that embeds a witness pay to public key
+    /// This is a segwit address type that looks familiar (as p2sh) to legacy clients
+    pub fn p2shwpkh(pk: &PublicKey, network: Network) -> Address {
+        let builder = script::Builder::new()
+            .push_int(0)
+            .push_slice(&Hash160::from_data(&pk.serialize()[..])[..]);
+        Address {
+            network: network,
+            payload: Payload::ScriptHash(Hash160::from_data(builder.into_script().as_bytes())),
+        }
+    }
+
+    /// Create a witness pay to script hash address
+    pub fn p2wsh(script: &script::Script, network: Network) -> Address {
+        use crypto::digest::Digest;
+        use crypto::sha2::Sha256;
+
+        let mut digest = Sha256::new();
+        digest.input(script.as_bytes());
+        let mut d = [0u8; 32];
+        digest.result(&mut d);
+
+        Address {
+            network: network,
+            payload: Payload::WitnessProgram(
+                // unwrap is safe as witness program is known to be correct as above
+                WitnessProgram::new(
+                    u5::try_from_u8(0).expect("0<32"),
+                    d.to_vec(),
+                    Address::bech_network(network),
+                )
+                .unwrap(),
+            ),
+        }
+    }
+
+    /// Create a pay to script address that embeds a witness pay to script hash address
+    /// This is a segwit address type that looks familiar (as p2sh) to legacy clients
+    pub fn p2shwsh(script: &script::Script, network: Network) -> Address {
+        use crypto::digest::Digest;
+        use crypto::sha2::Sha256;
+
+        let mut digest = Sha256::new();
+        digest.input(script.as_bytes());
+        let mut d = [0u8; 32];
+        digest.result(&mut d);
+        let ws = script::Builder::new()
+            .push_int(0)
+            .push_slice(&d)
+            .into_script();
+
+        Address {
+            network: network,
+            payload: Payload::ScriptHash(Hash160::from_data(ws.as_bytes())),
+        }
+    }
+
+    #[inline]
+    /// convert Network to bech32 network (this should go away soon)
+    fn bech_network(network: Network) -> bitcoin_bech32::constants::Network {
+        match network {
+            Network::Bitcoin => bitcoin_bech32::constants::Network::Bitcoin,
+            Network::Testnet => bitcoin_bech32::constants::Network::Testnet,
+            Network::Regtest => bitcoin_bech32::constants::Network::Regtest,
+            // this should never actually happen, Liquid does not have bech32 addresses
+            Network::Liquid | Network::LiquidV1 | Network::LiquidRegtest => {
+                bitcoin_bech32::constants::Network::Bitcoin
+            }
+        }
+    }
+
+    /// Generates a script pubkey spending to this address
+    pub fn script_pubkey(&self) -> script::Script {
+        match self.payload {
+            Payload::Pubkey(ref pk) => script::Builder::new()
+                .push_slice(&pk.serialize_uncompressed()[..])
+                .push_opcode(opcodes::All::OP_CHECKSIG),
+            Payload::PubkeyHash(ref hash) => script::Builder::new()
+                .push_opcode(opcodes::All::OP_DUP)
+                .push_opcode(opcodes::All::OP_HASH160)
+                .push_slice(&hash[..])
+                .push_opcode(opcodes::All::OP_EQUALVERIFY)
+                .push_opcode(opcodes::All::OP_CHECKSIG),
+            Payload::ScriptHash(ref hash) => script::Builder::new()
+                .push_opcode(opcodes::All::OP_HASH160)
+                .push_slice(&hash[..])
+                .push_opcode(opcodes::All::OP_EQUAL),
+            Payload::WitnessProgram(ref witprog) => script::Builder::new()
+                .push_int(witprog.version().to_u8() as i64)
+                .push_slice(witprog.program()),
+        }
+        .into_script()
+    }
+}
+
+impl Display for Address {
+    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
+        match self.payload {
+            // note: serialization for pay-to-pk is defined, but is irreversible
+            Payload::Pubkey(ref pk) => {
+                let hash = &Hash160::from_data(&pk.serialize_uncompressed()[..]);
+                let mut prefixed = [0; 21];
+                prefixed[0] = match self.network {
+                    Network::Bitcoin => 0,
+                    Network::Testnet | Network::Regtest => 111,
+                    Network::Liquid | Network::LiquidV1 => 57,
+                    Network::LiquidRegtest => 196,
+                };
+                prefixed[1..].copy_from_slice(&hash[..]);
+                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
+            }
+            Payload::PubkeyHash(ref hash) => {
+                let mut prefixed = [0; 21];
+                prefixed[0] = match self.network {
+                    Network::Bitcoin => 0,
+                    Network::Testnet | Network::Regtest => 111,
+                    Network::Liquid | Network::LiquidV1 => 57,
+                    Network::LiquidRegtest => 235,
+                };
+                prefixed[1..].copy_from_slice(&hash[..]);
+                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
+            }
+            Payload::ScriptHash(ref hash) => {
+                let mut prefixed = [0; 21];
+                prefixed[0] = match self.network {
+                    Network::Bitcoin => 5,
+                    Network::Testnet | Network::Regtest => 196,
+                    Network::Liquid | Network::LiquidV1 => 39,
+                    Network::LiquidRegtest => 75,
+                };
+                prefixed[1..].copy_from_slice(&hash[..]);
+                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
+            }
+            Payload::WitnessProgram(ref witprog) => fmt.write_str(&witprog.to_address()),
+        }
+    }
+}
+
+impl FromStr for Address {
+    type Err = encode::Error;
+
+    fn from_str(s: &str) -> Result<Address, encode::Error> {
+        // bech32 (note that upper or lowercase is allowed but NOT mixed case)
+        if s.starts_with("bc1")
+            || s.starts_with("BC1")
+            || s.starts_with("tb1")
+            || s.starts_with("TB1")
+            || s.starts_with("bcrt1")
+            || s.starts_with("BCRT1")
+        {
+            let witprog = WitnessProgram::from_address(s)?;
+            let network = match witprog.network() {
+                bitcoin_bech32::constants::Network::Bitcoin => Network::Bitcoin,
+                bitcoin_bech32::constants::Network::Testnet => Network::Testnet,
+                bitcoin_bech32::constants::Network::Regtest => Network::Regtest,
+                _ => panic!("unknown network"),
+            };
+            if witprog.version().to_u8() != 0 {
+                return Err(encode::Error::UnsupportedWitnessVersion(
+                    witprog.version().to_u8(),
+                ));
+            }
+            return Ok(Address {
+                network: network,
+                payload: Payload::WitnessProgram(witprog),
+            });
+        }
+
+        if s.len() > 50 {
+            return Err(encode::Error::Base58(base58::Error::InvalidLength(
+                s.len() * 11 / 15,
+            )));
+        }
+
+        // Base 58
+        let data = base58::from_check(s)?;
+
+        if data.len() != 21 {
+            return Err(encode::Error::Base58(base58::Error::InvalidLength(
+                data.len(),
+            )));
+        }
+
+        let (network, payload) = match data[0] {
+            0 => (
+                Network::Bitcoin,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            5 => (
+                Network::Bitcoin,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            111 => (
+                Network::Testnet,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            196 => (
+                Network::Testnet,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            57 => (
+                Network::LiquidV1,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            39 => (
+                Network::LiquidV1,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            235 => (
+                Network::LiquidRegtest,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            75 => (
+                Network::LiquidRegtest,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            x => {
+                return Err(encode::Error::Base58(base58::Error::InvalidVersion(vec![
+                    x,
+                ])))
+            }
+        };
+
+        Ok(Address {
+            network: network,
+            payload: payload,
+        })
+    }
+}
+
+impl ::std::fmt::Debug for Address {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}

src/utils/mod.rs

@@ -0,0 +1 @@
+pub mod address;