use base64; use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::blockdata::transaction::Transaction; use bitcoin::consensus::encode::{deserialize, serialize}; use bitcoin::network::constants::Network; use bitcoin::util::hash::BitcoinHash; use bitcoin::util::hash::Sha256dHash; use glob; use hex; use serde_json::{from_str, from_value, Value}; use std::collections::{HashMap, HashSet}; use std::io::{BufRead, BufReader, Lines, Write}; use std::net::{SocketAddr, TcpStream}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; use metrics::{HistogramOpts, HistogramVec, Metrics}; use signal::Waiter; use util::HeaderList; use errors::*; fn parse_hash(value: &Value) -> Result { Ok(Sha256dHash::from_hex( value .as_str() .chain_err(|| format!("non-string value: {}", value))?, ).chain_err(|| format!("non-hex value: {}", value))?) } fn header_from_value(value: Value) -> Result { let header_hex = value .as_str() .chain_err(|| format!("non-string header: {}", value))?; let header_bytes = hex::decode(header_hex).chain_err(|| "non-hex header")?; Ok( deserialize(&header_bytes) .chain_err(|| format!("failed to parse header {}", header_hex))?, ) } fn block_from_value(value: Value) -> Result { let block_hex = value.as_str().chain_err(|| "non-string block")?; let block_bytes = hex::decode(block_hex).chain_err(|| "non-hex block")?; Ok(deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))?) } fn tx_from_value(value: Value) -> Result { let tx_hex = value.as_str().chain_err(|| "non-string tx")?; let tx_bytes = hex::decode(tx_hex).chain_err(|| "non-hex tx")?; Ok(deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))?) } /// Parse JSONRPC error code, if exists. fn parse_error_code(err: &Value) -> Option { err.as_object()?.get("code")?.as_i64() } fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Result { if let Some(reply_obj) = reply.as_object_mut() { if let Some(err) = reply_obj.get("error") { if !err.is_null() { if let Some(code) = parse_error_code(&err) { match code { // RPC_IN_WARMUP -> retry by later reconnection -28 => bail!(ErrorKind::Connection(err.to_string())), _ => (), } } bail!("{} RPC error: {}", method, err); } } let id = reply_obj .get("id") .chain_err(|| format!("no id in reply: {:?}", reply_obj))? .clone(); if id != expected_id { bail!( "wrong {} response id {}, expected {}", method, id, expected_id ); } if let Some(result) = reply_obj.get_mut("result") { return Ok(result.take()); } bail!("no result in reply: {:?}", reply_obj); } bail!("non-object reply: {:?}", reply); } #[derive(Serialize, Deserialize, Debug)] pub struct BlockchainInfo { pub chain: String, pub blocks: u32, pub headers: u32, pub bestblockhash: String, pub pruned: bool, pub verificationprogress: f32, pub initialblockdownload: bool, } #[derive(Serialize, Deserialize, Debug)] struct NetworkInfo { version: u64, subversion: String, } pub struct MempoolEntry { fee: u64, // in satoshis vsize: u32, // in virtual bytes (= weight/4) fee_per_vbyte: f32, } impl MempoolEntry { fn new(fee: u64, vsize: u32) -> MempoolEntry { MempoolEntry { fee, vsize, fee_per_vbyte: fee as f32 / vsize as f32, } } pub fn fee_per_vbyte(&self) -> f32 { self.fee_per_vbyte } pub fn fee(&self) -> u64 { self.fee } pub fn vsize(&self) -> u32 { self.vsize } } pub trait CookieGetter: Send + Sync { fn get(&self) -> Result>; } struct Connection { tx: TcpStream, rx: Lines>, cookie_getter: Arc, addr: SocketAddr, signal: Waiter, } fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result { loop { match TcpStream::connect(addr) { Ok(conn) => return Ok(conn), Err(err) => { warn!("failed to connect daemon at {}: {}", addr, err); signal.wait(Duration::from_secs(3))?; continue; } } } } impl Connection { fn new( addr: SocketAddr, cookie_getter: Arc, signal: Waiter, ) -> Result { let conn = tcp_connect(addr, &signal)?; let reader = BufReader::new( conn.try_clone() .chain_err(|| format!("failed to clone {:?}", conn))?, ); Ok(Connection { tx: conn, rx: reader.lines(), cookie_getter, addr, signal, }) } fn reconnect(&self) -> Result { Connection::new(self.addr, self.cookie_getter.clone(), self.signal.clone()) } fn send(&mut self, request: &str) -> Result<()> { let cookie = &self.cookie_getter.get()?; let msg = format!( "POST / HTTP/1.1\nAuthorization: Basic {}\nContent-Length: {}\n\n{}", base64::encode(cookie), request.len(), request, ); self.tx.write_all(msg.as_bytes()).chain_err(|| { ErrorKind::Connection("disconnected from daemon while sending".to_owned()) }) } fn recv(&mut self) -> Result { // TODO: use proper HTTP parser. let mut in_header = true; let mut contents: Option = None; let iter = self.rx.by_ref(); let status = iter .next() .chain_err(|| { ErrorKind::Connection("disconnected from daemon while receiving".to_owned()) })?.chain_err(|| "failed to read status")?; let mut headers = HashMap::new(); for line in iter { let line = line.chain_err(|| ErrorKind::Connection("failed to read".to_owned()))?; if line.is_empty() { in_header = false; // next line should contain the actual response. } else if in_header { let parts: Vec<&str> = line.splitn(2, ": ").collect(); if parts.len() == 2 { headers.insert(parts[0].to_owned(), parts[1].to_owned()); } else { warn!("invalid header: {:?}", line); } } else { contents = Some(line); break; } } let contents = contents.chain_err(|| ErrorKind::Connection("no reply from daemon".to_owned()))?; let contents_length: &str = headers .get("Content-Length") .chain_err(|| format!("Content-Length is missing: {:?}", headers))?; let contents_length: usize = contents_length .parse() .chain_err(|| format!("invalid Content-Length: {:?}", contents_length))?; let expected_length = contents_length - 1; // trailing EOL is skipped if expected_length != contents.len() { bail!(ErrorKind::Connection(format!( "expected {} bytes, got {}", expected_length, contents.len() ))); } Ok(if status == "HTTP/1.1 200 OK" { contents } else if status == "HTTP/1.1 500 Internal Server Error" { warn!("HTTP status: {}", status); contents // the contents should have a JSONRPC error field } else { bail!( "request failed {:?}: {:?} = {:?}", status, headers, contents ); }) } } struct Counter { value: Mutex, } impl Counter { fn new() -> Self { Counter { value: Mutex::new(0), } } fn next(&self) -> u64 { let mut value = self.value.lock().unwrap(); *value += 1; *value } } pub struct Daemon { daemon_dir: PathBuf, network: Network, conn: Mutex, message_id: Counter, // for monotonic JSONRPC 'id' signal: Waiter, // monitoring latency: HistogramVec, size: HistogramVec, } impl Daemon { pub fn new( daemon_dir: &PathBuf, daemon_rpc_addr: SocketAddr, cookie_getter: Arc, network: Network, signal: Waiter, metrics: &Metrics, ) -> Result { let daemon = Daemon { daemon_dir: daemon_dir.clone(), network, conn: Mutex::new(Connection::new( daemon_rpc_addr, cookie_getter, signal.clone(), )?), message_id: Counter::new(), signal: signal.clone(), latency: metrics.histogram_vec( HistogramOpts::new("daemon_rpc", "Bitcoind RPC latency (in seconds)"), &["method"], ), size: metrics.histogram_vec( HistogramOpts::new("daemon_bytes", "Bitcoind RPC size (in bytes)"), &["method", "dir"], ), }; let network_info = daemon.getnetworkinfo()?; info!("{:?}", network_info); if network_info.version < 00_16_00_00 { bail!( "{} is not supported - please use bitcoind 0.16+", network_info.subversion, ) } let blockchain_info = daemon.getblockchaininfo()?; info!("{:?}", blockchain_info); if blockchain_info.pruned == true { bail!("pruned node is not supported (use '-prune=0' bitcoind flag)".to_owned()) } loop { if daemon.getblockchaininfo()?.initialblockdownload == false { break; } warn!("wait until bitcoind is synced (i.e. initialblockdownload = false)"); signal.wait(Duration::from_secs(3))?; } Ok(daemon) } pub fn reconnect(&self) -> Result { Ok(Daemon { daemon_dir: self.daemon_dir.clone(), network: self.network, conn: Mutex::new(self.conn.lock().unwrap().reconnect()?), message_id: Counter::new(), signal: self.signal.clone(), latency: self.latency.clone(), size: self.size.clone(), }) } pub fn list_blk_files(&self) -> Result> { let mut path = self.daemon_dir.clone(); path.push("blocks"); path.push("blk*.dat"); info!("listing block files at {:?}", path); let mut paths: Vec = glob::glob(path.to_str().unwrap()) .chain_err(|| "failed to list blk*.dat files")? .map(|res| res.unwrap()) .collect(); paths.sort(); Ok(paths) } pub fn magic(&self) -> u32 { self.network.magic() } fn call_jsonrpc(&self, method: &str, request: &Value) -> Result { let mut conn = self.conn.lock().unwrap(); let timer = self.latency.with_label_values(&[method]).start_timer(); let request = request.to_string(); conn.send(&request)?; self.size .with_label_values(&[method, "send"]) .observe(request.len() as f64); let response = conn.recv()?; let result: Value = from_str(&response).chain_err(|| "invalid JSON")?; timer.observe_duration(); self.size .with_label_values(&[method, "recv"]) .observe(response.len() as f64); Ok(result) } fn handle_request_batch(&self, method: &str, params_list: &[Value]) -> Result> { let id = self.message_id.next(); let reqs = params_list .iter() .map(|params| json!({"method": method, "params": params, "id": id})) .collect(); let mut results = vec![]; let mut replies = self.call_jsonrpc(method, &reqs)?; if let Some(replies_vec) = replies.as_array_mut() { for reply in replies_vec { results.push(parse_jsonrpc_reply(reply.take(), method, id)?) } return Ok(results); } bail!("non-array replies: {:?}", replies); } fn retry_request_batch(&self, method: &str, params_list: &[Value]) -> Result> { loop { match self.handle_request_batch(method, params_list) { Err(Error(ErrorKind::Connection(msg), _)) => { warn!("reconnecting to bitcoind: {}", msg); self.signal.wait(Duration::from_secs(3))?; let mut conn = self.conn.lock().unwrap(); *conn = conn.reconnect()?; continue; } result => return result, } } } fn request(&self, method: &str, params: Value) -> Result { let mut values = self.retry_request_batch(method, &[params])?; assert_eq!(values.len(), 1); Ok(values.remove(0)) } fn requests(&self, method: &str, params_list: &[Value]) -> Result> { self.retry_request_batch(method, params_list) } // bitcoind JSONRPC API: pub fn getblockchaininfo(&self) -> Result { let info: Value = self.request("getblockchaininfo", json!([]))?; Ok(from_value(info).chain_err(|| "invalid blockchain info")?) } fn getnetworkinfo(&self) -> Result { let info: Value = self.request("getnetworkinfo", json!([]))?; Ok(from_value(info).chain_err(|| "invalid network info")?) } pub fn getbestblockhash(&self) -> Result { parse_hash(&self.request("getbestblockhash", json!([]))?).chain_err(|| "invalid blockhash") } pub fn getblockheader(&self, blockhash: &Sha256dHash) -> Result { header_from_value(self.request( "getblockheader", json!([blockhash.be_hex_string(), /*verbose=*/ false]), )?) } pub fn getblockheaders(&self, heights: &[usize]) -> Result> { let heights: Vec = heights.iter().map(|height| json!([height])).collect(); let params_list: Vec = self .requests("getblockhash", &heights)? .into_iter() .map(|hash| json!([hash, /*verbose=*/ false])) .collect(); let mut result = vec![]; for h in self.requests("getblockheader", ¶ms_list)? { result.push(header_from_value(h)?); } Ok(result) } pub fn getblock(&self, blockhash: &Sha256dHash) -> Result { let block = block_from_value(self.request( "getblock", json!([blockhash.be_hex_string(), /*verbose=*/ false]), )?)?; assert_eq!(block.bitcoin_hash(), *blockhash); Ok(block) } pub fn getblocks(&self, blockhashes: &[Sha256dHash]) -> Result> { let params_list: Vec = blockhashes .iter() .map(|hash| json!([hash.be_hex_string(), /*verbose=*/ false])) .collect(); let values = self.requests("getblock", ¶ms_list)?; let mut blocks = vec![]; for value in values { blocks.push(block_from_value(value)?); } Ok(blocks) } pub fn gettransaction( &self, txhash: &Sha256dHash, blockhash: Option, ) -> Result { let mut args = json!([txhash.be_hex_string(), /*verbose=*/ false]); if let Some(blockhash) = blockhash { args.as_array_mut() .unwrap() .push(json!(blockhash.be_hex_string())); } tx_from_value(self.request("getrawtransaction", args)?) } pub fn gettransaction_raw( &self, txhash: &Sha256dHash, blockhash: Option, verbose: bool, ) -> Result { let mut args = json!([txhash.be_hex_string(), verbose]); if let Some(blockhash) = blockhash { args.as_array_mut() .unwrap() .push(json!(blockhash.be_hex_string())); } Ok(self.request("getrawtransaction", args)?) } pub fn gettransactions(&self, txhashes: &[&Sha256dHash]) -> Result> { let params_list: Vec = txhashes .iter() .map(|txhash| json!([txhash.be_hex_string(), /*verbose=*/ false])) .collect(); let values = self.requests("getrawtransaction", ¶ms_list)?; let mut txs = vec![]; for value in values { txs.push(tx_from_value(value)?); } assert_eq!(txhashes.len(), txs.len()); Ok(txs) } pub fn getmempooltxids(&self) -> Result> { let txids: Value = self.request("getrawmempool", json!([/*verbose=*/ false]))?; let mut result = HashSet::new(); for value in txids.as_array().chain_err(|| "non-array result")? { result.insert(parse_hash(&value).chain_err(|| "invalid txid")?); } Ok(result) } pub fn getmempoolentry(&self, txid: &Sha256dHash) -> Result { let entry = self.request("getmempoolentry", json!([txid.be_hex_string()]))?; let fee = (entry .get("fee") .chain_err(|| "missing fee")? .as_f64() .chain_err(|| "non-float fee")? * 100_000_000f64) as u64; let vsize = entry .get("size") .chain_err(|| "missing size")? .as_u64() .chain_err(|| "non-integer size")? as u32; Ok(MempoolEntry::new(fee, vsize)) } pub fn broadcast(&self, tx: &Transaction) -> Result { let tx = hex::encode(serialize(tx)); let txid = self.request("sendrawtransaction", json!([tx]))?; Ok( Sha256dHash::from_hex(txid.as_str().chain_err(|| "non-string txid")?) .chain_err(|| "failed to parse txid")?, ) } fn get_all_headers(&self, tip: &Sha256dHash) -> Result> { let info: Value = self.request("getblockheader", json!([tip.be_hex_string()]))?; let tip_height = info .get("height") .expect("missing height") .as_u64() .expect("non-numeric height") as usize; let all_heights: Vec = (0..tip_height + 1).collect(); let chunk_size = 100_000; let mut result = vec![]; let null_hash = Sha256dHash::default(); for heights in all_heights.chunks(chunk_size) { trace!("downloading {} block headers", heights.len()); let mut headers = self.getblockheaders(&heights)?; assert!(headers.len() == heights.len()); result.append(&mut headers); } let mut blockhash = null_hash; for header in &result { assert_eq!(header.prev_blockhash, blockhash); blockhash = header.bitcoin_hash(); } assert_eq!(blockhash, *tip); Ok(result) } // Returns a list of BlockHeaders in ascending height (i.e. the tip is last). pub fn get_new_headers( &self, indexed_headers: &HeaderList, bestblockhash: &Sha256dHash, ) -> Result> { // Iterate back over headers until known blockash is found: if indexed_headers.len() == 0 { return self.get_all_headers(bestblockhash); } debug!( "downloading new block headers ({} already indexed) from {}", indexed_headers.len(), bestblockhash, ); let mut new_headers = vec![]; let null_hash = Sha256dHash::default(); let mut blockhash = *bestblockhash; while blockhash != null_hash { if indexed_headers.header_by_blockhash(&blockhash).is_some() { break; } let header = self .getblockheader(&blockhash) .chain_err(|| format!("failed to get {} header", blockhash))?; new_headers.push(header); blockhash = header.prev_blockhash; } trace!("downloaded {} block headers", new_headers.len()); new_headers.reverse(); // so the tip is the last vector entry Ok(new_headers) } }