Compare commits

..

No commits in common. "mempool" and "update_libs" have entirely different histories.

60 changed files with 14569 additions and 12466 deletions

View File

@ -1 +0,0 @@
Dockerfile

View File

@ -1,6 +0,0 @@
# see https://editorconfig.org for more options, and setup instructions for yours editor
[*]
indent_style = space
indent_size = 4

View File

@ -1,50 +0,0 @@
name: CI Rust Setup
description: 'Sets up the environment for Rust jobs during CI workflow'
inputs:
cache-name:
description: 'Name of cache artifacts (same name is same cache key) empty to disable cache'
required: false
targets:
description: 'A comma separated list of extra targets you want to install'
required: false
components:
description: 'A comma separated list of extra components you want to install'
required: false
toolchain:
description: 'The toolchain to use. If not specified, the rust-toolchain file will be used'
required: false
runs:
using: composite
steps:
- name: Get toolchain from input OR rust-toolchain file
id: gettoolchain
shell: bash
run: |-
RUST_TOOLCHAIN="${{ inputs.toolchain }}"
if [ ! -f rust-toolchain ] && [ -z "${RUST_TOOLCHAIN}" ]; then
echo "***ERROR*** NEED toolchain INPUT OR rust-toolchain FILE IN ROOT OF REPOSITORY" >&2
exit 1
fi
if [ -z "${RUST_TOOLCHAIN}" ]; then
RUST_TOOLCHAIN="$(cat rust-toolchain)"
fi
echo "toolchain=\"${RUST_TOOLCHAIN}\"" >> $GITHUB_OUTPUT
- name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
id: toolchain
# Commit date is Nov 18, 2024
uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa
with:
toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
targets: ${{ inputs.targets }}
components: ${{ inputs.components }}
- name: Cache dependencies
uses: actions/cache@v3
if: inputs.cache-name != ''
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ inputs.cache-name }}-${{ steps.toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }}

View File

@ -1,107 +0,0 @@
on:
pull_request:
push:
branches:
- mempool
name: Compile Check and Lint
jobs:
check:
name: Compile Check
runs-on: mempool-ci
steps:
- uses: actions/checkout@v4
- name: Setup Rust
uses: './.github/actions/ci-rust-setup'
with:
cache-name: dev
- run: cargo check --all-features
fmt:
name: Formatter
runs-on: mempool-ci
steps:
- uses: actions/checkout@v4
- name: Setup Rust
uses: './.github/actions/ci-rust-setup'
with:
components: rustfmt
- run: cargo fmt --all -- --check
test:
name: Run Tests
runs-on: mempool-ci
steps:
- uses: actions/checkout@v4
- name: Setup Rust
uses: './.github/actions/ci-rust-setup'
with:
cache-name: test
- run: cargo test --lib --all-features
compile-freebsd:
runs-on: mempool-ci
name: Run Compile Checks in FreeBSD
env:
FREEBSD_VER: "14.3"
steps:
- uses: actions/checkout@v4
- name: Cache dependencies for FreeBSD
uses: actions/cache@v3
with:
path: |
.cargohome/registry
.cargohome/git
target
key: freebsd-${{ env.FREEBSD_VER }}-cargo-checks-${{ hashFiles('**/Cargo.lock') }}
- name: Compile Checks in FreeBSD
uses: vmactions/freebsd-vm@v1
with:
usesh: true
release: "${{ env.FREEBSD_VER }}"
arch: amd64
prepare: |
mkdir -p ~/.cargo/
mkdir -p ./.cargohome/registry/
mkdir -p ./.cargohome/git/
mv ./.cargohome/registry ~/.cargo/
mv ./.cargohome/git ~/.cargo/
rm -rf ./.cargohome
pkg install -y git rsync gmake llvm rust rocksdb cmake
run: |
cargo check --no-default-features
cargo check -F liquid
cargo check -F electrum-discovery
cargo check -F electrum-discovery,liquid
cargo build --release --bin electrs
rm -rf ./.cargohome
mkdir -p ~/.cargo/registry/
mkdir -p ~/.cargo/git/
mkdir -p ./.cargohome/
mv ~/.cargo/registry ./.cargohome/
mv ~/.cargo/git ./.cargohome/
clippy:
name: Linter
runs-on: mempool-ci
needs: [check]
strategy:
matrix: # Try all combinations of features. Some times weird things appear.
features: [
'',
'-F electrum-discovery',
'-F liquid',
'-F electrum-discovery,liquid',
]
steps:
- uses: actions/checkout@v4
- name: Setup Rust
uses: './.github/actions/ci-rust-setup'
with:
cache-name: dev
components: clippy
- name: Clippy with Features = ${{ matrix.features }}
run: cargo clippy ${{ matrix.features }} -- -D warnings

View File

@ -1,77 +0,0 @@
name: Docker build on tag
env:
DOCKER_CLI_EXPERIMENTAL: enabled
TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
DOCKER_BUILDKIT: 0
COMPOSE_DOCKER_CLI_BUILD: 0
on:
push:
tags:
- v[0-9]+.[0-9]+.[0-9]+
- v[0-9]+.[0-9]+.[0-9]+-*
permissions:
contents: read
jobs:
build:
runs-on: mempool-ci
timeout-minutes: 120
name: Build and push to DockerHub
strategy:
max-parallel: 1
matrix:
include:
- image: electrs
cargo_extra_args: ""
- image: electrs-liquid
cargo_extra_args: "--features liquid"
steps:
- name: Set env variables
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
- name: Show set environment variables
run: |
printf " TAG: %s\n" "$TAG"
- name: Add SHORT_SHA env property with commit short sha
run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
- name: Login to Docker for building
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
- name: Checkout project
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
id: qemu
- name: Setup Docker buildx action
uses: docker/setup-buildx-action@v3
id: buildx
- name: Available platforms
run: echo ${{ steps.buildx.outputs.platforms }}
- name: Cache Docker layers
uses: actions/cache@v3
id: cache
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx
restore-keys: |
${{ runner.os }}-buildx
- name: Run Docker buildx against tag
run: |
docker buildx build \
--cache-from "type=local,src=/tmp/.buildx-cache" \
--cache-to "type=local,dest=/tmp/.buildx-cache" \
--platform linux/amd64,linux/arm64 \
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:$TAG \
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:latest \
--output "type=registry" . \
--build-arg commitHash=$SHORT_SHA \
--build-arg CARGO_EXTRA_ARGS="${{ matrix.cargo_extra_args }}"

View File

@ -1,18 +0,0 @@
name: Project Board Automation
on:
pull_request:
types: [review_requested]
issues:
types: [opened]
jobs:
project-automation:
uses: mempool/.github/.github/workflows/project-board-automation.yml@master
with:
project-number: 8
secrets:
PROJECT_TOKEN: ${{ secrets.PROJECT_TOKEN }}
PROJECT_ID: ${{ secrets.PROJECT_ID }}
STATUS_FIELD_ID: ${{ secrets.STATUS_FIELD_ID }}
REVIEW_NEEDED_OPTION_ID: ${{ secrets.REVIEW_NEEDED_OPTION_ID }}

2
.gitignore vendored
View File

@ -4,5 +4,3 @@ target
*.sublime*
*~
*.pyc
.vscode
*.core

View File

@ -1,19 +0,0 @@
# electrs
## Rules
1. You are an expert Rust developer.
2. You are an expert Bitcoin developer.
3. If you are unsure of a change, ask the developer to make a choice proactively.
## Before testing
- Run cargo fmt (from root)
- command: `cargo fmt`
## Testing
- Run the checks script
- `./scripts/checks.sh`
- Run with tests only when a test is added or changed
- `INCLUDE_TESTS=1 ./scripts/checks.sh`

2525
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,69 +1,68 @@
[package]
name = "mempool-electrs"
version = "3.4.0-dev"
authors = [
"Roman Zeyde <me@romanzey.de>",
"Nadav Ivgi <nadav@shesek.info>",
"wiz <j@wiz.biz>",
"junderw <jonathan.underwood4649@gmail.com>"
]
name = "electrs"
version = "0.4.1"
authors = ["Roman Zeyde <me@romanzey.de>"]
description = "An efficient re-implementation of Electrum Server in Rust"
license = "MIT"
homepage = "https://github.com/mempool/electrs"
repository = "https://github.com/mempool/electrs"
publish = false
homepage = "https://github.com/romanz/electrs"
repository = "https://github.com/romanz/electrs"
keywords = ["bitcoin", "electrum", "server", "index", "database"]
documentation = "https://docs.rs/electrs/"
readme = "README.md"
edition = "2018"
[lib]
name = "electrs"
[features]
default = []
liquid = ["elements"]
electrum-discovery = ["electrum-client"]
[dependencies]
arrayref = "0.3.6"
base64 = "0.13.0"
bincode-do-not-use-directly = { version = "1.3.1", package = "bincode" }
bitcoin = { version = "0.32.8", features = [ "serde" ] }
bounded-vec-deque = "0.1.1"
clap = "2.33.3"
crossbeam-channel = "0.5.0"
dirs = "4.0.0"
elements = { version = "0.26.1", features = [ "serde" ], optional = true }
error-chain = "0.12.4"
arraydeque = "0.4"
arrayref = "0.3"
base64 = "0.10"
bincode = "1.0"
bitcoin-bech32 = "0.9.0"
chan = "0.1"
chan-signal = "0.3"
clap = "2.31"
dirs = "1.0"
error-chain = "0.12"
futures = "0.1"
glob = "0.3"
hex = "0.4.2"
itertools = "0.10"
lazy_static = "1.3.0"
hex = "0.3"
hyper = "0.12.27"
itertools = "0.8.0"
libc = "0.2"
log = "0.4.11"
socket2 = { version = "0.4", features = ["all"] }
num_cpus = "1.12.0"
page_size = "0.4.2"
prometheus = "0.13"
ppp = "2.3.0"
rayon = "1.5.0"
rocksdb = "0.24.0"
serde = "1.0.118"
serde_derive = "1.0.118"
serde_json = "1.0.60"
sha2 = "0.10.7"
signal-hook = "0.3"
stderrlog = "0.5.0"
time = { version = "0.3", features = ["formatting"] }
tiny_http = "0.11"
url = "2.2.0"
hyper = "0.14"
hyperlocal = "0.8"
# close to same tokio version as dependent by hyper v0.14 and hyperlocal 0.8 -- things can go awry if they mismatch
tokio = { version = "1", features = ["sync", "macros"] }
log = "0.4"
lru = "0.1"
lru-cache = "0.1.1"
num_cpus = "1.0"
page_size = "0.4"
prometheus = "0.5"
rayon = "1.0"
rocksdb = "0.12.1"
rust-crypto = "0.2"
secp256k1 = "0.12"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0"
stderrlog = "0.4.1"
sysconf = ">=0.3.4"
time = "0.1"
tiny_http = "0.6"
url = "1.0"
# optional dependencies for electrum-discovery
electrum-client = { version = "0.24.1", optional = true }
[dependencies.bitcoin]
version = "0.18"
features = ["serde"]
[dependencies.bitcoin_hashes]
version = "0.3"
features = ["serde"]
[dependencies.elements]
optional = true
version = "0.7.0"
features = ["serde-feature"] # Doesn't look to work by now
[dev-dependencies]
tempfile = "3.0"
@ -72,7 +71,3 @@ tempfile = "3.0"
lto = true
panic = 'abort'
codegen-units = 1
[patch.crates-io.electrum-client]
git = "https://github.com/mempool/rust-electrum-client"
rev = "4bbfc612d594fe23282c439d4bdc446cff01ba1c" # 0.24.1/add-peer branch

View File

@ -1,30 +1,21 @@
FROM debian:bookworm-slim AS base
FROM rust:latest
ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
RUN apt-get update
RUN apt-get install -y clang cmake
RUN apt update -qy && \
apt install -qy librocksdb-dev curl
RUN cargo install electrs
FROM base as build
RUN adduser --disabled-login --system --shell /bin/false --uid 1000 user
RUN apt install -qy git clang cmake
ENV RUSTUP_HOME=/rust
ENV CARGO_HOME=/cargo
ENV PATH=/cargo/bin:/rust/bin:$PATH
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
WORKDIR /build
COPY . .
ARG CARGO_EXTRA_ARGS=""
RUN cargo build --release --bin electrs ${CARGO_EXTRA_ARGS}
FROM base as deploy
COPY --from=build /build/target/release/electrs /bin/electrs
USER user
WORKDIR /home/user
# Electrum RPC
EXPOSE 50001
ENTRYPOINT ["/bin/electrs"]
# Prometheus monitoring
EXPOSE 4224
STOPSIGNAL SIGINT
CMD ["electrs", "-vvvv", "--timestamp"]

View File

@ -1,54 +1,54 @@
# Mempool - Electrs backend API
# Esplora - Electrs backend API
A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs) and [Blockstream/electrs](https://github.com/Blockstream/electrs).
A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs).
Used as the backend for the [mempool block explorer](https://github.com/mempool/mempool) powering [mempool.space](https://mempool.space/).
Used as the backend for the [Esplora block explorer](https://github.com/Blockstream/esplora) powering [blockstream.info](https://blockstream.info/).
API documentation [is available here](https://mempool.space/docs/api/rest).
API documentation [is available here](https://github.com/blockstream/esplora/blob/master/API.md).
Documentation for the database schema and indexing process [is available here](doc/schema.md).
### Installing & indexing
Install Rust, Bitcoin Core (no `txindex` needed) and the `clang` and `cmake` packages, then:
Install Rust, Bitcoin Core and the `clang` and `cmake` packages, then:
```bash
$ git clone https://github.com/mempool/electrs && cd electrs
$ git checkout mempool
$ git clone https://github.com/blockstream/electrs && cd electrs
$ git checkout new-index
$ cargo run --release --bin electrs -- -vvvv --daemon-dir ~/.bitcoin
# Or for liquid:
$ cargo run --features liquid --release --bin electrs -- -vvvv --network liquid --daemon-dir ~/.liquid
$ cargo run --features liquid --release --bin electrs -- -vvvv --daemon-dir ~/.liquid
```
See [electrs's original documentation](https://github.com/romanz/electrs/blob/master/doc/usage.md) for more detailed instructions.
Note that our indexes are incompatible with electrs's and has to be created separately.
The indexes require 1.3TB of storage after running compaction (as of October 2023), but you'll need to have
free space of about double that available during the index compaction process.
Creating the indexes should take a few hours on a beefy machine with high speed NVMe SSD(s).
The indexes require 440GB of storage after running compaction, but you'll need to have about 1TB
of free space available for the initial (non-compacted) indexing process.
Creating the indexes should take a few hours on a beefy machine with SSD.
### Light mode
> Note: the pre-v2 version supported a "light" indexing mode for personal use,
> where less data is kept on-disk in exchange for a performance hit and more reliance/load on bitcoind.
> This option is not currently available, but can be enabled with [an older esplora release](https://github.com/Blockstream/esplora/releases/tag/esplora_v1.67).
> We're hoping to eventually get this feature back. Let us know if you find this important!
For personal or low-volume use, you may set `--lightmode` to reduce disk storage requirements
by roughly 50% at the cost of slower and more expensive lookups.
With this option set, raw transactions and metadata associated with blocks will not be kept in rocksdb
(the `T`, `X` and `M` indexes),
but instead queried from bitcoind on demand.
To deploy with Docker, follow the [instructions here](https://github.com/Blockstream/esplora#how-to-build-the-docker-image).
### Notable changes from Electrs:
- HTTP REST API in addition to the Electrum JSON-RPC protocol, with extended transaction information
- HTTP REST API instead of the Electrum JSON-RPC protocol, with extended transaction information
(previous outputs, spending transactions, script asm and more).
- Extended indexes and database storage for improved performance under high load:
- A full transaction store mapping txids to raw transactions is kept in the database under the prefix `t`.
- An index of all spendable transaction outputs is kept under the prefix `O`.
- An index of all addresses (encoded as string) is kept under the prefix `a` to enable by-prefix address search.
This takes up ~200GB of extra storage.
- A map of blockhash to txids is kept in the database under the prefix `X`.
- Block stats metadata (number of transactions, size and weight) is kept in the database under the prefix `M`.
- The index with `T` prefix mapping txids to block heights now also includes the block hash.
This allows for quick reorg-aware transaction confirmation status lookups, by verifying the
current block at the recorded height still matches the recorded block hash.
With these new indexes, bitcoind is no longer queried to serve user requests and is only polled
periodically for new blocks and for syncing the mempool.
@ -61,22 +61,13 @@ but instead queried from bitcoind on demand.
In addition to electrs's original configuration options, a few new options are also available:
- `--http-addr <addr:port>` - HTTP server address/port to listen on (default: `127.0.0.1:3000`).
- `--lightmode` - enable light mode (see above)
- `--disable-prevout` - disable attaching previous output information to inputs.
This significantly reduces the amount of transaction lookups (and IO/CPU/memory usage),
at the cost of not knowing inputs amounts, their previous script/address, and the transaction fee.
- `--parent-network <network>` - the parent network this chain is pegged to (Elements/Liquid only).
- `--cors <origins>` - origins allowed to make cross-site request (optional, defaults to none).
- `--address-search` - enables the by-prefix address search index.
- `--index-unspendables` - enables indexing of provably unspendable outputs.
- `--utxos-limit <num>` - maximum number of utxos to return per address.
- `--electrum-txs-limit <num>` - maximum number of txs to return per address in the electrum server (does not apply for the http api).
- `--electrum-banner <text>` - welcome banner text for electrum server.
Additional options with the `liquid` feature:
- `--parent-network <network>` - the parent network this chain is pegged to.
Additional options with the `electrum-discovery` feature:
- `--electrum-hosts <json>` - a json map of the public hosts where the electrum server is reachable, in the [`server.features` format](https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server.features).
- `--electrum-announce` - announce the electrum server on the electrum p2p server discovery network.
See `$ cargo run --bin electrs -- --help` for the full list of options.
See `$ cargo run --release --bin electrs -- --help` for the full list of options.
## License

View File

@ -1,57 +0,0 @@
use std::{path::Path, process::Command};
fn main() {
// Specify re-run conditions
// 1. Rerun build script if we pass a new GIT_HASH
println!("cargo:rerun-if-env-changed=GIT_HASH");
// 2. Only do git based reruns if git directory exists
if Path::new(".git").exists() {
// If we change the branch, rerun
println!("cargo:rerun-if-changed=.git/HEAD");
if let Ok(r) = std::fs::read_to_string(".git/HEAD") {
if let Some(stripped) = r.strip_prefix("ref: ") {
// If the HEAD is detached it will be a commit hash
// so the HEAD changed directive above will pick it up,
// otherwise it will point to a ref in the refs directory
println!("cargo:rerun-if-changed=.git/{}", stripped);
}
}
}
// Getting git hash
// Don't fetch git hash if it's already in the ENV
let existing = std::env::var("GIT_HASH").unwrap_or_else(|_| String::new());
if !existing.is_empty() {
return;
}
// Get git hash from git and don't do anything if the command fails
if let Some(rev_parse) = cmd("git", &["rev-parse", "--short", "HEAD"]) {
// Add (dirty) to the GIT_HASH if the git status isn't clean
// This includes untracked files
let dirty = cmd("git", &["status", "--short"]).expect("git command works");
// Ignore Dockerfile deletion as it is expected in Docker buildx builds
let git_hash = if dirty.is_empty() || dirty.trim() == "D Dockerfile" {
rev_parse
} else {
format!("{}(dirty)", rev_parse.trim())
};
println!("cargo:rustc-env=GIT_HASH={}", git_hash.trim());
}
}
// Helper function, Command is verbose...
fn cmd(name: &str, args: &[&str]) -> Option<String> {
Command::new(name).args(args).output().ok().and_then(|o| {
if o.status.success() {
String::from_utf8(o.stdout).ok()
} else {
None
}
})
}

10692
contrib/popular-scripts.txt Normal file

File diff suppressed because it is too large Load Diff

View File

@ -37,38 +37,18 @@ Each output results in the following new row:
* `"O{txid}{vout}" → "{scriptpubkey}{value}"`
When the indexer is synced up to the tip of the chain, the hash of the tip is saved as following:
* `"t" → "{blockhash}"`
### `history`
Each funding output (except for provably unspendable ones when `--index-unspendables` is not enabled) results in the following new rows (`H` is for history, `F` is for funding):
Each funding output (except for provably unspendable ones) results in the following new row (`H` is for history, `F` is for funding):
* `"H{funding-scripthash}{funding-height}F{funding-txid:vout}{value}" → ""`
* `"a{funding-address-str}" → ""` (for prefix address search, only saved when `--address-search` is enabled)
* `"H{funding-scripthash}{funding-height}F{funding-txid:index}{value}" → ""`
Each spending input (except the coinbase) results in the following new rows (`S` is for spending):
* `"H{funding-scripthash}{spending-height}S{spending-txid:vin}{funding-txid:vout}{value}" → ""`
* `"H{funding-scripthash}{spending-height}S{spending-txid:index}{funding-txid:index}{value}" → ""`
* `"S{funding-txid:vout}{spending-txid:vin}" → ""`
* `"S{funding-txid:index}{spending-txid:index}" → ""`
#### Elements only
Assets (re)issuances results in the following new rows (only for user-issued assets):
* `"i{asset-id}" → "{issuing-txid:vin}{prev-txid:vout}{issuance}{reissuance_token}"`
* `"I{asset-id}{issuance-height}I{issuing-txid:vin}{is_reissuance}{amount}{tokens}" → ""`
Peg-ins/peg-outs results in the following new rows (only for the native asset, typically L-BTC):
* `"I{asset-id}{pegin-height}F{pegin-txid:vin}{value}" → ""`
* `"I{asset-id}{pegout-height}F{pegout-txid:vout}{value}" → ""`
Every burn (unspendable output) results in the following new row (both user-issued and native):
* `"I{asset-id}{burn-height}F{burning-txid:vout}{value}" → ""`
### `cache`
@ -83,11 +63,3 @@ If the `blockhash` was since orphaned, the cache is removed and re-computed.
* `"A{scripthash}" → "{stats}{blockhash}"` (where `stats` is composed of `tx_count`, `funded_txo_{count,sum}` and `spent_txo_{count,sum}`)
* `"U{scripthash}" → "{utxo}{blockhash}"` (where `utxo` is a set of `(txid,vout)` outpoints)
#### Elements only:
Stats for issued assets:
* `"z{asset-id}" → "{issued_stats}{blockhash}"` (where `issued_stats` is composed of `tx_count`, `issuance_count`, `issued_amount`, `burned_amount`, `has_blinded_issuances`, `reissuance_tokens`, `burned_reissuance_tokens`)
Stats for the native asset:
* `"z{issued-asset}" → "{native_stats}{blockhash}"` (where `native_stats` is composed of `tx_count`, `peg_in_count`, `peg_in_amount`, `peg_out_count`, `peg_out_amount`, `burn_count` and `burn_amount`)

View File

@ -1,24 +0,0 @@
version: '3.9'
services:
mempool-electrs:
build:
context: .
dockerfile: Dockerfile
restart: on-failure
ports:
- 50001:50001
entrypoint:
/bin/electrs
command: |
--address-search
--cookie mempool:mempool
--db-dir /electrs
--cors '*'
volumes:
- 'electrs_data:/electrs'
volumes:
electrs_data:

View File

View File

@ -1,24 +0,0 @@
#!/usr/bin/env zsh
export ZPOOL=nvm
export BITCOIN_HOME=/bitcoin
export BITCOIN_USER=bitcoin
export BITCOIN_GROUP=bitcoin
export ELEMENTS_HOME=/elements
export ELEMENTS_USER=elements
export ELEMENTS_GROUP=elements
# create /bitcoin/socket with custom ACL for electrs unix sockets
zfs create -o "mountpoint=${BITCOIN_HOME}/socket" "${ZPOOL}/bitcoin/socket"
# create /elements/socket with custom ACL for electrs unix sockets
zfs create -o "mountpoint=${ELEMENTS_HOME}/socket" "${ZPOOL}/elements/socket"
setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"

View File

@ -1 +0,0 @@
1.87

View File

@ -1,64 +0,0 @@
#!/bin/bash
set -e
# This script is used for running all the checks
# needed to make sure CI passes.
# See below for the reasoning behind testing
# clippy for all feature combinations.
# (Note: clippy reports compile errors too)
# You can pass extra args to cargo commands
# ./scripts/check.sh --release
# will run in release mode. You can use that
# if you already have release mode compiled, as
# it will be faster to use the version you already compiled
TESTNAME=""
cleanup() {
exit_code=$?
if [[ ${exit_code} -ne 0 ]]; then
echo -e "\n\n##### Failed on \"$TESTNAME\"" 1>&2
fi
exit $exit_code
}
trap cleanup EXIT
TESTNAME="Running cargo check"
echo "$TESTNAME"
cargo check $@ -q --all-features
TESTNAME="Running cargo fmt check"
echo "$TESTNAME"
cargo fmt $@ -q --all -- --check
# Testing all the combinations of clippy.
# There were many instances where a certain struct
# differed based on liquid or not(liquid) etc.
# and the clippy fixes would break the other
# feature combination.
#
# Be prepared to use #[allow(clippy::___)] attributes
# to "fix" contradictions between feature sets.
TESTNAME="Running cargo clippy check no features"
echo "$TESTNAME"
cargo clippy $@ -q
TESTNAME="Running cargo clippy check electrum-discovery"
echo "$TESTNAME"
cargo clippy $@ -q -F electrum-discovery
TESTNAME="Running cargo clippy check liquid"
echo "$TESTNAME"
cargo clippy $@ -q -F liquid
TESTNAME="Running cargo clippy check electrum-discovery + liquid"
echo "$TESTNAME"
cargo clippy $@ -q -F electrum-discovery,liquid
if [ $INCLUDE_TESTS ]; then
TESTNAME="Running cargo test with all features"
echo "$TESTNAME"
cargo test $@ -q --lib --all-features
fi

View File

@ -1,4 +1,4 @@
use bitcoin::hashes::sha256d::Hash as Sha256dHash;
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use std::sync::{Arc, Mutex};
use crate::{daemon, index, signal::Waiter, store};

View File

@ -1,3 +1,4 @@
extern crate bitcoin;
extern crate error_chain;
#[macro_use]
extern crate log;
@ -20,123 +21,75 @@ use electrs::{
signal::Waiter,
};
#[cfg(feature = "liquid")]
use electrs::elements::AssetRegistry;
fn fetch_from(config: &Config, store: &Store) -> FetchFrom {
let mut jsonrpc_import = config.jsonrpc_import;
if !jsonrpc_import {
// switch over to jsonrpc after the initial sync is done
jsonrpc_import = store.done_initial_sync();
jsonrpc_import = !store.is_empty();
}
match jsonrpc_import {
true => FetchFrom::Bitcoind, // slower, uses JSONRPC (good for incremental updates)
false => FetchFrom::BlkFiles, // faster, uses blk*.dat files (good for initial indexing)
}
}
if jsonrpc_import {
// slower, uses JSONRPC (good for incremental updates)
FetchFrom::Bitcoind
} else {
// faster, uses blk*.dat files (good for initial indexing)
FetchFrom::BlkFiles
fn finish_verification(daemon: &Daemon, signal: &Waiter) -> Result<()> {
loop {
let progress = daemon.getblockchaininfo()?.verificationprogress;
if progress > 0.9999 {
return Ok(());
}
warn!(
"waiting for verification to finish: {:.3}%",
progress * 100.0
);
signal.wait(Duration::from_secs(5))?;
}
}
fn run_server(config: Arc<Config>) -> Result<()> {
let signal = Waiter::start();
let signal = Waiter::new();
let metrics = Metrics::new(config.monitoring_addr);
metrics.start();
let daemon = Arc::new(Daemon::new(
config.daemon_dir.clone(),
config.blocks_dir.clone(),
&config.daemon_dir,
config.daemon_rpc_addr,
config.cookie_getter(),
config.network_type,
config.magic,
signal.clone(),
&metrics,
)?);
let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
let mut indexer = Indexer::open(
Arc::clone(&store),
fetch_from(&config, &store),
&config,
&metrics,
);
finish_verification(&daemon, &signal)?;
let store = Arc::new(Store::open(&config.db_path.join("newindex")));
let mut indexer = Indexer::open(Arc::clone(&store), fetch_from(&config, &store), &metrics);
let mut tip = indexer.update(&daemon)?;
let chain = Arc::new(ChainQuery::new(
Arc::clone(&store),
Arc::clone(&daemon),
&config,
&metrics,
));
let chain = Arc::new(ChainQuery::new(Arc::clone(&store), &metrics));
let mempool = Arc::new(RwLock::new(Mempool::new(
Arc::clone(&chain),
&metrics,
Arc::clone(&config),
)));
loop {
match Mempool::update(&mempool, &daemon) {
Ok(_) => break,
Err(e) => {
warn!(
"Error performing initial mempool update, trying again in 5 seconds: {}",
e.display_chain()
);
signal.wait(Duration::from_secs(5), false)?;
}
}
if let Some(ref precache_file) = config.precache_scripts {
let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
.expect("cannot load scripts to precache");
precache::precache(&chain, precache_scripthashes);
}
#[cfg(feature = "liquid")]
let asset_db = config.asset_db_path.as_ref().map(|db_dir| {
let asset_db = Arc::new(RwLock::new(AssetRegistry::new(db_dir.clone())));
AssetRegistry::spawn_sync(asset_db.clone());
asset_db
});
let mempool = Arc::new(RwLock::new(Mempool::new(Arc::clone(&chain), &metrics)));
mempool.write().unwrap().update(&daemon)?;
let query = Arc::new(Query::new(
Arc::clone(&chain),
Arc::clone(&mempool),
Arc::clone(&daemon),
Arc::clone(&config),
#[cfg(feature = "liquid")]
asset_db,
));
// TODO: configuration for which servers to start
let rest_server = rest::start(Arc::clone(&config), Arc::clone(&query), &metrics);
let electrum_server = ElectrumRPC::start(Arc::clone(&config), Arc::clone(&query), &metrics);
if let Some(ref precache_file) = config.precache_scripts {
let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
.expect("cannot load scripts to precache");
precache::precache(
Arc::clone(&chain),
precache_scripthashes,
config.precache_threads,
);
}
let rest_server = rest::run_server(Arc::clone(&config), Arc::clone(&query));
let electrum_server =
ElectrumRPC::start(config.electrum_rpc_addr, Arc::clone(&query), &metrics);
loop {
if let Err(err) = signal.wait(Duration::from_millis(config.main_loop_delay), true) {
if let Err(err) = signal.wait(Duration::from_secs(5)) {
info!("stopping server: {}", err);
electrs::util::spawn_thread("shutdown-thread-checker", || {
let mut counter = 40;
let interval_ms = 500;
while counter > 0 {
electrs::util::with_spawned_threads(|threads| {
debug!("Threads during shutdown: {:?}", threads);
});
std::thread::sleep(std::time::Duration::from_millis(interval_ms));
counter -= 1;
}
});
rest_server.stop();
// the electrum server is stopped when dropped
break;
}
@ -148,13 +101,7 @@ fn run_server(config: Arc<Config>) -> Result<()> {
};
// Update mempool
if let Err(e) = Mempool::update(&mempool, &daemon) {
// Log the error if the result is an Err
warn!(
"Error updating mempool, skipping mempool update: {}",
e.display_chain()
);
}
mempool.write().unwrap().update(&daemon)?;
// Update subscribed clients
electrum_server.notify();
@ -169,7 +116,4 @@ fn main() {
error!("server failed: {}", e.display_chain());
process::exit(1);
}
electrs::util::with_spawned_threads(|threads| {
debug!("Threads before closing: {:?}", threads);
});
}

View File

@ -1,183 +1,48 @@
extern crate electrs;
use std::{convert::TryInto, thread::ThreadId, time::Instant};
use bincode;
use electrs::{
config::Config,
new_index::{Store, TxHistoryKey},
};
use electrs::{config::Config, new_index::db::open_raw_db};
use lazy_static::lazy_static;
/*
// How to run:
export ELECTRS_DATA=/path/to/electrs
cargo run \
-q --release --bin popular-scripts -- \
--db-dir $ELECTRS_DATA/db \
> ./contrib/popular-scripts.txt
*/
type DB = rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>;
lazy_static! {
static ref HISTORY_DB: DB = {
let config = Config::from_args();
open_raw_db(
&config.db_path.join("newindex").join("history"),
electrs::new_index::db::OpenMode::ReadOnly,
)
};
}
// Dev note:
// Only use println for file output (lines for output)
// Use eprintln to print to stderr for dev notifications
fn main() {
let high_usage_threshold = std::env::var("HIGH_USAGE_THRESHOLD")
.ok()
.and_then(|s| s.parse::<u32>().ok())
.unwrap_or(4000);
let thread_count = std::env::var("JOB_THREAD_COUNT")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(4);
eprintln!(
"Seaching for scripts with history rows of {} or more...",
high_usage_threshold
);
let config = Config::from_args();
let store = Store::open(&config.db_path.join("newindex"));
let thread_pool = rayon::ThreadPoolBuilder::new()
.num_threads(thread_count)
.build()
.expect("Built threadpool");
let (sender, receiver) = crossbeam_channel::unbounded::<[u8; 32]>();
let increment = 256 / thread_count;
let bytes: Vec<u8> = (0u8..=255u8)
.filter(|n| *n % increment as u8 == 0)
.collect();
let now = Instant::now();
for i in 0..bytes.len() {
let sender = sender.clone();
let first_byte = bytes[i];
let second_byte = bytes.get(i + 1).copied();
thread_pool.spawn(move || {
let id = std::thread::current().id();
run_iterator(
id,
&HISTORY_DB,
high_usage_threshold,
first_byte,
second_byte,
sender,
now,
);
eprintln!("{id:?} Finished its job!");
})
}
// If we don't drop this sender
// the receiver will hang forever
drop(sender);
while let Ok(script) = receiver.recv() {
println!("{}", hex::encode(script));
}
eprintln!("Finished!!!!");
}
fn run_iterator(
thread_id: ThreadId,
db: &DB,
high_usage_threshold: u32,
first_byte: u8,
next_byte: Option<u8>,
sender: crossbeam_channel::Sender<[u8; 32]>,
now: Instant,
) {
let mut iter = db.raw_iterator();
eprintln!(
"Thread ({thread_id:?}) Seeking DB to beginning of tx histories for b'H' + {}",
hex::encode([first_byte])
);
let mut compare_vec: Vec<u8> = vec![b'H', first_byte];
iter.seek(&compare_vec); // Seek to beginning of our section
// Insert the byte of the next section for comparing
// This will tell us when to stop with a closure
type Checker<'a> = Box<dyn Fn(&[u8]) -> bool + 'a>;
let is_finished: Checker<'_> = if let Some(next) = next_byte {
// Modify the vec to what we're looking for next
// to indicate we left our section
compare_vec[1] = next;
Box::new(|key: &[u8]| -> bool { key.starts_with(&compare_vec) })
} else {
// Modify the vec to only have H so we know when we left H
compare_vec.remove(1);
Box::new(|key: &[u8]| -> bool { !key.starts_with(&compare_vec) })
};
eprintln!("Thread ({thread_id:?}) Seeking done");
let mut iter = store.history_db().raw_iterator();
iter.seek(b"H");
let mut curr_scripthash = [0u8; 32];
let mut total_entries: usize = 0;
let mut iter_index: usize = 1;
let mut total_entries = 0;
while iter.valid() {
let key = iter.key().unwrap();
if key.is_empty() || key[0] != b'H' || is_finished(key) {
// We have left the txhistory section,
// but we need to check the final scripthash
send_if_popular(
high_usage_threshold,
total_entries,
curr_scripthash,
&sender,
);
if !key.starts_with(b"H") {
break;
}
if iter_index % 10_000_000 == 0 {
let duration = now.elapsed().as_secs();
eprintln!(
"Thread ({thread_id:?}) Processing row #{iter_index}... {duration} seconds elapsed"
);
}
let entry: TxHistoryKey = bincode::config()
.big_endian()
.deserialize(&key)
.expect("failed to deserialize TxHistoryKey");
// We know that the TxHistory key is 1 byte "H" followed by
// 32 byte scripthash
let entry_hash: [u8; 32] = key[1..33].try_into().unwrap();
if curr_scripthash != entry.scripthash {
if total_entries > 100 {
println!("{} {}", hex::encode(&curr_scripthash), total_entries);
}
if curr_scripthash != entry_hash {
// We have rolled on to a new scripthash
// If the last scripthash was popular
// Collect for sorting
send_if_popular(
high_usage_threshold,
total_entries,
curr_scripthash,
&sender,
);
// After collecting, reset values for next scripthash
curr_scripthash = entry_hash;
curr_scripthash = entry.scripthash;
total_entries = 0;
}
total_entries += 1;
iter_index += 1;
total_entries = total_entries + 1;
iter.next();
}
}
#[inline]
fn send_if_popular(
high_usage_threshold: u32,
total_entries: usize,
curr_scripthash: [u8; 32],
sender: &crossbeam_channel::Sender<[u8; 32]>,
) {
if total_entries >= high_usage_threshold as usize {
sender.send(curr_scripthash).unwrap();
if total_entries > 100 {
println!("{} {}", hex::encode(&curr_scripthash), total_entries);
}
}

View File

@ -1,6 +1,4 @@
extern crate electrs;
#[cfg(not(feature = "liquid"))]
#[macro_use]
extern crate log;
@ -9,7 +7,7 @@ fn main() {
use std::collections::HashSet;
use std::sync::Arc;
use bitcoin::blockdata::script::ScriptBuf;
use bitcoin::blockdata::script::Script;
use bitcoin::consensus::encode::deserialize;
use electrs::{
chain::Transaction,
@ -21,30 +19,28 @@ fn main() {
util::has_prevout,
};
let signal = Waiter::start();
let signal = Waiter::new();
let config = Config::from_args();
let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
let store = Arc::new(Store::open(&config.db_path.join("newindex")));
let metrics = Metrics::new(config.monitoring_addr);
metrics.start();
let chain = ChainQuery::new(Arc::clone(&store), &metrics);
let daemon = Arc::new(
Daemon::new(
config.daemon_dir.clone(),
config.blocks_dir.clone(),
&config.daemon_dir,
config.daemon_rpc_addr,
config.cookie_getter(),
config.network_type,
config.magic,
signal,
signal.clone(),
&metrics,
)
.unwrap(),
);
let chain = ChainQuery::new(Arc::clone(&store), Arc::clone(&daemon), &config, &metrics);
let mut indexer = Indexer::open(Arc::clone(&store), FetchFrom::Bitcoind, &config, &metrics);
let mut indexer = Indexer::open(Arc::clone(&store), FetchFrom::Bitcoind, &metrics);
indexer.update(&daemon).unwrap();
let mut iter = store.txstore_db().raw_iterator();
@ -61,17 +57,17 @@ fn main() {
break;
}
let tx: Transaction = deserialize(value).expect("failed to parse Transaction");
let txid = tx.compute_txid();
iter.next();
let tx: Transaction = deserialize(&value).expect("failed to parse Transaction");
let txid = tx.txid();
// only consider transactions of exactly two outputs
if tx.output.len() != 2 {
continue;
}
// skip coinbase txs
if tx.is_coinbase() {
if tx.is_coin_base() {
continue;
}
@ -91,26 +87,12 @@ fn main() {
.collect(),
);
let total_out: u64 = tx.output.iter().map(|out| out.value.to_sat()).sum();
let small_out = tx
.output
.iter()
.map(|out| out.value.to_sat())
.min()
.unwrap();
let large_out = tx
.output
.iter()
.map(|out| out.value.to_sat())
.max()
.unwrap();
let total_out: u64 = tx.output.iter().map(|out| out.value).sum();
let small_out = tx.output.iter().map(|out| out.value).min().unwrap();
let large_out = tx.output.iter().map(|out| out.value).max().unwrap();
let total_in: u64 = prevouts.values().map(|out| out.value.to_sat()).sum();
let smallest_in = prevouts
.values()
.map(|out| out.value.to_sat())
.min()
.unwrap();
let total_in: u64 = prevouts.values().map(|out| out.value).sum();
let smallest_in = prevouts.values().map(|out| out.value).min().unwrap();
let fee = total_in - total_out;
@ -133,7 +115,7 @@ fn main() {
// test for sending back to one of the spent spks
let has_reuse = {
let prev_spks: HashSet<ScriptBuf> = prevouts
let prev_spks: HashSet<Script> = prevouts
.values()
.map(|out| out.script_pubkey.clone())
.collect();
@ -147,8 +129,8 @@ fn main() {
txid, blockid.height, tx.lock_time, uih, is_multi_spend as u8, has_reuse as u8
);
total += 1;
uih_totals[uih] += 1;
total = total + 1;
uih_totals[uih] = uih_totals[uih] + 1;
}
info!(
"processed {} total txs, UIH counts: {:?}",

View File

@ -1,65 +1,14 @@
use std::str::FromStr;
#[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures
pub use bitcoin::{
address,
block::Header as BlockHeader,
blockdata::{opcodes, script},
consensus::deserialize,
hashes, Block, BlockHash, OutPoint, ScriptBuf as Script, Transaction, TxIn, TxOut, Txid,
Witness,
};
pub use bitcoin::{Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut};
#[cfg(feature = "liquid")]
pub use {
crate::elements::asset,
elements::{
address, bitcoin::bech32::Hrp, confidential, encode::deserialize, hashes, opcodes, script,
Address, AssetId, Block, BlockHash, BlockHeader, OutPoint, Script, Transaction, TxIn,
TxInWitness as Witness, TxOut, Txid,
},
};
pub use elements::{confidential, Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut};
use bitcoin::blockdata::constants::genesis_block;
pub use bitcoin::Network as BNetwork;
// Extension trait for getting txid in a cross-compatible way
pub trait TxidCompat {
fn get_txid(&self) -> Txid;
}
#[cfg(not(feature = "liquid"))]
impl TxidCompat for Transaction {
fn get_txid(&self) -> Txid {
self.compute_txid()
}
}
#[cfg(feature = "liquid")]
impl TxidCompat for Transaction {
fn get_txid(&self) -> Txid {
self.txid()
}
}
// Extension trait for getting block size in a cross-compatible way
pub trait BlockSizeCompat {
fn get_block_size(&self) -> usize;
}
#[cfg(not(feature = "liquid"))]
impl BlockSizeCompat for Block {
fn get_block_size(&self) -> usize {
self.total_size()
}
}
#[cfg(feature = "liquid")]
impl BlockSizeCompat for Block {
fn get_block_size(&self) -> usize {
self.size()
}
}
use bitcoin::network::constants::Network as BNetwork;
use bitcoin::util::hash::BitcoinHash;
use bitcoin_bech32::constants::Network as B32Network;
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
#[cfg(not(feature = "liquid"))]
pub type Value = u64;
@ -68,92 +17,32 @@ pub use confidential::Value;
#[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize, Ord, PartialOrd, Eq)]
pub enum Network {
#[cfg(not(feature = "liquid"))]
Bitcoin,
#[cfg(not(feature = "liquid"))]
Testnet,
#[cfg(not(feature = "liquid"))]
Testnet4,
#[cfg(not(feature = "liquid"))]
Regtest,
#[cfg(not(feature = "liquid"))]
Signet,
#[cfg(feature = "liquid")]
Liquid,
#[cfg(feature = "liquid")]
LiquidTestnet,
#[cfg(feature = "liquid")]
LiquidRegtest,
}
#[cfg(feature = "liquid")]
pub const LIQUID_TESTNET_PARAMS: address::AddressParams = address::AddressParams {
p2pkh_prefix: 36,
p2sh_prefix: 19,
blinded_prefix: 23,
bech_hrp: Hrp::parse_unchecked("tex"),
blech_hrp: Hrp::parse_unchecked("tlq"),
};
/// Magic for testnet4, 0x1c163f28 (from BIP94) with flipped endianness.
#[cfg(not(feature = "liquid"))]
const TESTNET4_MAGIC: u32 = 0x283f161c;
impl Network {
#[cfg(not(feature = "liquid"))]
pub fn magic(self) -> u32 {
match self {
Self::Testnet4 => TESTNET4_MAGIC,
_ => {
let magic = BNetwork::from(self).magic();
u32::from_le_bytes(magic.to_bytes())
}
}
pub fn genesis_hash(&self) -> Sha256dHash {
let block = genesis_block(BNetwork::from(self));
block.bitcoin_hash()
}
#[cfg(feature = "liquid")]
pub fn magic(self) -> u32 {
pub fn magic(&self) -> u32 {
match self {
Network::Liquid | Network::LiquidRegtest => 0xDAB5_BFFA,
Network::LiquidTestnet => 0x62DD_0E41,
}
}
Network::Bitcoin => 0xD9B4BEF9,
Network::Testnet => 0x0709110B,
Network::Regtest => 0xDAB5BFFA,
pub fn is_regtest(self) -> bool {
match self {
#[cfg(not(feature = "liquid"))]
Network::Regtest => true,
#[cfg(feature = "liquid")]
Network::LiquidRegtest => true,
_ => false,
}
}
#[cfg(feature = "liquid")]
pub fn address_params(self) -> &'static address::AddressParams {
// Liquid regtest uses elements's address params
match self {
Network::Liquid => &address::AddressParams::LIQUID,
Network::LiquidRegtest => &address::AddressParams::ELEMENTS,
Network::LiquidTestnet => &LIQUID_TESTNET_PARAMS,
}
}
#[cfg(feature = "liquid")]
pub fn native_asset(self) -> &'static AssetId {
match self {
Network::Liquid => &asset::NATIVE_ASSET_ID,
Network::LiquidTestnet => &asset::NATIVE_ASSET_ID_TESTNET,
Network::LiquidRegtest => &asset::NATIVE_ASSET_ID_REGTEST,
}
}
#[cfg(feature = "liquid")]
pub fn pegged_asset(self) -> Option<&'static AssetId> {
match self {
Network::Liquid => Some(&*asset::NATIVE_ASSET_ID),
Network::LiquidTestnet | Network::LiquidRegtest => None,
Network::Liquid => 0xDAB5BFFA,
#[cfg(feature = "liquid")]
Network::LiquidRegtest => 0xDAB5BFFA,
}
}
@ -163,97 +52,29 @@ impl Network {
"mainnet".to_string(),
"testnet".to_string(),
"regtest".to_string(),
"signet".to_string(),
];
#[cfg(feature = "liquid")]
return vec![
"mainnet".to_string(),
"testnet".to_string(),
"regtest".to_string(),
"liquid".to_string(),
"liquidtestnet".to_string(),
"liquidregtest".to_string(),
];
}
}
pub fn genesis_hash(network: Network) -> BlockHash {
#[cfg(not(feature = "liquid"))]
return bitcoin_genesis_hash(network);
#[cfg(feature = "liquid")]
return liquid_genesis_hash(network);
}
pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash {
lazy_static! {
static ref BITCOIN_GENESIS: bitcoin::BlockHash =
genesis_block(BNetwork::Bitcoin).block_hash();
static ref TESTNET_GENESIS: bitcoin::BlockHash =
genesis_block(BNetwork::Testnet).block_hash();
static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str(
"00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043"
)
.unwrap();
static ref REGTEST_GENESIS: bitcoin::BlockHash =
genesis_block(BNetwork::Regtest).block_hash();
static ref SIGNET_GENESIS: bitcoin::BlockHash =
genesis_block(BNetwork::Signet).block_hash();
}
#[cfg(not(feature = "liquid"))]
match network {
Network::Bitcoin => *BITCOIN_GENESIS,
Network::Testnet => *TESTNET_GENESIS,
Network::Testnet4 => *TESTNET4_GENESIS,
Network::Regtest => *REGTEST_GENESIS,
Network::Signet => *SIGNET_GENESIS,
}
#[cfg(feature = "liquid")]
match network {
Network::Liquid => *BITCOIN_GENESIS,
Network::LiquidTestnet => *TESTNET_GENESIS,
Network::LiquidRegtest => *REGTEST_GENESIS,
}
}
#[cfg(feature = "liquid")]
pub fn liquid_genesis_hash(network: Network) -> elements::BlockHash {
lazy_static! {
static ref LIQUID_GENESIS: BlockHash =
"1466275836220db2944ca059a3a10ef6fd2ea684b0688d2c379296888a206003"
.parse()
.unwrap();
static ref ZERO_HASH: BlockHash =
"0000000000000000000000000000000000000000000000000000000000000000"
.parse()
.unwrap();
}
match network {
Network::Liquid => *LIQUID_GENESIS,
// The genesis block for liquid regtest chains varies based on the chain configuration.
// This instead uses an all zeroed-out hash, which doesn't matter in practice because its
// only used for Electrum server discovery, which isn't active on regtest.
_ => *ZERO_HASH,
}
}
impl From<&str> for Network {
fn from(network_name: &str) -> Self {
match network_name {
#[cfg(not(feature = "liquid"))]
"mainnet" => Network::Bitcoin,
#[cfg(not(feature = "liquid"))]
"testnet" => Network::Testnet,
#[cfg(not(feature = "liquid"))]
"testnet4" => Network::Testnet4,
#[cfg(not(feature = "liquid"))]
"regtest" => Network::Regtest,
#[cfg(not(feature = "liquid"))]
"signet" => Network::Signet,
#[cfg(feature = "liquid")]
"liquid" => Network::Liquid,
#[cfg(feature = "liquid")]
"liquidtestnet" => Network::LiquidTestnet,
#[cfg(feature = "liquid")]
"liquidregtest" => Network::LiquidRegtest,
_ => panic!("unsupported Bitcoin network: {:?}", network_name),
@ -261,28 +82,48 @@ impl From<&str> for Network {
}
}
#[cfg(not(feature = "liquid"))]
impl From<Network> for BNetwork {
fn from(network: Network) -> Self {
impl From<&Network> for BNetwork {
fn from(network: &Network) -> Self {
match network {
Network::Bitcoin => BNetwork::Bitcoin,
Network::Testnet => BNetwork::Testnet,
Network::Testnet4 => BNetwork::Testnet4,
Network::Regtest => BNetwork::Regtest,
Network::Signet => BNetwork::Signet,
#[cfg(feature = "liquid")]
Network::Liquid => BNetwork::Bitcoin, // @FIXME
#[cfg(feature = "liquid")]
Network::LiquidRegtest => BNetwork::Regtest, // @FIXME
}
}
}
#[cfg(not(feature = "liquid"))]
impl From<BNetwork> for Network {
fn from(network: BNetwork) -> Self {
impl From<&Network> for B32Network {
fn from(network: &Network) -> Self {
match network {
BNetwork::Bitcoin => Network::Bitcoin,
BNetwork::Testnet => Network::Testnet,
BNetwork::Testnet4 => Network::Testnet4,
BNetwork::Regtest => Network::Regtest,
BNetwork::Signet => Network::Signet,
Network::Bitcoin => B32Network::Bitcoin,
Network::Testnet => B32Network::Testnet,
Network::Regtest => B32Network::Regtest,
#[cfg(feature = "liquid")]
Network::Liquid => B32Network::Bitcoin, // @FIXME
#[cfg(feature = "liquid")]
Network::LiquidRegtest => B32Network::Regtest, // @FIXME
}
}
}
impl From<&BNetwork> for Network {
fn from(network: &BNetwork) -> Self {
match network {
#[cfg(not(feature = "liquid"))]
BNetwork::Bitcoin => Network::Bitcoin,
#[cfg(not(feature = "liquid"))]
BNetwork::Regtest => Network::Regtest,
#[cfg(feature = "liquid")]
BNetwork::Bitcoin => Network::Liquid, // @FIXME
#[cfg(feature = "liquid")]
BNetwork::Regtest => Network::LiquidRegtest, // @FIXME
BNetwork::Testnet => Network::Testnet, // @FIXME
}
}
}

View File

@ -1,109 +1,55 @@
use clap::{App, Arg};
use dirs::home_dir;
use num_cpus;
use std::fs;
use std::net::SocketAddr;
use std::net::ToSocketAddrs;
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock};
use std::sync::Arc;
use stderrlog;
#[cfg(feature = "liquid")]
use bitcoin_hashes::hex::ToHex;
use crate::chain::Network;
use crate::daemon::CookieGetter;
use crate::errors::*;
#[cfg(feature = "liquid")]
use bitcoin::Network as BNetwork;
pub(crate) const APP_NAME: &str = "mempool-electrs";
pub(crate) const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION");
pub(crate) const GIT_HASH: Option<&str> = option_env!("GIT_HASH");
// This will be set only once in the Daemon::new() constructor at startup
pub(crate) static BITCOIND_SUBVER: OnceLock<String> = OnceLock::new();
lazy_static! {
pub(crate) static ref VERSION_STRING: String = {
if let Some(hash) = GIT_HASH {
format!("{} {}-{}", APP_NAME, ELECTRS_VERSION, hash)
} else {
format!("{} {}", APP_NAME, ELECTRS_VERSION)
}
};
}
#[derive(Debug, Clone)]
pub struct Config {
// See below for the documentation of each field:
pub log: stderrlog::StdErrLog,
pub network_type: Network,
pub magic: Option<u32>,
pub db_path: PathBuf,
pub daemon_dir: PathBuf,
pub blocks_dir: PathBuf,
pub daemon_rpc_addr: SocketAddr,
pub cookie: Option<String>,
pub electrum_rpc_addr: SocketAddr,
pub http_addr: SocketAddr,
pub http_socket_file: Option<PathBuf>,
pub rpc_socket_file: Option<PathBuf>,
pub monitoring_addr: SocketAddr,
pub jsonrpc_import: bool,
pub light_mode: bool,
pub main_loop_delay: u64,
pub address_search: bool,
pub index_unspendables: bool,
pub index_batch_size: usize,
pub bulk_index_threads: usize,
pub tx_cache_size: usize,
pub prevout_enabled: bool,
pub cors: Option<String>,
pub precache_scripts: Option<String>,
pub precache_threads: usize,
pub utxos_limit: usize,
pub electrum_txs_limit: usize,
pub electrum_banner: String,
pub mempool_backlog_stats_ttl: u64,
pub mempool_recent_txs_size: usize,
pub rest_default_block_limit: usize,
pub rest_default_chain_txs_per_page: usize,
pub rest_default_max_mempool_txs: usize,
pub rest_default_max_address_summary_txs: usize,
pub rest_max_mempool_page_size: usize,
pub rest_max_mempool_txid_page_size: usize,
pub electrum_max_line_size: usize,
pub electrum_max_subscriptions: usize,
pub electrum_max_clients: usize,
pub electrum_idle_timeout: u64,
pub electrum_haproxy_depth: usize,
pub electrum_connections_per_client: usize,
pub electrum_public_hosts: Option<crate::electrum::ServerHosts>,
#[cfg(feature = "liquid")]
pub parent_network: BNetwork,
pub parent_network: Network,
#[cfg(feature = "liquid")]
pub asset_db_path: Option<PathBuf>,
#[cfg(feature = "electrum-discovery")]
pub electrum_announce: bool,
#[cfg(feature = "electrum-discovery")]
pub tor_proxy: Option<std::net::SocketAddr>,
}
fn str_to_socketaddr(address: &str, what: &str) -> SocketAddr {
address
.to_socket_addrs()
.unwrap_or_else(|_| panic!("unable to resolve {} address", what))
.collect::<Vec<_>>()
.pop()
.unwrap()
pub parent_genesis_hash: String,
}
impl Config {
pub fn from_args() -> Config {
let network_help = format!("Select network type ({})", Network::names().join(", "));
let network_help = format!(
"Select Bitcoin network type ({})",
Network::names().join(", ")
);
let args = App::new("Mempool Electrum Rust Server")
let args = App::new("Electrum Rust Server")
.version(crate_version!())
.arg(
Arg::with_name("version")
.long("version")
.help("Print out the version of this app and quit immediately."),
)
.arg(
Arg::with_name("verbosity")
.short("v")
@ -127,12 +73,6 @@ impl Config {
.help("Data directory of Bitcoind (default: ~/.bitcoin/)")
.takes_value(true),
)
.arg(
Arg::with_name("blocks_dir")
.long("blocks-dir")
.help("Analogous to bitcoind's -blocksdir option, this specifies the directory containing the raw blocks files (blk*.dat) (default: ~/.bitcoin/blocks/)")
.takes_value(true),
)
.arg(
Arg::with_name("cookie")
.long("cookie")
@ -145,12 +85,6 @@ impl Config {
.help(&network_help)
.takes_value(true),
)
.arg(
Arg::with_name("magic")
.long("magic")
.default_value("")
.takes_value(true),
)
.arg(
Arg::with_name("electrum_rpc_addr")
.long("electrum-rpc-addr")
@ -181,25 +115,27 @@ impl Config {
.help("Use JSONRPC instead of directly importing blk*.dat files. Useful for remote full node or low memory system"),
)
.arg(
Arg::with_name("light_mode")
.long("lightmode")
.help("Enable light mode for reduced storage")
Arg::with_name("index_batch_size")
.long("index-batch-size")
.help("Number of blocks to get in one JSONRPC request from bitcoind")
.default_value("100"),
)
.arg(
Arg::with_name("main_loop_delay")
.long("main-loop-delay")
.help("The number of milliseconds the main loop will wait between loops. (Can be shortened with SIGUSR1)")
.default_value("500")
Arg::with_name("bulk_index_threads")
.long("bulk-index-threads")
.help("Number of threads used for bulk indexing (default: use the # of CPUs)")
.default_value("0")
)
.arg(
Arg::with_name("address_search")
.long("address-search")
.help("Enable prefix address search")
Arg::with_name("tx_cache_size")
.long("tx-cache-size")
.help("Number of transactions to keep in for query LRU cache")
.default_value("10000") // should be enough for a small wallet.
)
.arg(
Arg::with_name("index_unspendables")
.long("index-unspendables")
.help("Enable indexing of provably unspendable outputs")
Arg::with_name("disable_prevout")
.long("disable-prevout")
.help("Don't attach previous output details to inputs")
)
.arg(
Arg::with_name("cors")
@ -212,329 +148,110 @@ impl Config {
.long("precache-scripts")
.help("Path to file with list of scripts to pre-cache")
.takes_value(true)
)
.arg(
Arg::with_name("precache_threads")
.long("precache-threads")
.help("Non-zero number of threads to use for precache threadpool. [default: 4 * CORE_COUNT]")
.takes_value(true)
)
.arg(
Arg::with_name("utxos_limit")
.long("utxos-limit")
.help("Maximum number of utxos to process per address. Lookups for addresses with more utxos will fail. Applies to the Electrum and HTTP APIs.")
.default_value("500")
)
.arg(
Arg::with_name("mempool_backlog_stats_ttl")
.long("mempool-backlog-stats-ttl")
.help("The number of seconds that need to pass before Mempool::update will update the latency histogram again.")
.default_value("10")
)
.arg(
Arg::with_name("mempool_recent_txs_size")
.long("mempool-recent-txs-size")
.help("The number of transactions that mempool will keep in its recents queue. This is returned by mempool/recent endpoint.")
.default_value("10")
)
.arg(
Arg::with_name("rest_default_block_limit")
.long("rest-default-block-limit")
.help("The default number of blocks returned from the blocks/[start_height] endpoint.")
.default_value("10")
)
.arg(
Arg::with_name("rest_default_chain_txs_per_page")
.long("rest-default-chain-txs-per-page")
.help("The default number of on-chain transactions returned by the txs endpoints.")
.default_value("25")
)
.arg(
Arg::with_name("rest_default_max_mempool_txs")
.long("rest-default-max-mempool-txs")
.help("The default number of mempool transactions returned by the txs endpoints.")
.default_value("50")
)
.arg(
Arg::with_name("rest_default_max_address_summary_txs")
.long("rest-default-max-address-summary-txs")
.help("The default number of transactions returned by the address summary endpoints.")
.default_value("5000")
)
.arg(
Arg::with_name("rest_max_mempool_page_size")
.long("rest-max-mempool-page-size")
.help("The maximum number of transactions returned by the paginated /internal/mempool/txs endpoint.")
.default_value("1000")
)
.arg(
Arg::with_name("rest_max_mempool_txid_page_size")
.long("rest-max-mempool-txid-page-size")
.help("The maximum number of transactions returned by the paginated /mempool/txids/page endpoint.")
.default_value("10000")
)
.arg(
Arg::with_name("electrum_txs_limit")
.long("electrum-txs-limit")
.help("Maximum number of transactions returned by Electrum history queries. Lookups with more results will fail.")
.default_value("500")
).arg(
Arg::with_name("electrum_banner")
.long("electrum-banner")
.help("Welcome banner for the Electrum server, shown in the console to clients.")
.takes_value(true)
).arg(
Arg::with_name("electrum_max_line_size")
.long("electrum-max-line-size")
.help("Maximum size of a single Electrum request line in bytes (default: 1 MiB).")
.default_value("1048576")
).arg(
Arg::with_name("electrum_max_subscriptions")
.long("electrum-max-subscriptions")
.help("Maximum number of scripthash subscriptions per client connection.")
.default_value("100")
).arg(
Arg::with_name("electrum_max_clients")
.long("electrum-max-clients")
.help("Maximum number of concurrent Electrum client connections.")
.default_value("10")
).arg(
Arg::with_name("electrum_idle_timeout")
.long("electrum-idle-timeout")
.help("Maximum idle time in seconds since the last client request before disconnecting the Electrum connection.")
.default_value("600")
).arg(
Arg::with_name("electrum_haproxy_depth")
.long("electrum-haproxy-depth")
.help("Which HAProxy PROXY-protocol header layer identifies the real client IP. 0 disables PROXY-protocol detection; 1 uses the first (outermost) address, 2 the second, and so on. If the requested layer or any PROXY header is absent, no client IP is associated with the connection.")
.default_value("0")
).arg(
Arg::with_name("electrum_connections_per_client")
.long("electrum-connections-per-client")
.help("Maximum number of concurrent Electrum connections allowed per client (keyed by the HAProxy-reported address when available, otherwise the peer IP). 0 disables the per-client limit.")
.default_value("10")
);
#[cfg(unix)]
let args = args.arg(
Arg::with_name("http_socket_file")
.long("http-socket-file")
.help("HTTP server 'unix socket file' to listen on (default disabled, enabling this disables the http server)")
.takes_value(true),
);
#[cfg(unix)]
let args = args.arg(
Arg::with_name("rpc_socket_file")
.long("rpc-socket-file")
.help("Electrum RPC 'unix socket file' to listen on (default disabled, enabling this ignores the electrum_rpc_addr arg)")
.takes_value(true),
);
#[cfg(feature = "liquid")]
let args = args
.arg(
Arg::with_name("parent_network")
.long("parent-network")
.help("Select parent network type (mainnet, testnet, regtest)")
.takes_value(true),
)
.arg(
Arg::with_name("asset_db_path")
.long("asset-db-path")
.help("Directory for liquid/elements asset db")
.takes_value(true),
);
#[cfg(feature = "electrum-discovery")]
let args = args.arg(
Arg::with_name("electrum_public_hosts")
.long("electrum-public-hosts")
.help("A dictionary of hosts where the Electrum server can be reached at. Required to enable server discovery. See https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-features")
.takes_value(true)
).arg(
Arg::with_name("electrum_announce")
.long("electrum-announce")
.help("Announce the Electrum server to other servers")
).arg(
Arg::with_name("tor_proxy")
.long("tor-proxy")
.help("ip:addr of socks proxy for accessing onion hosts")
Arg::with_name("parent_network")
.long("parent-network")
.help("Select parent network type (mainnet, testnet, regtest)")
.takes_value(true),
);
let m = args.get_matches();
if m.is_present("version") {
eprintln!("{}", *VERSION_STRING);
std::process::exit(0);
}
let network_name = m.value_of("network").unwrap_or("mainnet");
let network_type = Network::from(network_name);
let magic: Option<u32> = m
.value_of("magic")
.filter(|s| !s.is_empty())
.map(|s| u32::from_str_radix(s, 16).expect("invalid network magic"));
let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db"));
let db_path = db_dir.join(network_name);
#[cfg(feature = "liquid")]
let parent_network = m
.value_of("parent_network")
.map(|s| s.parse().expect("invalid parent network"))
.unwrap_or_else(|| match network_type {
Network::Liquid => BNetwork::Bitcoin,
// XXX liquid testnet/regtest don't have a parent chain
Network::LiquidTestnet | Network::LiquidRegtest => BNetwork::Regtest,
});
let parent_network = Network::from(m.value_of("parent_network").unwrap_or("mainnet"));
#[cfg(feature = "liquid")]
let asset_db_path = m.value_of("asset_db_path").map(PathBuf::from);
let parent_genesis_hash = parent_network.genesis_hash().to_hex();
let default_daemon_port = match network_type {
#[cfg(not(feature = "liquid"))]
Network::Bitcoin => 8332,
#[cfg(not(feature = "liquid"))]
Network::Testnet => 18332,
#[cfg(not(feature = "liquid"))]
Network::Regtest => 18443,
#[cfg(not(feature = "liquid"))]
Network::Signet => 38332,
#[cfg(not(feature = "liquid"))]
Network::Testnet4 => 48332,
#[cfg(feature = "liquid")]
Network::Liquid => 7041,
#[cfg(feature = "liquid")]
Network::LiquidTestnet | Network::LiquidRegtest => 7040,
Network::LiquidRegtest => 7041,
};
let default_electrum_port = match network_type {
#[cfg(not(feature = "liquid"))]
Network::Bitcoin => 50001,
#[cfg(not(feature = "liquid"))]
Network::Testnet => 60001,
#[cfg(not(feature = "liquid"))]
Network::Testnet4 => 40001,
#[cfg(not(feature = "liquid"))]
Network::Regtest => 60401,
#[cfg(not(feature = "liquid"))]
Network::Signet => 60601,
#[cfg(feature = "liquid")]
Network::Liquid => 51000,
#[cfg(feature = "liquid")]
Network::LiquidTestnet => 51301,
#[cfg(feature = "liquid")]
Network::LiquidRegtest => 51401,
};
let default_http_port = match network_type {
#[cfg(not(feature = "liquid"))]
Network::Bitcoin => 3000,
#[cfg(not(feature = "liquid"))]
Network::Testnet => 3001,
#[cfg(not(feature = "liquid"))]
Network::Regtest => 3002,
#[cfg(not(feature = "liquid"))]
Network::Signet => 3003,
#[cfg(not(feature = "liquid"))]
Network::Testnet4 => 3004,
#[cfg(feature = "liquid")]
Network::Liquid => 3000,
#[cfg(feature = "liquid")]
Network::LiquidTestnet => 3001,
#[cfg(feature = "liquid")]
Network::LiquidRegtest => 3002,
};
let default_monitoring_port = match network_type {
#[cfg(not(feature = "liquid"))]
Network::Bitcoin => 4224,
#[cfg(not(feature = "liquid"))]
Network::Testnet => 14224,
#[cfg(not(feature = "liquid"))]
Network::Regtest => 24224,
#[cfg(not(feature = "liquid"))]
Network::Testnet4 => 44224,
#[cfg(not(feature = "liquid"))]
Network::Signet => 54224,
#[cfg(feature = "liquid")]
Network::Liquid => 34224,
#[cfg(feature = "liquid")]
Network::LiquidTestnet => 44324,
#[cfg(feature = "liquid")]
Network::LiquidRegtest => 44224,
};
let daemon_rpc_addr: SocketAddr = str_to_socketaddr(
m.value_of("daemon_rpc_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_daemon_port)),
"Bitcoin RPC",
);
let electrum_rpc_addr: SocketAddr = str_to_socketaddr(
m.value_of("electrum_rpc_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_electrum_port)),
"Electrum RPC",
);
let http_addr: SocketAddr = str_to_socketaddr(
m.value_of("http_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_http_port)),
"HTTP Server",
);
let http_socket_file: Option<PathBuf> = m.value_of("http_socket_file").map(PathBuf::from);
let rpc_socket_file: Option<PathBuf> = m.value_of("rpc_socket_file").map(PathBuf::from);
let monitoring_addr: SocketAddr = str_to_socketaddr(
m.value_of("monitoring_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port)),
"Prometheus monitoring",
);
let daemon_rpc_addr: SocketAddr = m
.value_of("daemon_rpc_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_daemon_port))
.parse()
.expect("invalid Bitcoind RPC address");
let electrum_rpc_addr: SocketAddr = m
.value_of("electrum_rpc_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_electrum_port))
.parse()
.expect("invalid Electrum RPC address");
let http_addr: SocketAddr = m
.value_of("http_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_http_port))
.parse()
.expect("invalid HTTP server address");
let monitoring_addr: SocketAddr = m
.value_of("monitoring_addr")
.unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port))
.parse()
.expect("invalid Prometheus monitoring address");
let mut daemon_dir = m
.value_of("daemon_dir")
.map(PathBuf::from)
.map(|p| PathBuf::from(p))
.unwrap_or_else(|| {
let mut default_dir = home_dir().expect("no homedir");
default_dir.push(".bitcoin");
default_dir
});
match network_type {
#[cfg(not(feature = "liquid"))]
Network::Bitcoin => (),
#[cfg(not(feature = "liquid"))]
Network::Testnet => daemon_dir.push("testnet3"),
#[cfg(not(feature = "liquid"))]
Network::Testnet4 => daemon_dir.push("testnet4"),
#[cfg(not(feature = "liquid"))]
Network::Regtest => daemon_dir.push("regtest"),
#[cfg(not(feature = "liquid"))]
Network::Signet => daemon_dir.push("signet"),
#[cfg(feature = "liquid")]
Network::Liquid => daemon_dir.push("liquidv1"),
#[cfg(feature = "liquid")]
Network::LiquidTestnet => daemon_dir.push("liquidtestnet"),
#[cfg(feature = "liquid")]
Network::LiquidRegtest => daemon_dir.push("liquidregtest"),
}
let blocks_dir = m
.value_of("blocks_dir")
.map(PathBuf::from)
.unwrap_or_else(|| daemon_dir.join("blocks"));
let cookie = m.value_of("cookie").map(|s| s.to_owned());
let electrum_banner = m
.value_of("electrum_banner")
.map_or_else(|| format!("Welcome to {}", *VERSION_STRING), |s| s.into());
#[cfg(feature = "electrum-discovery")]
let electrum_public_hosts = m
.value_of("electrum_public_hosts")
.map(|s| serde_json::from_str(s).expect("invalid --electrum-public-hosts"));
#[cfg(not(feature = "electrum-discovery"))]
let electrum_public_hosts: Option<crate::electrum::ServerHosts> = None;
let mut log = stderrlog::new();
log.verbosity(m.occurrences_of("verbosity") as usize);
log.timestamp(if m.is_present("timestamp") {
@ -543,97 +260,37 @@ impl Config {
stderrlog::Timestamp::Off
});
log.init().expect("logging initialization failed");
let mut bulk_index_threads = value_t_or_exit!(m, "bulk_index_threads", usize);
if bulk_index_threads == 0 {
bulk_index_threads = num_cpus::get();
}
let config = Config {
log,
network_type,
magic,
db_path,
daemon_dir,
blocks_dir,
daemon_rpc_addr,
cookie,
utxos_limit: value_t_or_exit!(m, "utxos_limit", usize),
electrum_rpc_addr,
electrum_txs_limit: value_t_or_exit!(m, "electrum_txs_limit", usize),
electrum_banner,
http_addr,
http_socket_file,
rpc_socket_file,
monitoring_addr,
mempool_backlog_stats_ttl: value_t_or_exit!(m, "mempool_backlog_stats_ttl", u64),
mempool_recent_txs_size: value_t_or_exit!(m, "mempool_recent_txs_size", usize),
rest_default_block_limit: value_t_or_exit!(m, "rest_default_block_limit", usize),
rest_default_chain_txs_per_page: value_t_or_exit!(
m,
"rest_default_chain_txs_per_page",
usize
),
rest_default_max_mempool_txs: value_t_or_exit!(
m,
"rest_default_max_mempool_txs",
usize
),
rest_default_max_address_summary_txs: value_t_or_exit!(
m,
"rest_default_max_address_summary_txs",
usize
),
rest_max_mempool_page_size: value_t_or_exit!(m, "rest_max_mempool_page_size", usize),
rest_max_mempool_txid_page_size: value_t_or_exit!(
m,
"rest_max_mempool_txid_page_size",
usize
),
electrum_max_line_size: value_t_or_exit!(m, "electrum_max_line_size", usize),
electrum_max_subscriptions: value_t_or_exit!(m, "electrum_max_subscriptions", usize),
electrum_max_clients: value_t_or_exit!(m, "electrum_max_clients", usize),
electrum_idle_timeout: value_t_or_exit!(m, "electrum_idle_timeout", u64),
electrum_haproxy_depth: value_t_or_exit!(m, "electrum_haproxy_depth", usize),
electrum_connections_per_client: value_t_or_exit!(
m,
"electrum_connections_per_client",
usize
),
jsonrpc_import: m.is_present("jsonrpc_import"),
light_mode: m.is_present("light_mode"),
main_loop_delay: value_t_or_exit!(m, "main_loop_delay", u64),
address_search: m.is_present("address_search"),
index_unspendables: m.is_present("index_unspendables"),
index_batch_size: value_t_or_exit!(m, "index_batch_size", usize),
bulk_index_threads,
tx_cache_size: value_t_or_exit!(m, "tx_cache_size", usize),
prevout_enabled: !m.is_present("disable_prevout"),
cors: m.value_of("cors").map(|s| s.to_string()),
precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()),
precache_threads: m.value_of("precache_threads").map_or_else(
|| {
std::thread::available_parallelism()
.expect("Can't get core count")
.get()
* 4
},
|s| match s.parse::<usize>() {
Ok(v) if v > 0 => v,
_ => clap::Error::value_validation_auto(format!(
"The argument '{}' isn't a valid value",
s
))
.exit(),
},
),
#[cfg(feature = "liquid")]
parent_network,
#[cfg(feature = "liquid")]
asset_db_path,
electrum_public_hosts,
#[cfg(feature = "electrum-discovery")]
electrum_announce: m.is_present("electrum_announce"),
#[cfg(feature = "electrum-discovery")]
tor_proxy: m.value_of("tor_proxy").map(|s| s.parse().unwrap()),
parent_genesis_hash,
};
eprintln!("{:?}", config);
config
}
pub fn cookie_getter(&self) -> Arc<dyn CookieGetter> {
pub fn cookie_getter(&self) -> Arc<CookieGetter> {
if let Some(ref value) = self.cookie {
Arc::new(StaticCookie {
value: value.as_bytes().to_vec(),

View File

@ -1,41 +1,32 @@
use base64;
use bitcoin::consensus::encode::{deserialize, serialize};
use bitcoin::util::hash::BitcoinHash;
use bitcoin_hashes::hex::{FromHex, ToHex};
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use glob;
use hex;
use serde_json::{from_str, from_value, Value};
use std::collections::{HashMap, HashSet};
use std::io::{BufRead, BufReader, Lines, Write};
use std::net::{SocketAddr, TcpStream};
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use base64;
use bitcoin::hashes::Hash;
use glob;
use hex;
use itertools::Itertools;
use serde_json::{from_str, from_value, Value};
#[cfg(not(feature = "liquid"))]
use bitcoin::consensus::encode::{deserialize, serialize};
#[cfg(feature = "liquid")]
use elements::encode::{deserialize, serialize};
use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid};
use crate::config::BITCOIND_SUBVER;
use crate::chain::{Block, BlockHeader, Network, Transaction};
use crate::metrics::{HistogramOpts, HistogramVec, Metrics};
use crate::signal::Waiter;
use crate::util::HeaderList;
use crate::errors::*;
fn parse_hash<T>(value: &Value) -> Result<T>
where
T: FromStr,
<T as FromStr>::Err: std::fmt::Debug,
{
value
.as_str()
.chain_err(|| format!("non-string value: {}", value))?
.parse::<T>()
.map_err(|e| format!("failed to parse hash: {:?}", e).into())
fn parse_hash(value: &Value) -> Result<Sha256dHash> {
Ok(Sha256dHash::from_hex(
value
.as_str()
.chain_err(|| format!("non-string value: {}", value))?,
)
.chain_err(|| format!("non-hex value: {}", value))?)
}
fn header_from_value(value: Value) -> Result<BlockHeader> {
@ -43,19 +34,22 @@ fn header_from_value(value: Value) -> Result<BlockHeader> {
.as_str()
.chain_err(|| format!("non-string header: {}", value))?;
let header_bytes = hex::decode(header_hex).chain_err(|| "non-hex header")?;
deserialize(&header_bytes).chain_err(|| format!("failed to parse header {}", header_hex))
Ok(
deserialize(&header_bytes)
.chain_err(|| format!("failed to parse header {}", header_hex))?,
)
}
fn block_from_value(value: Value) -> Result<Block> {
let block_hex = value.as_str().chain_err(|| "non-string block")?;
let block_bytes = hex::decode(block_hex).chain_err(|| "non-hex block")?;
deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))
Ok(deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))?)
}
fn tx_from_value(value: Value) -> Result<Transaction> {
let tx_hex = value.as_str().chain_err(|| "non-string tx")?;
let tx_bytes = hex::decode(tx_hex).chain_err(|| "non-hex tx")?;
deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))
Ok(deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))?)
}
/// Parse JSONRPC error code, if exists.
@ -67,13 +61,14 @@ fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Resu
if let Some(reply_obj) = reply.as_object_mut() {
if let Some(err) = reply_obj.get("error") {
if !err.is_null() {
if let Some(code) = parse_error_code(err) {
if let Some(code) = parse_error_code(&err) {
match code {
// RPC_IN_WARMUP -> retry by later reconnection
-28 => bail!(ErrorKind::Connection(err.to_string())),
_ => bail!("{} RPC error: {}", method, err),
_ => (),
}
}
bail!("{} RPC error: {}", method, err);
}
}
let id = reply_obj
@ -107,64 +102,38 @@ pub struct BlockchainInfo {
pub initialblockdownload: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MempoolInfo {
pub loaded: bool,
}
#[derive(Serialize, Deserialize, Debug)]
struct NetworkInfo {
version: u64,
subversion: String,
relayfee: f64, // in BTC/kB
}
#[derive(Serialize, Deserialize, Debug)]
struct MempoolFees {
base: f64,
#[serde(rename = "effective-feerate")]
effective_feerate: f64,
#[serde(rename = "effective-includes")]
effective_includes: Vec<String>,
pub struct MempoolEntry {
fee: u64, // in satoshis
vsize: u32, // in virtual bytes (= weight/4)
fee_per_vbyte: f32,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MempoolAcceptResult {
txid: String,
wtxid: String,
allowed: Option<bool>,
vsize: Option<u32>,
fees: Option<MempoolFees>,
#[serde(rename = "reject-reason")]
reject_reason: Option<String>,
}
impl MempoolEntry {
fn new(fee: u64, vsize: u32) -> MempoolEntry {
MempoolEntry {
fee,
vsize,
fee_per_vbyte: fee as f32 / vsize as f32,
}
}
#[derive(Serialize, Deserialize, Debug)]
struct MempoolFeesSubmitPackage {
base: f64,
#[serde(rename = "effective-feerate")]
effective_feerate: Option<f64>,
#[serde(rename = "effective-includes")]
effective_includes: Option<Vec<String>>,
}
pub fn fee_per_vbyte(&self) -> f32 {
self.fee_per_vbyte
}
#[derive(Serialize, Deserialize, Debug)]
pub struct SubmitPackageResult {
package_msg: String,
#[serde(rename = "tx-results")]
tx_results: HashMap<String, TxResult>,
#[serde(rename = "replaced-transactions")]
replaced_transactions: Option<Vec<String>>,
}
pub fn fee(&self) -> u64 {
self.fee
}
#[derive(Serialize, Deserialize, Debug)]
pub struct TxResult {
txid: String,
#[serde(rename = "other-wtxid")]
other_wtxid: Option<String>,
vsize: Option<u32>,
fees: Option<MempoolFeesSubmitPackage>,
error: Option<String>,
pub fn vsize(&self) -> u32 {
self.vsize
}
}
pub trait CookieGetter: Send + Sync {
@ -174,7 +143,7 @@ pub trait CookieGetter: Send + Sync {
struct Connection {
tx: TcpStream,
rx: Lines<BufReader<TcpStream>>,
cookie_getter: Arc<dyn CookieGetter>,
cookie_getter: Arc<CookieGetter>,
addr: SocketAddr,
signal: Waiter,
}
@ -185,7 +154,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
Ok(conn) => return Ok(conn),
Err(err) => {
warn!("failed to connect daemon at {}: {}", addr, err);
signal.wait(Duration::from_secs(3), false)?;
signal.wait(Duration::from_secs(3))?;
continue;
}
}
@ -195,7 +164,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
impl Connection {
fn new(
addr: SocketAddr,
cookie_getter: Arc<dyn CookieGetter>,
cookie_getter: Arc<CookieGetter>,
signal: Waiter,
) -> Result<Connection> {
let conn = tcp_connect(addr, &signal)?;
@ -239,7 +208,7 @@ impl Connection {
.chain_err(|| {
ErrorKind::Connection("disconnected from daemon while receiving".to_owned())
})?
.chain_err(|| ErrorKind::Connection("failed to read status".to_owned()))?;
.chain_err(|| "failed to read status")?;
let mut headers = HashMap::new();
for line in iter {
let line = line.chain_err(|| ErrorKind::Connection("failed to read".to_owned()))?;
@ -312,9 +281,7 @@ impl Counter {
pub struct Daemon {
daemon_dir: PathBuf,
blocks_dir: PathBuf,
network: Network,
magic: Option<u32>,
conn: Mutex<Connection>,
message_id: Counter, // for monotonic JSONRPC 'id'
signal: Waiter,
@ -325,22 +292,17 @@ pub struct Daemon {
}
impl Daemon {
#[allow(clippy::too_many_arguments)]
pub fn new(
daemon_dir: PathBuf,
blocks_dir: PathBuf,
daemon_dir: &PathBuf,
daemon_rpc_addr: SocketAddr,
cookie_getter: Arc<dyn CookieGetter>,
cookie_getter: Arc<CookieGetter>,
network: Network,
magic: Option<u32>,
signal: Waiter,
metrics: &Metrics,
) -> Result<Daemon> {
let daemon = Daemon {
daemon_dir,
blocks_dir,
daemon_dir: daemon_dir.clone(),
network,
magic,
conn: Mutex::new(Connection::new(
daemon_rpc_addr,
cookie_getter,
@ -359,42 +321,32 @@ impl Daemon {
};
let network_info = daemon.getnetworkinfo()?;
info!("{:?}", network_info);
if network_info.version < 16_00_00 {
if network_info.version < 00_16_00_00 {
bail!(
"{} is not supported - please use bitcoind 0.16+",
network_info.subversion,
)
}
// Insert the subversion (/Satoshi xx.xx.xx(comment)/) string from bitcoind
_ = BITCOIND_SUBVER.set(network_info.subversion);
let blockchain_info = daemon.getblockchaininfo()?;
info!("{:?}", blockchain_info);
if blockchain_info.pruned {
if blockchain_info.pruned == true {
bail!("pruned node is not supported (use '-prune=0' bitcoind flag)".to_owned())
}
loop {
let info = daemon.getblockchaininfo()?;
let mempool = daemon.getmempoolinfo()?;
let ibd_done = if network.is_regtest() {
info.blocks == info.headers
} else {
!info.initialblockdownload.unwrap_or(false)
// initialblockdownload is unavailable on the 0.14-based elements
let synced = match info.initialblockdownload {
Some(ibd) => !ibd,
None => info.verificationprogress == 1.0,
};
if mempool.loaded && ibd_done && info.blocks == info.headers {
if synced {
break;
}
warn!(
"waiting for bitcoind sync and mempool load to finish: {}/{} blocks, verification progress: {:.3}%, mempool loaded: {}",
info.blocks,
info.headers,
info.verificationprogress * 100.0,
mempool.loaded
);
signal.wait(Duration::from_secs(5), false)?;
warn!("wait until bitcoind is synced");
signal.wait(Duration::from_secs(3))?;
}
Ok(daemon)
}
@ -402,9 +354,7 @@ impl Daemon {
pub fn reconnect(&self) -> Result<Daemon> {
Ok(Daemon {
daemon_dir: self.daemon_dir.clone(),
blocks_dir: self.blocks_dir.clone(),
network: self.network,
magic: self.magic,
conn: Mutex::new(self.conn.lock().unwrap().reconnect()?),
message_id: Counter::new(),
signal: self.signal.clone(),
@ -414,7 +364,9 @@ impl Daemon {
}
pub fn list_blk_files(&self) -> Result<Vec<PathBuf>> {
let path = self.blocks_dir.join("blk*.dat");
let mut path = self.daemon_dir.clone();
path.push("blocks");
path.push("blk*.dat");
debug!("listing block files at {:?}", path);
let mut paths: Vec<PathBuf> = glob::glob(path.to_str().unwrap())
.chain_err(|| "failed to list blk*.dat files")?
@ -425,7 +377,7 @@ impl Daemon {
}
pub fn magic(&self) -> u32 {
self.magic.unwrap_or_else(|| self.network.magic())
self.network.magic()
}
fn call_jsonrpc(&self, method: &str, request: &Value) -> Result<Value> {
@ -445,66 +397,29 @@ impl Daemon {
Ok(result)
}
fn handle_request_batch(
&self,
method: &str,
params_list: &[Value],
failure_threshold: f64,
) -> Result<Vec<Value>> {
fn handle_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
let id = self.message_id.next();
let chunks = params_list
let reqs = params_list
.iter()
.map(|params| json!({"method": method, "params": params, "id": id}))
.chunks(50_000); // Max Amount of batched requests
.collect();
let mut results = vec![];
let total_requests = params_list.len();
let mut failed_requests: u64 = 0;
let threshold = (failure_threshold * total_requests as f64).round() as u64;
let mut n = 0;
for chunk in &chunks {
let reqs = chunk.collect();
let mut replies = self.call_jsonrpc(method, &reqs)?;
if let Some(replies_vec) = replies.as_array_mut() {
for reply in replies_vec {
n += 1;
match parse_jsonrpc_reply(reply.take(), method, id) {
Ok(parsed_reply) => results.push(parsed_reply),
Err(e) => {
failed_requests += 1;
warn!(
"batch request {} {}/{} failed: {}",
method,
n,
total_requests,
e.to_string()
);
// abort and return the last error once a threshold number of requests have failed
if failed_requests > threshold {
return Err(e);
}
}
}
}
} else {
bail!("non-array replies: {:?}", replies);
let mut replies = self.call_jsonrpc(method, &reqs)?;
if let Some(replies_vec) = replies.as_array_mut() {
for reply in replies_vec {
results.push(parse_jsonrpc_reply(reply.take(), method, id)?)
}
return Ok(results);
}
Ok(results)
bail!("non-array replies: {:?}", replies);
}
fn retry_request_batch(
&self,
method: &str,
params_list: &[Value],
failure_threshold: f64,
) -> Result<Vec<Value>> {
fn retry_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
loop {
match self.handle_request_batch(method, params_list, failure_threshold) {
match self.handle_request_batch(method, params_list) {
Err(Error(ErrorKind::Connection(msg), _)) => {
warn!("reconnecting to bitcoind: {}", msg);
self.signal.wait(Duration::from_secs(3), false)?;
self.signal.wait(Duration::from_secs(3))?;
let mut conn = self.conn.lock().unwrap();
*conn = conn.reconnect()?;
continue;
@ -515,40 +430,35 @@ impl Daemon {
}
fn request(&self, method: &str, params: Value) -> Result<Value> {
let mut values = self.retry_request_batch(method, &[params], 0.0)?;
let mut values = self.retry_request_batch(method, &[params])?;
assert_eq!(values.len(), 1);
Ok(values.remove(0))
}
fn requests(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
self.retry_request_batch(method, params_list, 0.0)
self.retry_request_batch(method, params_list)
}
// bitcoind JSONRPC API:
pub fn getblockchaininfo(&self) -> Result<BlockchainInfo> {
let info: Value = self.request("getblockchaininfo", json!([]))?;
from_value(info).chain_err(|| "invalid blockchain info")
}
fn getmempoolinfo(&self) -> Result<MempoolInfo> {
let info: Value = self.request("getmempoolinfo", json!([]))?;
from_value(info).chain_err(|| "invalid mempool info")
Ok(from_value(info).chain_err(|| "invalid blockchain info")?)
}
fn getnetworkinfo(&self) -> Result<NetworkInfo> {
let info: Value = self.request("getnetworkinfo", json!([]))?;
from_value(info).chain_err(|| "invalid network info")
Ok(from_value(info).chain_err(|| "invalid network info")?)
}
pub fn getbestblockhash(&self) -> Result<BlockHash> {
parse_hash(&self.request("getbestblockhash", json!([]))?)
pub fn getbestblockhash(&self) -> Result<Sha256dHash> {
parse_hash(&self.request("getbestblockhash", json!([]))?).chain_err(|| "invalid blockhash")
}
pub fn getblockheader(&self, blockhash: &BlockHash) -> Result<BlockHeader> {
pub fn getblockheader(&self, blockhash: &Sha256dHash) -> Result<BlockHeader> {
header_from_value(self.request(
"getblockheader",
json!([blockhash.to_string(), /*verbose=*/ false]),
json!([blockhash.to_hex(), /*verbose=*/ false]),
)?)
}
@ -566,23 +476,22 @@ impl Daemon {
Ok(result)
}
pub fn getblock(&self, blockhash: &BlockHash) -> Result<Block> {
let block = block_from_value(self.request(
"getblock",
json!([blockhash.to_string(), /*verbose=*/ false]),
)?)?;
assert_eq!(block.block_hash(), *blockhash);
pub fn getblock(&self, blockhash: &Sha256dHash) -> Result<Block> {
let block = block_from_value(
self.request("getblock", json!([blockhash.to_hex(), /*verbose=*/ false]))?,
)?;
assert_eq!(block.bitcoin_hash(), *blockhash);
Ok(block)
}
pub fn getblock_raw(&self, blockhash: &BlockHash, verbose: u32) -> Result<Value> {
self.request("getblock", json!([blockhash.to_string(), verbose]))
pub fn getblock_raw(&self, blockhash: &Sha256dHash, verbose: u32) -> Result<Value> {
self.request("getblock", json!([blockhash.to_hex(), verbose]))
}
pub fn getblocks(&self, blockhashes: &[BlockHash]) -> Result<Vec<Block>> {
pub fn getblocks(&self, blockhashes: &[Sha256dHash]) -> Result<Vec<Block>> {
let params_list: Vec<Value> = blockhashes
.iter()
.map(|hash| json!([hash.to_string(), /*verbose=*/ false]))
.map(|hash| json!([hash.to_hex(), /*verbose=*/ false]))
.collect();
let values = self.requests("getblock", &params_list)?;
let mut blocks = vec![];
@ -592,145 +501,104 @@ impl Daemon {
Ok(blocks)
}
pub fn gettransactions(&self, txhashes: &[&Txid]) -> Result<Vec<Transaction>> {
pub fn gettransactions(&self, txhashes: &[&Sha256dHash]) -> Result<Vec<Transaction>> {
let params_list: Vec<Value> = txhashes
.iter()
.map(|txhash| json!([txhash.to_string(), /*verbose=*/ false]))
.map(|txhash| json!([txhash.to_hex(), /*verbose=*/ false]))
.collect();
let values = self.retry_request_batch("getrawtransaction", &params_list, 0.25)?;
let values = self.requests("getrawtransaction", &params_list)?;
let mut txs = vec![];
for value in values {
txs.push(tx_from_value(value)?);
}
// missing transactions are skipped, so the number of txs returned may be less than the number of txids requested
assert_eq!(txhashes.len(), txs.len());
Ok(txs)
}
pub fn gettransaction_raw(
&self,
txid: &Txid,
blockhash: &BlockHash,
verbose: bool,
) -> Result<Value> {
self.request(
"getrawtransaction",
json!([txid.to_string(), verbose, blockhash]),
)
}
pub fn getmempooltx(&self, txhash: &Txid) -> Result<Transaction> {
pub fn getmempooltx(&self, txhash: &Sha256dHash) -> Result<Transaction> {
let value = self.request(
"getrawtransaction",
json!([txhash.to_string(), /*verbose=*/ false]),
json!([txhash.to_hex(), /*verbose=*/ false]),
)?;
tx_from_value(value)
}
pub fn getmempooltxids(&self) -> Result<HashSet<Txid>> {
let res = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
serde_json::from_value(res).chain_err(|| "invalid getrawmempool reply")
pub fn getmempooltxids(&self) -> Result<HashSet<Sha256dHash>> {
let txids: Value = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
let mut result = HashSet::new();
for value in txids.as_array().chain_err(|| "non-array result")? {
result.insert(parse_hash(&value).chain_err(|| "invalid txid")?);
}
Ok(result)
}
pub fn broadcast(&self, tx: &Transaction) -> Result<Txid> {
pub fn getmempoolentry(&self, txid: &Sha256dHash) -> Result<MempoolEntry> {
let entry = self.request("getmempoolentry", json!([txid.to_hex()]))?;
let fee = (entry
.get("fee")
.chain_err(|| "missing fee")?
.as_f64()
.chain_err(|| "non-float fee")?
* 100_000_000f64) as u64;
let vsize = entry
.get("size")
.chain_err(|| "missing size")?
.as_u64()
.chain_err(|| "non-integer size")? as u32;
Ok(MempoolEntry::new(fee, vsize))
}
pub fn broadcast(&self, tx: &Transaction) -> Result<Sha256dHash> {
self.broadcast_raw(&hex::encode(serialize(tx)))
}
pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
pub fn broadcast_raw(&self, txhex: &String) -> Result<Sha256dHash> {
let txid = self.request("sendrawtransaction", json!([txhex]))?;
txid.as_str()
.chain_err(|| "non-string txid")?
.parse::<Txid>()
.map_err(|e| format!("failed to parse txid: {:?}", e).into())
Ok(
Sha256dHash::from_hex(txid.as_str().chain_err(|| "non-string txid")?)
.chain_err(|| "failed to parse txid")?,
)
}
pub fn test_mempool_accept(
&self,
txhex: Vec<String>,
maxfeerate: Option<f64>,
) -> Result<Vec<MempoolAcceptResult>> {
let params = match maxfeerate {
Some(rate) => json!([txhex, format!("{:.8}", rate)]),
None => json!([txhex]),
};
let result = self.request("testmempoolaccept", params)?;
serde_json::from_value::<Vec<MempoolAcceptResult>>(result)
.chain_err(|| "invalid testmempoolaccept reply")
pub fn estimatesmartfee(&self, conf_target: u16) -> Result<f32> {
let feerate = self
.request("estimatesmartfee", json!([conf_target]))?
.get("feerate")
.chain_err(|| "missing feerate")?
.as_f64()
.chain_err(|| "invalid feerate")?;
if feerate == -1f64 {
bail!("not enough data to estimate");
}
// from BTC/kB to sat/b
Ok((feerate * 100_000f64) as f32)
}
pub fn submit_package(
&self,
txhex: Vec<String>,
maxfeerate: Option<f64>,
maxburnamount: Option<f64>,
) -> Result<SubmitPackageResult> {
let params = match (maxfeerate, maxburnamount) {
(Some(rate), Some(burn)) => {
json!([txhex, format!("{:.8}", rate), format!("{:.8}", burn)])
}
(Some(rate), None) => json!([txhex, format!("{:.8}", rate)]),
(None, Some(burn)) => json!([txhex, null, format!("{:.8}", burn)]),
(None, None) => json!([txhex]),
};
let result = self.request("submitpackage", params)?;
serde_json::from_value::<SubmitPackageResult>(result)
.chain_err(|| "invalid submitpackage reply")
}
// Get estimated feerates for the provided confirmation targets using a batch RPC request
// Missing estimates are logged but do not cause a failure, whatever is available is returned
#[allow(clippy::float_cmp)]
pub fn estimatesmartfee_batch(&self, conf_targets: &[u16]) -> Result<HashMap<u16, f64>> {
let params_list: Vec<Value> = conf_targets.iter().map(|t| json!([t])).collect();
Ok(self
.requests("estimatesmartfee", &params_list)?
.iter()
.zip(conf_targets)
.filter_map(|(reply, target)| {
if !reply["errors"].is_null() {
warn!(
"failed estimating fee for target {}: {:?}",
target, reply["errors"]
);
return None;
}
let feerate = reply["feerate"]
.as_f64()
.unwrap_or_else(|| panic!("invalid estimatesmartfee response: {:?}", reply));
if feerate == -1f64 {
warn!("not enough data to estimate fee for target {}", target);
return None;
}
// from BTC/kB to sat/b
Some((*target, feerate * 100_000f64))
})
.collect())
}
fn get_all_headers(&self, tip: &BlockHash) -> Result<Vec<BlockHeader>> {
let info: Value = self.request("getblockheader", json!([tip.to_string()]))?;
fn get_all_headers(&self, tip: &Sha256dHash) -> Result<Vec<BlockHeader>> {
let info: Value = self.request("getblockheader", json!([tip.to_hex()]))?;
let tip_height = info
.get("height")
.expect("missing height")
.as_u64()
.expect("non-numeric height") as usize;
let all_heights: Vec<usize> = (0..=tip_height).collect();
let all_heights: Vec<usize> = (0..tip_height + 1).collect();
let chunk_size = 100_000;
let mut result = vec![];
let null_hash = Sha256dHash::default();
for heights in all_heights.chunks(chunk_size) {
trace!("downloading {} block headers", heights.len());
let mut headers = self.getblockheaders(heights)?;
let mut headers = self.getblockheaders(&heights)?;
assert!(headers.len() == heights.len());
result.append(&mut headers);
}
let mut blockhash = BlockHash::all_zeros();
let mut blockhash = null_hash;
for header in &result {
assert_eq!(header.prev_blockhash, blockhash);
blockhash = header.block_hash();
blockhash = header.bitcoin_hash();
}
assert_eq!(blockhash, *tip);
Ok(result)
@ -740,10 +608,10 @@ impl Daemon {
pub fn get_new_headers(
&self,
indexed_headers: &HeaderList,
bestblockhash: &BlockHash,
bestblockhash: &Sha256dHash,
) -> Result<Vec<BlockHeader>> {
// Iterate back over headers until known blockash is found:
if indexed_headers.is_empty() {
if indexed_headers.len() == 0 {
debug!("downloading all block headers up to {}", bestblockhash);
return self.get_all_headers(bestblockhash);
}
@ -753,7 +621,7 @@ impl Daemon {
bestblockhash,
);
let mut new_headers = vec![];
let null_hash = BlockHash::all_zeros();
let null_hash = Sha256dHash::default();
let mut blockhash = *bestblockhash;
while blockhash != null_hash {
if indexed_headers.header_by_blockhash(&blockhash).is_some() {
@ -769,11 +637,4 @@ impl Daemon {
new_headers.reverse(); // so the tip is the last vector entry
Ok(new_headers)
}
pub fn get_relayfee(&self) -> Result<f64> {
let relayfee = self.getnetworkinfo()?.relayfee;
// from BTC/kB to sat/b
Ok(relayfee * 100_000f64)
}
}

630
src/electrum.rs Normal file
View File

@ -0,0 +1,630 @@
use bitcoin::consensus::encode::serialize;
use bitcoin_hashes::hex::{FromHex, ToHex};
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use crypto::digest::Digest;
use crypto::sha2::Sha256;
use error_chain::ChainedError;
use hex;
use serde_json::{from_str, Value};
use std::collections::HashMap;
use std::io::{BufRead, BufReader, Write};
use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
use std::sync::mpsc::{Sender, SyncSender, TrySendError};
use std::sync::{Arc, Mutex};
use std::thread;
use crate::errors::*;
use crate::metrics::{Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics};
use crate::new_index::Query;
use crate::util::{
full_hash, get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof, spawn_thread,
BlockId, Channel, FullHash, HeaderEntry, SyncChannel,
};
// TODO: Sha256dHash should be a generic hash-container (since script hash is single SHA256)
fn hash_from_value(val: Option<&Value>) -> Result<Sha256dHash> {
let script_hash = val.chain_err(|| "missing hash")?;
let script_hash = script_hash.as_str().chain_err(|| "non-string hash")?;
let script_hash = Sha256dHash::from_hex(script_hash).chain_err(|| "non-hex hash")?;
Ok(script_hash)
}
fn usize_from_value(val: Option<&Value>, name: &str) -> Result<usize> {
let val = val.chain_err(|| format!("missing {}", name))?;
let val = val.as_u64().chain_err(|| format!("non-integer {}", name))?;
Ok(val as usize)
}
fn usize_from_value_or(val: Option<&Value>, name: &str, default: usize) -> Result<usize> {
if val.is_none() {
return Ok(default);
}
usize_from_value(val, name)
}
fn bool_from_value(val: Option<&Value>, name: &str) -> Result<bool> {
let val = val.chain_err(|| format!("missing {}", name))?;
let val = val.as_bool().chain_err(|| format!("not a bool {}", name))?;
Ok(val)
}
fn bool_from_value_or(val: Option<&Value>, name: &str, default: bool) -> Result<bool> {
if val.is_none() {
return Ok(default);
}
bool_from_value(val, name)
}
// FIXME: implement caching and delta updates
// FIXME: ensure stable ordering
fn get_status_hash(txs: Vec<(Sha256dHash, Option<BlockId>)>) -> Option<FullHash> {
if txs.is_empty() {
None
} else {
let mut hash = FullHash::default();
let mut sha2 = Sha256::new();
for (txid, blockid) in txs {
// TODO: use height of 0 to indicate a mempool tx with confirmed inputs
let part = format!(
"{}:{}:",
txid.to_hex(),
blockid.map_or(-1, |b| b.height as isize)
);
sha2.input(part.as_bytes());
}
sha2.result(&mut hash);
Some(hash)
}
}
struct Connection {
query: Arc<Query>,
last_header_entry: Option<HeaderEntry>,
status_hashes: HashMap<Sha256dHash, Value>, // ScriptHash -> StatusHash
stream: TcpStream,
addr: SocketAddr,
chan: SyncChannel<Message>,
stats: Arc<Stats>,
}
impl Connection {
pub fn new(
query: Arc<Query>,
stream: TcpStream,
addr: SocketAddr,
stats: Arc<Stats>,
) -> Connection {
Connection {
query,
last_header_entry: None, // disable header subscription for now
status_hashes: HashMap::new(),
stream,
addr,
chan: SyncChannel::new(10),
stats,
}
}
fn blockchain_headers_subscribe(&mut self) -> Result<Value> {
let entry = self.query.chain().best_header();
let hex_header = hex::encode(serialize(entry.header()));
let result = json!({"hex": hex_header, "height": entry.height()});
self.last_header_entry = Some(entry);
Ok(result)
}
fn server_version(&self) -> Result<Value> {
// TODO dynamic version
Ok(json!(["electrs-esplora", "1.4"]))
}
fn server_banner(&self) -> Result<Value> {
// TODO dynamic banner
Ok(json!("Welcome to electrs-esplora"))
}
fn server_donation_address(&self) -> Result<Value> {
Ok(Value::Null)
}
fn server_peers_subscribe(&self) -> Result<Value> {
Ok(json!([]))
}
fn mempool_get_fee_histogram(&self) -> Result<Value> {
Ok(json!(&self.query.mempool().backlog_stats().fee_histogram))
}
fn blockchain_block_header(&self, params: &[Value]) -> Result<Value> {
let height = usize_from_value(params.get(0), "height")?;
let cp_height = usize_from_value_or(params.get(1), "cp_height", 0)?;
let raw_header_hex: String = self
.query
.chain()
.header_by_height(height)
.map(|entry| hex::encode(&serialize(entry.header())))
.chain_err(|| "missing header")?;
if cp_height == 0 {
return Ok(json!(raw_header_hex));
}
let (branch, root) = get_header_merkle_proof(self.query.chain(), height, cp_height)?;
let branch_vec: Vec<String> = branch.into_iter().map(|b| b.to_hex()).collect();
return Ok(json!({
"header": raw_header_hex,
"root": root.to_hex(),
"branch": branch_vec
}));
}
fn blockchain_block_headers(&self, params: &[Value]) -> Result<Value> {
let start_height = usize_from_value(params.get(0), "start_height")?;
let count = usize_from_value(params.get(1), "count")?;
let cp_height = usize_from_value_or(params.get(2), "cp_height", 0)?;
let heights: Vec<usize> = (start_height..(start_height + count)).collect();
let headers: Vec<String> = heights
.into_iter()
.map(|height| {
self.query
.chain()
.header_by_height(height)
.map(|entry| hex::encode(&serialize(entry.header())))
})
.collect::<Option<Vec<String>>>()
.chain_err(|| "requested block headers not found")?;
if count == 0 || cp_height == 0 {
return Ok(json!({
"count": headers.len(),
"hex": headers.join(""),
"max": 2016,
}));
}
let (branch, root) =
get_header_merkle_proof(self.query.chain(), start_height + (count - 1), cp_height)?;
let branch_vec: Vec<String> = branch.into_iter().map(|b| b.to_hex()).collect();
Ok(json!({
"count": headers.len(),
"hex": headers.join(""),
"max": 2016,
"root": root.to_hex(),
"branch" : branch_vec
}))
}
fn blockchain_estimatefee(&self, params: &[Value]) -> Result<Value> {
let conf_target = usize_from_value(params.get(0), "blocks_count")?;
let fee_rate = self
.query
.estimate_fee(conf_target as u16)
.chain_err(|| format!("cannot estimate fee for {} blocks", conf_target))?;
// convert from sat/b to BTC/kB, as expected by Electrum clients
Ok(json!(fee_rate / 100_000f32))
}
fn blockchain_relayfee(&self) -> Result<Value> {
// TODO
Ok(json!(0.0)) // allow sending transactions with any fee.
}
fn blockchain_scripthash_subscribe(&mut self, params: &[Value]) -> Result<Value> {
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
let history_txids = self.query.history_txids(&script_hash[..]);
let status_hash = get_status_hash(history_txids)
.map_or(Value::Null, |h| json!(hex::encode(full_hash(&h[..]))));
self.status_hashes.insert(script_hash, status_hash.clone());
Ok(status_hash)
}
#[cfg(not(feature = "liquid"))]
fn blockchain_scripthash_get_balance(&self, params: &[Value]) -> Result<Value> {
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]);
Ok(
json!({ "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, "unconfirmed": mempool_stats.funded_txo_sum - mempool_stats.spent_txo_sum}),
)
}
fn blockchain_scripthash_get_history(&self, params: &[Value]) -> Result<Value> {
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
let history_txids = self.query.history_txids(&script_hash[..]);
Ok(json!(Value::Array(
history_txids
.into_iter()
.map(|(txid, blockid)| json!({"height": blockid.map_or(0, |b| b.height), "tx_hash": txid.to_hex()}))
.collect()
)))
}
fn blockchain_scripthash_listunspent(&self, params: &[Value]) -> Result<Value> {
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
let utxos = self.query.utxo(&script_hash[..]);
Ok(json!(Value::Array(
utxos
.into_iter()
.map(|utxo| json!({
"height": utxo.confirmed.map_or(0, |b| b.height),
"tx_pos": utxo.vout,
"tx_hash": utxo.txid.to_hex(),
"value": utxo.value,
}))
.collect()
)))
}
fn blockchain_transaction_broadcast(&self, params: &[Value]) -> Result<Value> {
let tx = params.get(0).chain_err(|| "missing tx")?;
let tx = tx.as_str().chain_err(|| "non-string tx")?.to_string();
let txid = self.query.broadcast_raw(&tx)?;
if let Err(e) = self.chan.sender().try_send(Message::PeriodicUpdate) {
warn!("failed to issue PeriodicUpdate after broadcast: {}", e);
}
Ok(json!(txid.to_hex()))
}
fn blockchain_transaction_get(&self, params: &[Value]) -> Result<Value> {
let tx_hash = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
let verbose = match params.get(1) {
Some(value) => value.as_bool().chain_err(|| "non-bool verbose value")?,
None => false,
};
// FIXME: implement verbose support
if verbose {
bail!("verbose transactions are currently unsupported");
}
let tx = self
.query
.lookup_raw_txn(&tx_hash)
.chain_err(|| "missing transaction")?;
Ok(json!(hex::encode(tx)))
}
fn blockchain_transaction_get_merkle(&self, params: &[Value]) -> Result<Value> {
let txid = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
let height = usize_from_value(params.get(1), "height")?;
let blockid = self
.query
.chain()
.tx_confirming_block(&txid)
.ok_or_else(|| "tx not found or is unconfirmed")?;
if blockid.height != height {
bail!("invalid confirmation height provided");
}
let (merkle, pos) = get_tx_merkle_proof(self.query.chain(), &txid, &blockid.hash)
.chain_err(|| "cannot create merkle proof")?;
let merkle: Vec<String> = merkle.into_iter().map(|txid| txid.to_hex()).collect();
Ok(json!({
"block_height": blockid.height,
"merkle": merkle,
"pos": pos}))
}
fn blockchain_transaction_id_from_pos(&self, params: &[Value]) -> Result<Value> {
let height = usize_from_value(params.get(0), "height")?;
let tx_pos = usize_from_value(params.get(1), "tx_pos")?;
let want_merkle = bool_from_value_or(params.get(2), "merkle", false)?;
let (txid, merkle) = get_id_from_pos(self.query.chain(), height, tx_pos, want_merkle)?;
if !want_merkle {
return Ok(json!(txid.to_hex()));
}
let merkle_vec: Vec<String> = merkle.into_iter().map(|entry| entry.to_hex()).collect();
Ok(json!({
"tx_hash" : txid.to_hex(),
"merkle" : merkle_vec}))
}
fn handle_command(&mut self, method: &str, params: &[Value], id: &Value) -> Result<Value> {
let timer = self
.stats
.latency
.with_label_values(&[method])
.start_timer();
let result = match method {
"blockchain.block.header" => self.blockchain_block_header(&params),
"blockchain.block.headers" => self.blockchain_block_headers(&params),
"blockchain.estimatefee" => self.blockchain_estimatefee(&params),
"blockchain.headers.subscribe" => self.blockchain_headers_subscribe(),
"blockchain.relayfee" => self.blockchain_relayfee(),
#[cfg(not(feature = "liquid"))]
"blockchain.scripthash.get_balance" => self.blockchain_scripthash_get_balance(&params),
"blockchain.scripthash.get_history" => self.blockchain_scripthash_get_history(&params),
"blockchain.scripthash.listunspent" => self.blockchain_scripthash_listunspent(&params),
"blockchain.scripthash.subscribe" => self.blockchain_scripthash_subscribe(&params),
"blockchain.transaction.broadcast" => self.blockchain_transaction_broadcast(&params),
"blockchain.transaction.get" => self.blockchain_transaction_get(&params),
"blockchain.transaction.get_merkle" => self.blockchain_transaction_get_merkle(&params),
"blockchain.transaction.id_from_pos" => {
self.blockchain_transaction_id_from_pos(&params)
}
"mempool.get_fee_histogram" => self.mempool_get_fee_histogram(),
"server.banner" => self.server_banner(),
"server.donation_address" => self.server_donation_address(),
"server.peers.subscribe" => self.server_peers_subscribe(),
"server.ping" => Ok(Value::Null),
"server.version" => self.server_version(),
&_ => bail!("unknown method {} {:?}", method, params),
};
timer.observe_duration();
// TODO: return application errors should be sent to the client
Ok(match result {
Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}),
Err(e) => {
warn!(
"rpc #{} {} {:?} failed: {}",
id,
method,
params,
e.display_chain()
);
json!({"jsonrpc": "2.0", "id": id, "error": format!("{}", e)})
}
})
}
fn update_subscriptions(&mut self) -> Result<Vec<Value>> {
let timer = self
.stats
.latency
.with_label_values(&["periodic_update"])
.start_timer();
let mut result = vec![];
if let Some(ref mut last_entry) = self.last_header_entry {
let entry = self.query.chain().best_header();
if *last_entry != entry {
*last_entry = entry;
let hex_header = hex::encode(serialize(last_entry.header()));
let header = json!({"hex": hex_header, "height": last_entry.height()});
result.push(json!({
"jsonrpc": "2.0",
"method": "blockchain.headers.subscribe",
"params": [header]}));
}
}
for (script_hash, status_hash) in self.status_hashes.iter_mut() {
let history_txids = self.query.history_txids(&script_hash[..]);
let new_status_hash = get_status_hash(history_txids)
.map_or(Value::Null, |h| json!(hex::encode(full_hash(&h[..]))));
if new_status_hash == *status_hash {
continue;
}
result.push(json!({
"jsonrpc": "2.0",
"method": "blockchain.scripthash.subscribe",
"params": [script_hash.to_hex(), new_status_hash]}));
*status_hash = new_status_hash;
}
timer.observe_duration();
self.stats
.subscriptions
.set(self.status_hashes.len() as i64);
Ok(result)
}
fn send_values(&mut self, values: &[Value]) -> Result<()> {
for value in values {
let line = value.to_string() + "\n";
self.stream
.write_all(line.as_bytes())
.chain_err(|| format!("failed to send {}", value))?;
}
Ok(())
}
fn handle_replies(&mut self) -> Result<()> {
let empty_params = json!([]);
loop {
let msg = self.chan.receiver().recv().chain_err(|| "channel closed")?;
trace!("RPC {:?}", msg);
match msg {
Message::Request(line) => {
let cmd: Value = from_str(&line).chain_err(|| "invalid JSON format")?;
let reply = match (
cmd.get("method"),
cmd.get("params").unwrap_or_else(|| &empty_params),
cmd.get("id"),
) {
(
Some(&Value::String(ref method)),
&Value::Array(ref params),
Some(ref id),
) => self.handle_command(method, params, id)?,
_ => bail!("invalid command: {}", cmd),
};
self.send_values(&[reply])?
}
Message::PeriodicUpdate => {
let values = self
.update_subscriptions()
.chain_err(|| "failed to update subscriptions")?;
self.send_values(&values)?
}
Message::Done => return Ok(()),
}
}
}
fn handle_requests(mut reader: BufReader<TcpStream>, tx: SyncSender<Message>) -> Result<()> {
loop {
let mut line = Vec::<u8>::new();
reader
.read_until(b'\n', &mut line)
.chain_err(|| "failed to read a request")?;
if line.is_empty() {
tx.send(Message::Done).chain_err(|| "channel closed")?;
return Ok(());
} else {
if line.starts_with(&[22, 3, 1]) {
// (very) naive SSL handshake detection
let _ = tx.send(Message::Done);
bail!("invalid request - maybe SSL-encrypted data?: {:?}", line)
}
match String::from_utf8(line) {
Ok(req) => tx
.send(Message::Request(req))
.chain_err(|| "channel closed")?,
Err(err) => {
let _ = tx.send(Message::Done);
bail!("invalid UTF8: {}", err)
}
}
}
}
}
pub fn run(mut self) {
let reader = BufReader::new(self.stream.try_clone().expect("failed to clone TcpStream"));
let tx = self.chan.sender();
let child = spawn_thread("reader", || Connection::handle_requests(reader, tx));
if let Err(e) = self.handle_replies() {
error!(
"[{}] connection handling failed: {}",
self.addr,
e.display_chain().to_string()
);
}
debug!("[{}] shutting down connection", self.addr);
let _ = self.stream.shutdown(Shutdown::Both);
if let Err(err) = child.join().expect("receiver panicked") {
error!("[{}] receiver failed: {}", self.addr, err);
}
}
}
#[derive(Debug)]
pub enum Message {
Request(String),
PeriodicUpdate,
Done,
}
pub enum Notification {
Periodic,
Exit,
}
pub struct RPC {
notification: Sender<Notification>,
server: Option<thread::JoinHandle<()>>, // so we can join the server while dropping this ojbect
}
struct Stats {
latency: HistogramVec,
subscriptions: Gauge,
}
impl RPC {
fn start_notifier(
notification: Channel<Notification>,
senders: Arc<Mutex<Vec<SyncSender<Message>>>>,
acceptor: Sender<Option<(TcpStream, SocketAddr)>>,
) {
spawn_thread("notification", move || {
for msg in notification.receiver().iter() {
let mut senders = senders.lock().unwrap();
match msg {
Notification::Periodic => {
for sender in senders.split_off(0) {
if let Err(TrySendError::Disconnected(_)) =
sender.try_send(Message::PeriodicUpdate)
{
continue;
}
senders.push(sender);
}
}
Notification::Exit => acceptor.send(None).unwrap(),
}
}
});
}
fn start_acceptor(addr: SocketAddr) -> Channel<Option<(TcpStream, SocketAddr)>> {
let chan = Channel::new();
let acceptor = chan.sender();
spawn_thread("acceptor", move || {
let listener = TcpListener::bind(addr).expect(&format!("bind({}) failed", addr));
info!("Electrum RPC server running on {}", addr);
loop {
let (stream, addr) = listener.accept().expect("accept failed");
stream
.set_nonblocking(false)
.expect("failed to set connection as blocking");
acceptor.send(Some((stream, addr))).expect("send failed");
}
});
chan
}
pub fn start(addr: SocketAddr, query: Arc<Query>, metrics: &Metrics) -> RPC {
let stats = Arc::new(Stats {
latency: metrics.histogram_vec(
HistogramOpts::new("electrum_rpc", "Electrum RPC latency (seconds)"),
&["method"],
),
subscriptions: metrics.gauge(MetricOpts::new(
"electrum_subscriptions",
"# of Electrum subscriptions",
)),
});
let notification = Channel::new();
let handle = RPC {
notification: notification.sender(),
server: Some(spawn_thread("rpc", move || {
let senders = Arc::new(Mutex::new(Vec::<SyncSender<Message>>::new()));
let acceptor = RPC::start_acceptor(addr);
RPC::start_notifier(notification, senders.clone(), acceptor.sender());
let mut children = vec![];
while let Some((stream, addr)) = acceptor.receiver().recv().unwrap() {
let query = query.clone();
let senders = senders.clone();
let stats = stats.clone();
children.push(spawn_thread("peer", move || {
info!("[{}] connected peer", addr);
let conn = Connection::new(query, stream, addr, stats);
senders.lock().unwrap().push(conn.chan.sender());
conn.run();
info!("[{}] disconnected peer", addr);
}));
}
trace!("closing {} RPC connections", senders.lock().unwrap().len());
for sender in senders.lock().unwrap().iter() {
let _ = sender.send(Message::Done);
}
trace!("waiting for {} RPC handling threads", children.len());
for child in children {
let _ = child.join();
}
trace!("RPC connections are closed");
})),
};
handle
}
pub fn notify(&self) {
self.notification.send(Notification::Periodic).unwrap();
}
}
impl Drop for RPC {
fn drop(&mut self) {
trace!("stop accepting new RPCs");
self.notification.send(Notification::Exit).unwrap();
self.server.take().map(|t| t.join().unwrap());
trace!("RPC server is stopped");
}
}

View File

@ -1,41 +0,0 @@
use std::collections::HashMap;
use std::convert::TryFrom;
use bitcoin::hashes::sha256d;
use bitcoin::hashes::Hash;
pub use electrum_client::client::Client;
pub use electrum_client::ServerFeaturesRes;
use crate::chain::BlockHash;
use crate::electrum::ServerFeatures;
use crate::errors::{Error, ResultExt};
// Convert from electrum-client's server features struct to ours. We're using a different struct because
// the electrum-client's one doesn't support the "hosts" key.
impl TryFrom<ServerFeaturesRes> for ServerFeatures {
type Error = Error;
fn try_from(mut features: ServerFeaturesRes) -> Result<Self, Self::Error> {
features.genesis_hash.reverse();
Ok(ServerFeatures {
// electrum-client doesn't retain the hosts map data, but we already have it from the add_peer request
hosts: HashMap::new(),
genesis_hash: BlockHash::from_raw_hash(sha256d::Hash::from_byte_array(
features.genesis_hash,
)),
server_version: features.server_version,
protocol_min: features
.protocol_min
.parse()
.chain_err(|| "invalid protocol_min")?,
protocol_max: features
.protocol_max
.parse()
.chain_err(|| "invalid protocol_max")?,
pruning: features.pruning.map(|pruning| pruning as usize),
hash_function: features
.hash_function
.chain_err(|| "missing hash_function")?,
})
}
}

View File

@ -1,603 +0,0 @@
use std::cmp::Ordering;
use std::collections::{hash_map::Entry, BinaryHeap, HashMap, HashSet};
use std::convert::TryInto;
use std::fmt;
use std::net::{IpAddr, SocketAddr, ToSocketAddrs};
use std::str::FromStr;
use std::sync::{Arc, RwLock};
use std::thread;
use std::time::{Duration, Instant};
use electrum_client::ElectrumApi;
use crate::chain::Network;
use crate::electrum::{Client, Hostname, Port, ProtocolVersion, ServerFeatures};
use crate::errors::{Result, ResultExt};
use crate::util::spawn_thread;
mod default_servers;
use default_servers::add_default_servers;
const HEALTH_CHECK_FREQ: Duration = Duration::from_secs(3600); // check servers every hour
const JOB_INTERVAL: Duration = Duration::from_secs(1); // run one health check job every second
const MAX_CONSECUTIVE_FAILURES: usize = 24; // drop servers after 24 consecutive failing attempts (~24 hours) (~24 hours)
const MAX_QUEUE_SIZE: usize = 500; // refuse accepting new servers if we have that many health check jobs
const MAX_SERVERS_PER_REQUEST: usize = 3; // maximum number of server hosts added per server.add_peer call
const MAX_SERVICES_PER_REQUEST: usize = 6; // maximum number of services added per server.add_peer call
#[derive(Debug)]
pub struct DiscoveryManager {
/// A queue of scheduled health check jobs, including for healthy, unhealthy and untested servers
queue: RwLock<BinaryHeap<HealthCheck>>,
/// A list of servers that were found to be healthy on their last health check
healthy: RwLock<HashMap<ServerAddr, Server>>,
/// Used to test for protocol version compatibility
our_version: ProtocolVersion,
/// So that we don't list ourselves
our_addrs: HashSet<ServerAddr>,
/// For advertising ourself to other servers
our_features: ServerFeatures,
/// Whether we should announce ourselves to the servers we're connecting to
announce: bool,
/// Optional, will not support onion hosts without this
tor_proxy: Option<SocketAddr>,
}
/// A Server corresponds to a single IP address or onion hostname, with one or more services
/// exposed on different ports.
#[derive(Debug)]
struct Server {
services: HashSet<Service>,
hostname: Hostname,
features: ServerFeatures,
// the `ServerAddr` isn't kept here directly, but is also available next to `Server` as the key for
// the `healthy` field on `DiscoveryManager`
}
#[derive(Eq, PartialEq, Hash, Clone, Debug)]
enum ServerAddr {
Clearnet(IpAddr),
Onion(Hostname),
}
#[derive(Eq, PartialEq, Hash, Copy, Clone, Debug)]
pub enum Service {
Tcp(Port),
Ssl(Port),
// unimplemented: Ws and Wss
}
/// A queued health check job, one per service/port (and not per server)
#[derive(Eq, Debug)]
struct HealthCheck {
addr: ServerAddr,
hostname: Hostname,
service: Service,
is_default: bool,
#[allow(dead_code)]
added_by: Option<IpAddr>,
last_check: Option<Instant>,
last_healthy: Option<Instant>,
consecutive_failures: usize,
}
/// The server entry format returned from server.peers.subscribe
#[derive(Serialize)]
pub struct ServerEntry(ServerAddr, Hostname, Vec<String>);
impl DiscoveryManager {
pub fn new(
our_network: Network,
our_features: ServerFeatures,
our_version: ProtocolVersion,
announce: bool,
tor_proxy: Option<SocketAddr>,
) -> Self {
let our_addrs = our_features
.hosts
.keys()
.filter_map(|hostname| {
ServerAddr::resolve(hostname)
.map_err(|e| warn!("failed resolving own hostname {}: {:?}", hostname, e))
.ok()
})
.collect();
let discovery = Self {
our_addrs,
our_version,
our_features,
announce,
tor_proxy,
healthy: Default::default(),
queue: Default::default(),
};
add_default_servers(&discovery, our_network);
discovery
}
/// Add a server requested via `server.add_peer`
pub fn add_server_request(&self, added_by: IpAddr, features: ServerFeatures) -> Result<()> {
self.verify_compatibility(&features)?;
let mut queue = self.queue.write().unwrap();
ensure!(queue.len() < MAX_QUEUE_SIZE, "queue size exceeded");
// TODO optimize
let mut existing_services: HashMap<ServerAddr, HashSet<Service>> = HashMap::new();
for job in queue.iter() {
existing_services
.entry(job.addr.clone())
.or_default()
.insert(job.service);
}
// collect HealthChecks for candidate services
let jobs = features
.hosts
.iter()
.take(MAX_SERVERS_PER_REQUEST)
.filter_map(|(hostname, ports)| {
let hostname = hostname.to_lowercase();
if hostname.len() > 100 {
warn!("skipping invalid hostname");
return None;
}
let addr = match ServerAddr::resolve(&hostname) {
Ok(addr) => addr,
Err(e) => {
warn!("failed resolving {}: {:?}", hostname, e);
return None;
}
};
if !is_remote_addr(&addr) || self.our_addrs.contains(&addr) {
warn!("skipping own or non-remote server addr");
return None;
}
// ensure the server address matches the ip that advertised it to us.
// onion hosts are exempt.
if let ServerAddr::Clearnet(ip) = addr {
if ip != added_by {
warn!(
"server ip does not match source ip ({}, {} != {})",
hostname, ip, added_by
);
return None;
}
}
Some((addr, hostname, ports))
})
.flat_map(|(addr, hostname, ports)| {
let tcp_service = ports.tcp_port.into_iter().map(Service::Tcp);
let ssl_service = ports.ssl_port.into_iter().map(Service::Ssl);
let services = tcp_service.chain(ssl_service).collect::<HashSet<Service>>();
services
.into_iter()
.filter(|service| {
existing_services
.get(&addr)
.is_none_or(|s| !s.contains(service))
})
.map(|service| {
HealthCheck::new(addr.clone(), hostname.clone(), service, Some(added_by))
})
.collect::<Vec<_>>()
})
.take(MAX_SERVICES_PER_REQUEST)
.collect::<Vec<_>>();
ensure!(
queue.len() + jobs.len() <= MAX_QUEUE_SIZE,
"queue size exceeded"
);
queue.extend(jobs);
Ok(())
}
/// Add a default server. Default servers are exempt from limits and given more leniency
/// before being removed due to unavailability.
pub fn add_default_server(&self, hostname: Hostname, services: Vec<Service>) -> Result<()> {
let addr = ServerAddr::resolve(&hostname)?;
let mut queue = self.queue.write().unwrap();
queue.extend(
services
.into_iter()
.map(|service| HealthCheck::new(addr.clone(), hostname.clone(), service, None)),
);
Ok(())
}
/// Get the list of healthy servers formatted for `servers.peers.subscribe`
pub fn get_servers(&self) -> Vec<ServerEntry> {
// XXX return a random sample instead of everything?
self.healthy
.read()
.unwrap()
.iter()
.map(|(addr, server)| {
ServerEntry(addr.clone(), server.hostname.clone(), server.feature_strs())
})
.collect()
}
pub fn our_features(&self) -> &ServerFeatures {
&self.our_features
}
/// Run the next health check in the queue (a single one)
fn run_health_check(&self) -> Result<()> {
// abort if there are no entries in the queue, or its still too early for the next one up
if self.queue.read().unwrap().peek().is_none_or(|next| {
next.last_check
.is_some_and(|t| t.elapsed() < HEALTH_CHECK_FREQ)
}) {
return Ok(());
}
let mut job = self.queue.write().unwrap().pop().unwrap();
debug!("processing {:?}", job);
let was_healthy = job.is_healthy();
match self.check_server(&job.addr, &job.hostname, job.service) {
Ok(features) => {
debug!("{} {:?} is available", job.hostname, job.service);
if !was_healthy {
self.save_healthy_service(&job, features);
}
// XXX update features?
job.last_check = Some(Instant::now());
job.last_healthy = job.last_check;
job.consecutive_failures = 0;
// schedule the next health check
self.queue.write().unwrap().push(job);
Ok(())
}
Err(e) => {
debug!("{} {:?} is unavailable: {:?}", job.hostname, job.service, e);
if was_healthy {
// XXX should we assume the server's other services are down too?
self.remove_unhealthy_service(&job);
}
job.last_check = Some(Instant::now());
job.consecutive_failures += 1;
if job.should_retry() {
self.queue.write().unwrap().push(job);
} else {
debug!("giving up on {:?}", job);
}
Err(e)
}
}
}
/// Upsert the server/service into the healthy set
fn save_healthy_service(&self, job: &HealthCheck, features: ServerFeatures) {
let addr = job.addr.clone();
let mut healthy = self.healthy.write().unwrap();
healthy
.entry(addr)
.or_insert_with(|| Server::new(job.hostname.clone(), features))
.services
.insert(job.service);
}
/// Remove the service, and remove the server entirely if it has no other reamining healthy services
fn remove_unhealthy_service(&self, job: &HealthCheck) {
let addr = job.addr.clone();
let mut healthy = self.healthy.write().unwrap();
if let Entry::Occupied(mut entry) = healthy.entry(addr) {
let server = entry.get_mut();
assert!(server.services.remove(&job.service));
if server.services.is_empty() {
entry.remove_entry();
}
} else {
unreachable!("missing expected server, corrupted state");
}
}
fn check_server(
&self,
addr: &ServerAddr,
hostname: &Hostname,
service: Service,
) -> Result<ServerFeatures> {
debug!("checking service {:?} {:?}", addr, service);
let server_url = match (addr, service) {
(ServerAddr::Clearnet(ip), Service::Tcp(port)) => format!("tcp://{}:{}", ip, port),
(ServerAddr::Clearnet(_), Service::Ssl(port)) => format!("ssl://{}:{}", hostname, port),
(ServerAddr::Onion(onion_host), Service::Tcp(port)) => {
format!("tcp://{}:{}", onion_host, port)
}
(ServerAddr::Onion(onion_host), Service::Ssl(port)) => {
format!("ssl://{}:{}", onion_host, port)
}
};
let mut config = electrum_client::ConfigBuilder::new();
if let ServerAddr::Onion(_) = addr {
let socks = electrum_client::Socks5Config::new(
self.tor_proxy
.chain_err(|| "no tor proxy configured, onion hosts are unsupported")?,
);
config = config.socks5(Some(socks))
}
let client = Client::from_config(&server_url, config.build())?;
let features = client.server_features()?.try_into()?;
self.verify_compatibility(&features)?;
if self.announce {
// XXX should we require the other side to reciprocate?
ensure!(
client.server_add_peer(&self.our_features)?,
"server does not reciprocate"
);
}
Ok(features)
}
fn verify_compatibility(&self, features: &ServerFeatures) -> Result<()> {
ensure!(
features.genesis_hash == self.our_features.genesis_hash,
"incompatible networks"
);
ensure!(
features.protocol_min <= self.our_version && features.protocol_max >= self.our_version,
"incompatible protocol versions"
);
ensure!(
features.hash_function == "sha256",
"incompatible hash function"
);
Ok(())
}
pub fn spawn_jobs_thread(manager: Arc<DiscoveryManager>) {
spawn_thread("discovery-jobs", move || loop {
if let Err(e) = manager.run_health_check() {
debug!("health check failed: {:?}", e);
}
// XXX use a dynamic JOB_INTERVAL, adjusted according to the queue size and HEALTH_CHECK_FREQ?
thread::sleep(JOB_INTERVAL);
});
}
}
impl Server {
fn new(hostname: Hostname, features: ServerFeatures) -> Self {
Server {
hostname,
features,
services: HashSet::new(),
}
}
/// Get server features and services in the compact string array format used for `servers.peers.subscribe`
fn feature_strs(&self) -> Vec<String> {
let mut strs = Vec::with_capacity(self.services.len() + 1);
strs.push(format!("v{}", self.features.protocol_max));
if let Some(pruning) = self.features.pruning {
strs.push(format!("p{}", pruning));
}
strs.extend(self.services.iter().map(|s| s.to_string()));
strs
}
}
impl ServerAddr {
fn resolve(host: &str) -> Result<Self> {
Ok(if host.ends_with(".onion") {
ServerAddr::Onion(host.into())
} else if let Ok(ip) = IpAddr::from_str(host) {
ServerAddr::Clearnet(ip)
} else {
let ip = format!("{}:1", host)
.to_socket_addrs()
.chain_err(|| "hostname resolution failed")?
.next()
.chain_err(|| "hostname resolution failed")?
.ip();
ServerAddr::Clearnet(ip)
})
}
}
impl fmt::Display for ServerAddr {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
ServerAddr::Clearnet(ip) => write!(f, "{}", ip),
ServerAddr::Onion(hostname) => write!(f, "{}", hostname),
}
}
}
impl serde::Serialize for ServerAddr {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.to_string())
}
}
impl HealthCheck {
fn new(
addr: ServerAddr,
hostname: Hostname,
service: Service,
added_by: Option<IpAddr>,
) -> Self {
HealthCheck {
addr,
hostname,
service,
is_default: added_by.is_none(),
added_by,
last_check: None,
last_healthy: None,
consecutive_failures: 0,
}
}
fn is_healthy(&self) -> bool {
match (self.last_check, self.last_healthy) {
(Some(last_check), Some(last_healthy)) => last_check == last_healthy,
_ => false,
}
}
// allow the server to fail up to MAX_CONSECTIVE_FAILURES time before giving up on it.
// if its a non-default server and the very first attempt fails, give up immediatly.
fn should_retry(&self) -> bool {
(self.last_healthy.is_some() || self.is_default)
&& self.consecutive_failures < MAX_CONSECUTIVE_FAILURES
}
}
impl PartialEq for HealthCheck {
fn eq(&self, other: &Self) -> bool {
self.hostname == other.hostname && self.service == other.service
}
}
impl Ord for HealthCheck {
fn cmp(&self, other: &Self) -> Ordering {
self.last_check.cmp(&other.last_check).reverse()
}
}
impl PartialOrd for HealthCheck {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl fmt::Display for Service {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Service::Tcp(port) => write!(f, "t{}", port),
Service::Ssl(port) => write!(f, "s{}", port),
}
}
}
fn is_remote_addr(addr: &ServerAddr) -> bool {
match addr {
ServerAddr::Onion(_) => true,
ServerAddr::Clearnet(ip) => {
!ip.is_loopback()
&& !ip.is_unspecified()
&& !ip.is_multicast()
&& !match ip {
IpAddr::V4(ipv4) => ipv4.is_private(),
IpAddr::V6(_) => false,
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::chain::genesis_hash;
use crate::chain::Network;
use std::time;
use crate::config::VERSION_STRING;
const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::new(1, 4);
#[test]
#[ignore = "This test requires external connection to server that no longer exists"]
fn test() -> Result<()> {
stderrlog::new().verbosity(4).init().unwrap();
#[cfg(feature = "liquid")]
let testnet = Network::LiquidTestnet;
#[cfg(not(feature = "liquid"))]
let testnet = Network::Testnet;
let features = ServerFeatures {
hosts: serde_json::from_str("{\"test.foobar.example\":{\"tcp_port\":60002}}").unwrap(),
server_version: VERSION_STRING.clone(),
genesis_hash: genesis_hash(testnet),
protocol_min: PROTOCOL_VERSION,
protocol_max: PROTOCOL_VERSION,
hash_function: "sha256".into(),
pruning: None,
};
let discovery = Arc::new(DiscoveryManager::new(
testnet,
features,
PROTOCOL_VERSION,
false,
None,
));
discovery
.add_default_server(
"electrum.blockstream.info".into(),
vec![Service::Tcp(60001)],
)
.unwrap();
discovery
.add_default_server("testnet.hsmiths.com".into(), vec![Service::Ssl(53012)])
.unwrap();
discovery
.add_default_server(
"tn.not.fyi".into(),
vec![Service::Tcp(55001), Service::Ssl(55002)],
)
.unwrap();
discovery
.add_default_server(
"electrum.blockstream.info".into(),
vec![Service::Tcp(60001), Service::Ssl(60002)],
)
.unwrap();
discovery
.add_default_server(
"explorerzydxu5ecjrkwceayqybizmpjjznk5izmitf2modhcusuqlid.onion".into(),
vec![Service::Tcp(143)],
)
.unwrap();
debug!("{:#?}", discovery);
for _ in 0..12 {
discovery
.run_health_check()
.map_err(|e| warn!("{:?}", e))
.ok();
thread::sleep(time::Duration::from_secs(1));
}
debug!("{:#?}", discovery);
info!("{}", json!(discovery.get_servers()));
Ok(())
}
}

View File

@ -1,449 +0,0 @@
use crate::chain::Network;
#[allow(unused_imports)]
use crate::electrum::discovery::{DiscoveryManager, Service};
#[allow(unused_variables)]
pub fn add_default_servers(discovery: &DiscoveryManager, network: Network) {
match network {
#[cfg(not(feature = "liquid"))]
Network::Bitcoin => {
discovery
.add_default_server(
"3smoooajg7qqac2y.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"81-7-10-251.blue.kundencontroller.de".into(),
vec![Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"E-X.not.fyi".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"VPS.hsmiths.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"b.ooze.cc".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"bauerjda5hnedjam.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"bauerjhejlv6di7s.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"bitcoin.corgi.party".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"bitcoin3nqy3db7c.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"bitcoins.sk".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"btc.cihar.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"btc.xskyx.net".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"currentlane.lovebitco.in".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"daedalus.bauerj.eu".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum.jochen-hoenicke.de".into(),
vec![Service::Tcp(50003), Service::Ssl(50005)],
)
.ok();
discovery
.add_default_server(
"dragon085.startdedicated.de".into(),
vec![Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"e-1.claudioboxx.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"e.keff.org".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum-server.ninja".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum-unlimited.criptolayer.net".into(),
vec![Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum.eff.ro".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum.festivaldelhumor.org".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum.hsmiths.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum.leblancnet.us".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("electrum.mindspot.org".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server(
"electrum.qtornado.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("electrum.taborsky.cz".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server(
"electrum.villocq.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum2.eff.ro".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum2.villocq.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrumx.bot.nu".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrumx.ddns.net".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("electrumx.ftp.sh".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server(
"electrumx.ml".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrumx.soon.it".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("electrumxhqdsmlu.onion".into(), vec![Service::Tcp(50001)])
.ok();
discovery
.add_default_server(
"elx01.knas.systems".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"enode.duckdns.org".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"fedaykin.goip.de".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"fn.48.org".into(),
vec![Service::Tcp(50003), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"helicarrier.bauerj.eu".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"hsmiths4fyqlw5xw.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"hsmiths5mjk6uijs.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"icarus.tetradrachm.net".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"electrum.emzy.de".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"ndnd.selfhost.eu".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("ndndword5lpb7eex.onion".into(), vec![Service::Tcp(50001)])
.ok();
discovery
.add_default_server(
"orannis.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"ozahtqwp25chjdjd.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"qtornadoklbgdyww.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("rbx.curalle.ovh".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server("s7clinmo4cazmhul.onion".into(), vec![Service::Tcp(50001)])
.ok();
discovery
.add_default_server(
"tardis.bauerj.eu".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("technetium.network".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server(
"tomscryptos.com".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"ulrichard.ch".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"vmd27610.contaboserver.net".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"vmd30612.contaboserver.net".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"wsw6tua3xl24gsmi264zaep6seppjyrkyucpsmuxnjzyt3f3j6swshad.onion".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"xray587.startdedicated.de".into(),
vec![Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"yuio.top".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"bitcoin.dragon.zone".into(),
vec![Service::Tcp(50003), Service::Ssl(50004)],
)
.ok();
discovery
.add_default_server(
"ecdsa.net".into(),
vec![Service::Tcp(50001), Service::Ssl(110)],
)
.ok();
discovery
.add_default_server("btc.usebsv.com".into(), vec![Service::Ssl(50006)])
.ok();
discovery
.add_default_server(
"e2.keff.org".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server("electrum.hodlister.co".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server("electrum3.hodlister.co".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server("electrum5.hodlister.co".into(), vec![Service::Ssl(50002)])
.ok();
discovery
.add_default_server(
"electrumx.electricnewyear.net".into(),
vec![Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"fortress.qtornado.com".into(),
vec![Service::Tcp(50001), Service::Ssl(443)],
)
.ok();
discovery
.add_default_server(
"green-gold.westeurope.cloudapp.azure.com".into(),
vec![Service::Tcp(56001), Service::Ssl(56002)],
)
.ok();
discovery
.add_default_server(
"electrumx.erbium.eu".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
}
#[cfg(not(feature = "liquid"))]
Network::Testnet => {
discovery
.add_default_server(
"hsmithsxurybd7uh.onion".into(),
vec![Service::Tcp(53011), Service::Ssl(53012)],
)
.ok();
discovery
.add_default_server(
"testnet.hsmiths.com".into(),
vec![Service::Tcp(53011), Service::Ssl(53012)],
)
.ok();
discovery
.add_default_server(
"testnet.qtornado.com".into(),
vec![Service::Tcp(51001), Service::Ssl(51002)],
)
.ok();
discovery
.add_default_server(
"testnet1.bauerj.eu".into(),
vec![Service::Tcp(50001), Service::Ssl(50002)],
)
.ok();
discovery
.add_default_server(
"tn.not.fyi".into(),
vec![Service::Tcp(55001), Service::Ssl(55002)],
)
.ok();
discovery
.add_default_server(
"bitcoin.cluelessperson.com".into(),
vec![Service::Tcp(51001), Service::Ssl(51002)],
)
.ok();
}
_ => (),
}
}

View File

@ -1,119 +0,0 @@
mod server;
pub use server::RPC;
#[cfg(feature = "electrum-discovery")]
mod client;
#[cfg(feature = "electrum-discovery")]
mod discovery;
#[cfg(feature = "electrum-discovery")]
pub use {client::Client, discovery::DiscoveryManager};
use std::cmp::Ordering;
use std::collections::HashMap;
use std::str::FromStr;
use serde::{de, Deserialize, Deserializer, Serialize};
use crate::chain::BlockHash;
use crate::errors::ResultExt;
use crate::util::BlockId;
pub fn get_electrum_height(blockid: Option<BlockId>, has_unconfirmed_parents: bool) -> isize {
match (blockid, has_unconfirmed_parents) {
(Some(blockid), _) => blockid.height as isize,
(None, false) => 0,
(None, true) => -1,
}
}
pub type Port = u16;
pub type Hostname = String;
pub type ServerHosts = HashMap<Hostname, ServerPorts>;
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ServerFeatures {
pub hosts: ServerHosts,
pub genesis_hash: BlockHash,
pub server_version: String,
pub protocol_min: ProtocolVersion,
pub protocol_max: ProtocolVersion,
pub pruning: Option<usize>,
pub hash_function: String,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ServerPorts {
tcp_port: Option<Port>,
ssl_port: Option<Port>,
}
#[derive(Eq, PartialEq, Debug, Clone, Default)]
pub struct ProtocolVersion {
major: usize,
minor: usize,
}
impl ProtocolVersion {
pub const fn new(major: usize, minor: usize) -> Self {
Self { major, minor }
}
}
impl Ord for ProtocolVersion {
fn cmp(&self, other: &Self) -> Ordering {
self.major
.cmp(&other.major)
.then_with(|| self.minor.cmp(&other.minor))
}
}
impl PartialOrd for ProtocolVersion {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl FromStr for ProtocolVersion {
type Err = crate::errors::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut iter = s.split('.');
Ok(Self {
major: iter
.next()
.chain_err(|| "missing major")?
.parse()
.chain_err(|| "invalid major")?,
minor: iter
.next()
.chain_err(|| "missing minor")?
.parse()
.chain_err(|| "invalid minor")?,
})
}
}
impl std::fmt::Display for ProtocolVersion {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}.{}", self.major, self.minor)
}
}
impl Serialize for ProtocolVersion {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.collect_str(&self)
}
}
impl<'de> Deserialize<'de> for ProtocolVersion {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
FromStr::from_str(&s).map_err(de::Error::custom)
}
}

File diff suppressed because it is too large Load Diff

View File

@ -1,662 +0,0 @@
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, RwLock, RwLockReadGuard};
use bitcoin::hashes::{sha256, Hash};
use elements::confidential::{Asset, Value};
use elements::encode::{deserialize, serialize};
use elements::secp256k1_zkp::ZERO_TWEAK;
use elements::{issuance::ContractHash, AssetId, AssetIssuance, OutPoint, Transaction, TxIn};
use crate::chain::{BNetwork, BlockHash, Network, Txid};
use crate::elements::peg::{get_pegin_data, get_pegout_data, PeginInfo, PegoutInfo};
use crate::elements::registry::{AssetMeta, AssetRegistry};
use crate::errors::*;
use crate::new_index::schema::{Operation, TxHistoryInfo, TxHistoryKey, TxHistoryRow};
use crate::new_index::{db::DBFlush, ChainQuery, DBRow, Mempool, Query};
use crate::util::{
bincode_util, full_hash, Bytes, FullHash, IsProvablyUnspendable, TransactionStatus, TxInput,
};
lazy_static! {
pub static ref NATIVE_ASSET_ID: AssetId =
"6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d"
.parse()
.unwrap();
pub static ref NATIVE_ASSET_ID_TESTNET: AssetId =
"144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49"
.parse()
.unwrap();
pub static ref NATIVE_ASSET_ID_REGTEST: AssetId =
"5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225"
.parse()
.unwrap();
}
fn parse_asset_id(sl: &[u8]) -> AssetId {
AssetId::from_slice(sl).expect("failed to parse AssetId")
}
#[derive(Serialize)]
#[serde(untagged)]
#[allow(clippy::large_enum_variant)]
pub enum LiquidAsset {
Issued(IssuedAsset),
Native(PeggedAsset),
}
#[derive(Serialize)]
pub struct PeggedAsset {
pub asset_id: AssetId,
pub chain_stats: PeggedAssetStats,
pub mempool_stats: PeggedAssetStats,
}
#[derive(Serialize)]
pub struct IssuedAsset {
pub asset_id: AssetId,
pub issuance_txin: TxInput,
#[serde(serialize_with = "crate::util::serialize_outpoint")]
pub issuance_prevout: OutPoint,
pub reissuance_token: AssetId,
#[serde(skip_serializing_if = "Option::is_none")]
pub contract_hash: Option<ContractHash>,
// the confirmation status of the initial issuance transaction
pub status: TransactionStatus,
pub chain_stats: IssuedAssetStats,
pub mempool_stats: IssuedAssetStats,
// optional metadata from registry
#[serde(flatten)]
pub meta: Option<AssetMeta>,
}
// DB representation (issued assets only)
#[derive(Serialize, Deserialize, Debug)]
pub struct AssetRow {
pub issuance_txid: FullHash,
pub issuance_vin: u32,
pub prev_txid: FullHash,
pub prev_vout: u32,
pub issuance: Bytes, // bincode does not like dealing with AssetIssuance, deserialization fails with "invalid type: sequence, expected a struct"
pub reissuance_token: FullHash,
}
impl IssuedAsset {
pub fn new(
asset_id: &AssetId,
asset: &AssetRow,
(chain_stats, mempool_stats): (IssuedAssetStats, IssuedAssetStats),
meta: Option<AssetMeta>,
status: TransactionStatus,
) -> Self {
let issuance: AssetIssuance =
deserialize(&asset.issuance).expect("failed parsing AssetIssuance");
let reissuance_token = parse_asset_id(&asset.reissuance_token);
let contract_hash = if issuance.asset_entropy != [0u8; 32] {
Some(ContractHash::from_byte_array(issuance.asset_entropy))
} else {
None
};
Self {
asset_id: *asset_id,
issuance_txin: TxInput {
txid: deserialize(&asset.issuance_txid).unwrap(),
vin: asset.issuance_vin,
},
issuance_prevout: OutPoint {
txid: deserialize(&asset.prev_txid).unwrap(),
vout: asset.prev_vout,
},
contract_hash,
reissuance_token,
status,
chain_stats,
mempool_stats,
meta,
}
}
}
impl LiquidAsset {
pub fn supply(&self) -> Option<u64> {
match self {
LiquidAsset::Native(asset) => Some(
asset.chain_stats.peg_in_amount
- asset.chain_stats.peg_out_amount
- asset.chain_stats.burned_amount
+ asset.mempool_stats.peg_in_amount
- asset.mempool_stats.peg_out_amount
- asset.mempool_stats.burned_amount,
),
LiquidAsset::Issued(asset) => {
if asset.chain_stats.has_blinded_issuances
|| asset.mempool_stats.has_blinded_issuances
{
None
} else {
Some(
asset.chain_stats.issued_amount - asset.chain_stats.burned_amount
+ asset.mempool_stats.issued_amount
- asset.mempool_stats.burned_amount,
)
}
}
}
}
pub fn precision(&self) -> u8 {
match self {
LiquidAsset::Native(_) => 8,
LiquidAsset::Issued(asset) => asset.meta.as_ref().map_or(0, |m| m.precision),
}
}
}
#[derive(Serialize, Deserialize, Debug)]
#[cfg_attr(test, derive(PartialEq, Eq))]
pub struct IssuingInfo {
pub txid: FullHash,
pub vin: u32,
pub is_reissuance: bool,
// None for blinded issuances
pub issued_amount: Option<u64>,
pub token_amount: Option<u64>,
}
#[derive(Serialize, Deserialize, Debug)]
#[cfg_attr(test, derive(PartialEq, Eq))]
pub struct BurningInfo {
pub txid: FullHash,
pub vout: u32,
pub value: u64,
}
// Index confirmed transaction issuances and save as db rows
pub fn index_confirmed_tx_assets(
tx: &Transaction,
confirmed_height: u32,
tx_position: u16,
network: Network,
parent_network: BNetwork,
rows: &mut Vec<DBRow>,
op: &Operation,
) {
let (history, issuances) = index_tx_assets(tx, network, parent_network);
rows.extend(history.into_iter().map(|(asset_id, info)| {
let history_row = asset_history_row(&asset_id, confirmed_height, tx_position, info);
if let Operation::DeleteBlocksWithHistory(tx) = op {
tx.send(history_row.key.hash)
.expect("unbounded channel won't fail");
}
history_row.into_row()
}));
// the initial issuance is kept twice: once in the history index under I<asset><height><txid:vin>,
// and once separately under i<asset> for asset lookup with some more associated metadata.
// reissuances are only kept under the history index.
rows.extend(issuances.into_iter().map(|(asset_id, asset_row)| DBRow {
key: [b"i", &asset_id.into_inner()[..]].concat(),
value: bincode_util::serialize_little(&asset_row).unwrap(),
}));
}
// Index mempool transaction issuances and save to in-memory store
pub fn index_mempool_tx_assets(
tx: &Transaction,
network: Network,
parent_network: BNetwork,
asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
asset_issuance: &mut HashMap<AssetId, AssetRow>,
) {
let (history, issuances) = index_tx_assets(tx, network, parent_network);
for (asset_id, info) in history {
asset_history.entry(asset_id).or_default().push(info);
}
for (asset_id, issuance) in issuances {
asset_issuance.insert(asset_id, issuance);
}
}
// Remove mempool transaction issuances from in-memory store
pub fn remove_mempool_tx_assets(
to_remove: &HashSet<&Txid>,
asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
asset_issuance: &mut HashMap<AssetId, AssetRow>,
) {
// TODO optimize
asset_history.retain(|_assethash, entries| {
entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
!entries.is_empty()
});
asset_issuance.retain(|_assethash, issuance| {
let txid: Txid = deserialize(&issuance.issuance_txid).unwrap();
!to_remove.contains(&txid)
});
}
type HistoryAndIssuances = (Vec<(AssetId, TxHistoryInfo)>, Vec<(AssetId, AssetRow)>);
// Internal utility function, index a transaction and return its history entries and issuances
fn index_tx_assets(
tx: &Transaction,
network: Network,
parent_network: BNetwork,
) -> HistoryAndIssuances {
let mut history = vec![];
let mut issuances = vec![];
let txid = full_hash(&tx.txid()[..]);
for (txo_index, txo) in tx.output.iter().enumerate() {
if let Some(pegout) = get_pegout_data(txo, network, parent_network) {
history.push((
pegout.asset.explicit().unwrap(),
TxHistoryInfo::Pegout(PegoutInfo {
txid,
vout: txo_index as u32,
value: pegout.value,
}),
));
} else if txo.script_pubkey.is_provably_unspendable_() && !txo.is_fee() {
if let (Asset::Explicit(asset_id), Value::Explicit(value)) = (txo.asset, txo.value) {
if value > 0 {
history.push((
asset_id,
TxHistoryInfo::Burning(BurningInfo {
txid,
vout: txo_index as u32,
value,
}),
));
}
}
}
}
for (txi_index, txi) in tx.input.iter().enumerate() {
if let Some(pegin) = get_pegin_data(txi, network) {
history.push((
pegin.asset,
TxHistoryInfo::Pegin(PeginInfo {
txid,
vin: txi_index as u32,
value: pegin.value,
}),
));
} else if txi.has_issuance() {
let is_reissuance = txi.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
let asset_entropy = get_issuance_entropy(txi).expect("invalid issuance");
let asset_id = AssetId::from_entropy(asset_entropy);
let issued_amount = match txi.asset_issuance.amount {
Value::Explicit(amount) => Some(amount),
Value::Null => Some(0),
_ => None,
};
let token_amount = match txi.asset_issuance.inflation_keys {
Value::Explicit(amount) => Some(amount),
Value::Null => Some(0),
_ => None,
};
history.push((
asset_id,
TxHistoryInfo::Issuing(IssuingInfo {
txid,
vin: txi_index as u32,
is_reissuance,
issued_amount,
token_amount,
}),
));
if !is_reissuance {
let is_confidential =
matches!(txi.asset_issuance.inflation_keys, Value::Confidential(..));
let reissuance_token =
AssetId::reissuance_token_from_entropy(asset_entropy, is_confidential);
issuances.push((
asset_id,
AssetRow {
issuance_txid: txid,
issuance_vin: txi_index as u32,
prev_txid: full_hash(&txi.previous_output.txid[..]),
prev_vout: txi.previous_output.vout,
issuance: serialize(&txi.asset_issuance),
reissuance_token: full_hash(&reissuance_token.into_inner()[..]),
},
));
}
}
}
(history, issuances)
}
fn asset_history_row(
asset_id: &AssetId,
confirmed_height: u32,
tx_position: u16,
txinfo: TxHistoryInfo,
) -> TxHistoryRow {
let key = TxHistoryKey {
code: b'I',
hash: full_hash(&asset_id.into_inner()[..]),
confirmed_height,
tx_position,
txinfo,
};
TxHistoryRow { key }
}
pub enum AssetRegistryLock<'a> {
RwLock(&'a Arc<RwLock<AssetRegistry>>),
RwLockReadGuard(&'a RwLockReadGuard<'a, AssetRegistry>),
}
pub fn lookup_asset(
query: &Query,
registry: Option<AssetRegistryLock>,
asset_id: &AssetId,
meta: Option<&AssetMeta>, // may optionally be provided if already known
) -> Result<Option<LiquidAsset>> {
if query.network().pegged_asset() == Some(asset_id) {
let (chain_stats, mempool_stats) = pegged_asset_stats(query, asset_id);
return Ok(Some(LiquidAsset::Native(PeggedAsset {
asset_id: *asset_id,
chain_stats,
mempool_stats,
})));
}
let history_db = query.chain().store().history_db();
let mempool = query.mempool();
let mempool_issuances = &mempool.asset_issuance;
let chain_row = history_db
.get(&[b"i", &asset_id.into_inner()[..]].concat())
.map(|row| {
bincode_util::deserialize_little::<AssetRow>(&row).expect("failed parsing AssetRow")
});
let row = chain_row
.as_ref()
.or_else(|| mempool_issuances.get(asset_id));
Ok(if let Some(row) = row {
let reissuance_token = parse_asset_id(&row.reissuance_token);
let meta = meta.cloned().or_else(|| match registry {
Some(AssetRegistryLock::RwLock(rwlock)) => {
rwlock.read().unwrap().get(asset_id).cloned()
}
Some(AssetRegistryLock::RwLockReadGuard(guard)) => guard.get(asset_id).cloned(),
None => None,
});
let stats = issued_asset_stats(query.chain(), &mempool, asset_id, &reissuance_token);
let status = query.get_tx_status(&deserialize(&row.issuance_txid).unwrap());
let asset = IssuedAsset::new(asset_id, row, stats, meta, status);
Some(LiquidAsset::Issued(asset))
} else {
None
})
}
pub fn get_issuance_entropy(txin: &TxIn) -> Result<sha256::Midstate> {
if !txin.has_issuance() {
bail!("input has no issuance");
}
let is_reissuance = txin.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
Ok(if !is_reissuance {
let contract_hash = ContractHash::from_slice(&txin.asset_issuance.asset_entropy)
.chain_err(|| "invalid entropy (contract hash)")?;
AssetId::generate_asset_entropy(txin.previous_output, contract_hash)
} else {
sha256::Midstate::from_slice(&txin.asset_issuance.asset_entropy)
.chain_err(|| "invalid entropy (reissuance)")?
})
}
//
// Asset stats
//
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct IssuedAssetStats {
pub tx_count: usize,
pub issuance_count: usize,
pub issued_amount: u64,
pub burned_amount: u64,
pub has_blinded_issuances: bool,
pub reissuance_tokens: Option<u64>, // none if confidential
pub burned_reissuance_tokens: u64,
}
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct PeggedAssetStats {
pub tx_count: usize,
pub peg_in_count: usize,
pub peg_in_amount: u64,
pub peg_out_count: usize,
pub peg_out_amount: u64,
pub burn_count: usize,
pub burned_amount: u64,
}
type AssetStatApplyFn<T> = fn(&TxHistoryInfo, &mut T, &mut HashSet<Txid>);
fn asset_cache_key(asset_id: &AssetId) -> Bytes {
[b"z", &asset_id.into_inner()[..]].concat()
}
fn asset_cache_row<T>(asset_id: &AssetId, stats: &T, blockhash: &BlockHash) -> DBRow
where
T: serde::Serialize,
{
DBRow {
key: asset_cache_key(asset_id),
value: bincode_util::serialize_little(&(stats, blockhash)).unwrap(),
}
}
// Get stats for the network's pegged asset
fn pegged_asset_stats(query: &Query, asset_id: &AssetId) -> (PeggedAssetStats, PeggedAssetStats) {
(
chain_asset_stats(query.chain(), asset_id, apply_pegged_asset_stats),
mempool_asset_stats(&query.mempool(), asset_id, apply_pegged_asset_stats),
)
}
// Get stats for issued assets
fn issued_asset_stats(
chain: &ChainQuery,
mempool: &Mempool,
asset_id: &AssetId,
reissuance_token: &AssetId,
) -> (IssuedAssetStats, IssuedAssetStats) {
let afn = apply_issued_asset_stats;
let mut chain_stats = chain_asset_stats(chain, asset_id, afn);
chain_stats.burned_reissuance_tokens =
chain_asset_stats(chain, reissuance_token, afn).burned_amount;
let mut mempool_stats = mempool_asset_stats(mempool, asset_id, afn);
mempool_stats.burned_reissuance_tokens =
mempool_asset_stats(mempool, reissuance_token, afn).burned_amount;
(chain_stats, mempool_stats)
}
// Get on-chain confirmed asset stats (issued or the pegged asset)
fn chain_asset_stats<T>(chain: &ChainQuery, asset_id: &AssetId, apply_fn: AssetStatApplyFn<T>) -> T
where
T: Default + serde::Serialize + serde::de::DeserializeOwned,
{
// get the last known stats and the blockhash they are updated for.
// invalidates the cache if the block was orphaned.
let cache: Option<(T, usize)> = chain
.store()
.cache_db()
.get(&asset_cache_key(asset_id))
.map(|c| bincode_util::deserialize_little(&c).unwrap())
.and_then(|(stats, blockhash)| {
chain
.height_by_hash(&blockhash)
.map(|height| (stats, height))
});
// update stats with new transactions since
let (newstats, lastblock) = cache.map_or_else(
|| chain_asset_stats_delta(chain, asset_id, T::default(), 0, apply_fn),
|(oldstats, blockheight)| {
chain_asset_stats_delta(chain, asset_id, oldstats, blockheight + 1, apply_fn)
},
);
// save updated stats to cache
if let Some(lastblock) = lastblock {
chain.store().cache_db().write(
vec![asset_cache_row(asset_id, &newstats, &lastblock)],
DBFlush::Enable,
);
}
newstats
}
// Update the asset stats with the delta of confirmed txs since start_height
fn chain_asset_stats_delta<T>(
chain: &ChainQuery,
asset_id: &AssetId,
init_stats: T,
start_height: usize,
apply_fn: AssetStatApplyFn<T>,
) -> (T, Option<BlockHash>) {
let history_iter = chain
.history_iter_scan(b'I', &asset_id.into_inner()[..], start_height)
.map(TxHistoryRow::from_row)
.filter_map(|history| {
chain
.tx_confirming_block(&history.get_txid())
.map(|blockid| (history, blockid))
});
let mut stats = init_stats;
let mut seen_txids = HashSet::new();
let mut lastblock = None;
for (row, blockid) in history_iter {
if lastblock != Some(blockid.hash) {
seen_txids.clear();
}
apply_fn(&row.key.txinfo, &mut stats, &mut seen_txids);
lastblock = Some(blockid.hash);
}
(stats, lastblock)
}
// Get mempool asset stats (issued or the pegged asset)
pub fn mempool_asset_stats<T>(
mempool: &Mempool,
asset_id: &AssetId,
apply_fn: AssetStatApplyFn<T>,
) -> T
where
T: Default,
{
let mut stats = T::default();
if let Some(history) = mempool.asset_history.get(asset_id) {
let mut seen_txids = HashSet::new();
for info in history {
apply_fn(info, &mut stats, &mut seen_txids)
}
}
stats
}
fn apply_issued_asset_stats(
info: &TxHistoryInfo,
stats: &mut IssuedAssetStats,
seen_txids: &mut HashSet<Txid>,
) {
if seen_txids.insert(info.get_txid()) {
stats.tx_count += 1;
}
match info {
TxHistoryInfo::Issuing(issuance) => {
stats.issuance_count += 1;
match issuance.issued_amount {
Some(amount) => stats.issued_amount += amount,
None => stats.has_blinded_issuances = true,
}
if !issuance.is_reissuance {
stats.reissuance_tokens = issuance.token_amount;
}
}
TxHistoryInfo::Burning(info) => {
stats.burned_amount += info.value;
}
TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
// we don't keep funding/spending entries for assets
unreachable!();
}
TxHistoryInfo::Pegin(_) | TxHistoryInfo::Pegout(_) => {
// issued assets cannot have pegins/pegouts
unreachable!();
}
}
}
fn apply_pegged_asset_stats(
info: &TxHistoryInfo,
stats: &mut PeggedAssetStats,
seen_txids: &mut HashSet<Txid>,
) {
if seen_txids.insert(info.get_txid()) {
stats.tx_count += 1;
}
match info {
TxHistoryInfo::Pegin(info) => {
stats.peg_in_count += 1;
stats.peg_in_amount += info.value;
}
TxHistoryInfo::Pegout(info) => {
stats.peg_out_count += 1;
stats.peg_out_amount += info.value;
}
TxHistoryInfo::Burning(info) => {
stats.burn_count += 1;
stats.burned_amount += info.value;
}
TxHistoryInfo::Issuing(_) => {
warn!("encountered issuance of native asset, ignoring (possibly freeinitialcoins?)");
}
TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
// these history entries variants are never kept for native assets
unreachable!();
}
}
}

View File

@ -1,77 +0,0 @@
use bitcoin::hashes::Hash;
use elements::hex::ToHex;
use elements::secp256k1_zkp::ZERO_TWEAK;
use elements::{confidential::Value, encode::serialize, issuance::ContractHash, AssetId, TxIn};
pub mod asset;
pub mod peg;
mod registry;
use asset::get_issuance_entropy;
pub use asset::{lookup_asset, LiquidAsset};
pub use registry::{AssetMeta, AssetRegistry, AssetSorting};
#[derive(Serialize, Deserialize, Clone)]
pub struct IssuanceValue {
pub asset_id: String,
pub is_reissuance: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub asset_blinding_nonce: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub contract_hash: Option<String>,
pub asset_entropy: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub assetamount: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub assetamountcommitment: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tokenamount: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tokenamountcommitment: Option<String>,
}
impl From<&TxIn> for IssuanceValue {
fn from(txin: &TxIn) -> Self {
let issuance = &txin.asset_issuance;
let is_reissuance = issuance.asset_blinding_nonce != ZERO_TWEAK;
let asset_entropy = get_issuance_entropy(txin).expect("invalid issuance");
let asset_id = AssetId::from_entropy(asset_entropy);
let contract_hash = if !is_reissuance {
Some(ContractHash::from_slice(&issuance.asset_entropy).expect("invalid asset entropy"))
} else {
None
};
IssuanceValue {
asset_id: asset_id.to_hex(),
asset_entropy: asset_entropy.to_hex(),
contract_hash: contract_hash.map(|h| h.to_hex()),
is_reissuance,
asset_blinding_nonce: if is_reissuance {
Some(hex::encode(issuance.asset_blinding_nonce.as_ref()))
} else {
None
},
assetamount: match issuance.amount {
Value::Explicit(value) => Some(value),
Value::Null => Some(0),
Value::Confidential(..) => None,
},
assetamountcommitment: match issuance.amount {
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
_ => None,
},
tokenamount: match issuance.inflation_keys {
Value::Explicit(value) => Some(value),
Value::Null => Some(0),
Value::Confidential(..) => None,
},
tokenamountcommitment: match issuance.inflation_keys {
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys))),
_ => None,
},
}
}
}

View File

@ -1,84 +0,0 @@
use elements::hex::ToHex;
use elements::{confidential::Asset, PeginData, PegoutData, TxIn, TxOut};
use crate::chain::{bitcoin_genesis_hash, BNetwork, Network};
use crate::util::FullHash;
pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option<PeginData<'_>> {
let pegged_asset_id = network.pegged_asset()?;
txout.pegin_data().and_then(|pegin| {
if pegin.asset == *pegged_asset_id {
Some(pegin)
} else {
None
}
})
}
pub fn get_pegout_data(
txout: &TxOut,
network: Network,
parent_network: BNetwork,
) -> Option<PegoutData<'_>> {
let pegged_asset_id = network.pegged_asset()?;
txout.pegout_data().and_then(|pegout| {
if pegout.asset == Asset::Explicit(*pegged_asset_id)
&& pegout.genesis_hash
== bitcoin_genesis_hash(match parent_network {
BNetwork::Bitcoin => Network::Liquid,
BNetwork::Testnet | BNetwork::Testnet4 => Network::LiquidTestnet,
BNetwork::Signet => return None,
BNetwork::Regtest => Network::LiquidRegtest,
})
{
Some(pegout)
} else {
None
}
})
}
// API representation of pegout data assocaited with an output
#[derive(Serialize, Deserialize, Clone)]
pub struct PegoutValue {
pub genesis_hash: String,
pub scriptpubkey: bitcoin::ScriptBuf,
pub scriptpubkey_asm: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub scriptpubkey_address: Option<String>,
}
impl PegoutValue {
pub fn from_txout(txout: &TxOut, network: Network, parent_network: BNetwork) -> Option<Self> {
let pegoutdata = get_pegout_data(txout, network, parent_network)?;
// pending https://github.com/ElementsProject/rust-elements/pull/69 is merged
let scriptpubkey = bitcoin::ScriptBuf::from(pegoutdata.script_pubkey.into_bytes());
let address = bitcoin::Address::from_script(&scriptpubkey, parent_network).ok();
Some(PegoutValue {
genesis_hash: pegoutdata.genesis_hash.to_hex(),
scriptpubkey_asm: scriptpubkey.to_asm_string(),
scriptpubkey_address: address.map(|s| s.to_string()),
scriptpubkey,
})
}
}
// Inner type for the indexer TxHistoryInfo::Pegin variant
#[derive(Serialize, Deserialize, Debug)]
#[cfg_attr(test, derive(PartialEq, Eq))]
pub struct PeginInfo {
pub txid: FullHash,
pub vin: u32,
pub value: u64,
}
// Inner type for the indexer TxHistoryInfo::Pegout variant
#[derive(Serialize, Deserialize, Debug)]
#[cfg_attr(test, derive(PartialEq, Eq))]
pub struct PegoutInfo {
pub txid: FullHash,
pub vout: u32,
pub value: u64,
}

View File

@ -1,298 +0,0 @@
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::{Arc, RwLock};
use std::time::{Duration, SystemTime};
use std::{cmp, fs, path, thread};
use serde_json::Value as JsonValue;
use elements::AssetId;
use crate::errors::*;
// length of asset id prefix to use for sub-directory partitioning
// (in number of hex characters, not bytes)
const DIR_PARTITION_LEN: usize = 2;
const SEARCH_SORT_CANDIDATE_LIMIT: usize = 2000;
pub struct AssetRegistry {
directory: path::PathBuf,
assets_cache: HashMap<AssetId, (SystemTime, AssetMeta)>,
}
pub type AssetEntry<'a> = (&'a AssetId, &'a AssetMeta);
impl AssetRegistry {
pub fn new(directory: path::PathBuf) -> Self {
Self {
directory,
assets_cache: Default::default(),
}
}
pub fn get(&self, asset_id: &AssetId) -> Option<&AssetMeta> {
self.assets_cache
.get(asset_id)
.map(|(_, metadata)| metadata)
}
pub fn list(
&self,
start_index: usize,
limit: usize,
sorting: AssetSorting,
) -> (usize, Vec<AssetEntry<'_>>) {
let mut assets: Vec<AssetEntry> = self
.assets_cache
.iter()
.map(|(asset_id, (_, metadata))| (asset_id, metadata))
.collect();
assets.sort_by(sorting.as_comparator());
(
assets.len(),
assets.into_iter().skip(start_index).take(limit).collect(),
)
}
pub fn search(&self, query: &str, limit: usize) -> Vec<AssetEntry<'_>> {
let query = query.trim();
if query.is_empty() || limit == 0 {
return vec![];
}
let (mut results, candidates) = search_by(
self.assets_cache
.iter()
.map(|(asset_id, (_, metadata))| (asset_id, metadata)),
query,
limit,
|metadata| metadata.ticker.as_deref(),
);
if results.len() < limit {
let (name_matches, candidates) =
search_by(candidates, query, limit - results.len(), |metadata| {
Some(&metadata.name)
});
results.extend(name_matches);
if results.len() < limit {
let (domain_matches, _) =
search_by(candidates, query, limit - results.len(), AssetMeta::domain);
results.extend(domain_matches);
}
}
results.truncate(limit);
results
}
pub fn fs_sync(&mut self) -> Result<()> {
for entry in fs::read_dir(&self.directory).chain_err(|| "failed reading asset dir")? {
let entry = entry.chain_err(|| "invalid fh")?;
let filetype = entry.file_type().chain_err(|| "failed getting file type")?;
if !filetype.is_dir() || entry.file_name().len() != DIR_PARTITION_LEN {
continue;
}
for file_entry in
fs::read_dir(entry.path()).chain_err(|| "failed reading asset subdir")?
{
let file_entry = file_entry.chain_err(|| "invalid fh")?;
let path = file_entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let asset_id = AssetId::from_str(
path.file_stem()
.unwrap() // cannot fail if extension() succeeded
.to_str()
.chain_err(|| "invalid filename")?,
)
.chain_err(|| "invalid filename")?;
let modified = file_entry
.metadata()
.chain_err(|| "failed reading metadata")?
.modified()
.chain_err(|| "metadata modified failed")?;
if let Some((last_update, _)) = self.assets_cache.get(&asset_id) {
if *last_update == modified {
continue;
}
}
let metadata: AssetMeta = serde_json::from_str(
&fs::read_to_string(path).chain_err(|| "failed reading file")?,
)
.chain_err(|| "failed parsing file")?;
self.assets_cache.insert(asset_id, (modified, metadata));
}
}
Ok(())
}
pub fn spawn_sync(asset_db: Arc<RwLock<AssetRegistry>>) -> thread::JoinHandle<()> {
crate::util::spawn_thread("asset-registry", move || loop {
if let Err(e) = asset_db.write().unwrap().fs_sync() {
error!("registry fs_sync failed: {:?}", e);
}
thread::sleep(Duration::from_secs(15));
// TODO handle shutdowm
})
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct AssetMeta {
#[serde(skip_serializing_if = "JsonValue::is_null")]
pub contract: JsonValue,
#[serde(skip_serializing_if = "JsonValue::is_null")]
pub entity: JsonValue,
pub precision: u8,
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub ticker: Option<String>,
}
impl AssetMeta {
pub(crate) fn domain(&self) -> Option<&str> {
self.entity["domain"].as_str()
}
}
pub struct AssetSorting(AssetSortField, AssetSortDir);
pub enum AssetSortField {
Name,
Domain,
Ticker,
}
pub enum AssetSortDir {
Descending,
Ascending,
}
type Comparator = Box<dyn Fn(&AssetEntry, &AssetEntry) -> cmp::Ordering>;
impl AssetSorting {
#[allow(clippy::wrong_self_convention)]
fn as_comparator(self) -> Comparator {
let sort_fn: Comparator = match self.0 {
AssetSortField::Name => {
// Order by name first, use asset id as a tie breaker. the other sorting fields
// don't require this because they're guaranteed to be unique.
Box::new(|a, b| lc_cmp(&a.1.name, &b.1.name).then_with(|| a.0.cmp(b.0)))
}
AssetSortField::Domain => Box::new(|a, b| a.1.domain().cmp(&b.1.domain())),
AssetSortField::Ticker => Box::new(|a, b| lc_cmp_opt(&a.1.ticker, &b.1.ticker)),
};
match self.1 {
AssetSortDir::Ascending => sort_fn,
AssetSortDir::Descending => Box::new(move |a, b| sort_fn(a, b).reverse()),
}
}
pub fn from_query_params(query: &HashMap<String, String>) -> Result<Self> {
let field = match query.get("sort_field").map(String::as_str) {
None => AssetSortField::Ticker,
Some("name") => AssetSortField::Name,
Some("domain") => AssetSortField::Domain,
Some("ticker") => AssetSortField::Ticker,
_ => bail!("invalid sort field"),
};
let dir = match query.get("sort_dir").map(String::as_str) {
None => AssetSortDir::Ascending,
Some("asc") => AssetSortDir::Ascending,
Some("desc") => AssetSortDir::Descending,
_ => bail!("invalid sort direction"),
};
Ok(Self(field, dir))
}
}
fn lc_cmp(a: &str, b: &str) -> cmp::Ordering {
a.to_lowercase().cmp(&b.to_lowercase())
}
fn lc_cmp_opt(a: &Option<String>, b: &Option<String>) -> cmp::Ordering {
a.as_ref()
.map(|a| a.to_lowercase())
.cmp(&b.as_ref().map(|b| b.to_lowercase()))
}
fn search_by<'a, I, F>(
candidates: I,
query: &str,
limit: usize,
field: F,
) -> (Vec<AssetEntry<'a>>, Vec<AssetEntry<'a>>)
where
I: IntoIterator<Item = AssetEntry<'a>>,
F: Fn(&AssetMeta) -> Option<&str>,
{
let mut matches = vec![];
let mut remaining = vec![];
for (asset_id, metadata) in candidates {
let position = field(metadata).and_then(|field| {
// registry fields are ascii, so we don't need full unicode case-folding
ascii_ci_find(field, query).map(|position| (position, field))
});
if let Some((position, field)) = position {
if matches.len() >= SEARCH_SORT_CANDIDATE_LIMIT {
continue;
}
matches.push((position, field, asset_id, metadata));
} else {
remaining.push((asset_id, metadata));
}
}
matches.sort_unstable_by(|a, b| {
a.0.cmp(&b.0)
.then_with(|| ascii_ci_cmp(a.1, b.1))
.then_with(|| a.2.cmp(b.2))
});
(
matches
.into_iter()
.take(limit)
.map(|(_, _, asset_id, metadata)| (asset_id, metadata))
.collect(),
remaining,
)
}
// zero-allocation case-insensitive ASCII substring search
// returns the byte offset of the first match
fn ascii_ci_find(haystack: &str, needle: &str) -> Option<usize> {
let (haystack, needle) = (haystack.as_bytes(), needle.as_bytes());
if needle.is_empty() {
return Some(0);
}
haystack
.windows(needle.len())
.position(|window| window.eq_ignore_ascii_case(needle))
}
// zero-allocation case-insensitive ASCII string comparison
fn ascii_ci_cmp(a: &str, b: &str) -> cmp::Ordering {
let (a, b) = (a.as_bytes(), b.as_bytes());
for i in 0..a.len().min(b.len()) {
match a[i].to_ascii_lowercase().cmp(&b[i].to_ascii_lowercase()) {
cmp::Ordering::Equal => continue,
ord => return ord,
}
}
a.len().cmp(&b.len())
}

View File

@ -1,4 +1,5 @@
#![allow(unexpected_cfgs)]
use chan_signal::Signal;
error_chain! {
types {
Error, ErrorKind, ResultExt, Result;
@ -10,33 +11,9 @@ error_chain! {
display("Connection error: {}", msg)
}
Interrupt(sig: i32) {
Interrupt(signal: Signal) {
description("Interruption by external signal")
display("Iterrupted by signal {}", sig)
display("Iterrupted by SIG{:?}", signal)
}
TooManyUtxos(limit: usize) {
description("Too many unspent transaction outputs. Contact support to raise limits.")
display("Too many unspent transaction outputs (>{}). Contact support to raise limits.", limit)
}
TooManyTxs(limit: usize) {
description("Too many history transactions. Contact support to raise limits.")
display("Too many history transactions (>{}). Contact support to raise limits.", limit)
}
#[cfg(feature = "electrum-discovery")]
ElectrumClient(e: electrum_client::Error) {
description("Electrum client error")
display("Electrum client error: {:?}", e)
}
}
}
#[cfg(feature = "electrum-discovery")]
impl From<electrum_client::Error> for Error {
fn from(e: electrum_client::Error) -> Self {
Error::from(ErrorKind::ElectrumClient(e))
}
}

View File

@ -1,7 +1,38 @@
#![recursion_limit = "1024"]
extern crate base64;
extern crate bincode;
extern crate bitcoin;
extern crate bitcoin_bech32;
extern crate bitcoin_hashes;
extern crate chan_signal;
extern crate crypto;
extern crate dirs;
extern crate glob;
extern crate hex;
extern crate hyper;
extern crate itertools;
extern crate libc;
extern crate lru;
extern crate lru_cache;
extern crate num_cpus;
extern crate page_size;
extern crate prometheus;
extern crate rayon;
extern crate rocksdb;
extern crate secp256k1;
extern crate serde;
extern crate stderrlog;
extern crate sysconf;
extern crate time;
extern crate tiny_http;
extern crate url;
#[cfg(feature = "liquid")]
extern crate elements;
#[macro_use]
extern crate chan;
#[macro_use]
extern crate clap;
#[macro_use]
@ -15,19 +46,14 @@ extern crate serde_derive;
#[macro_use]
extern crate serde_json;
#[macro_use]
extern crate lazy_static;
pub mod chain;
pub mod config;
pub mod daemon;
pub mod electrum;
pub mod errors;
pub mod metrics;
pub mod new_index;
pub mod rest;
pub mod signal;
pub mod util;
#[cfg(feature = "liquid")]
pub mod elements;
pub mod new_index;

View File

@ -5,6 +5,7 @@ use std::io;
use std::net::SocketAddr;
use std::thread;
use std::time::Duration;
use sysconf;
use tiny_http;
pub use prometheus::{
@ -66,9 +67,11 @@ impl Metrics {
}
pub fn start(&self) {
let server = tiny_http::Server::http(self.addr)
.unwrap_or_else(|_| panic!("failed to start monitoring HTTP server at {}", self.addr));
start_process_exporter(self);
let server = tiny_http::Server::http(self.addr).expect(&format!(
"failed to start monitoring HTTP server at {}",
self.addr
));
start_process_exporter(&self);
let reg = self.reg.clone();
spawn_thread("metrics", move || loop {
if let Err(e) = handle_request(&reg, server.recv()) {
@ -97,14 +100,6 @@ struct Stats {
fds: usize,
}
fn get_ticks_per_second() -> Result<f64> {
// Safety: This code is taken directly from sysconf
match unsafe { libc::sysconf(libc::_SC_CLK_TCK) } {
-1 => Err("Clock Tick unsupported".into()),
ret => Ok(ret as f64),
}
}
fn parse_stats() -> Result<Stats> {
if cfg!(target_os = "macos") {
return Ok(Stats {
@ -116,14 +111,15 @@ fn parse_stats() -> Result<Stats> {
let value = fs::read_to_string("/proc/self/stat").chain_err(|| "failed to read stats")?;
let parts: Vec<&str> = value.split_whitespace().collect();
let page_size = page_size::get() as u64;
let ticks_per_second = get_ticks_per_second().expect("failed to get _SC_CLK_TCK");
let ticks_per_second = sysconf::raw::sysconf(sysconf::raw::SysconfVariable::ScClkTck)
.expect("failed to get _SC_CLK_TCK") as f64;
let parse_part = |index: usize, name: &str| -> Result<u64> {
parts
Ok(parts
.get(index)
.chain_err(|| format!("missing {}: {:?}", name, parts))?
.parse::<u64>()
.chain_err(|| format!("invalid {}: {:?}", name, parts))
.chain_err(|| format!("invalid {}: {:?}", name, parts))?)
};
// For details, see '/proc/[pid]/stat' section at `man 5 proc`:
@ -148,7 +144,7 @@ fn start_process_exporter(metrics: &Metrics) {
spawn_thread("exporter", move || loop {
match parse_stats() {
Ok(stats) => {
cpu.with_label_values(&["utime"]).set(stats.utime);
cpu.with_label_values(&["utime"]).set(stats.utime as f64);
rss.set(stats.rss as i64);
fds.set(stats.fds as i64);
}

View File

@ -2,14 +2,9 @@ use rocksdb;
use std::path::Path;
use crate::config::Config;
use crate::util::{bincode_util, Bytes};
use crate::util::Bytes;
/// Each version will break any running instance with a DB that has a differing version.
/// It will also break if light mode is enabled or disabled.
// 1 = Original DB (since fork from Blockstream)
// 2 = Add tx position to TxHistory rows and place Spending before Funding
static DB_VERSION: u32 = 2;
static DB_VERSION: u32 = 1;
#[derive(Debug, Eq, PartialEq)]
pub struct DBRow {
@ -23,14 +18,14 @@ pub struct ScanIterator<'a> {
done: bool,
}
impl Iterator for ScanIterator<'_> {
impl<'a> Iterator for ScanIterator<'a> {
type Item = DBRow;
fn next(&mut self) -> Option<DBRow> {
if self.done {
return None;
}
let (key, value) = self.iter.next().map(Result::ok)??;
let (key, value) = self.iter.next()?;
if !key.starts_with(&self.prefix) {
self.done = true;
return None;
@ -48,7 +43,7 @@ pub struct ReverseScanIterator<'a> {
done: bool,
}
impl Iterator for ReverseScanIterator<'_> {
impl<'a> Iterator for ReverseScanIterator<'a> {
type Item = DBRow;
fn next(&mut self) -> Option<DBRow> {
@ -61,76 +56,11 @@ impl Iterator for ReverseScanIterator<'_> {
self.done = true;
return None;
}
let row = DBRow {
key: key.into(),
value: self.iter.value().unwrap().into(),
};
let value = self.iter.value().unwrap();
self.iter.prev();
Some(row)
}
}
pub struct ReverseScanGroupIterator<'a> {
iters: Vec<ReverseScanIterator<'a>>,
next_rows: Vec<Option<DBRow>>,
value_offset: usize,
done: bool,
}
impl<'a> ReverseScanGroupIterator<'a> {
pub fn new(
mut iters: Vec<ReverseScanIterator<'a>>,
value_offset: usize,
) -> ReverseScanGroupIterator<'a> {
let mut next_rows: Vec<Option<DBRow>> = Vec::with_capacity(iters.len());
for iter in &mut iters {
let next = iter.next();
next_rows.push(next);
}
let done = next_rows.iter().all(|row| row.is_none());
ReverseScanGroupIterator {
iters,
next_rows,
value_offset,
done,
}
}
}
impl Iterator for ReverseScanGroupIterator<'_> {
type Item = DBRow;
fn next(&mut self) -> Option<DBRow> {
if self.done {
return None;
}
let best_index = self
.next_rows
.iter()
.enumerate()
.max_by(|(a_index, a_opt), (b_index, b_opt)| match (a_opt, b_opt) {
(None, None) => a_index.cmp(b_index),
(Some(_), None) => std::cmp::Ordering::Greater,
(None, Some(_)) => std::cmp::Ordering::Less,
(Some(a), Some(b)) => a.key[self.value_offset..].cmp(&(b.key[self.value_offset..])),
})
.map(|(index, _)| index)
.unwrap_or(0);
let best = self.next_rows[best_index].take();
self.next_rows[best_index] = self.iters.get_mut(best_index)?.next();
if self.next_rows.iter().all(|row| row.is_none()) {
self.done = true;
}
best
Some(DBRow { key, value })
}
}
@ -146,11 +76,28 @@ pub enum DBFlush {
}
impl DB {
pub fn open(path: &Path, config: &Config) -> DB {
pub fn open(path: &Path) -> DB {
debug!("opening DB at {:?}", path);
let mut db_opts = rocksdb::Options::default();
db_opts.create_if_missing(true);
db_opts.set_max_open_files(-1); // TODO: make sure to `ulimit -n` this process correctly
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
db_opts.set_target_file_size_base(256 << 20);
db_opts.set_write_buffer_size(256 << 20);
db_opts.set_disable_auto_compactions(true); // for initial bulk load
// db_opts.set_advise_random_on_open(???);
db_opts.set_compaction_readahead_size(1 << 20);
db_opts.increase_parallelism(2);
// let mut block_opts = rocksdb::BlockBasedOptions::default();
// block_opts.set_block_size(???);
let db = DB {
db: open_raw_db(path, OpenMode::ReadWrite),
db: rocksdb::DB::open(&db_opts, path).expect("failed to open RocksDB"),
};
db.verify_compatibility(config);
db.verify_compatibility();
db
}
@ -166,11 +113,11 @@ impl DB {
self.db.set_options(&opts).unwrap();
}
pub fn raw_iterator(&self) -> rocksdb::DBRawIterator<'_> {
pub fn raw_iterator(&self) -> rocksdb::DBRawIterator {
self.db.raw_iterator()
}
pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator<'_> {
pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator {
ScanIterator {
prefix: prefix.to_vec(),
iter: self.db.prefix_iterator(prefix),
@ -178,7 +125,7 @@ impl DB {
}
}
pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator<'_> {
pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator {
let iter = self.db.iterator(rocksdb::IteratorMode::From(
start_at,
rocksdb::Direction::Forward,
@ -190,7 +137,7 @@ impl DB {
}
}
pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator<'_> {
pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator {
let mut iter = self.db.raw_iterator();
iter.seek_for_prev(prefix_max);
@ -201,25 +148,6 @@ impl DB {
}
}
pub fn iter_scan_group_reverse(
&self,
prefixes: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
value_offset: usize,
) -> ReverseScanGroupIterator<'_> {
let iters = prefixes
.map(|(prefix, prefix_max)| {
let mut iter = self.db.raw_iterator();
iter.seek_for_prev(prefix_max);
ReverseScanIterator {
prefix: prefix.to_vec(),
iter,
done: false,
}
})
.collect();
ReverseScanGroupIterator::new(iters, value_offset)
}
pub fn write(&self, mut rows: Vec<DBRow>, flush: DBFlush) {
debug!(
"writing {} rows to {:?}, flush={:?}",
@ -230,7 +158,7 @@ impl DB {
rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
let mut batch = rocksdb::WriteBatch::default();
for row in rows {
batch.put(&row.key, &row.value);
batch.put(&row.key, &row.value).unwrap();
}
let do_flush = match flush {
DBFlush::Enable => true,
@ -242,43 +170,16 @@ impl DB {
self.db.write_opt(batch, &opts).unwrap();
}
pub fn delete(&self, keys: Vec<Vec<u8>>) {
debug!("deleting {} rows from {:?}", keys.len(), self.db);
for key in keys {
let _ = self.db.delete(key).inspect_err(|err| {
warn!("Error while deleting DB row: {err}");
});
}
}
pub fn flush(&self) {
self.db.flush().unwrap();
}
pub fn put(&self, key: &[u8], value: &[u8]) {
self.db.put(key, value).unwrap();
}
pub fn put_sync(&self, key: &[u8], value: &[u8]) {
let mut opts = rocksdb::WriteOptions::new();
opts.set_sync(true);
self.db.put_opt(key, value, &opts).unwrap();
}
pub fn get(&self, key: &[u8]) -> Option<Bytes> {
self.db.get(key).unwrap().map(|v| v.to_vec())
}
fn verify_compatibility(&self, config: &Config) {
let mut compatibility_bytes = bincode_util::serialize_little(&DB_VERSION).unwrap();
if config.light_mode {
// append a byte to indicate light_mode is enabled.
// we're not letting bincode serialize this so that the compatiblity bytes won't change
// (and require a reindex) when light_mode is disabled. this should be chagned the next
// time we bump DB_VERSION and require a re-index anyway.
compatibility_bytes.push(1);
}
fn verify_compatibility(&self) {
let compatibility_bytes = bincode::serialize(&DB_VERSION).unwrap();
match self.get(b"V") {
None => self.put(b"V", &compatibility_bytes),
@ -290,41 +191,34 @@ impl DB {
}
}
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[repr(u8)]
pub enum OpenMode {
ReadOnly,
ReadWrite,
}
#[cfg(test)]
mod tests {
pub fn open_raw_db<T: rocksdb::ThreadMode>(
path: &Path,
read_mode: OpenMode,
) -> rocksdb::DBWithThreadMode<T> {
debug!("opening DB at {:?}", path);
let mut db_opts = rocksdb::Options::default();
db_opts.create_if_missing(true);
db_opts.set_max_open_files(100_000); // TODO: make sure to `ulimit -n` this process correctly
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
db_opts.set_compression_type(rocksdb::DBCompressionType::None);
db_opts.set_target_file_size_base(1_073_741_824);
db_opts.set_write_buffer_size(256 << 20);
db_opts.set_disable_auto_compactions(true); // for initial bulk load
use crate::new_index::db::{DBRow, DB};
use tempfile;
// db_opts.set_advise_random_on_open(???);
db_opts.set_compaction_readahead_size(1 << 20);
db_opts.increase_parallelism(2);
// let mut block_opts = rocksdb::BlockBasedOptions::default();
// block_opts.set_block_size(???);
match read_mode {
OpenMode::ReadOnly => {
rocksdb::DBWithThreadMode::<T>::open_for_read_only(&db_opts, path, false)
.expect("failed to open RocksDB (READ ONLY)")
}
OpenMode::ReadWrite => {
rocksdb::DBWithThreadMode::<T>::open(&db_opts, path).expect("failed to open RocksDB")
#[test]
fn test_db_iterator() {
let dir = tempfile::tempdir().unwrap();
let db = DB::open(dir.path());
let rows = vec![
DBRow {
key: b"X11".to_vec(),
value: b"a".to_vec(),
},
DBRow {
key: b"X22".to_vec(),
value: b"b".to_vec(),
},
DBRow {
key: b"X33".to_vec(),
value: b"c".to_vec(),
},
];
for row in &rows {
db.put(&row.key, &row.value);
}
let actual: Vec<DBRow> = db.iter_scan(b"X").collect();
assert_eq!(rows, actual);
}
}

View File

@ -1,17 +1,17 @@
use rayon::prelude::*;
use crate::chain::Block;
#[cfg(not(feature = "liquid"))]
use bitcoin::consensus::encode::{deserialize, Decodable};
#[cfg(feature = "liquid")]
use elements::encode::{deserialize, Decodable};
use bitcoin::consensus::encode::{deserialize, serialize, Decodable};
use bitcoin::util::hash::BitcoinHash;
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use rayon::prelude::*;
use std::collections::HashMap;
use std::fs;
use std::io::Cursor;
use std::io::{Cursor, Seek, SeekFrom};
use std::path::PathBuf;
use std::sync::mpsc::Receiver;
use std::thread;
use crate::chain::{Block, BlockHash, BlockSizeCompat};
use crate::daemon::Daemon;
use crate::errors::*;
use crate::util::{spawn_thread, HeaderEntry, SyncChannel};
@ -42,70 +42,19 @@ pub struct BlockEntry {
type SizedBlock = (Block, u32);
pub struct SequentialFetcher<T> {
fetcher: Box<dyn FnOnce() -> Vec<Vec<T>>>,
}
impl<T> SequentialFetcher<T> {
fn from<F: FnOnce() -> Vec<Vec<T>> + 'static>(pre_func: F) -> Self {
SequentialFetcher {
fetcher: Box::new(pre_func),
}
}
pub fn map<FN>(self, mut func: FN)
where
FN: FnMut(Vec<T>),
{
for item in (self.fetcher)() {
func(item);
}
}
}
pub fn bitcoind_sequential_fetcher(
daemon: &Daemon,
new_headers: Vec<HeaderEntry>,
) -> Result<SequentialFetcher<BlockEntry>> {
let daemon = daemon.reconnect()?;
Ok(SequentialFetcher::from(move || {
new_headers
.chunks(100)
.map(|entries| {
let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
let blocks = daemon
.getblocks(&blockhashes)
.expect("failed to get blocks from bitcoind");
assert_eq!(blocks.len(), entries.len());
let block_entries: Vec<BlockEntry> = blocks
.into_iter()
.zip(entries)
.map(|(block, entry)| BlockEntry {
entry: entry.clone(), // TODO: remove this clone()
size: block.get_block_size() as u32,
block,
})
.collect();
assert_eq!(block_entries.len(), entries.len());
block_entries
})
.collect()
}))
}
pub struct Fetcher<T> {
receiver: crossbeam_channel::Receiver<T>,
receiver: Receiver<T>,
thread: thread::JoinHandle<()>,
}
impl<T> Fetcher<T> {
fn from(receiver: crossbeam_channel::Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
fn from(receiver: Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
Fetcher { receiver, thread }
}
pub fn map<F>(self, mut func: F)
where
F: FnMut(T),
F: FnMut(T) -> (),
{
for item in self.receiver {
func(item);
@ -118,17 +67,17 @@ fn bitcoind_fetcher(
daemon: &Daemon,
new_headers: Vec<HeaderEntry>,
) -> Result<Fetcher<Vec<BlockEntry>>> {
if let Some(tip) = new_headers.last() {
debug!("{:?} ({} left to index)", tip, new_headers.len());
};
new_headers.last().map(|tip| {
debug!("{:?} ({} new blocks to fetch)", tip, new_headers.len());
});
let daemon = daemon.reconnect()?;
let chan = SyncChannel::new(1);
let sender = chan.sender();
Ok(Fetcher::from(
chan.into_receiver(),
spawn_thread("bitcoind_fetcher", move || {
spawn_thread("bitcoind_fetcher", move || -> () {
for entries in new_headers.chunks(100) {
let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
let blockhashes: Vec<Sha256dHash> = entries.iter().map(|e| *e.hash()).collect();
let blocks = daemon
.getblocks(&blockhashes)
.expect("failed to get blocks from bitcoind");
@ -137,8 +86,8 @@ fn bitcoind_fetcher(
.into_iter()
.zip(entries)
.map(|(block, entry)| BlockEntry {
entry: entry.clone(), // TODO: remove this clone()
size: block.get_block_size() as u32,
entry: entry.clone(), // TODO: remove this clone()
size: serialize(&block).len() as u32, // TODO: avoid re-serializing
block,
})
.collect();
@ -161,18 +110,18 @@ fn blkfiles_fetcher(
let chan = SyncChannel::new(1);
let sender = chan.sender();
let mut entry_map: HashMap<BlockHash, HeaderEntry> =
let mut entry_map: HashMap<Sha256dHash, HeaderEntry> =
new_headers.into_iter().map(|h| (*h.hash(), h)).collect();
let parser = blkfiles_parser(blkfiles_reader(blk_files), magic);
Ok(Fetcher::from(
chan.into_receiver(),
spawn_thread("blkfiles_fetcher", move || {
spawn_thread("blkfiles_fetcher", move || -> () {
parser.map(|sizedblocks| {
let block_entries: Vec<BlockEntry> = sizedblocks
.into_iter()
.filter_map(|(block, size)| {
let blockhash = block.block_hash();
let blockhash = block.bitcoin_hash();
entry_map
.remove(&blockhash)
.map(|entry| BlockEntry { block, entry, size })
@ -200,36 +149,16 @@ fn blkfiles_fetcher(
fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
let chan = SyncChannel::new(1);
let sender = chan.sender();
let xor_key = blk_files.first().and_then(|p| {
let xor_file = p
.parent()
.expect("blk.dat files must exist in a directory")
.join("xor.dat");
if xor_file.exists() {
Some(fs::read(xor_file).expect("xor.dat exists"))
} else {
None
}
});
Fetcher::from(
chan.into_receiver(),
spawn_thread("blkfiles_reader", move || {
spawn_thread("blkfiles_reader", move || -> () {
for path in blk_files {
trace!("reading {:?}", path);
let mut blob = fs::read(&path)
.unwrap_or_else(|e| panic!("failed to read {:?}: {:?}", path, e));
// If the xor.dat exists. Use it to decrypt the block files.
if let Some(xor_key) = &xor_key {
for (&key, byte) in xor_key.iter().cycle().zip(blob.iter_mut()) {
*byte ^= key;
}
}
let blob = fs::read(&path).expect(&format!("failed to read {:?}", path));
sender
.send(blob)
.unwrap_or_else(|_| panic!("failed to send {:?} contents", path));
.expect(&format!("failed to send {:?} contents", path));
}
}),
)
@ -241,7 +170,7 @@ fn blkfiles_parser(blobs: Fetcher<Vec<u8>>, magic: u32) -> Fetcher<Vec<SizedBloc
Fetcher::from(
chan.into_receiver(),
spawn_thread("blkfiles_parser", move || {
spawn_thread("blkfiles_parser", move || -> () {
blobs.map(|blob| {
trace!("parsing {} bytes", blob.len());
let blocks = parse_blocks(blob, magic).expect("failed to parse blk*.dat file");
@ -257,40 +186,34 @@ fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<SizedBlock>> {
let mut cursor = Cursor::new(&blob);
let mut slices = vec![];
let max_pos = blob.len() as u64;
while cursor.position() < max_pos {
let offset = cursor.position();
match u32::consensus_decode(&mut cursor) {
Ok(value) => {
if magic != value {
cursor.set_position(offset + 1);
cursor
.seek(SeekFrom::Current(-3))
.expect("failed to seek back");
continue;
}
}
Err(_) => break, // EOF
};
let block_size = u32::consensus_decode(&mut cursor).chain_err(|| "no block size")?;
let start = cursor.position();
let end = start + block_size as u64;
let start = cursor.position() as usize;
cursor
.seek(SeekFrom::Current(block_size as i64))
.chain_err(|| format!("seek {} failed", block_size))?;
let end = cursor.position() as usize;
// If Core's WriteBlockToDisk ftell fails, only the magic bytes and size will be written
// and the block body won't be written to the blk*.dat file.
// Since the first 4 bytes should contain the block's version, we can skip such blocks
// by peeking the cursor (and skipping previous `magic` and `block_size`).
match u32::consensus_decode(&mut cursor) {
Ok(value) => {
if magic == value {
cursor.set_position(start);
continue;
}
}
Err(_) => break, // EOF
}
slices.push((&blob[start as usize..end as usize], block_size));
cursor.set_position(end);
slices.push((&blob[start..end], block_size));
}
Ok(super::THREAD_POOL.install(|| {
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(0) // CPU-bound
.thread_name(|i| format!("parse-blocks-{}", i))
.build()
.unwrap();
Ok(pool.install(|| {
slices
.into_par_iter()
.map(|(slice, size)| (deserialize(slice).expect("failed to parse Block"), size))

View File

@ -1,58 +1,46 @@
use bounded_vec_deque::BoundedVecDeque;
use arraydeque::{ArrayDeque, Wrapping};
use bitcoin::consensus::encode::serialize;
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use itertools::Itertools;
#[cfg(not(feature = "liquid"))]
use bitcoin::consensus::encode::serialize;
#[cfg(feature = "liquid")]
use elements::{encode::serialize, AssetId};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::collections::{BTreeSet, HashMap, HashSet};
use std::iter::FromIterator;
use std::ops::Bound::{Excluded, Unbounded};
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::chain::{deserialize, Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
use crate::config::Config;
use crate::chain::{OutPoint, Transaction, TxOut};
use crate::daemon::Daemon;
use crate::errors::*;
use crate::metrics::{GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics};
use crate::new_index::{
compute_script_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats, SpendingInfo,
SpendingInput, TxHistoryInfo, Utxo,
compute_script_hash, parse_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats,
SpendingInfo, SpendingInput, TxHistoryInfo, Utxo,
};
use crate::util::fees::{make_fee_histogram, TxFeeInfo};
use crate::util::{extract_tx_prevouts, full_hash, has_prevout, is_spendable, Bytes};
use crate::util::{full_hash, has_prevout, is_spendable, Bytes};
#[cfg(feature = "liquid")]
use crate::elements::asset;
const RECENT_TXS_SIZE: usize = 10;
const BACKLOG_STATS_TTL: u64 = 10;
pub struct Mempool {
chain: Arc<ChainQuery>,
config: Arc<Config>,
txstore: BTreeMap<Txid, Transaction>,
feeinfo: HashMap<Txid, TxFeeInfo>,
txstore: HashMap<Sha256dHash, Transaction>,
feeinfo: HashMap<Sha256dHash, TxFeeInfo>,
history: HashMap<FullHash, Vec<TxHistoryInfo>>, // ScriptHash -> {history_entries}
edges: HashMap<OutPoint, (Txid, u32)>, // OutPoint -> (spending_txid, spending_vin)
recent: BoundedVecDeque<TxOverview>, // The N most recent txs to enter the mempool
edges: HashMap<OutPoint, (Sha256dHash, u32)>, // OutPoint -> (spending_txid, spending_vin)
recent: ArrayDeque<[TxOverview; RECENT_TXS_SIZE], Wrapping>, // The N most recent txs to enter the mempool
backlog_stats: (BacklogStats, Instant),
// monitoring
latency: HistogramVec, // mempool requests latency
delta: HistogramVec, // # of added/removed txs
count: GaugeVec, // current state of the mempool
// elements only
#[cfg(feature = "liquid")]
pub asset_history: HashMap<AssetId, Vec<TxHistoryInfo>>,
#[cfg(feature = "liquid")]
pub asset_issuance: HashMap<AssetId, asset::AssetRow>,
}
// A simplified transaction view used for the list of most recent transactions
#[derive(Serialize)]
pub struct TxOverview {
txid: Txid,
txid: Sha256dHash,
fee: u64,
vsize: u32,
#[cfg(not(feature = "liquid"))]
@ -60,17 +48,17 @@ pub struct TxOverview {
}
impl Mempool {
pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics, config: Arc<Config>) -> Self {
pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics) -> Self {
Mempool {
chain,
txstore: BTreeMap::new(),
txstore: HashMap::new(),
feeinfo: HashMap::new(),
history: HashMap::new(),
edges: HashMap::new(),
recent: BoundedVecDeque::new(config.mempool_recent_txs_size),
recent: ArrayDeque::new(),
backlog_stats: (
BacklogStats::default(),
Instant::now() - Duration::from_secs(config.mempool_backlog_stats_ttl),
Instant::now() - Duration::from_secs(BACKLOG_STATS_TTL),
),
latency: metrics.histogram_vec(
HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"),
@ -84,24 +72,14 @@ impl Mempool {
MetricOpts::new("mempool_count", "# of elements currently at the mempool"),
&["type"],
),
#[cfg(feature = "liquid")]
asset_history: HashMap::new(),
#[cfg(feature = "liquid")]
asset_issuance: HashMap::new(),
config,
}
}
pub fn network(&self) -> Network {
self.config.network_type
pub fn lookup_txn(&self, txid: &Sha256dHash) -> Option<Transaction> {
self.txstore.get(txid).map(|item| item.clone())
}
pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
self.txstore.get(txid).cloned()
}
pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
pub fn lookup_raw_txn(&self, txid: &Sha256dHash) -> Option<Bytes> {
self.txstore.get(txid).map(serialize)
}
@ -117,122 +95,30 @@ impl Mempool {
self.edges.contains_key(outpoint)
}
pub fn get_tx_fee(&self, txid: &Txid) -> Option<u64> {
Some(self.feeinfo.get(txid)?.fee)
}
pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
let tx = match self.txstore.get(txid) {
Some(tx) => tx,
None => return false,
};
tx.input
.iter()
.any(|txin| self.txstore.contains_key(&txin.previous_output.txid))
}
pub fn history(
&self,
scripthash: &[u8],
last_seen_txid: Option<&Txid>,
limit: usize,
) -> Vec<Transaction> {
// TODO: return as Vec<(Transaction,Option<BlockId>)>?
pub fn history(&self, scripthash: &[u8], limit: usize) -> Vec<Transaction> {
let _timer = self.latency.with_label_values(&["history"]).start_timer();
self.history
.get(scripthash)
.map_or_else(std::vec::Vec::new, |entries| {
self._history(entries, last_seen_txid, limit)
})
match self.history.get(scripthash) {
None => return vec![],
Some(entries) => entries
.iter()
.map(get_entry_txid)
.unique()
.take(limit)
.map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
.cloned()
.collect(),
}
}
pub fn history_txids_iter<'a>(&'a self, scripthash: &[u8]) -> impl Iterator<Item = Txid> + 'a {
self.history
.get(scripthash)
.into_iter()
.flat_map(|v| v.iter().map(|e| e.get_txid()).unique())
}
fn _history(
&self,
entries: &[TxHistoryInfo],
last_seen_txid: Option<&Txid>,
limit: usize,
) -> Vec<Transaction> {
entries
.iter()
.map(|e| e.get_txid())
.unique()
// TODO seek directly to last seen tx without reading earlier rows
.skip_while(|txid| {
// skip until we reach the last_seen_txid
last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
})
.skip(match last_seen_txid {
Some(_) => 1, // skip the last_seen_txid itself
None => 0,
})
.take(limit)
.map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
.cloned()
.collect()
}
pub fn history_group(
&self,
scripthashes: &[[u8; 32]],
last_seen_txid: Option<&Txid>,
limit: usize,
) -> Vec<Transaction> {
let _timer = self
.latency
.with_label_values(&["history_group"])
.start_timer();
scripthashes
.iter()
.filter_map(|scripthash| self.history.get(&scripthash[..]))
.flat_map(|entries| entries.iter())
.map(|e| e.get_txid())
.unique()
// TODO seek directly to last seen tx without reading earlier rows
.skip_while(|txid| {
// skip until we reach the last_seen_txid
last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
})
.skip(match last_seen_txid {
Some(_) => 1, // skip the last_seen_txid itself
None => 0,
})
.take(limit)
.map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
.cloned()
.collect()
}
pub fn history_txids_iter_group<'a>(
&'a self,
scripthashes: &'a [[u8; 32]],
) -> impl Iterator<Item = Txid> + 'a {
scripthashes
.iter()
.filter_map(move |scripthash| self.history.get(&scripthash[..]))
.flat_map(|entries| entries.iter())
.map(|entry| entry.get_txid())
.unique()
}
pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<Txid> {
pub fn history_txids(&self, scripthash: &[u8]) -> Vec<Sha256dHash> {
let _timer = self
.latency
.with_label_values(&["history_txids"])
.start_timer();
match self.history.get(scripthash) {
None => vec![],
Some(entries) => entries
.iter()
.map(|e| e.get_txid())
.unique()
.take(limit)
.collect(),
None => return vec![],
Some(entries) => entries.iter().map(get_entry_txid).unique().collect(),
}
}
@ -244,33 +130,15 @@ impl Mempool {
};
entries
.iter()
.into_iter()
.filter_map(|entry| match entry {
TxHistoryInfo::Funding(info) => {
// Liquid requires some additional information from the txo that's not available in the TxHistoryInfo index.
#[cfg(feature = "liquid")]
let txo = self.lookup_txo(&entry.get_funded_outpoint())?;
Some(Utxo {
txid: deserialize(&info.txid).expect("invalid txid"),
vout: info.vout,
value: info.value,
confirmed: None,
#[cfg(feature = "liquid")]
asset: txo.asset,
#[cfg(feature = "liquid")]
nonce: txo.nonce,
#[cfg(feature = "liquid")]
witness: txo.witness,
})
}
TxHistoryInfo::Spending(_) => None,
#[cfg(feature = "liquid")]
TxHistoryInfo::Issuing(_)
| TxHistoryInfo::Burning(_)
| TxHistoryInfo::Pegin(_)
| TxHistoryInfo::Pegout(_) => unreachable!(),
TxHistoryInfo::Funding(info) => Some(Utxo {
txid: parse_hash(&info.txid),
vout: info.vout as u32,
value: info.value,
confirmed: None,
}),
TxHistoryInfo::Spending(..) => None,
})
.filter(|utxo| !self.has_spend(&OutPoint::from(utxo)))
.collect()
@ -288,7 +156,7 @@ impl Mempool {
};
for entry in entries {
if seen_txids.insert(entry.get_txid()) {
if seen_txids.insert(get_entry_txid(entry)) {
stats.tx_count += 1;
}
@ -298,27 +166,20 @@ impl Mempool {
stats.funded_txo_count += 1;
stats.funded_txo_sum += info.value;
}
#[cfg(feature = "liquid")]
TxHistoryInfo::Funding(_) => {
stats.funded_txo_count += 1;
}
#[cfg(not(feature = "liquid"))]
TxHistoryInfo::Spending(info) => {
stats.spent_txo_count += 1;
stats.spent_txo_sum += info.value;
}
// Elements
#[cfg(feature = "liquid")]
TxHistoryInfo::Funding(_) => {
stats.funded_txo_count += 1;
}
#[cfg(feature = "liquid")]
TxHistoryInfo::Spending(_) => {
stats.spent_txo_count += 1;
}
#[cfg(feature = "liquid")]
TxHistoryInfo::Issuing(_)
| TxHistoryInfo::Burning(_)
| TxHistoryInfo::Pegin(_)
| TxHistoryInfo::Pegout(_) => unreachable!(),
};
}
@ -326,54 +187,11 @@ impl Mempool {
}
// Get all txids in the mempool
pub fn txids(&self) -> Vec<&Txid> {
pub fn txids(&self) -> Vec<&Sha256dHash> {
let _timer = self.latency.with_label_values(&["txids"]).start_timer();
self.txstore.keys().collect()
}
// Get n txids after the given txid in the mempool
pub fn txids_page(&self, n: usize, start: Option<Txid>) -> Vec<&Txid> {
let _timer = self
.latency
.with_label_values(&["txids_page"])
.start_timer();
let start_bound = match start {
Some(txid) => Excluded(txid),
None => Unbounded,
};
self.txstore
.range((start_bound, Unbounded))
.take(n)
.map(|(k, _v)| k)
.collect()
}
// Get all txs in the mempool
pub fn txs(&self) -> Vec<Transaction> {
let _timer = self.latency.with_label_values(&["txs"]).start_timer();
self.txstore.values().cloned().collect()
}
// Get n txs after the given txid in the mempool
pub fn txs_page(&self, n: usize, start: Option<Txid>) -> Vec<Transaction> {
let _timer = self.latency.with_label_values(&["txs_page"]).start_timer();
let mut page = Vec::with_capacity(n);
let start_bound = match start {
Some(txid) => Excluded(txid),
None => Unbounded,
};
self.txstore
.range((start_bound, Unbounded))
.take(n)
.for_each(|(_, value)| {
page.push(value.clone());
});
page
}
// Get an overview of the most recent transactions
pub fn recent_txs_overview(&self) -> Vec<&TxOverview> {
// We don't bother ever deleting elements from the recent list.
@ -386,150 +204,100 @@ impl Mempool {
&self.backlog_stats.0
}
pub fn unique_txids(&self) -> HashSet<Txid> {
HashSet::from_iter(self.txstore.keys().cloned())
}
pub fn update(mempool: &RwLock<Mempool>, daemon: &Daemon) -> Result<()> {
// 1. Start the metrics timer and get the current mempool txids
// [LOCK] Takes read lock for whole scope.
let (_timer, old_txids) = {
let mempool = mempool.read().unwrap();
(
mempool.latency.with_label_values(&["update"]).start_timer(),
mempool.unique_txids(),
)
};
// 2. Get all the mempool txids from the RPC.
// [LOCK] No lock taken. Wait for RPC request. Get lists of remove/add txes.
let all_txids = daemon
pub fn update(&mut self, daemon: &Daemon) -> Result<()> {
let _timer = self.latency.with_label_values(&["update"]).start_timer();
let new_txids = daemon
.getmempooltxids()
.chain_err(|| "failed to update mempool from daemon")?;
let txids_to_remove: HashSet<&Txid> = old_txids.difference(&all_txids).collect();
let txids_to_add: Vec<&Txid> = all_txids.difference(&old_txids).collect();
let old_txids = HashSet::from_iter(self.txstore.keys().cloned());
let to_remove: HashSet<&Sha256dHash> = old_txids.difference(&new_txids).collect();
// 3. Remove missing transactions. Even if we are unable to download new transactions from
// the daemon, we still want to remove the transactions that are no longer in the mempool.
// [LOCK] Write lock is released at the end of the call to remove().
mempool.write().unwrap().remove(txids_to_remove);
// 4. Download the new transactions from the daemon's mempool
// [LOCK] No lock taken, waiting for RPC response.
let txs_to_add = daemon
.gettransactions(&txids_to_add)
.chain_err(|| format!("failed to get {} transactions", txids_to_add.len()))?;
// 4. Update local mempool to match daemon's state
// [LOCK] Takes Write lock for whole scope.
{
let mut mempool = mempool.write().unwrap();
// Add new transactions
if txs_to_add.len() > mempool.add(txs_to_add) {
debug!("Mempool update added less transactions than expected");
// Download and add new transactions from bitcoind's mempool
let txids: Vec<&Sha256dHash> = new_txids.difference(&old_txids).collect();
let to_add = match daemon.gettransactions(&txids) {
Ok(txs) => txs,
Err(err) => {
warn!("failed to get transactions {:?}: {}", txids, err); // e.g. new block or RBF
return Ok(()); // keep the mempool until next update()
}
};
// Add new transactions
self.add(to_add);
// Remove missing transactions
self.remove(to_remove);
mempool
.count
.with_label_values(&["txs"])
.set(mempool.txstore.len() as f64);
self.count
.with_label_values(&["txs"])
.set(self.txstore.len() as f64);
// Update cached backlog stats (if expired)
if mempool.backlog_stats.1.elapsed()
> Duration::from_secs(mempool.config.mempool_backlog_stats_ttl)
{
let _timer = mempool
.latency
.with_label_values(&["update_backlog_stats"])
.start_timer();
mempool.backlog_stats = (BacklogStats::new(&mempool.feeinfo), Instant::now());
}
Ok(())
// Update cached backlog stats (if expired)
if self.backlog_stats.1.elapsed() > Duration::from_secs(BACKLOG_STATS_TTL) {
let _timer = self
.latency
.with_label_values(&["update_backlog_stats"])
.start_timer();
self.backlog_stats = (BacklogStats::new(&self.feeinfo), Instant::now());
}
}
pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) -> Result<()> {
if !self.txstore.contains_key(txid) {
if let Ok(tx) = daemon.getmempooltx(txid) {
if self.add(vec![tx]) == 0 {
return Err(format!(
"Unable to add {txid} to mempool likely due to missing parents."
)
.into());
}
}
}
Ok(())
}
/// Add transactions to the mempool.
///
/// The return value is the number of transactions processed.
#[must_use = "Must deal with [[input vec's length]] > [[result]]."]
fn add(&mut self, txs: Vec<Transaction>) -> usize {
pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Sha256dHash) {
if let Ok(tx) = daemon.getmempooltx(&txid) {
self.add(vec![tx])
}
}
fn add(&mut self, txs: Vec<Transaction>) {
self.delta
.with_label_values(&["add"])
.observe(txs.len() as f64);
let _timer = self.latency.with_label_values(&["add"]).start_timer();
let txlen = txs.len();
if txlen == 0 {
return 0;
}
debug!("Adding {} transactions to Mempool", txlen);
let mut txids = Vec::with_capacity(txs.len());
let mut txids = vec![];
// Phase 1: add to txstore
for tx in txs {
let txid = tx.get_txid();
// Only push if it doesn't already exist.
// This is important now that update doesn't lock during
// the entire function body.
if self.txstore.insert(txid, tx).is_none() {
txids.push(txid);
}
let txid = tx.txid();
txids.push(txid);
self.txstore.insert(txid, tx);
}
// Phase 2: index history and spend edges (some txos can be missing)
let txos = self.lookup_txos(&self.get_prevouts(&txids));
// Count how many transactions were actually processed.
let mut processed_count = 0;
// Phase 3: Iterate over the transactions and do the following:
// 1. Find all of the TxOuts of each input parent using `txos`
// 2. If any parent wasn't found, skip parsing this transaction
// 3. Insert TxFeeInfo into info.
// 4. Push TxOverview into recent tx queue.
// 5. Create the Spend and Fund TxHistory structs for inputs + outputs
// 6. Insert all TxHistory into history.
// 7. Insert the tx edges into edges (HashMap of (Outpoint, (Txid, vin)))
// 8. (Liquid only) Parse assets of tx.
// Phase 2: index history and spend edges (can fail if some txos cannot be found)
let txos = match self.lookup_txos(&self.get_prevouts(&txids)) {
Ok(txos) => txos,
Err(err) => {
warn!("lookup txouts failed: {}", err);
// TODO: should we remove txids from txstore?
return;
}
};
for txid in txids {
let tx = self.txstore.get(&txid).expect("missing tx from txstore");
let prevouts = match extract_tx_prevouts(tx, &txos) {
Ok(v) => v,
Err(e) => {
warn!("Skipping tx {txid} missing parent error: {e}");
continue;
}
};
let tx = self.txstore.get(&txid).expect("missing mempool tx");
let txid_bytes = full_hash(&txid[..]);
// Get feeinfo for caching and recent tx overview
let feeinfo = TxFeeInfo::new(tx, &prevouts, self.config.network_type);
let prevouts: HashMap<u32, &TxOut> = tx
.input
.iter()
.enumerate()
.filter(|(_, txi)| has_prevout(txi))
.map(|(index, txi)| {
(
index as u32,
txos.get(&txi.previous_output)
.expect(&format!("missing outpoint {:?}", txi.previous_output)),
)
})
.collect();
// recent is an BoundedVecDeque that automatically evicts the oldest elements
// Get feeinfo for caching and recent tx overview
let feeinfo = TxFeeInfo::new(&tx, &prevouts);
// recent is an ArrayDeque that automatically evicts the oldest elements
self.recent.push_front(TxOverview {
txid,
txid: txid,
fee: feeinfo.fee,
vsize: feeinfo.vsize,
#[cfg(not(feature = "liquid"))]
value: prevouts
.values()
.map(|prevout| prevout.value.to_sat())
.sum(),
value: prevouts.values().map(|prevout| prevout.value).sum(),
});
self.feeinfo.insert(txid, feeinfo);
@ -537,82 +305,49 @@ impl Mempool {
// An iterator over (ScriptHash, TxHistoryInfo)
let spending = prevouts.into_iter().map(|(input_index, prevout)| {
let txi = tx.input.get(input_index as usize).unwrap();
#[cfg(not(feature = "liquid"))]
let value = prevout.value.to_sat();
#[cfg(feature = "liquid")]
let value = prevout.value;
(
compute_script_hash(&prevout.script_pubkey),
TxHistoryInfo::Spending(SpendingInfo {
txid: txid_bytes,
vin: input_index,
vin: input_index as u16,
prev_txid: full_hash(&txi.previous_output.txid[..]),
prev_vout: txi.previous_output.vout,
value,
prev_vout: txi.previous_output.vout as u16,
value: prevout.value,
}),
)
});
let config = &self.config;
// An iterator over (ScriptHash, TxHistoryInfo)
let funding = tx
.output
.iter()
.enumerate()
.filter(|(_, txo)| is_spendable(txo) || config.index_unspendables)
.filter(|(_, txo)| is_spendable(txo))
.map(|(index, txo)| {
#[cfg(not(feature = "liquid"))]
let value = txo.value.to_sat();
#[cfg(feature = "liquid")]
let value = txo.value;
(
compute_script_hash(&txo.script_pubkey),
TxHistoryInfo::Funding(FundingInfo {
txid: txid_bytes,
vout: index as u32,
value,
vout: index as u16,
value: txo.value,
}),
)
});
// Index funding/spending history entries and spend edges
for (scripthash, entry) in funding.chain(spending) {
self.history.entry(scripthash).or_default().push(entry);
self.history
.entry(scripthash)
.or_insert_with(|| Vec::new())
.push(entry);
}
for (i, txi) in tx.input.iter().enumerate() {
self.edges.insert(txi.previous_output, (txid, i as u32));
}
// Index issued assets & native asset pegins/pegouts/burns
#[cfg(feature = "liquid")]
asset::index_mempool_tx_assets(
tx,
self.config.network_type,
self.config.parent_network,
&mut self.asset_history,
&mut self.asset_issuance,
);
processed_count += 1;
}
processed_count
}
/// Returns None if the lookup fails (mempool transaction RBF-ed etc.)
pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option<TxOut> {
let mut outpoints = BTreeSet::new();
outpoints.insert(*outpoint);
// This can possibly be None now
self.lookup_txos(&outpoints).remove(outpoint)
}
/// For a given set of OutPoints, return a HashMap<OutPoint, TxOut>
///
/// Not all OutPoints from mempool transactions are guaranteed to be there.
/// Ensure you deal with the None case in your logic.
pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
let _timer = self
.latency
.with_label_values(&["lookup_txos"])
@ -623,29 +358,21 @@ impl Mempool {
let mempool_txos = outpoints
.iter()
.filter(|outpoint| !confirmed_txos.contains_key(outpoint))
.flat_map(|outpoint| {
.map(|outpoint| {
self.txstore
.get(&outpoint.txid)
.and_then(|tx| tx.output.get(outpoint.vout as usize).cloned())
.map(|txout| (*outpoint, txout))
.or_else(|| {
warn!("missing outpoint {:?}", outpoint);
None
})
.chain_err(|| format!("missing outpoint {:?}", outpoint))
})
.collect::<HashMap<OutPoint, TxOut>>();
.collect::<Result<HashMap<OutPoint, TxOut>>>()?;
let mut txos = confirmed_txos;
txos.extend(mempool_txos);
txos
Ok(txos)
}
fn get_prevouts(&self, txids: &[Txid]) -> BTreeSet<OutPoint> {
let _timer = self
.latency
.with_label_values(&["get_prevouts"])
.start_timer();
fn get_prevouts(&self, txids: &[Sha256dHash]) -> BTreeSet<OutPoint> {
txids
.iter()
.map(|txid| self.txstore.get(txid).expect("missing mempool tx"))
@ -658,7 +385,7 @@ impl Mempool {
.collect()
}
fn remove(&mut self, to_remove: HashSet<&Txid>) {
fn remove(&mut self, to_remove: HashSet<&Sha256dHash>) {
self.delta
.with_label_values(&["remove"])
.observe(to_remove.len() as f64);
@ -667,43 +394,20 @@ impl Mempool {
for txid in &to_remove {
self.txstore
.remove(*txid)
.unwrap_or_else(|| panic!("missing mempool tx {}", txid));
self.feeinfo.remove(*txid).or_else(|| {
warn!("missing mempool tx feeinfo {}", txid);
None
});
.expect(&format!("missing mempool tx {}", txid));
self.feeinfo
.remove(*txid)
.expect(&format!("missing mempool tx feeinfo {}", txid));
}
// TODO: make it more efficient (currently it takes O(|mempool|) time)
self.history.retain(|_scripthash, entries| {
entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
entries.retain(|entry| !to_remove.contains(&get_entry_txid(entry)));
!entries.is_empty()
});
#[cfg(feature = "liquid")]
asset::remove_mempool_tx_assets(
&to_remove,
&mut self.asset_history,
&mut self.asset_issuance,
);
self.edges
.retain(|_outpoint, (txid, _vin)| !to_remove.contains(txid));
}
#[cfg(feature = "liquid")]
pub fn asset_history(&self, asset_id: &AssetId, limit: usize) -> Vec<Transaction> {
let _timer = self
.latency
.with_label_values(&["asset_history"])
.start_timer();
self.asset_history
.get(asset_id)
.map_or_else(std::vec::Vec::new, |entries| {
self._history(entries, None, limit)
})
}
}
#[derive(Serialize)]
@ -724,7 +428,7 @@ impl BacklogStats {
}
}
fn new(feeinfo: &HashMap<Txid, TxFeeInfo>) -> Self {
fn new(feeinfo: &HashMap<Sha256dHash, TxFeeInfo>) -> Self {
let (count, vsize, total_fee) = feeinfo
.values()
.fold((0, 0, 0), |(count, vsize, fee), feeinfo| {
@ -739,3 +443,10 @@ impl BacklogStats {
}
}
}
fn get_entry_txid(entry: &TxHistoryInfo) -> Sha256dHash {
match entry {
TxHistoryInfo::Funding(info) => parse_hash(&info.txid),
TxHistoryInfo::Spending(info) => parse_hash(&info.txid),
}
}

View File

@ -1,25 +1,14 @@
pub mod db;
mod db;
mod fetch;
mod mempool;
pub mod precache;
mod query;
pub mod schema;
mod schema;
use std::sync::LazyLock;
pub(crate) static THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
rayon::ThreadPoolBuilder::new()
.num_threads(0) // 0 = use number of logical CPUs
.thread_name(|i| format!("electrs-worker-{}", i))
.build()
.expect("failed to create global rayon thread pool")
});
pub use self::db::{DBRow, DB};
pub use self::fetch::{BlockEntry, FetchFrom};
pub use self::mempool::Mempool;
pub use self::query::Query;
pub use self::schema::{
compute_script_hash, parse_hash, ChainQuery, FundingInfo, Indexer, ScriptStats, SpendingInfo,
SpendingInput, Store, TxHistoryInfo, TxHistoryKey, TxHistoryRow, Utxo,
SpendingInput, Store, TxHistoryInfo, TxHistoryKey, Utxo,
};

View File

@ -1,59 +1,37 @@
use crate::errors::*;
use crate::new_index::ChainQuery;
use crate::util::{full_hash, FullHash};
use crate::util::{full_hash, Address, FullHash};
use crypto::digest::Digest;
use crypto::sha2::Sha256;
use rayon::prelude::*;
use hex;
use std::fs::File;
use std::io;
use std::io::prelude::*;
use std::sync::{atomic::AtomicUsize, Arc};
use std::time::Instant;
use std::str::FromStr;
pub fn precache(chain: Arc<ChainQuery>, scripthashes: Vec<FullHash>, threads: usize) {
pub fn precache(chain: &ChainQuery, scripthashes: Vec<FullHash>) {
let total = scripthashes.len();
info!(
"Pre-caching stats and utxo set on {} threads for {} scripthashes",
threads, total
);
info!("Pre-caching stats and utxo set for {} scripthashes", total);
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(threads)
.num_threads(16)
.thread_name(|i| format!("precache-{}", i))
.build()
.unwrap();
let now = Instant::now();
let counter = AtomicUsize::new(0);
std::thread::spawn(move || {
pool.install(|| {
scripthashes
.par_iter()
.for_each(|scripthash| {
// First, cache
chain.stats(&scripthash[..], crate::new_index::db::DBFlush::Disable);
let _ = chain.utxo(&scripthash[..], usize::MAX, crate::new_index::db::DBFlush::Disable);
// Then, increment the counter
let pre_increment = counter.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
let post_increment_counter = pre_increment + 1;
// Then, log
if post_increment_counter % 500 == 0 {
let now_millis = now.elapsed().as_millis();
info!("{post_increment_counter}/{total} Processed in {now_millis} ms running pre-cache for scripthash");
}
// Every 10k counts, flush the DB to disk
if post_increment_counter % 10000 == 0 {
info!("Flushing cache_db... {post_increment_counter}");
chain.store().cache_db().flush();
info!("Done Flushing cache_db!!! {post_increment_counter}");
}
})
});
// After everything is done, flush the cache
chain.store().cache_db().flush();
pool.install(|| {
scripthashes
.par_iter()
.enumerate()
.for_each(|(i, scripthash)| {
if i % 5 == 0 {
debug!("Starting pre-cache for scripthash {}/{}", i + 1, total);
}
chain.stats(&scripthash[..]);
//chain.utxo(&scripthash[..]);
})
});
}
@ -64,7 +42,34 @@ pub fn scripthashes_from_file(path: String) -> Result<Vec<FullHash>> {
.lines()
.map(|line| {
let line = line.chain_err(|| "cannot read scripthash line")?;
Ok(full_hash(&hex::decode(line).chain_err(|| "invalid hex")?))
let cols: Vec<&str> = line.split(",").collect();
to_scripthash(cols[0], cols[1])
})
.collect()
}
fn to_scripthash(script_type: &str, script_str: &str) -> Result<FullHash> {
match script_type {
"address" => address_to_scripthash(script_str),
"scripthash" => Ok(full_hash(
&hex::decode(script_str).chain_err(|| "invalid hex")?,
)),
"scriptpubkey" => Ok(compute_script_hash(
&hex::decode(script_str).chain_err(|| "invalid hex")?,
)),
_ => bail!("Invalid script type".to_string()),
}
}
fn address_to_scripthash(addr: &str) -> Result<FullHash> {
let addr = Address::from_str(addr).chain_err(|| "invalid address")?;
Ok(compute_script_hash(&addr.script_pubkey().as_bytes()))
}
pub fn compute_script_hash(data: &[u8]) -> FullHash {
let mut hash = FullHash::default();
let mut sha2 = Sha256::new();
sha2.input(data);
sha2.result(&mut hash);
hash
}

View File

@ -1,55 +1,31 @@
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use rayon::prelude::*;
use std::collections::{BTreeSet, HashMap};
use std::sync::{Arc, RwLock, RwLockReadGuard};
use std::time::{Duration, Instant};
use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
use crate::config::Config;
use crate::daemon::{Daemon, MempoolAcceptResult, SubmitPackageResult};
use crate::chain::{OutPoint, Transaction, TxOut};
use crate::daemon::Daemon;
use crate::errors::*;
use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo};
use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus};
#[cfg(feature = "liquid")]
use crate::{
chain::{asset::AssetRegistryLock, AssetId},
elements::{lookup_asset, AssetMeta, AssetRegistry, AssetSorting, LiquidAsset},
};
const FEE_ESTIMATES_TTL: u64 = 60; // seconds
const CONF_TARGETS: [u16; 28] = [
1u16, 2u16, 3u16, 4u16, 5u16, 6u16, 7u16, 8u16, 9u16, 10u16, 11u16, 12u16, 13u16, 14u16, 15u16,
16u16, 17u16, 18u16, 19u16, 20u16, 21u16, 22u16, 23u16, 24u16, 25u16, 144u16, 504u16, 1008u16,
const CONF_TARGETS: [u16; 9] = [
2u16, 3u16, 4u16, 6u16, 10u16, 20u16, 144u16, 504u16, 1008u16,
];
pub struct Query {
chain: Arc<ChainQuery>, // TODO: should be used as read-only
mempool: Arc<RwLock<Mempool>>,
daemon: Arc<Daemon>,
config: Arc<Config>,
cached_estimates: RwLock<(HashMap<u16, f64>, Option<Instant>)>,
cached_relayfee: RwLock<Option<f64>>,
#[cfg(feature = "liquid")]
asset_db: Option<Arc<RwLock<AssetRegistry>>>,
}
impl Query {
#[cfg(not(feature = "liquid"))]
pub fn new(
chain: Arc<ChainQuery>,
mempool: Arc<RwLock<Mempool>>,
daemon: Arc<Daemon>,
config: Arc<Config>,
) -> Self {
pub fn new(chain: Arc<ChainQuery>, mempool: Arc<RwLock<Mempool>>, daemon: Arc<Daemon>) -> Self {
Query {
chain,
mempool,
daemon,
config,
cached_estimates: RwLock::new((HashMap::new(), None)),
cached_relayfee: RwLock::new(None),
}
}
@ -57,73 +33,37 @@ impl Query {
&self.chain
}
pub fn config(&self) -> &Config {
&self.config
}
pub fn network(&self) -> Network {
self.config.network_type
}
pub fn mempool(&self) -> RwLockReadGuard<'_, Mempool> {
pub fn mempool(&self) -> RwLockReadGuard<Mempool> {
self.mempool.read().unwrap()
}
pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
let txid = self.daemon.broadcast_raw(txhex)?;
// The important part is whether we succeeded in broadcasting.
// Ignore errors in adding to the cache and show an internal warning.
if let Err(e) = self
.mempool
pub fn broadcast_raw(&self, txhex: &String) -> Result<Sha256dHash> {
let txid = self.daemon.broadcast_raw(&txhex)?;
self.mempool
.write()
.unwrap()
.add_by_txid(&self.daemon, &txid)
{
warn!(
"broadcast_raw of {txid} succeeded to broadcast \
but failed to add to mempool-electrs Mempool cache: {e}"
);
}
.add_by_txid(&self.daemon, &txid);
Ok(txid)
}
pub fn test_mempool_accept(
&self,
txhex: Vec<String>,
maxfeerate: Option<f64>,
) -> Result<Vec<MempoolAcceptResult>> {
self.daemon.test_mempool_accept(txhex, maxfeerate)
}
pub fn submit_package(
&self,
txhex: Vec<String>,
maxfeerate: Option<f64>,
maxburnamount: Option<f64>,
) -> Result<SubmitPackageResult> {
self.daemon.submit_package(txhex, maxfeerate, maxburnamount)
}
pub fn utxo(&self, scripthash: &[u8]) -> Result<Vec<Utxo>> {
let mut utxos = self.chain.utxo(
scripthash,
self.config.utxos_limit,
super::db::DBFlush::Enable,
)?;
pub fn utxo(&self, scripthash: &[u8]) -> Vec<Utxo> {
let mut utxos = self.chain.utxo(scripthash);
let mempool = self.mempool();
utxos.retain(|utxo| !mempool.has_spend(&OutPoint::from(utxo)));
utxos.extend(mempool.utxo(scripthash));
Ok(utxos)
utxos
}
pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<(Txid, Option<BlockId>)> {
let confirmed_txids = self.chain.history_txids(scripthash, limit);
let confirmed_len = confirmed_txids.len();
let confirmed_txids = confirmed_txids.into_iter().map(|(tx, b)| (tx, Some(b)));
pub fn history_txids(&self, scripthash: &[u8]) -> Vec<(Sha256dHash, Option<BlockId>)> {
let confirmed_txids = self
.chain
.history_txids(scripthash)
.into_iter()
.map(|(tx, b)| (tx, Some(b)));
let mempool_txids = self
.mempool()
.history_txids(scripthash, limit - confirmed_len)
.history_txids(scripthash)
.into_iter()
.map(|tx| (tx, None));
@ -132,26 +72,27 @@ impl Query {
pub fn stats(&self, scripthash: &[u8]) -> (ScriptStats, ScriptStats) {
(
self.chain.stats(scripthash, super::db::DBFlush::Enable),
self.chain.stats(scripthash),
self.mempool().stats(scripthash),
)
}
pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
pub fn lookup_txn(&self, txid: &Sha256dHash) -> Option<Transaction> {
self.chain
.lookup_txn(txid, None)
.lookup_txn(txid)
.or_else(|| self.mempool().lookup_txn(txid))
}
pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
pub fn lookup_raw_txn(&self, txid: &Sha256dHash) -> Option<Bytes> {
self.chain
.lookup_raw_txn(txid, None)
.lookup_raw_txn(txid)
.or_else(|| self.mempool().lookup_raw_txn(txid))
}
/// Not all OutPoints from mempool transactions are guaranteed to be included in the result
pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
// the mempool lookup_txos() internally looks up confirmed txos as well
self.mempool().lookup_txos(outpoints)
self.mempool()
.lookup_txos(outpoints)
.expect("failed loading txos")
}
pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option<SpendingInput> {
@ -161,7 +102,7 @@ impl Query {
}
pub fn lookup_tx_spends(&self, tx: Transaction) -> Vec<Option<SpendingInput>> {
let txid = tx.get_txid();
let txid = tx.txid();
tx.output
.par_iter()
@ -179,155 +120,23 @@ impl Query {
.collect()
}
pub fn get_tx_status(&self, txid: &Txid) -> TransactionStatus {
pub fn get_tx_status(&self, txid: &Sha256dHash) -> TransactionStatus {
TransactionStatus::from(self.chain.tx_confirming_block(txid))
}
pub fn get_mempool_tx_fee(&self, txid: &Txid) -> Option<u64> {
self.mempool().get_tx_fee(txid)
// TODO cache, only allow getting estimatess for cached items
pub fn estimate_fee(&self, conf_target: u16) -> Option<f32> {
self.daemon.estimatesmartfee(conf_target).ok()
}
pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
self.mempool().has_unconfirmed_parents(txid)
}
pub fn estimate_fee(&self, conf_target: u16) -> Option<f64> {
if self.config.network_type.is_regtest() {
return self.get_relayfee().ok();
}
if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
return cache.get(&conf_target).copied();
}
}
self.update_fee_estimates();
self.cached_estimates
.read()
.unwrap()
.0
.get(&conf_target)
.copied()
}
pub fn estimate_fee_map(&self) -> HashMap<u16, f64> {
if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
return cache.clone();
}
}
self.update_fee_estimates();
self.cached_estimates.read().unwrap().0.clone()
}
fn update_fee_estimates(&self) {
match self.daemon.estimatesmartfee_batch(&CONF_TARGETS) {
Ok(estimates) => {
*self.cached_estimates.write().unwrap() = (estimates, Some(Instant::now()));
}
Err(err) => {
warn!("failed estimating feerates: {:?}", err);
}
}
}
pub fn get_relayfee(&self) -> Result<f64> {
if let Some(cached) = *self.cached_relayfee.read().unwrap() {
return Ok(cached);
}
let relayfee = self.daemon.get_relayfee()?;
self.cached_relayfee.write().unwrap().replace(relayfee);
Ok(relayfee)
}
#[cfg(feature = "liquid")]
pub fn new(
chain: Arc<ChainQuery>,
mempool: Arc<RwLock<Mempool>>,
daemon: Arc<Daemon>,
config: Arc<Config>,
asset_db: Option<Arc<RwLock<AssetRegistry>>>,
) -> Self {
Query {
chain,
mempool,
daemon,
config,
asset_db,
cached_estimates: RwLock::new((HashMap::new(), None)),
cached_relayfee: RwLock::new(None),
}
}
#[cfg(feature = "liquid")]
pub fn lookup_asset(&self, asset_id: &AssetId) -> Result<Option<LiquidAsset>> {
lookup_asset(
self,
self.asset_db.as_ref().map(AssetRegistryLock::RwLock),
asset_id,
None,
)
}
#[cfg(feature = "liquid")]
pub fn lookup_registry_asset(&self, asset_id: &AssetId) -> Result<Option<AssetMeta>> {
let asset_db = self
.asset_db
.as_ref()
.chain_err(|| "asset registry unavailable")?;
Ok(asset_db.read().unwrap().get(asset_id).cloned())
}
#[cfg(feature = "liquid")]
pub fn list_registry_assets(
&self,
start_index: usize,
limit: usize,
sorting: AssetSorting,
) -> Result<(usize, Vec<LiquidAsset>)> {
let asset_db = match &self.asset_db {
None => return Ok((0, vec![])),
Some(db) => db.read().unwrap(),
};
let (total_num, results) = asset_db.list(start_index, limit, sorting);
// Attach on-chain information alongside the registry metadata
let results = results
.into_iter()
.map(|(asset_id, metadata)| {
lookup_asset(
self,
Some(AssetRegistryLock::RwLockReadGuard(&asset_db)),
asset_id,
Some(metadata),
)?
.chain_err(|| "missing registered asset")
// TODO cache
pub fn estimate_fee_targets(&self) -> HashMap<u16, f32> {
CONF_TARGETS
.iter()
.filter_map(|conf_target| {
self.estimate_fee(*conf_target)
.map(|feerate| (*conf_target, feerate))
})
.collect::<Result<Vec<_>>>()?;
Ok((total_num, results))
}
#[cfg(feature = "liquid")]
pub fn search_registry_assets<T, F>(
&self,
search: &str,
limit: usize,
mut map: F,
) -> Result<Vec<T>>
where
F: FnMut(&AssetId, &AssetMeta) -> T,
{
let asset_db = self
.asset_db
.as_ref()
.chain_err(|| "asset registry unavailable")?;
Ok(asset_db
.read()
.unwrap()
.search(search, limit)
.into_iter()
.map(|(asset_id, metadata)| map(asset_id, metadata))
.collect())
.collect()
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,58 +1,34 @@
use crossbeam_channel as channel;
use crossbeam_channel::RecvTimeoutError;
use std::time::{Duration, Instant};
use signal_hook::consts::{SIGINT, SIGTERM, SIGUSR1};
use chan;
use chan_signal;
use std::time::Duration;
use crate::errors::*;
#[derive(Clone)] // so multiple threads could wait on signals
pub struct Waiter {
receiver: channel::Receiver<i32>,
}
fn notify(signals: &[i32]) -> channel::Receiver<i32> {
let (s, r) = channel::bounded(1);
let mut signals =
signal_hook::iterator::Signals::new(signals).expect("failed to register signal hook");
crate::util::spawn_thread("signal-notifier", move || {
for signal in signals.forever() {
s.send(signal)
.unwrap_or_else(|_| panic!("failed to send signal {}", signal));
}
});
r
signal: chan::Receiver<chan_signal::Signal>,
}
impl Waiter {
pub fn start() -> Waiter {
pub fn new() -> Waiter {
Waiter {
receiver: notify(&[
SIGINT, SIGTERM,
SIGUSR1, // allow external triggering (e.g. via bitcoind `blocknotify`)
]),
signal: chan_signal::notify(&[chan_signal::Signal::INT, chan_signal::Signal::TERM]),
}
}
pub fn wait(&self, duration: Duration, accept_sigusr: bool) -> Result<()> {
// Determine the deadline time based on the duration, so that it doesn't
// get pushed back when wait_deadline() recurses
self.wait_deadline(Instant::now() + duration, accept_sigusr)
}
fn wait_deadline(&self, deadline: Instant, accept_sigusr: bool) -> Result<()> {
match self.receiver.recv_deadline(deadline) {
Ok(sig) if sig == SIGUSR1 => {
trace!("notified via SIGUSR1");
if accept_sigusr {
Ok(())
} else {
self.wait_deadline(deadline, accept_sigusr)
pub fn wait(&self, duration: Duration) -> Result<()> {
let signal = &self.signal;
let timeout = chan::after(duration);
chan_select! {
signal.recv() -> s => {
if let Some(sig) = s {
bail!(ErrorKind::Interrupt(sig));
}
}
Ok(sig) => bail!(ErrorKind::Interrupt(sig)),
Err(RecvTimeoutError::Timeout) => Ok(()),
Err(RecvTimeoutError::Disconnected) => bail!("signal hook channel disconnected"),
},
timeout.recv() => {},
}
Ok(())
}
pub fn poll(&self) -> Result<()> {
self.wait(Duration::from_secs(0))
}
}

320
src/util/address.rs Normal file
View File

@ -0,0 +1,320 @@
// Rust Bitcoin Library
// Written in 2014 by
// Andrew Poelstra <apoelstra@wpsoftware.net>
// To the extent possible under law, the author(s) have dedicated all
// copyright and related and neighboring rights to this software to
// the public domain worldwide. This software is distributed without
// any warranty.
//
// You should have received a copy of the CC0 Public Domain Dedication
// along with this software.
// If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
//
//! Addresses
//!
//! Support for ordinary base58 Bitcoin addresses and private keys
//!
use std::fmt::{self, Display, Formatter};
use std::str::FromStr;
use bitcoin_bech32::{self, u5, WitnessProgram};
use bitcoin_hashes::{hash160, Hash};
use crate::chain::Network;
use bitcoin::blockdata::opcodes;
use bitcoin::blockdata::script;
use bitcoin::consensus::encode;
use bitcoin::util::base58;
use bitcoin::util::key;
/// The method used to produce an address
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Payload {
/// pay-to-pkhash address
PubkeyHash(hash160::Hash),
/// P2SH address
ScriptHash(hash160::Hash),
/// Segwit address
WitnessProgram(WitnessProgram),
}
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
/// A Bitcoin address
pub struct Address {
/// The type of the address
pub payload: Payload,
/// The network on which this address is usable
pub network: Network,
}
impl Address {
/// Creates a pay to (compressed) public key hash address from a public key
/// This is the preferred non-witness type address
#[inline]
pub fn p2pkh(pk: &key::PublicKey, network: Network) -> Address {
let mut hash_engine = hash160::Hash::engine();
pk.write_into(&mut hash_engine);
Address {
network: network,
payload: Payload::PubkeyHash(hash160::Hash::from_engine(hash_engine)),
}
}
/// Creates a pay to script hash P2SH address from a script
/// This address type was introduced with BIP16 and is the popular type to implement multi-sig these days.
#[inline]
pub fn p2sh(script: &script::Script, network: Network) -> Address {
Address {
network: network,
payload: Payload::ScriptHash(hash160::Hash::hash(&script[..])),
}
}
/// Create a witness pay to public key address from a public key
/// This is the native segwit address type for an output redeemable with a single signature
pub fn p2wpkh(pk: &key::PublicKey, network: Network) -> Address {
let mut hash_engine = hash160::Hash::engine();
pk.write_into(&mut hash_engine);
Address {
network: network,
payload: Payload::WitnessProgram(
// unwrap is safe as witness program is known to be correct as above
WitnessProgram::new(
u5::try_from_u8(0).expect("0<32"),
hash160::Hash::from_engine(hash_engine)[..].to_vec(),
Address::bech_network(network),
)
.unwrap(),
),
}
}
/// Create a pay to script address that embeds a witness pay to public key
/// This is a segwit address type that looks familiar (as p2sh) to legacy clients
pub fn p2shwpkh(pk: &key::PublicKey, network: Network) -> Address {
let mut hash_engine = hash160::Hash::engine();
pk.write_into(&mut hash_engine);
let builder = script::Builder::new()
.push_int(0)
.push_slice(&hash160::Hash::from_engine(hash_engine)[..]);
Address {
network: network,
payload: Payload::ScriptHash(hash160::Hash::hash(builder.into_script().as_bytes())),
}
}
/// Create a witness pay to script hash address
pub fn p2wsh(script: &script::Script, network: Network) -> Address {
use bitcoin_hashes::sha256;
use bitcoin_hashes::Hash;
Address {
network: network,
payload: Payload::WitnessProgram(
// unwrap is safe as witness program is known to be correct as above
WitnessProgram::new(
u5::try_from_u8(0).expect("0<32"),
sha256::Hash::hash(&script[..])[..].to_vec(),
Address::bech_network(network),
)
.unwrap(),
),
}
}
/// Create a pay to script address that embeds a witness pay to script hash address
/// This is a segwit address type that looks familiar (as p2sh) to legacy clients
pub fn p2shwsh(script: &script::Script, network: Network) -> Address {
use bitcoin_hashes::hash160;
use bitcoin_hashes::sha256;
use bitcoin_hashes::Hash;
let ws = script::Builder::new()
.push_int(0)
.push_slice(&sha256::Hash::hash(&script[..])[..])
.into_script();
Address {
network: network,
payload: Payload::ScriptHash(hash160::Hash::hash(&ws[..])),
}
}
#[inline]
/// convert Network to bech32 network (this should go away soon)
fn bech_network(network: Network) -> bitcoin_bech32::constants::Network {
match network {
Network::Bitcoin => bitcoin_bech32::constants::Network::Bitcoin,
Network::Testnet => bitcoin_bech32::constants::Network::Testnet,
Network::Regtest => bitcoin_bech32::constants::Network::Regtest,
// this should never actually happen, Liquid does not have bech32 addresses
#[cfg(feature = "liquid")]
Network::Liquid | Network::LiquidRegtest => bitcoin_bech32::constants::Network::Bitcoin,
}
}
/// Generates a script pubkey spending to this address
pub fn script_pubkey(&self) -> script::Script {
match self.payload {
Payload::PubkeyHash(ref hash) => script::Builder::new()
.push_opcode(opcodes::all::OP_DUP)
.push_opcode(opcodes::all::OP_HASH160)
.push_slice(&hash[..])
.push_opcode(opcodes::all::OP_EQUALVERIFY)
.push_opcode(opcodes::all::OP_CHECKSIG),
Payload::ScriptHash(ref hash) => script::Builder::new()
.push_opcode(opcodes::all::OP_HASH160)
.push_slice(&hash[..])
.push_opcode(opcodes::all::OP_EQUAL),
Payload::WitnessProgram(ref witprog) => script::Builder::new()
.push_int(witprog.version().to_u8() as i64)
.push_slice(witprog.program()),
}
.into_script()
}
}
impl Display for Address {
fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
match self.payload {
Payload::PubkeyHash(ref hash) => {
let mut prefixed = [0; 21];
prefixed[0] = match self.network {
Network::Bitcoin => 0,
Network::Testnet | Network::Regtest => 111,
#[cfg(feature = "liquid")]
Network::Liquid => 57,
#[cfg(feature = "liquid")]
Network::LiquidRegtest => 235,
};
prefixed[1..].copy_from_slice(&hash[..]);
base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
}
Payload::ScriptHash(ref hash) => {
let mut prefixed = [0; 21];
prefixed[0] = match self.network {
Network::Bitcoin => 5,
Network::Testnet | Network::Regtest => 196,
#[cfg(feature = "liquid")]
Network::Liquid => 39,
#[cfg(feature = "liquid")]
Network::LiquidRegtest => 75,
};
prefixed[1..].copy_from_slice(&hash[..]);
base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
}
Payload::WitnessProgram(ref witprog) => fmt.write_str(&witprog.to_address()),
}
}
}
impl FromStr for Address {
type Err = encode::Error;
fn from_str(s: &str) -> Result<Address, encode::Error> {
// bech32 (note that upper or lowercase is allowed but NOT mixed case)
if s.starts_with("bc1")
|| s.starts_with("BC1")
|| s.starts_with("tb1")
|| s.starts_with("TB1")
|| s.starts_with("bcrt1")
|| s.starts_with("BCRT1")
{
let witprog = WitnessProgram::from_address(s)?;
let network = match witprog.network() {
bitcoin_bech32::constants::Network::Bitcoin => Network::Bitcoin,
bitcoin_bech32::constants::Network::Testnet => Network::Testnet,
bitcoin_bech32::constants::Network::Regtest => Network::Regtest,
_ => panic!("unknown network"),
};
if witprog.version().to_u8() != 0 {
return Err(encode::Error::UnsupportedWitnessVersion(
witprog.version().to_u8(),
));
}
return Ok(Address {
network: network,
payload: Payload::WitnessProgram(witprog),
});
}
if s.len() > 50 {
return Err(encode::Error::Base58(base58::Error::InvalidLength(
s.len() * 11 / 15,
)));
}
// Base 58
let data = base58::from_check(s)?;
if data.len() != 21 {
return Err(encode::Error::Base58(base58::Error::InvalidLength(
data.len(),
)));
}
let (network, payload) = match data[0] {
0 => (
Network::Bitcoin,
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
5 => (
Network::Bitcoin,
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
111 => (
Network::Testnet,
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
196 => (
Network::Testnet,
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
#[cfg(feature = "liquid")]
57 => (
Network::Liquid,
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
#[cfg(feature = "liquid")]
39 => (
Network::Liquid,
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
#[cfg(feature = "liquid")]
235 => (
Network::LiquidRegtest,
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
#[cfg(feature = "liquid")]
75 => (
Network::LiquidRegtest,
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
),
x => {
return Err(encode::Error::Base58(base58::Error::InvalidVersion(vec![
x,
])));
}
};
Ok(Address {
network: network,
payload: payload,
})
}
}
impl ::std::fmt::Debug for Address {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
write!(f, "{}", self.to_string())
}
}

View File

@ -1,112 +0,0 @@
/*
The tests below show us the following defaults for each method of using bincode.
1. Using bincode::[de]serialize() directly: "function"
2. Using bincode::config().[de]serialize(): "Config" (deprecated)
3. Using bincode::options().[de]serialize(): "Options" (currently recommended for v1.3.3)
```
+----------+--------+------------+----------------+------------+
| | Endian | Int Length | Allow Trailing | Byte Limit |
+----------+--------+------------+----------------+------------+
| function | little | fixed | allow | unlimited |
| Config | little | fixed | allow | unlimited |
| Options | little | variable * | reject * | unlimited |
+----------+--------+------------+----------------+------------+
```
Thus we only need to change the int length from variable to fixed,
and allow trailing to allow in order to match the previous behavior.
(note: TxHistory was using Big Endian by explicitly setting it to big.)
*/
use bincode_do_not_use_directly as bincode;
#[test]
fn bincode_settings() {
let value = TestStruct::new();
let mut large = [0_u8; 4096];
let decoded = [
8_u8, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 8, 7, 6, 5, 4, 3, 2, 1, 1,
2, 3, 4, 5, 6, 7, 8, 12, 0, 0, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114,
108, 100, 33,
];
large[0..56].copy_from_slice(&decoded);
// Using functions: Little endian, Fixint, Allow trailing, Unlimited
assert_eq!(bincode::serialize(&value).unwrap(), &decoded);
assert_eq!(bincode::deserialize::<TestStruct>(&large).unwrap(), value);
// Using Config (deprecated)
// Little endian, fixint, Allow trailing, Unlimited
#[allow(deprecated)]
{
assert_eq!(bincode::config().serialize(&value).unwrap(), &decoded);
assert_eq!(
bincode::config().deserialize::<TestStruct>(&large).unwrap(),
value
);
}
// Using Options
// Little endian, VARINT (different), Reject trailing (different), unlimited
use bincode::Options;
assert_eq!(
bincode::options()
.with_fixint_encoding()
.allow_trailing_bytes()
.serialize(&value)
.unwrap(),
&decoded
);
assert_eq!(
bincode::options()
.with_fixint_encoding()
.allow_trailing_bytes()
.deserialize::<TestStruct>(&large)
.unwrap(),
value
);
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct TestStruct {
a: u64,
b: [u8; 8],
c: TestData,
d: String,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
enum TestData {
Foo(FooStruct),
Bar(BarStruct),
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct FooStruct {
a: u64,
b: [u8; 8],
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct BarStruct {
a: u64,
b: [u8; 8],
}
impl TestStruct {
fn new() -> Self {
Self {
a: 0x0102030405060708,
b: [1, 2, 3, 4, 5, 6, 7, 8],
c: TestData::Foo(FooStruct {
a: 0x0102030405060708,
b: [1, 2, 3, 4, 5, 6, 7, 8],
}),
d: String::from("Hello World!"),
}
}
}

View File

@ -1,70 +0,0 @@
//! This module creates two sets of serialize and deserialize for bincode.
//! They explicitly spell out the bincode settings so that switching to
//! new versions in the future is less error prone.
//!
//! This is a list of all the row types and their settings for bincode.
//! +--------------+--------+------------+----------------+------------+
//! | | Endian | Int Length | Allow Trailing | Byte Limit |
//! +--------------+--------+------------+----------------+------------+
//! | TxHistoryRow | big | fixed | allow | unlimited |
//! | All others | little | fixed | allow | unlimited |
//! +--------------+--------+------------+----------------+------------+
// We only want people to use bincode_util
use bincode::Options;
use bincode_do_not_use_directly as bincode;
pub fn serialize_big<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
where
T: ?Sized + serde::Serialize,
{
big_endian().serialize(value)
}
pub fn deserialize_big<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
where
T: serde::Deserialize<'a>,
{
big_endian().deserialize(bytes)
}
pub fn serialize_little<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
where
T: ?Sized + serde::Serialize,
{
little_endian().serialize(value)
}
pub fn deserialize_little<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
where
T: serde::Deserialize<'a>,
{
little_endian().deserialize(bytes)
}
/// This is the default settings for Options,
/// but all explicitly spelled out, except for endianness.
/// The following functions will add endianness.
#[inline]
fn options() -> impl Options {
bincode::options()
.with_fixint_encoding()
.with_no_limit()
.allow_trailing_bytes()
}
/// Adding the endian flag for big endian
#[inline]
fn big_endian() -> impl Options {
options().with_big_endian()
}
/// Adding the endian flag for little endian
#[inline]
fn little_endian() -> impl Options {
options().with_little_endian()
}
#[cfg(test)]
#[path = "./bincode_tests.rs"]
mod bincode_tests;

View File

@ -1,22 +1,21 @@
use bitcoin::hashes::Hash;
use crate::chain::{BlockHash, BlockHeader};
use crate::chain::{Block, BlockHeader};
use crate::errors::*;
use crate::new_index::BlockEntry;
use bitcoin::consensus::encode::serialize;
use bitcoin::util::hash::BitcoinHash;
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use std::collections::HashMap;
use std::fmt;
use std::iter::FromIterator;
use std::slice;
use time::format_description::well_known::Rfc3339;
use time::OffsetDateTime as DateTime;
const MTP_SPAN: usize = 11;
use time;
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct BlockId {
pub height: usize,
pub hash: BlockHash,
pub hash: Sha256dHash,
pub time: u32,
}
@ -33,12 +32,12 @@ impl From<&HeaderEntry> for BlockId {
#[derive(Eq, PartialEq, Clone)]
pub struct HeaderEntry {
height: usize,
hash: BlockHash,
hash: Sha256dHash,
header: BlockHeader,
}
impl HeaderEntry {
pub fn hash(&self) -> &BlockHash {
pub fn hash(&self) -> &Sha256dHash {
&self.hash
}
@ -53,21 +52,23 @@ impl HeaderEntry {
impl fmt::Debug for HeaderEntry {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let last_block_time = DateTime::from_unix_timestamp(self.header().time as i64).unwrap();
let last_block_time = time::at_utc(time::Timespec::new(self.header().time as i64, 0))
.rfc3339()
.to_string();
write!(
f,
"hash={} height={} @ {}",
"best={} height={} @ {}",
self.hash(),
self.height(),
last_block_time.format(&Rfc3339).unwrap(),
last_block_time,
)
}
}
pub struct HeaderList {
headers: Vec<HeaderEntry>,
heights: HashMap<BlockHash, usize>,
tip: BlockHash,
heights: HashMap<Sha256dHash, usize>,
tip: Sha256dHash,
}
impl HeaderList {
@ -75,57 +76,19 @@ impl HeaderList {
HeaderList {
headers: vec![],
heights: HashMap::new(),
tip: BlockHash::all_zeros(),
tip: Sha256dHash::default(),
}
}
pub fn new(
mut headers_map: HashMap<BlockHash, BlockHeader>,
tip_hash: BlockHash,
) -> HeaderList {
trace!(
"processing {} headers, tip at {:?}",
headers_map.len(),
tip_hash
);
let mut blockhash = tip_hash;
let mut headers_chain: Vec<BlockHeader> = vec![];
let null_hash = BlockHash::all_zeros();
while blockhash != null_hash {
let header = headers_map.remove(&blockhash).unwrap_or_else(|| {
panic!(
"missing expected blockhash in headers map: {:?}, pointed from: {:?}",
blockhash,
headers_chain.last().map(|h| h.block_hash())
)
});
blockhash = header.prev_blockhash;
headers_chain.push(header);
}
headers_chain.reverse();
trace!(
"{} chained headers ({} orphan blocks left)",
headers_chain.len(),
headers_map.len()
);
let mut headers = HeaderList::empty();
headers.apply(headers.order(headers_chain));
headers
}
pub fn order(&self, new_headers: Vec<BlockHeader>) -> Vec<HeaderEntry> {
// header[i] -> header[i-1] (i.e. header.last() is the tip)
struct HashedHeader {
blockhash: BlockHash,
blockhash: Sha256dHash,
header: BlockHeader,
}
let hashed_headers =
Vec::<HashedHeader>::from_iter(new_headers.into_iter().map(|header| HashedHeader {
blockhash: header.block_hash(),
blockhash: header.bitcoin_hash(),
header,
}));
for i in 1..hashed_headers.len() {
@ -138,31 +101,26 @@ impl HeaderList {
Some(h) => h.header.prev_blockhash,
None => return vec![], // hashed_headers is empty
};
let null_hash = BlockHash::all_zeros();
let null_hash = Sha256dHash::default();
let new_height: usize = if prev_blockhash == null_hash {
0
} else {
self.header_by_blockhash(&prev_blockhash)
.unwrap_or_else(|| panic!("{} is not part of the blockchain", prev_blockhash))
.expect(&format!("{} is not part of the blockchain", prev_blockhash))
.height()
+ 1
};
(new_height..)
.zip(hashed_headers)
.zip(hashed_headers.into_iter())
.map(|(height, hashed_header)| HeaderEntry {
height,
height: height,
hash: hashed_header.blockhash,
header: hashed_header.header,
})
.collect()
}
/// Returns any rolled back blocks in order from old tip first and first block in the fork is last
/// It also returns the blockhash of the post-rollback tip.
pub fn apply(
&mut self,
new_headers: Vec<HeaderEntry>,
) -> (Vec<HeaderEntry>, Option<BlockHash>) {
pub fn apply(&mut self, new_headers: Vec<HeaderEntry>) {
// new_headers[i] -> new_headers[i - 1] (i.e. new_headers.last() is the tip)
for i in 1..new_headers.len() {
assert_eq!(new_headers[i - 1].height() + 1, new_headers[i].height());
@ -177,27 +135,19 @@ impl HeaderList {
let expected_prev_blockhash = if height > 0 {
*self.headers[height - 1].hash()
} else {
BlockHash::all_zeros()
Sha256dHash::default()
};
assert_eq!(entry.header().prev_blockhash, expected_prev_blockhash);
height
}
None => return (vec![], None),
None => return,
};
debug!(
"applying {} new headers from height {}",
new_headers.len(),
new_height
);
let mut removed = self.headers.split_off(new_height); // keep [0..new_height) entries
// If we reorged, we should return the last blockhash before adding the new chain's blockheaders.
let reorged_tip = if !removed.is_empty() {
self.headers.last().map(|be| be.hash()).cloned()
} else {
None
};
self.headers.split_off(new_height); // keep [0..new_height) entries
for new_header in new_headers {
let height = new_header.height();
assert_eq!(height, self.headers.len());
@ -205,11 +155,9 @@ impl HeaderList {
self.headers.push(new_header);
self.heights.insert(self.tip, height);
}
removed.reverse();
(removed, reorged_tip)
}
pub fn header_by_blockhash(&self, blockhash: &BlockHash) -> Option<&HeaderEntry> {
pub fn header_by_blockhash(&self, blockhash: &Sha256dHash) -> Option<&HeaderEntry> {
let height = self.heights.get(blockhash)?;
let header = self.headers.get(*height)?;
if *blockhash == *header.hash() {
@ -220,8 +168,9 @@ impl HeaderList {
}
pub fn header_by_height(&self, height: usize) -> Option<&HeaderEntry> {
self.headers.get(height).inspect(|entry| {
self.headers.get(height).map(|entry| {
assert_eq!(entry.height(), height);
entry
})
}
@ -229,13 +178,13 @@ impl HeaderList {
self.headers.last() == other.headers.last()
}
pub fn tip(&self) -> &BlockHash {
pub fn tip(&self) -> &Sha256dHash {
assert_eq!(
self.tip,
self.headers
.last()
.map(|h| *h.hash())
.unwrap_or(BlockHash::all_zeros())
.unwrap_or(Sha256dHash::default())
);
&self.tip
}
@ -244,41 +193,20 @@ impl HeaderList {
self.headers.len()
}
pub fn is_empty(&self) -> bool {
self.headers.is_empty()
}
pub fn iter(&self) -> slice::Iter<'_, HeaderEntry> {
pub fn iter(&self) -> slice::Iter<HeaderEntry> {
self.headers.iter()
}
/// Get the Median Time Past
pub fn get_mtp(&self, height: usize) -> u32 {
// Use the timestamp as the mtp of the genesis block.
// Matches bitcoind's behaviour: bitcoin-cli getblock `bitcoin-cli getblockhash 0` | jq '.time == .mediantime'
if height == 0 {
self.headers.first().unwrap().header.time
} else if height > self.len() - 1 {
0
} else {
let mut timestamps = (height.saturating_sub(MTP_SPAN - 1)..=height)
.map(|p_height| self.headers.get(p_height).unwrap().header.time)
.collect::<Vec<_>>();
timestamps.sort_unstable();
timestamps[timestamps.len() / 2]
}
}
}
#[derive(Serialize, Deserialize)]
pub struct BlockStatus {
pub in_best_chain: bool,
pub height: Option<usize>,
pub next_best: Option<BlockHash>,
pub next_best: Option<Sha256dHash>,
}
impl BlockStatus {
pub fn confirmed(height: usize, next_best: Option<BlockHash>) -> BlockStatus {
pub fn confirmed(height: usize, next_best: Option<Sha256dHash>) -> BlockStatus {
BlockStatus {
in_best_chain: true,
height: Some(height),
@ -297,7 +225,6 @@ impl BlockStatus {
#[derive(Serialize, Deserialize, Debug)]
pub struct BlockMeta {
#[serde(alias = "nTx")]
pub tx_count: u32,
pub size: u32,
pub weight: u32,
@ -306,18 +233,23 @@ pub struct BlockMeta {
pub struct BlockHeaderMeta {
pub header_entry: HeaderEntry,
pub meta: BlockMeta,
pub mtp: u32,
}
impl From<&Block> for BlockMeta {
fn from(block: &Block) -> BlockMeta {
BlockMeta {
tx_count: block.txdata.len() as u32,
weight: block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
size: serialize(block).len() as u32,
}
}
}
impl From<&BlockEntry> for BlockMeta {
fn from(b: &BlockEntry) -> BlockMeta {
#[cfg(not(feature = "liquid"))]
let weight = b.block.weight().to_wu() as u32;
#[cfg(feature = "liquid")]
let weight = b.block.weight() as u32;
BlockMeta {
tx_count: b.block.txdata.len() as u32,
weight,
weight: b.block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
size: b.size,
}
}

130
src/util/elements.rs Normal file
View File

@ -0,0 +1,130 @@
use bitcoin::blockdata::script::Instruction::PushBytes;
use bitcoin::consensus::encode::serialize;
use bitcoin::Script;
use elements::confidential::Value;
use elements::{AssetIssuance, Proof};
use hex;
use crate::chain::Network;
use crate::util::{get_script_asm, script_to_address};
#[derive(Serialize, Deserialize)]
pub struct BlockProofValue {
challenge: Script,
challenge_asm: String,
solution: Script,
solution_asm: String,
}
impl From<&Proof> for BlockProofValue {
fn from(proof: &Proof) -> Self {
BlockProofValue {
challenge_asm: get_script_asm(&proof.challenge),
challenge: proof.challenge.clone(),
solution_asm: get_script_asm(&proof.solution),
solution: proof.solution.clone(),
}
}
}
#[derive(Serialize, Deserialize, Clone)]
pub struct IssuanceValue {
pub is_reissuance: bool,
pub asset_blinding_nonce: Option<String>,
pub asset_entropy: Option<String>,
pub assetamount: Option<u64>,
pub assetamountcommitment: Option<String>,
pub tokenamount: Option<u64>,
pub tokenamountcommitment: Option<String>,
}
impl From<&AssetIssuance> for IssuanceValue {
fn from(issuance: &AssetIssuance) -> Self {
let zero = [0u8; 32];
let is_reissuance = issuance.asset_blinding_nonce != zero;
IssuanceValue {
is_reissuance,
asset_blinding_nonce: if is_reissuance {
Some(hex::encode(issuance.asset_blinding_nonce))
} else {
None
},
asset_entropy: if issuance.asset_entropy != zero {
Some(hex::encode(issuance.asset_entropy))
} else {
None
},
assetamount: match issuance.amount {
Value::Explicit(value) => Some(value),
_ => None,
},
assetamountcommitment: match issuance.amount {
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
_ => None,
},
tokenamount: match issuance.inflation_keys {
Value::Explicit(value) => Some(value / 100000000), // https://github.com/ElementsProject/rust-elements/issues/7
_ => None,
},
tokenamountcommitment: match issuance.inflation_keys {
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys))),
_ => None,
},
}
}
}
#[derive(Serialize, Deserialize, Clone)]
pub struct PegOutRequest {
pub genesis_hash: String,
pub scriptpubkey: Script,
pub scriptpubkey_asm: String,
pub scriptpubkey_address: Option<String>,
}
impl PegOutRequest {
pub fn parse(
script: &Script,
parent_network: &Network,
parent_genesis_hash: &str,
) -> Option<PegOutRequest> {
if !script.is_op_return() {
return None;
}
let nulldata: Vec<_> = script.iter(true).skip(1).collect();
if nulldata.len() < 2 {
return None;
}
let genesis_hash = if let PushBytes(data) = nulldata[0] {
let mut data = data.to_vec();
data.reverse();
hex::encode(data)
} else {
return None;
};
let scriptpubkey = if let PushBytes(data) = nulldata[1] {
Script::from(data.to_vec())
} else {
return None;
};
if genesis_hash != parent_genesis_hash {
return None;
}
let scriptpubkey_asm = get_script_asm(&scriptpubkey);
let scriptpubkey_address = script_to_address(&scriptpubkey, parent_network);
Some(PegOutRequest {
genesis_hash,
scriptpubkey,
scriptpubkey_asm,
scriptpubkey_address,
})
}
}

View File

@ -1,4 +1,6 @@
use crate::chain::{Network, Transaction, TxOut};
#[cfg(feature = "liquid")]
use crate::chain::Value;
use crate::chain::{Transaction, TxOut};
use std::collections::HashMap;
const VSIZE_BIN_WIDTH: u32 = 50_000; // in vbytes
@ -10,12 +12,32 @@ pub struct TxFeeInfo {
}
impl TxFeeInfo {
pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, network: Network) -> Self {
let fee = get_tx_fee(tx, prevouts, network);
#[cfg(not(feature = "liquid"))]
let vsize = tx.weight().to_wu() / 4;
#[cfg(feature = "liquid")]
let vsize = tx.weight() / 4;
#[cfg(not(feature = "liquid"))]
pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>) -> Self {
let total_in: u64 = prevouts.values().map(|prevout| prevout.value).sum();
let total_out: u64 = tx.output.iter().map(|vout| vout.value).sum();
let fee = total_in - total_out;
let vsize = tx.get_weight() / 4;
TxFeeInfo {
fee,
vsize: vsize as u32,
fee_per_vbyte: fee as f32 / vsize as f32,
}
}
#[cfg(feature = "liquid")]
pub fn new(tx: &Transaction, _prevouts: &HashMap<u32, &TxOut>) -> Self {
let fee = tx
.output
.iter()
.find(|vout| vout.is_fee())
.map_or(0, |vout| match vout.value {
Value::Explicit(value) => value,
_ => 0u64,
});
let vsize = tx.get_weight() / 4;
TxFeeInfo {
fee,
@ -25,42 +47,23 @@ impl TxFeeInfo {
}
}
#[cfg(not(feature = "liquid"))]
pub fn get_tx_fee(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, _network: Network) -> u64 {
if tx.is_coinbase() {
return 0;
}
let total_in: u64 = prevouts
.values()
.map(|prevout| prevout.value.to_sat())
.sum();
let total_out: u64 = tx.output.iter().map(|vout| vout.value.to_sat()).sum();
total_in - total_out
}
#[cfg(feature = "liquid")]
pub fn get_tx_fee(tx: &Transaction, _prevouts: &HashMap<u32, &TxOut>, network: Network) -> u64 {
tx.fee_in(*network.native_asset())
}
pub fn make_fee_histogram(mut entries: Vec<&TxFeeInfo>) -> Vec<(f32, u32)> {
entries.sort_unstable_by(|e1, e2| e1.fee_per_vbyte.partial_cmp(&e2.fee_per_vbyte).unwrap());
let mut histogram = vec![];
let mut bin_size = 0;
let mut last_fee_rate = 0.0;
let mut last_fee_rate = None;
for e in entries.iter().rev() {
if bin_size > VSIZE_BIN_WIDTH && last_fee_rate != e.fee_per_vbyte {
// vsize of transactions paying >= last_fee_rate
histogram.push((last_fee_rate, bin_size));
last_fee_rate = Some(e.fee_per_vbyte);
bin_size += e.vsize;
if bin_size > VSIZE_BIN_WIDTH {
// vsize of transactions paying >= e.fee_per_vbyte
histogram.push((e.fee_per_vbyte, bin_size));
bin_size = 0;
}
last_fee_rate = e.fee_per_vbyte;
bin_size += e.vsize;
}
if bin_size > 0 {
histogram.push((last_fee_rate, bin_size));
if let Some(fee_rate) = last_fee_rate {
histogram.push((fee_rate, bin_size));
}
histogram
}

View File

@ -1,22 +1,20 @@
use crate::chain::{BlockHash, Txid};
use bitcoin_hashes::{sha256d::Hash as Sha256dHash, Hash};
use crate::errors::*;
use crate::new_index::ChainQuery;
use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash};
pub fn get_tx_merkle_proof(
chain: &ChainQuery,
tx_hash: &Txid,
block_hash: &BlockHash,
tx_hash: &Sha256dHash,
block_hash: &Sha256dHash,
) -> Result<(Vec<Sha256dHash>, usize)> {
let txids = chain
.get_block_txids(block_hash)
.get_block_txids(&block_hash)
.chain_err(|| format!("missing block txids for #{}", block_hash))?;
let pos = txids
.iter()
.position(|txid| txid == tx_hash)
.chain_err(|| format!("missing txid {}", tx_hash))?;
let txids = txids.into_iter().map(Sha256dHash::from).collect();
let (branch, _root) = create_merkle_branch_and_root(txids, pos);
Ok((branch, pos))
}
@ -39,14 +37,12 @@ pub fn get_header_merkle_proof(
);
}
let heights: Vec<usize> = (0..=cp_height).collect();
let header_hashes: Vec<BlockHash> = heights
let heights: Vec<usize> = (0..cp_height + 1).collect();
let header_hashes: Vec<Sha256dHash> = heights
.into_iter()
.map(|height| chain.hash_by_height(height))
.collect::<Option<Vec<BlockHash>>>()
.collect::<Option<Vec<Sha256dHash>>>()
.chain_err(|| "missing block headers")?;
let header_hashes = header_hashes.into_iter().map(Sha256dHash::from).collect();
Ok(create_merkle_branch_and_root(header_hashes, height))
}
@ -55,7 +51,7 @@ pub fn get_id_from_pos(
height: usize,
tx_pos: usize,
want_merkle: bool,
) -> Result<(Txid, Vec<Sha256dHash>)> {
) -> Result<(Sha256dHash, Vec<Sha256dHash>)> {
let header_hash = chain
.hash_by_height(height)
.chain_err(|| format!("missing block #{}", height))?;
@ -68,8 +64,6 @@ pub fn get_id_from_pos(
.get(tx_pos)
.chain_err(|| format!("No tx in position #{} in block #{}", tx_pos, height))?;
let txids = txids.into_iter().map(Sha256dHash::from).collect();
let branch = if want_merkle {
create_merkle_branch_and_root(txids, tx_pos).0
} else {
@ -90,12 +84,12 @@ fn create_merkle_branch_and_root(
let mut merkle = vec![];
while hashes.len() > 1 {
if hashes.len() % 2 != 0 {
let last = *hashes.last().unwrap();
let last = hashes.last().unwrap().clone();
hashes.push(last);
}
index = if index % 2 == 0 { index + 1 } else { index - 1 };
merkle.push(hashes[index]);
index /= 2;
index = index / 2;
hashes = hashes
.chunks(2)
.map(|pair| merklize(pair[0], pair[1]))

View File

@ -1,31 +1,33 @@
mod address;
mod block;
mod merkle;
mod script;
mod transaction;
pub mod bincode_util;
pub mod electrum_merkle;
pub mod fees;
#[cfg(feature = "liquid")]
mod elements;
pub use self::address::{Address, Payload};
pub use self::block::{BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, HeaderEntry, HeaderList};
pub use self::fees::get_tx_fee;
pub use self::script::{
get_innerscripts, IsProvablyUnspendable, ScriptToAddr, ScriptToAsm, SegwitDetection,
};
pub use self::transaction::{
extract_tx_prevouts, has_prevout, is_coinbase, is_spendable, serialize_outpoint,
sigops::transaction_sigop_count, TransactionStatus, TxInput,
};
pub use self::merkle::{get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof};
pub use self::script::{get_script_asm, script_to_address};
pub use self::transaction::{has_prevout, is_coinbase, is_spendable, TransactionStatus};
#[cfg(feature = "liquid")]
pub use self::elements::{BlockProofValue, IssuanceValue, PegOutRequest};
#[cfg(feature = "liquid")]
pub const REGTEST_INITIAL_ISSUANCE_PREVOUT: &str =
"cb4953dfe64aba4687d0de08e5ce17d8f7640deeef63be4ac3bac3afdefdfd78";
use std::collections::HashMap;
use std::sync::atomic::AtomicUsize;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::Mutex;
use std::thread::{self, ThreadId};
use std::sync::mpsc::{channel, sync_channel, Receiver, Sender, SyncSender};
use std::thread;
use crate::chain::BlockHeader;
use bitcoin::hashes::sha256d::Hash as Sha256dHash;
use socket2::{Domain, Protocol, Socket, Type};
use std::net::SocketAddr;
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
pub type Bytes = Vec<u8>;
pub type HeaderMap = HashMap<Sha256dHash, BlockHeader>;
@ -36,42 +38,30 @@ const HASH_LEN: usize = 32;
pub type FullHash = [u8; HASH_LEN];
pub fn full_hash(hash: &[u8]) -> FullHash {
*array_ref![hash, 0, HASH_LEN]
array_ref![hash, 0, HASH_LEN].clone()
}
pub struct SyncChannel<T> {
tx: Option<crossbeam_channel::Sender<T>>,
rx: Option<crossbeam_channel::Receiver<T>>,
tx: SyncSender<T>,
rx: Receiver<T>,
}
impl<T> SyncChannel<T> {
pub fn new(size: usize) -> SyncChannel<T> {
let (tx, rx) = crossbeam_channel::bounded(size);
SyncChannel {
tx: Some(tx),
rx: Some(rx),
}
let (tx, rx) = sync_channel(size);
SyncChannel { tx, rx }
}
pub fn sender(&self) -> crossbeam_channel::Sender<T> {
self.tx.as_ref().expect("No Sender").clone()
pub fn sender(&self) -> SyncSender<T> {
self.tx.clone()
}
pub fn receiver(&self) -> &crossbeam_channel::Receiver<T> {
self.rx.as_ref().expect("No Receiver")
pub fn receiver(&self) -> &Receiver<T> {
&self.rx
}
pub fn into_receiver(self) -> crossbeam_channel::Receiver<T> {
self.rx.expect("No Receiver")
}
/// This drops the sender and receiver, causing all other methods to panic.
///
/// Use only when you know that the channel will no longer be used.
/// ie. shutdown.
pub fn close(&mut self) -> Option<crossbeam_channel::Receiver<T>> {
self.tx.take();
self.rx.take()
pub fn into_receiver(self) -> Receiver<T> {
self.rx
}
}
@ -81,7 +71,7 @@ pub struct Channel<T> {
}
impl<T> Channel<T> {
pub fn unbounded() -> Self {
pub fn new() -> Channel<T> {
let (tx, rx) = channel();
Channel { tx, rx }
}
@ -99,131 +89,14 @@ impl<T> Channel<T> {
}
}
/// This static HashMap contains all the threads spawned with [`spawn_thread`] with their name
#[inline]
pub fn with_spawned_threads<F>(f: F)
where
F: FnOnce(&mut HashMap<ThreadId, String>),
{
lazy_static! {
static ref SPAWNED_THREADS: Mutex<HashMap<ThreadId, String>> = Mutex::new(HashMap::new());
}
let mut lock = match SPAWNED_THREADS.lock() {
Ok(threads) => threads,
// There's no possible broken state
Err(threads) => {
warn!("SPAWNED_THREADS is in a poisoned state! Be wary of incorrect logs!");
threads.into_inner()
}
};
f(&mut lock)
}
pub fn spawn_thread<F, T>(prefix: &str, do_work: F) -> thread::JoinHandle<T>
pub fn spawn_thread<F, T>(name: &str, f: F) -> thread::JoinHandle<T>
where
F: FnOnce() -> T,
F: Send + 'static,
T: Send + 'static,
{
static THREAD_COUNTER: AtomicUsize = AtomicUsize::new(0);
let counter = THREAD_COUNTER.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
thread::Builder::new()
.name(format!("{}-{}", prefix, counter))
.spawn(move || {
let thread = std::thread::current();
let name = thread.name().unwrap();
let id = thread.id();
trace!("[THREAD] GETHASHMAP INSERT | {name} {id:?}");
with_spawned_threads(|threads| {
threads.insert(id, name.to_owned());
});
trace!("[THREAD] START WORK | {name} {id:?}");
let result = do_work();
trace!("[THREAD] FINISHED WORK | {name} {id:?}");
trace!("[THREAD] GETHASHMAP REMOVE | {name} {id:?}");
with_spawned_threads(|threads| {
threads.remove(&id);
});
trace!("[THREAD] HASHMAP REMOVED | {name} {id:?}");
result
})
.name(name.to_owned())
.spawn(f)
.unwrap()
}
// Similar to https://doc.rust-lang.org/std/primitive.bool.html#method.then (nightly only),
// but with a function that returns an `Option<T>` instead of `T`. Adding something like
// this to std is being discussed: https://github.com/rust-lang/rust/issues/64260
pub trait BoolThen {
fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T>;
}
impl BoolThen for bool {
fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T> {
if self {
f()
} else {
None
}
}
}
pub fn create_socket(addr: &SocketAddr) -> Socket {
let domain = match &addr {
SocketAddr::V4(_) => Domain::IPV4,
SocketAddr::V6(_) => Domain::IPV6,
};
let socket =
Socket::new(domain, Type::STREAM, Some(Protocol::TCP)).expect("creating socket failed");
#[cfg(unix)]
socket
.set_reuse_port(true)
.expect("cannot enable SO_REUSEPORT");
socket.bind(&(*addr).into()).expect("cannot bind");
socket
}
/// A module used for serde serialization of bytes in hexadecimal format.
///
/// The module is compatible with the serde attribute.
///
/// Copied from https://github.com/rust-bitcoin/rust-bitcoincore-rpc/blob/master/json/src/lib.rs
pub mod serde_hex {
use bitcoin::hashes::hex::FromHex;
use serde::de::Error;
use serde::{Deserializer, Serializer};
pub fn serialize<S: Serializer>(b: &[u8], s: S) -> Result<S::Ok, S::Error> {
s.serialize_str(&hex::encode(b))
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
let hex_str: String = ::serde::Deserialize::deserialize(d)?;
FromHex::from_hex(&hex_str).map_err(D::Error::custom)
}
pub mod opt {
use bitcoin::hashes::hex::FromHex;
use serde::de::Error;
use serde::{Deserializer, Serializer};
pub fn serialize<S: Serializer>(b: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
match *b {
None => s.serialize_none(),
Some(ref b) => s.serialize_str(&hex::encode(b)),
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
let hex_str: String = ::serde::Deserialize::deserialize(d)?;
Ok(Some(FromHex::from_hex(&hex_str).map_err(D::Error::custom)?))
}
}
}

View File

@ -1,196 +1,54 @@
#[cfg(feature = "liquid")]
use elements::address as elements_address;
use crate::chain::Network;
use crate::util::{Address, Payload};
use bitcoin::Script;
use bitcoin_bech32::constants::Network as B32Network;
use bitcoin_bech32::{u5, WitnessProgram};
use bitcoin_hashes::{hash160::Hash as Hash160, Hash};
use crate::chain::{script, Network, Script, TxIn, TxOut};
use script::Instruction::PushBytes;
pub struct InnerScripts {
pub redeem_script: Option<Script>,
pub witness_script: Option<Script>,
}
pub trait IsProvablyUnspendable {
fn is_provably_unspendable_(&self) -> bool;
}
#[cfg(not(feature = "liquid"))]
impl IsProvablyUnspendable for bitcoin::Script {
// is_provably_unspendable() is deprecated in rust-bitcoin
// so we re-implement it here. Copy pasted.
fn is_provably_unspendable_(&self) -> bool {
use bitcoin::blockdata::opcodes::{
Class::{IllegalOp, ReturnOp},
ClassifyContext, Opcode,
};
match self.as_bytes().first() {
Some(b) => {
let first = Opcode::from(*b);
let class = first.classify(ClassifyContext::Legacy);
class == ReturnOp || class == IllegalOp
}
None => false,
}
}
}
#[cfg(feature = "liquid")]
impl IsProvablyUnspendable for elements::Script {
#[inline(always)]
fn is_provably_unspendable_(&self) -> bool {
// Not deprecated yet
self.is_provably_unspendable()
}
}
// Extension trait for segwit script detection that works across bitcoin and elements
pub trait SegwitDetection {
fn segwit_is_p2wpkh(&self) -> bool;
fn segwit_is_p2wsh(&self) -> bool;
fn segwit_is_p2tr(&self) -> bool;
}
#[cfg(not(feature = "liquid"))]
impl SegwitDetection for bitcoin::Script {
fn segwit_is_p2wpkh(&self) -> bool {
self.is_p2wpkh()
}
fn segwit_is_p2wsh(&self) -> bool {
self.is_p2wsh()
}
fn segwit_is_p2tr(&self) -> bool {
self.is_p2tr()
}
}
#[cfg(not(feature = "liquid"))]
impl SegwitDetection for bitcoin::ScriptBuf {
fn segwit_is_p2wpkh(&self) -> bool {
self.is_p2wpkh()
}
fn segwit_is_p2wsh(&self) -> bool {
self.is_p2wsh()
}
fn segwit_is_p2tr(&self) -> bool {
self.is_p2tr()
}
}
#[cfg(feature = "liquid")]
impl SegwitDetection for elements::Script {
fn segwit_is_p2wpkh(&self) -> bool {
self.is_v0_p2wpkh()
}
fn segwit_is_p2wsh(&self) -> bool {
self.is_v0_p2wsh()
}
fn segwit_is_p2tr(&self) -> bool {
self.is_v1_p2tr()
}
}
pub trait ScriptToAsm: std::fmt::Debug {
fn to_asm(&self) -> String {
let asm = format!("{:?}", self);
asm[7..asm.len() - 1].to_string()
}
}
impl ScriptToAsm for bitcoin::Script {}
impl ScriptToAsm for bitcoin::ScriptBuf {}
#[cfg(feature = "liquid")]
impl ScriptToAsm for elements::Script {}
pub trait ScriptToAddr {
fn to_address_str(&self, network: Network) -> Option<String>;
}
#[cfg(not(feature = "liquid"))]
impl ScriptToAddr for bitcoin::Script {
fn to_address_str(&self, network: Network) -> Option<String> {
bitcoin::Address::from_script(self, bitcoin::Network::from(network))
.ok()
.map(|s| s.to_string())
}
}
#[cfg(feature = "liquid")]
impl ScriptToAddr for elements::Script {
fn to_address_str(&self, network: Network) -> Option<String> {
elements_address::Address::from_script(self, None, network.address_params())
.map(|a| a.to_string())
}
}
// Returns the witnessScript in the case of p2wsh, or the redeemScript in the case of p2sh.
pub fn get_innerscripts(txin: &TxIn, prevout: &TxOut) -> InnerScripts {
// Wrapped redeemScript for P2SH spends
let redeem_script = if prevout.script_pubkey.is_p2sh() {
if let Some(Ok(PushBytes(redeemscript))) = txin.script_sig.instructions().last() {
#[cfg(not(feature = "liquid"))]
let bytes = redeemscript.as_bytes().to_vec();
#[cfg(feature = "liquid")]
let bytes = redeemscript.to_vec();
Some(Script::from(bytes))
} else {
None
}
// @XXX we can't use any of the Address:p2{...}h utility methods, since they expect the pre-image data, which we don't have.
// we must instead create the Payload manually, which results in code duplication with the p2{...}h methods, especially for witness programs.
// ideally, this should be implemented as part of the rust-bitcoin lib.
pub fn script_to_address(script: &Script, network: &Network) -> Option<String> {
let payload = if script.is_p2pkh() {
Some(Payload::PubkeyHash(
Hash160::from_slice(&script[3..23]).ok()?,
))
} else if script.is_p2sh() {
Some(Payload::ScriptHash(
Hash160::from_slice(&script[2..22]).ok()?,
))
} else if script.is_v0_p2wpkh() {
Some(Payload::WitnessProgram(
WitnessProgram::new(
u5::try_from_u8(0).expect("0<32"),
script[2..22].to_vec(),
B32Network::from(network),
)
.unwrap(),
))
} else if script.is_v0_p2wsh() {
Some(Payload::WitnessProgram(
WitnessProgram::new(
u5::try_from_u8(0).expect("0<32"),
script[2..34].to_vec(),
B32Network::from(network),
)
.unwrap(),
))
} else {
None
};
// Wrapped witnessScript for P2WSH or P2SH-P2WSH spends
let witness_script = if prevout.script_pubkey.segwit_is_p2wsh()
|| prevout.script_pubkey.segwit_is_p2tr()
|| redeem_script.as_ref().is_some_and(|s| s.segwit_is_p2wsh())
{
let witness = &txin.witness;
#[cfg(feature = "liquid")]
let witness = &witness.script_witness;
// rust-bitcoin returns witness items as a [u8] slice, while rust-elements returns a Vec<u8>
#[cfg(not(feature = "liquid"))]
let wit_to_vec = Vec::from;
#[cfg(feature = "liquid")]
let wit_to_vec = Clone::clone;
let inner_script_slice = if prevout.script_pubkey.segwit_is_p2tr() {
// Witness stack is potentially very large
// so we avoid to_vec() or iter().collect() for performance
let w_len = witness.len();
witness
.last()
// Get the position of the script spend script (if it exists)
.map(|last_elem| {
// From BIP341:
// If there are at least two witness elements, and the first byte of
// the last element is 0x50, this last element is called annex a
// and is removed from the witness stack.
if w_len >= 2 && last_elem.first().filter(|&&v| v == 0x50).is_some() {
// account for the extra item removed from the end
3
} else {
// otherwise script is 2nd from last
2
}
})
// Convert to None if not script spend
// Note: Option doesn't have filter_map() method
.filter(|&script_pos_from_last| w_len >= script_pos_from_last)
.and_then(|script_pos_from_last| {
// Can't use second_to_last() since it might be 3rd to last
#[allow(clippy::iter_nth)]
witness.iter().nth(w_len - script_pos_from_last)
})
} else {
witness.last()
};
inner_script_slice.map(wit_to_vec).map(Script::from)
} else {
None
};
InnerScripts {
redeem_script,
witness_script,
}
Some(
Address {
payload: payload?,
network: *network,
}
.to_string(),
)
}
pub fn get_script_asm(script: &Script) -> String {
let asm = format!("{:?}", script);
(&asm[7..asm.len() - 1]).to_string()
}

View File

@ -1,29 +1,18 @@
use crate::chain::{BlockHash, OutPoint, Transaction, TxIn, TxOut, Txid};
use crate::errors;
use crate::util::{BlockId, IsProvablyUnspendable};
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
use std::collections::HashMap;
use crate::chain::{TxIn, TxOut};
use crate::util::BlockId;
#[cfg(feature = "liquid")]
lazy_static! {
static ref REGTEST_INITIAL_ISSUANCE_PREVOUT: Txid =
"50cdc410c9d0d61eeacc531f52d2c70af741da33af127c364e52ac1ee7c030a5"
.parse()
.unwrap();
static ref TESTNET_INITIAL_ISSUANCE_PREVOUT: Txid =
"0c52d2526a5c9f00e9fb74afd15dd3caaf17c823159a514f929ae25193a43a52"
.parse()
.unwrap();
}
use crate::util::REGTEST_INITIAL_ISSUANCE_PREVOUT;
#[cfg(feature = "liquid")]
use bitcoin_hashes::hex::ToHex;
#[derive(Serialize, Deserialize)]
pub struct TransactionStatus {
pub confirmed: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub block_height: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub block_hash: Option<BlockHash>,
#[serde(skip_serializing_if = "Option::is_none")]
pub block_hash: Option<Sha256dHash>,
pub block_time: Option<u32>,
}
@ -32,7 +21,7 @@ impl From<Option<BlockId>> for TransactionStatus {
match blockid {
Some(b) => TransactionStatus {
confirmed: true,
block_height: Some(b.height),
block_height: Some(b.height as usize),
block_hash: Some(b.hash),
block_time: Some(b.time),
},
@ -46,12 +35,6 @@ impl From<Option<BlockId>> for TransactionStatus {
}
}
#[derive(Serialize, Deserialize)]
pub struct TxInput {
pub txid: Txid,
pub vin: u32,
}
pub fn is_coinbase(txin: &TxIn) -> bool {
#[cfg(not(feature = "liquid"))]
return txin.previous_output.is_null();
@ -65,328 +48,12 @@ pub fn has_prevout(txin: &TxIn) -> bool {
#[cfg(feature = "liquid")]
return !txin.is_coinbase()
&& !txin.is_pegin
&& txin.previous_output.txid != *REGTEST_INITIAL_ISSUANCE_PREVOUT
&& txin.previous_output.txid != *TESTNET_INITIAL_ISSUANCE_PREVOUT;
&& txin.previous_output.txid.to_hex() != REGTEST_INITIAL_ISSUANCE_PREVOUT;
}
pub fn is_spendable(txout: &TxOut) -> bool {
#[cfg(not(feature = "liquid"))]
return !txout.script_pubkey.is_provably_unspendable_();
return !txout.script_pubkey.is_provably_unspendable();
#[cfg(feature = "liquid")]
return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable_();
}
/// Extract the previous TxOuts of a Transaction's TxIns
///
/// # Errors
///
/// This function MUST NOT return an error variant when allow_missing is true.
/// If allow_missing is false, it will return an error when any Outpoint is
/// missing from the keys of the txos argument's HashMap.
pub fn extract_tx_prevouts<'a>(
tx: &Transaction,
txos: &'a HashMap<OutPoint, TxOut>,
) -> Result<HashMap<u32, &'a TxOut>, errors::Error> {
tx.input
.iter()
.enumerate()
.filter(|(_, txi)| has_prevout(txi))
.map(|(index, txi)| {
Ok((
index as u32,
match txos.get(&txi.previous_output) {
Some(txo) => txo,
None => {
return Err(format!("missing outpoint {:?}", txi.previous_output).into());
}
},
))
})
.collect()
}
pub fn serialize_outpoint<S>(outpoint: &OutPoint, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
use serde::ser::SerializeStruct;
let mut s = serializer.serialize_struct("OutPoint", 2)?;
s.serialize_field("txid", &outpoint.txid)?;
s.serialize_field("vout", &outpoint.vout)?;
s.end()
}
pub(super) mod sigops {
use crate::chain::{
opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
script::{self, Instruction},
Transaction, TxOut, Witness,
};
#[cfg(not(feature = "liquid"))]
use bitcoin::opcodes::Opcode;
#[cfg(feature = "liquid")]
use elements::opcodes::All as Opcode;
use std::collections::HashMap;
/// Get sigop count for transaction. prevout_map must have all the prevouts.
pub fn transaction_sigop_count(
tx: &Transaction,
prevout_map: &HashMap<u32, &TxOut>,
) -> Result<usize, script::Error> {
let input_count = tx.input.len();
let mut prevouts = Vec::with_capacity(input_count);
#[cfg(not(feature = "liquid"))]
let is_coinbase_or_pegin = tx.is_coinbase();
#[cfg(feature = "liquid")]
let is_coinbase_or_pegin = tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin);
if !is_coinbase_or_pegin {
for idx in 0..input_count {
prevouts.push(
*prevout_map
.get(&(idx as u32))
.ok_or(script::Error::EarlyEndOfScript)?,
);
}
}
// coinbase tx won't use prevouts so it can be empty.
get_sigop_cost(tx, &prevouts, true, true)
}
fn decode_pushnum(op: &Opcode) -> Option<u8> {
// 81 = OP_1, 96 = OP_16
// 81 -> 1, so... 81 - 80 -> 1
#[cfg(not(feature = "liquid"))]
let self_u8 = op.to_u8();
#[cfg(feature = "liquid")]
let self_u8 = op.into_u8();
match self_u8 {
81..=96 => Some(self_u8 - 80),
_ => None,
}
}
fn count_sigops(script: &script::Script, accurate: bool) -> usize {
let mut n = 0;
let mut pushnum_cache = None;
for inst in script.instructions() {
match inst {
Ok(Instruction::Op(opcode)) => {
match opcode {
OP_CHECKSIG | OP_CHECKSIGVERIFY => {
n += 1;
}
OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => {
match (accurate, pushnum_cache) {
(true, Some(pushnum)) => {
// Add the number of pubkeys in the multisig as sigop count
n += usize::from(pushnum);
}
_ => {
// MAX_PUBKEYS_PER_MULTISIG from Bitcoin Core
// https://github.com/bitcoin/bitcoin/blob/v25.0/src/script/script.h#L29-L30
n += 20;
}
}
}
_ => {
pushnum_cache = decode_pushnum(&opcode);
}
}
}
// We ignore errors as well as pushdatas
_ => {
pushnum_cache = None;
}
}
}
n
}
/// Get the sigop count for legacy transactions
fn get_legacy_sigop_count(tx: &Transaction) -> usize {
let mut n = 0;
for input in &tx.input {
n += count_sigops(&input.script_sig, false);
}
for output in &tx.output {
n += count_sigops(&output.script_pubkey, false);
}
n
}
fn get_p2sh_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
#[cfg(not(feature = "liquid"))]
if tx.is_coinbase() {
return 0;
}
#[cfg(feature = "liquid")]
if tx.is_coinbase() {
return 0;
}
let mut n = 0;
for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
if prevout.script_pubkey.is_p2sh() {
if let Some(Ok(script::Instruction::PushBytes(redeem))) =
input.script_sig.instructions().last()
{
#[cfg(not(feature = "liquid"))]
let script = script::Script::from_bytes(redeem.as_bytes());
#[cfg(feature = "liquid")]
let script = script::Script::from(redeem.to_vec());
#[allow(clippy::needless_borrow)]
{
n += count_sigops(&script, true);
}
}
}
}
n
}
fn get_witness_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
let mut n = 0;
#[inline]
fn is_push_only(script: &script::Script) -> bool {
for inst in script.instructions() {
match inst {
Err(_) => return false,
Ok(Instruction::Op(_)) => return false,
Ok(Instruction::PushBytes(_)) => {}
}
}
true
}
#[inline]
fn last_pushdata(script: &script::Script) -> Option<&[u8]> {
match script.instructions().last() {
#[cfg(not(feature = "liquid"))]
Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes.as_bytes()),
#[cfg(feature = "liquid")]
Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes),
_ => None,
}
}
#[inline]
fn count_with_prevout(
prevout: &TxOut,
script_sig: &script::Script,
witness: &Witness,
) -> usize {
let mut n = 0;
let script_owned;
let script: &script::Script = if prevout.script_pubkey.is_witness_program() {
&prevout.script_pubkey
} else if prevout.script_pubkey.is_p2sh()
&& is_push_only(script_sig)
&& !script_sig.is_empty()
{
#[cfg(not(feature = "liquid"))]
{
script_owned =
script::ScriptBuf::from(last_pushdata(script_sig).unwrap().to_vec());
}
#[cfg(feature = "liquid")]
{
script_owned =
script::Script::from(last_pushdata(script_sig).unwrap().to_vec());
}
&script_owned
} else {
return 0;
};
#[cfg(not(feature = "liquid"))]
if script.is_p2wsh() {
let bytes = script.as_bytes();
n += sig_ops(witness, bytes[0], &bytes[2..]);
} else if script.is_p2wpkh() {
n += 1;
}
#[cfg(feature = "liquid")]
if script.is_v0_p2wsh() {
let bytes = script.as_bytes();
n += sig_ops(witness, bytes[0], &bytes[2..]);
} else if script.is_v0_p2wpkh() {
n += 1;
}
n
}
for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
n += count_with_prevout(prevout, &input.script_sig, &input.witness);
}
n
}
/// Get the sigop cost for this transaction.
fn get_sigop_cost(
tx: &Transaction,
previous_outputs: &[&TxOut],
verify_p2sh: bool,
verify_witness: bool,
) -> Result<usize, script::Error> {
let mut n_sigop_cost = get_legacy_sigop_count(tx) * 4;
#[cfg(not(feature = "liquid"))]
if tx.is_coinbase() {
return Ok(n_sigop_cost);
}
#[cfg(feature = "liquid")]
if tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin) {
return Ok(n_sigop_cost);
}
if tx.input.len() != previous_outputs.len() {
return Err(script::Error::EarlyEndOfScript);
}
if verify_witness && !verify_p2sh {
return Err(script::Error::EarlyEndOfScript);
}
if verify_p2sh {
n_sigop_cost += get_p2sh_sigop_count(tx, previous_outputs) * 4;
}
if verify_witness {
n_sigop_cost += get_witness_sigop_count(tx, previous_outputs);
}
Ok(n_sigop_cost)
}
/// Get sigops for the Witness
///
/// witness_version is the raw opcode. OP_0 is 0, OP_1 is 81, etc.
#[allow(clippy::redundant_closure)]
fn sig_ops(witness: &Witness, witness_version: u8, witness_program: &[u8]) -> usize {
#[cfg(feature = "liquid")]
let last_witness = witness.script_witness.last();
#[cfg(not(feature = "liquid"))]
let last_witness = witness.last();
match (witness_version, witness_program.len()) {
(0, 20) => 1,
(0, 32) => {
#[cfg(not(feature = "liquid"))]
{
#[allow(clippy::needless_borrow)]
last_witness
.map(|sl| script::Script::from_bytes(sl))
.map(|s| count_sigops(s, true))
.unwrap_or_default()
}
#[cfg(feature = "liquid")]
{
last_witness
.map(|sl| script::Script::from(sl.clone()))
.map(|s| count_sigops(&s, true))
.unwrap_or_default()
}
}
_ => 0,
}
}
return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable();
}

274
start
View File

@ -1,274 +0,0 @@
#!/usr/bin/env zsh
# initialize variables
DAEMON=bitcoin
NETWORK=mainnet
FEATURES=default
DB_FOLDER=/electrs
ASSET_DB_ARGS=()
NODENAME=$(hostname|cut -d . -f1)
LOCATION=$(hostname|cut -d . -f2)
USAGE="Usage: $0 (mainnet|testnet|signet|liquid|liquidtestnet) [popular-scripts]"
# load rust if necessary
if [ -e "${HOME}/.cargo/env" ];then
source "${HOME}/.cargo/env"
export PATH="${HOME}/.cargo/bin:${PATH}"
fi
# which OS?
case "$(uname -s)" in
FreeBSD)
OS=FreeBSD
NPROC=$(sysctl -n hw.ncpu)
export CC=/usr/local/bin/clang17
export CXX=/usr/local/bin/clang++17
export CPP=/usr/local/bin/clang-cpp17
export RUSTFLAGS="-C linker=clang17"
;;
Darwin)
OS=Darwin
NPROC=$(sysctl -n hw.ncpu)
;;
Linux)
OS=Linux
NPROC=$(grep -c proc /proc/cpuinfo)
;;
*)
OS=Unknown
NPROC=4
;;
esac
# which network?
case "${1}" in
mainnet)
THREADS=$((NPROC / 8))
CRONJOB_TIMING="20 4 * * *"
;;
testnet)
NETWORK=testnet
THREADS=$((NPROC / 8))
CRONJOB_TIMING="2 4 * * *"
;;
testnet4)
NETWORK=testnet4
THREADS=$((NPROC / 8))
CRONJOB_TIMING="17 4 * * *"
;;
signet)
NETWORK=signet
THREADS=$((NPROC / 8))
CRONJOB_TIMING="9 4 * * *"
;;
liquid)
DAEMON=elements
NETWORK=liquid
FEATURES=liquid
ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_db)
THREADS=$((NPROC / 8))
CRONJOB_TIMING="12 4 * * *"
;;
liquidtestnet)
DAEMON=elements
NETWORK=liquidtestnet
FEATURES=liquid
ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_testnet_db)
THREADS=$((NPROC / 8))
CRONJOB_TIMING="17 4 * * *"
;;
*)
echo "${USAGE}"
exit 1
;;
esac
# Run the popular address txt file generator before each run
POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}"
POPULAR_SCRIPTS_FILE_RAW="${POPULAR_SCRIPTS_FOLDER}/popular-scripts-raw.txt"
POPULAR_SCRIPTS_FILE="${POPULAR_SCRIPTS_FOLDER}/popular-scripts.txt"
# This function runs the job for generating the popular scripts text file for the precache arg
generate_popular_scripts() {
mkdir -p "${POPULAR_SCRIPTS_FOLDER}"
## Use nproc * 4 threads to generate the txt file (lots of iowait, so 2x~4x core count is ok)
## Only pick up addresses with 101 history events or more
## (Without lowering MIN_HISTORY_ITEMS_TO_CACHE this is the lowest we can go)
## It prints out progress to STDERR
echo "[*] Generating popular-scripts using ${THREADS} threads..."
cd "${HOME}/electrs"
HIGH_USAGE_THRESHOLD=101 \
JOB_THREAD_COUNT=${THREADS} \
nice cargo run \
--release \
--bin popular-scripts \
--features "${FEATURES}" \
-- \
--network "${NETWORK}" \
--db-dir "${DB_FOLDER}" \
> "${POPULAR_SCRIPTS_FILE_RAW}"
## Only overwrite the existing file if the popular-scripts cargo run succeeded
if [ "$?" = "0" ];then
## Sorted and deduplicated just in case
echo "Sorting popular scripts for final results..."
sort "${POPULAR_SCRIPTS_FILE_RAW}" | uniq > "${POPULAR_SCRIPTS_FILE}"
fi
rm "${POPULAR_SCRIPTS_FILE_RAW}"
}
# This function is for inserting the cronjob for generating the popular scripts
CRONJOB_CMD="\"${HOME}/electrs/start\" \"${NETWORK}\" popular-scripts"
echo "${CRONJOB_TIMING} ${CRONJOB_CMD}"
case "${2}" in
popular-scripts)
echo "[*] Only generate popular-scripts, then exit"
generate_popular_scripts
exit 0
;;
version)
echo "[*] Only print versions, then exit"
cargo run --bin electrs --release -- --version
cargo run --bin popular-scripts --release -- --version
exit 0
;;
"")
# If the 2nd arg isn't passed, just run the normal electrs script as-is
;;
*)
echo "${USAGE}"
exit 1
;;
esac
# run in loop in case of crash
until false
do
# reset CWD
cd "${HOME}/electrs"
# disable making electrs.core files
ulimit -c 0
# prepare run-time variables
UTXOS_LIMIT=500
ELECTRUM_TXS_LIMIT=500
ELECTRUM_MAX_LINE_SIZE=1048576 # 1 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100
ELECTRUM_MAX_CLIENTS=1000
MAIN_LOOP_DELAY=500
DAEMON_CONF="${HOME}/${DAEMON}.conf"
HTTP_SOCKET_FILE="${HOME}/socket/esplora-${DAEMON}-${NETWORK}"
RPC_SOCKET_FILE="${HOME}/socket/electrum-${DAEMON}-${NETWORK}"
# get RPC credentials from bitcoin.conf or elements.conf directly
echo "[*] Getting RPC credentials from ${DAEMON_CONF}"
RPC_USER=$(grep 'rpcuser=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
RPC_PASS=$(grep 'rpcpassword=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
# override limits based on hostname
if [ "${NODENAME}" = "node201" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
MAIN_LOOP_DELAY=14000
fi
if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "sg1" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "hnl" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${NODENAME}" = "node206" ] && [ "${LOCATION}" = "tk7" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${NODENAME}" = "node211" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${NODENAME}" = "node212" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${NODENAME}" = "node213" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${NODENAME}" = "node214" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${NETWORK}" = "testnet4" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ "${LOCATION}" = "fmt" ];then
UTXOS_LIMIT=9000
ELECTRUM_TXS_LIMIT=9000
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
ELECTRUM_MAX_SUBSCRIPTIONS=100000
ELECTRUM_MAX_CLIENTS=10000
fi
if [ ! -e "${POPULAR_SCRIPTS_FILE}" ];then
generate_popular_scripts
fi
# Run the electrs process (Note: db-dir is used in both commands)
nice cargo run \
--release \
--bin electrs \
--features "${FEATURES}" \
-- \
--network "${NETWORK}" \
--daemon-dir "${HOME}" \
--db-dir "${DB_FOLDER}" \
"${ASSET_DB_ARGS[@]}" \
--main-loop-delay "${MAIN_LOOP_DELAY}" \
--rpc-socket-file "${RPC_SOCKET_FILE}" \
--http-socket-file "${HTTP_SOCKET_FILE}" \
--precache-scripts "${POPULAR_SCRIPTS_FILE}" \
--precache-threads "${THREADS}" \
--cookie "${RPC_USER}:${RPC_PASS}" \
--cors '*' \
--address-search \
--utxos-limit "${UTXOS_LIMIT}" \
--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
--electrum-max-line-size "${ELECTRUM_MAX_LINE_SIZE}" \
--electrum-max-subscriptions "${ELECTRUM_MAX_SUBSCRIPTIONS}" \
--electrum-max-clients "${ELECTRUM_MAX_CLIENTS}" \
-vv
sleep 1
done