Merge pull request #152 from mempool/mononaut/enable-liquid-asset-registry

enable liquid asset registry in start script

.dockerignore

@@ -0,0 +1 @@
+Dockerfile

.editorconfig

@@ -0,0 +1,6 @@
+# see https://editorconfig.org for more options, and setup instructions for yours editor
+
+[*]
+indent_style = space
+indent_size = 4
+

.github/actions/ci-rust-setup/action.yml

@@ -0,0 +1,50 @@
+name: CI Rust Setup
+description: 'Sets up the environment for Rust jobs during CI workflow'
+
+inputs:
+  cache-name:
+    description: 'Name of cache artifacts (same name is same cache key) empty to disable cache'
+    required: false
+  targets:
+    description: 'A comma separated list of extra targets you want to install'
+    required: false
+  components:
+    description: 'A comma separated list of extra components you want to install'
+    required: false
+  toolchain:
+    description: 'The toolchain to use. If not specified, the rust-toolchain file will be used'
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - name: Get toolchain from input OR rust-toolchain file
+      id: gettoolchain
+      shell: bash
+      run: |-
+        RUST_TOOLCHAIN="${{ inputs.toolchain }}"
+        if [ ! -f rust-toolchain ] && [ -z "${RUST_TOOLCHAIN}" ]; then
+          echo "***ERROR*** NEED toolchain INPUT OR rust-toolchain FILE IN ROOT OF REPOSITORY" >&2
+          exit 1
+        fi
+        if [ -z "${RUST_TOOLCHAIN}" ]; then
+          RUST_TOOLCHAIN="$(cat rust-toolchain)"
+        fi
+        echo "toolchain=\"${RUST_TOOLCHAIN}\"" >> $GITHUB_OUTPUT
+    - name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
+      id: toolchain
+      # Commit date is Nov 18, 2024
+      uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa
+      with:
+        toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
+        targets: ${{ inputs.targets }}
+        components: ${{ inputs.components }}
+    - name: Cache dependencies
+      uses: actions/cache@v3
+      if: inputs.cache-name != ''
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-cargo-${{ inputs.cache-name }}-${{ steps.toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }}

.github/workflows/ci.yml

@@ -0,0 +1,107 @@
+on:
+  pull_request:
+  push:
+    branches:
+      - mempool
+
+
+name: Compile Check and Lint
+
+jobs:
+  check:
+    name: Compile Check
+    runs-on: mempool-ci
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          cache-name: dev
+      - run: cargo check --all-features
+
+  fmt:
+    name: Formatter
+    runs-on: mempool-ci
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          components: rustfmt
+      - run: cargo fmt --all -- --check
+
+  test:
+    name: Run Tests
+    runs-on: mempool-ci
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          cache-name: test
+      - run: cargo test --lib --all-features
+
+  compile-freebsd:
+    runs-on: mempool-ci
+    name: Run Compile Checks in FreeBSD
+    env:
+      FREEBSD_VER: "14.3"
+    steps:
+    - uses: actions/checkout@v4
+    - name: Cache dependencies for FreeBSD
+      uses: actions/cache@v3
+      with:
+        path: |
+          .cargohome/registry
+          .cargohome/git
+          target
+        key: freebsd-${{ env.FREEBSD_VER }}-cargo-checks-${{ hashFiles('**/Cargo.lock') }}
+    - name: Compile Checks in FreeBSD
+      uses: vmactions/freebsd-vm@v1
+      with:
+        usesh: true
+        release: "${{ env.FREEBSD_VER }}"
+        arch: amd64
+        prepare: |
+          mkdir -p ~/.cargo/
+          mkdir -p ./.cargohome/registry/
+          mkdir -p ./.cargohome/git/
+          mv ./.cargohome/registry ~/.cargo/
+          mv ./.cargohome/git ~/.cargo/
+          rm -rf ./.cargohome
+          pkg install -y git rsync gmake llvm rust rocksdb cmake
+
+        run: |
+          cargo check --no-default-features
+          cargo check -F liquid
+          cargo check -F electrum-discovery
+          cargo check -F electrum-discovery,liquid
+          cargo build --release --bin electrs
+          rm -rf ./.cargohome
+          mkdir -p ~/.cargo/registry/
+          mkdir -p ~/.cargo/git/
+          mkdir -p ./.cargohome/
+          mv ~/.cargo/registry ./.cargohome/
+          mv ~/.cargo/git ./.cargohome/
+
+  clippy:
+    name: Linter
+    runs-on: mempool-ci
+    needs: [check]
+    strategy:
+      matrix: # Try all combinations of features. Some times weird things appear.
+        features: [
+          '',
+          '-F electrum-discovery',
+          '-F liquid',
+          '-F electrum-discovery,liquid',
+        ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          cache-name: dev
+          components: clippy
+      - name: Clippy with Features = ${{ matrix.features }}
+        run: cargo clippy ${{ matrix.features }} -- -D warnings

.github/workflows/on-tag.yml

@@ -0,0 +1,77 @@
+name: Docker build on tag
+env:
+  DOCKER_CLI_EXPERIMENTAL: enabled
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
+  DOCKER_BUILDKIT: 0
+  COMPOSE_DOCKER_CLI_BUILD: 0
+
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+      - v[0-9]+.[0-9]+.[0-9]+-*
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: mempool-ci
+    timeout-minutes: 120
+    name: Build and push to DockerHub
+    strategy:
+      max-parallel: 1
+      matrix:
+        include:
+          - image: electrs
+            cargo_extra_args: ""
+          - image: electrs-liquid
+            cargo_extra_args: "--features liquid"
+    steps:
+      - name: Set env variables
+        run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
+
+      - name: Show set environment variables
+        run: |
+          printf "      TAG: %s\n"  "$TAG"
+
+      - name: Add SHORT_SHA env property with commit short sha
+        run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
+
+      - name: Login to Docker for building
+        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+
+      - name: Checkout project
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        id: qemu
+
+      - name: Setup Docker buildx action
+        uses: docker/setup-buildx-action@v3
+        id: buildx
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx
+          restore-keys: |
+            ${{ runner.os }}-buildx
+
+      - name: Run Docker buildx against tag
+        run: |
+          docker buildx build \
+          --cache-from "type=local,src=/tmp/.buildx-cache" \
+          --cache-to "type=local,dest=/tmp/.buildx-cache" \
+          --platform linux/amd64,linux/arm64 \
+          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:$TAG \
+          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:latest \
+          --output "type=registry" . \
+          --build-arg commitHash=$SHORT_SHA \
+          --build-arg CARGO_EXTRA_ARGS="${{ matrix.cargo_extra_args }}"

.github/workflows/project-review-status.yml

@@ -0,0 +1,18 @@
+name: Project Board Automation
+
+on:
+  pull_request:
+    types: [review_requested]
+  issues:
+    types: [opened]
+
+jobs:
+  project-automation:
+    uses: mempool/.github/.github/workflows/project-board-automation.yml@master
+    with:
+      project-number: 8
+    secrets:
+      PROJECT_TOKEN: ${{ secrets.PROJECT_TOKEN }}
+      PROJECT_ID: ${{ secrets.PROJECT_ID }}
+      STATUS_FIELD_ID: ${{ secrets.STATUS_FIELD_ID }}
+      REVIEW_NEEDED_OPTION_ID: ${{ secrets.REVIEW_NEEDED_OPTION_ID }}

.gitignore

@@ -4,3 +4,5 @@ target
 *.sublime*
 *~
 *.pyc
+.vscode
+*.core

AGENTS.md

@@ -0,0 +1,19 @@
+# electrs
+
+## Rules
+
+1. You are an expert Rust developer.
+2. You are an expert Bitcoin developer.
+3. If you are unsure of a change, ask the developer to make a choice proactively.
+
+## Before testing
+
+- Run cargo fmt (from root)
+  - command: `cargo fmt`
+
+## Testing
+
+- Run the checks script
+  - `./scripts/checks.sh`
+- Run with tests only when a test is added or changed
+  - `INCLUDE_TESTS=1 ./scripts/checks.sh`

Cargo.toml

@@ -1,68 +1,69 @@
 [package]
-name = "electrs"
-version = "0.4.1"
-authors = ["Roman Zeyde <me@romanzey.de>"]
+name = "mempool-electrs"
+version = "3.4.0-dev"
+authors = [
+    "Roman Zeyde <me@romanzey.de>",
+    "Nadav Ivgi <nadav@shesek.info>",
+    "wiz <j@wiz.biz>",
+    "junderw <jonathan.underwood4649@gmail.com>"
+]
 description = "An efficient re-implementation of Electrum Server in Rust"
 license = "MIT"
-homepage = "https://github.com/romanz/electrs"
-repository = "https://github.com/romanz/electrs"
+homepage = "https://github.com/mempool/electrs"
+repository = "https://github.com/mempool/electrs"
+publish = false
 keywords = ["bitcoin", "electrum", "server", "index", "database"]
-documentation = "https://docs.rs/electrs/"
 readme = "README.md"
 edition = "2018"
 
+[lib]
+name = "electrs"
+
 [features]
 default = []
 liquid = ["elements"]
+electrum-discovery = ["electrum-client"]
 
 [dependencies]
-arraydeque = "0.4"
-arrayref = "0.3"
-base64 = "0.10"
-bincode = "1.0"
-bitcoin-bech32 = "0.9.0"
-chan = "0.1"
-chan-signal = "0.3"
-clap = "2.31"
-dirs = "1.0"
-error-chain = "0.12"
-futures = "0.1"
+arrayref = "0.3.6"
+base64 = "0.13.0"
+bincode-do-not-use-directly = { version = "1.3.1", package = "bincode" }
+bitcoin = { version = "0.32.8", features = [ "serde" ] }
+bounded-vec-deque = "0.1.1"
+clap = "2.33.3"
+crossbeam-channel = "0.5.0"
+dirs = "4.0.0"
+elements = { version = "0.26.1", features = [ "serde" ], optional = true }
+error-chain = "0.12.4"
 glob = "0.3"
-hex = "0.3"
-hyper = "0.12.27"
-itertools = "0.8.0"
+hex = "0.4.2"
+itertools = "0.10"
+lazy_static = "1.3.0"
 libc = "0.2"
-log = "0.4"
-lru = "0.1"
-lru-cache = "0.1.1"
-num_cpus = "1.0"
-page_size = "0.4"
-prometheus = "0.5"
-rayon = "1.0"
-rocksdb = "0.12.1"
-rust-crypto = "0.2"
-secp256k1 = "0.12"
-serde = "1.0"
-serde_derive = "1.0"
-serde_json = "1.0"
-stderrlog = "0.4.1"
-sysconf = ">=0.3.4"
-time = "0.1"
-tiny_http = "0.6"
-url = "1.0"
+log = "0.4.11"
+socket2 = { version = "0.4", features = ["all"] }
+num_cpus = "1.12.0"
+page_size = "0.4.2"
+prometheus = "0.13"
+ppp = "2.3.0"
+rayon = "1.5.0"
+rocksdb = "0.24.0"
+serde = "1.0.118"
+serde_derive = "1.0.118"
+serde_json = "1.0.60"
+sha2 = "0.10.7"
+signal-hook = "0.3"
+stderrlog = "0.5.0"
+time = { version = "0.3", features = ["formatting"] }
+tiny_http = "0.11"
+url = "2.2.0"
+hyper = "0.14"
+hyperlocal = "0.8"
+# close to same tokio version as dependent by hyper v0.14 and hyperlocal 0.8 -- things can go awry if they mismatch
+tokio = { version = "1", features = ["sync", "macros"] }
 
-[dependencies.bitcoin]
-version = "0.18"
-features = ["serde"]
-
-[dependencies.bitcoin_hashes]
-version = "0.3"
-features = ["serde"]
-
-[dependencies.elements]
-optional = true
-version = "0.7.0"
-features = ["serde-feature"] # Doesn't look to work by now
+# optional dependencies for electrum-discovery
+electrum-client = { version = "0.24.1", optional = true }
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -71,3 +72,7 @@ tempfile = "3.0"
 lto = true
 panic = 'abort'
 codegen-units = 1
+
+[patch.crates-io.electrum-client]
+git = "https://github.com/mempool/rust-electrum-client"
+rev = "4bbfc612d594fe23282c439d4bdc446cff01ba1c" # 0.24.1/add-peer branch

Dockerfile

@@ -1,21 +1,30 @@
-FROM rust:latest
+FROM debian:bookworm-slim AS base
 
-RUN apt-get update
-RUN apt-get install -y clang cmake
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 
-RUN cargo install electrs
+RUN apt update -qy && \
+    apt install -qy librocksdb-dev curl
 
-RUN adduser --disabled-login --system --shell /bin/false --uid 1000 user
+FROM base as build
 
-USER user
-WORKDIR /home/user
+RUN apt install -qy git clang cmake
+
+ENV RUSTUP_HOME=/rust
+ENV CARGO_HOME=/cargo 
+ENV PATH=/cargo/bin:/rust/bin:$PATH
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+WORKDIR /build
+COPY . .
+
+ARG CARGO_EXTRA_ARGS=""
+RUN cargo build --release --bin electrs ${CARGO_EXTRA_ARGS}
+
+FROM base as deploy
+
+COPY --from=build /build/target/release/electrs /bin/electrs
 
-# Electrum RPC
 EXPOSE 50001
 
-# Prometheus monitoring
-EXPOSE 4224
-
-STOPSIGNAL SIGINT
-
-CMD ["electrs", "-vvvv", "--timestamp"]
+ENTRYPOINT ["/bin/electrs"]

README.md

@@ -1,54 +1,54 @@
-# Esplora - Electrs backend API
+# Mempool - Electrs backend API
 
-A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs).
+A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs) and [Blockstream/electrs](https://github.com/Blockstream/electrs).
 
-Used as the backend for the [Esplora block explorer](https://github.com/Blockstream/esplora) powering [blockstream.info](https://blockstream.info/).
+Used as the backend for the [mempool block explorer](https://github.com/mempool/mempool) powering [mempool.space](https://mempool.space/).
 
-API documentation [is available here](https://github.com/blockstream/esplora/blob/master/API.md).
+API documentation [is available here](https://mempool.space/docs/api/rest).
 
 Documentation for the database schema and indexing process [is available here](doc/schema.md).
 
 ### Installing & indexing
 
-Install Rust, Bitcoin Core and the `clang` and `cmake` packages, then:
+Install Rust, Bitcoin Core (no `txindex` needed) and the `clang` and `cmake` packages, then:
 
 ```bash
-$ git clone https://github.com/blockstream/electrs && cd electrs
-$ git checkout new-index
+$ git clone https://github.com/mempool/electrs && cd electrs
+$ git checkout mempool
 $ cargo run --release --bin electrs -- -vvvv --daemon-dir ~/.bitcoin
 
 # Or for liquid:
-$ cargo run --features liquid --release --bin electrs -- -vvvv --daemon-dir ~/.liquid
+$ cargo run --features liquid --release --bin electrs -- -vvvv --network liquid --daemon-dir ~/.liquid
 ```
 
 See [electrs's original documentation](https://github.com/romanz/electrs/blob/master/doc/usage.md) for more detailed instructions.
 Note that our indexes are incompatible with electrs's and has to be created separately.
 
-The indexes require 440GB of storage after running compaction, but you'll need to have about 1TB
-of free space available for the initial (non-compacted) indexing process.
-Creating the indexes should take a few hours on a beefy machine with SSD.
+The indexes require 1.3TB of storage after running compaction (as of October 2023), but you'll need to have
+free space of about double that available during the index compaction process.
+Creating the indexes should take a few hours on a beefy machine with high speed NVMe SSD(s).
 
-> Note: the pre-v2 version supported a "light" indexing mode for personal use,
-> where less data is kept on-disk in exchange for a performance hit and more reliance/load on bitcoind.
-> This option is not currently available, but can be enabled with [an older esplora release](https://github.com/Blockstream/esplora/releases/tag/esplora_v1.67).
-> We're hoping to eventually get this feature back. Let us know if you find this important!
+### Light mode
 
-To deploy with Docker, follow the [instructions here](https://github.com/Blockstream/esplora#how-to-build-the-docker-image).
+For personal or low-volume use, you may set `--lightmode` to reduce disk storage requirements
+by roughly 50% at the cost of slower and more expensive lookups.
+
+With this option set, raw transactions and metadata associated with blocks will not be kept in rocksdb
+(the `T`, `X` and `M` indexes),
+but instead queried from bitcoind on demand.
 
 ### Notable changes from Electrs:
 
-- HTTP REST API instead of the Electrum JSON-RPC protocol, with extended transaction information
+- HTTP REST API in addition to the Electrum JSON-RPC protocol, with extended transaction information
   (previous outputs, spending transactions, script asm and more).
 
 - Extended indexes and database storage for improved performance under high load:
 
   - A full transaction store mapping txids to raw transactions is kept in the database under the prefix `t`.
-    This takes up ~200GB of extra storage.
+  - An index of all spendable transaction outputs is kept under the prefix `O`.
+  - An index of all addresses (encoded as string) is kept under the prefix `a` to enable by-prefix address search.
   - A map of blockhash to txids is kept in the database under the prefix `X`.
   - Block stats metadata (number of transactions, size and weight) is kept in the database under the prefix `M`.
-  - The index with `T` prefix mapping txids to block heights now also includes the block hash.
-    This allows for quick reorg-aware transaction confirmation status lookups, by verifying the
-    current block at the recorded height still matches the recorded block hash.
 
   With these new indexes, bitcoind is no longer queried to serve user requests and is only polled
   periodically for new blocks and for syncing the mempool.
@@ -61,13 +61,22 @@ To deploy with Docker, follow the [instructions here](https://github.com/Blockst
 In addition to electrs's original configuration options, a few new options are also available:
 
 - `--http-addr <addr:port>` - HTTP server address/port to listen on (default: `127.0.0.1:3000`).
-- `--disable-prevout` - disable attaching previous output information to inputs.
-  This significantly reduces the amount of transaction lookups (and IO/CPU/memory usage),
-  at the cost of not knowing inputs amounts, their previous script/address, and the transaction fee.
-- `--parent-network <network>` - the parent network this chain is pegged to (Elements/Liquid only).
+- `--lightmode` - enable light mode (see above)
 - `--cors <origins>` - origins allowed to make cross-site request (optional, defaults to none).
+- `--address-search` - enables the by-prefix address search index.
+- `--index-unspendables` - enables indexing of provably unspendable outputs.
+- `--utxos-limit <num>` - maximum number of utxos to return per address.
+- `--electrum-txs-limit <num>` - maximum number of txs to return per address in the electrum server (does not apply for the http api).
+- `--electrum-banner <text>` - welcome banner text for electrum server.
 
-See `$ cargo run --release --bin electrs -- --help` for the full list of options.
+Additional options with the `liquid` feature:
+- `--parent-network <network>` - the parent network this chain is pegged to.
+
+Additional options with the `electrum-discovery` feature:
+- `--electrum-hosts <json>` - a json map of the public hosts where the electrum server is reachable, in the [`server.features` format](https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server.features).
+- `--electrum-announce` - announce the electrum server on the electrum p2p server discovery network.
+
+See `$ cargo run --bin electrs -- --help` for the full list of options.
 
 ## License
 

build.rs

@@ -0,0 +1,57 @@
+use std::{path::Path, process::Command};
+
+fn main() {
+    // Specify re-run conditions
+
+    // 1. Rerun build script if we pass a new GIT_HASH
+    println!("cargo:rerun-if-env-changed=GIT_HASH");
+
+    // 2. Only do git based reruns if git directory exists
+    if Path::new(".git").exists() {
+        // If we change the branch, rerun
+        println!("cargo:rerun-if-changed=.git/HEAD");
+        if let Ok(r) = std::fs::read_to_string(".git/HEAD") {
+            if let Some(stripped) = r.strip_prefix("ref: ") {
+                // If the HEAD is detached it will be a commit hash
+                // so the HEAD changed directive above will pick it up,
+                // otherwise it will point to a ref in the refs directory
+                println!("cargo:rerun-if-changed=.git/{}", stripped);
+            }
+        }
+    }
+
+    // Getting git hash
+
+    // Don't fetch git hash if it's already in the ENV
+    let existing = std::env::var("GIT_HASH").unwrap_or_else(|_| String::new());
+    if !existing.is_empty() {
+        return;
+    }
+
+    // Get git hash from git and don't do anything if the command fails
+    if let Some(rev_parse) = cmd("git", &["rev-parse", "--short", "HEAD"]) {
+        // Add (dirty) to the GIT_HASH if the git status isn't clean
+        // This includes untracked files
+        let dirty = cmd("git", &["status", "--short"]).expect("git command works");
+
+        // Ignore Dockerfile deletion as it is expected in Docker buildx builds
+        let git_hash = if dirty.is_empty() || dirty.trim() == "D Dockerfile" {
+            rev_parse
+        } else {
+            format!("{}(dirty)", rev_parse.trim())
+        };
+
+        println!("cargo:rustc-env=GIT_HASH={}", git_hash.trim());
+    }
+}
+
+// Helper function, Command is verbose...
+fn cmd(name: &str, args: &[&str]) -> Option<String> {
+    Command::new(name).args(args).output().ok().and_then(|o| {
+        if o.status.success() {
+            String::from_utf8(o.stdout).ok()
+        } else {
+            None
+        }
+    })
+}

doc/schema.md

@@ -37,18 +37,38 @@ Each output results in the following new row:
 
  * `"O{txid}{vout}" → "{scriptpubkey}{value}"`
 
+When the indexer is synced up to the tip of the chain, the hash of the tip is saved as following:
+
+ * `"t" →  "{blockhash}"`
+
 ### `history`
 
-Each funding output (except for provably unspendable ones) results in the following new row (`H` is for history, `F` is for funding):
+Each funding output (except for provably unspendable ones when `--index-unspendables` is not enabled) results in the following new rows (`H` is for history, `F` is for funding):
 
- * `"H{funding-scripthash}{funding-height}F{funding-txid:index}{value}" → ""`
+ * `"H{funding-scripthash}{funding-height}F{funding-txid:vout}{value}" → ""`
+ * `"a{funding-address-str}" → ""` (for prefix address search, only saved when `--address-search` is enabled)
 
 Each spending input (except the coinbase) results in the following new rows (`S` is for spending):
 
- * `"H{funding-scripthash}{spending-height}S{spending-txid:index}{funding-txid:index}{value}" → ""`
+ * `"H{funding-scripthash}{spending-height}S{spending-txid:vin}{funding-txid:vout}{value}" → ""`
 
- * `"S{funding-txid:index}{spending-txid:index}" → ""`
+ * `"S{funding-txid:vout}{spending-txid:vin}" → ""`
 
+#### Elements only
+
+Assets (re)issuances results in the following new rows (only for user-issued assets):
+
+ * `"i{asset-id}" → "{issuing-txid:vin}{prev-txid:vout}{issuance}{reissuance_token}"`
+ * `"I{asset-id}{issuance-height}I{issuing-txid:vin}{is_reissuance}{amount}{tokens}" → ""`
+
+Peg-ins/peg-outs results in the following new rows (only for the native asset, typically L-BTC):
+
+ * `"I{asset-id}{pegin-height}F{pegin-txid:vin}{value}" → ""`
+ * `"I{asset-id}{pegout-height}F{pegout-txid:vout}{value}" → ""`
+
+Every burn (unspendable output) results in the following new row (both user-issued and native):
+
+ * `"I{asset-id}{burn-height}F{burning-txid:vout}{value}" → ""`
 
 ### `cache`
 
@@ -63,3 +83,11 @@ If the `blockhash` was since orphaned, the cache is removed and re-computed.
  * `"A{scripthash}" → "{stats}{blockhash}"` (where `stats` is composed of `tx_count`, `funded_txo_{count,sum}` and `spent_txo_{count,sum}`)
 
  * `"U{scripthash}" → "{utxo}{blockhash}"` (where `utxo` is a set of `(txid,vout)` outpoints)
+
+#### Elements only:
+
+Stats for issued assets:
+ * `"z{asset-id}" → "{issued_stats}{blockhash}"` (where `issued_stats` is composed of `tx_count`, `issuance_count`, `issued_amount`, `burned_amount`, `has_blinded_issuances`, `reissuance_tokens`, `burned_reissuance_tokens`)
+
+Stats for the native asset:
+ * `"z{issued-asset}" → "{native_stats}{blockhash}"` (where `native_stats` is composed of `tx_count`, `peg_in_count`, `peg_in_amount`, `peg_out_count`, `peg_out_amount`, `burn_count` and `burn_amount`)

docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.9'
+services:
+
+  mempool-electrs:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: on-failure
+
+    ports:
+      - 50001:50001
+
+    entrypoint:
+      /bin/electrs
+    command: |
+      --address-search
+      --cookie mempool:mempool
+      --db-dir /electrs
+      --cors '*'
+    volumes:
+    - 'electrs_data:/electrs'
+
+volumes:
+  electrs_data:

init-electrs-sockets

@@ -0,0 +1,24 @@
+#!/usr/bin/env zsh
+export ZPOOL=nvm
+
+export BITCOIN_HOME=/bitcoin
+export BITCOIN_USER=bitcoin
+export BITCOIN_GROUP=bitcoin
+
+export ELEMENTS_HOME=/elements
+export ELEMENTS_USER=elements
+export ELEMENTS_GROUP=elements
+
+# create /bitcoin/socket with custom ACL for electrs unix sockets
+zfs create -o "mountpoint=${BITCOIN_HOME}/socket" "${ZPOOL}/bitcoin/socket"
+
+# create /elements/socket with custom ACL for electrs unix sockets
+zfs create -o "mountpoint=${ELEMENTS_HOME}/socket" "${ZPOOL}/elements/socket"
+
+setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
+
+chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
+
+setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
+
+chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"

rust-toolchain

@@ -0,0 +1 @@
+1.87

scripts/checks.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+# This script is used for running all the checks
+# needed to make sure CI passes.
+
+# See below for the reasoning behind testing
+# clippy for all feature combinations.
+# (Note: clippy reports compile errors too)
+
+# You can pass extra args to cargo commands
+#   ./scripts/check.sh --release
+# will run in release mode. You can use that
+# if you already have release mode compiled, as
+# it will be faster to use the version you already compiled
+
+TESTNAME=""
+cleanup() {
+    exit_code=$?
+    if [[ ${exit_code} -ne 0 ]]; then
+        echo -e "\n\n##### Failed on \"$TESTNAME\"" 1>&2
+    fi
+    exit $exit_code
+}
+trap cleanup EXIT
+
+TESTNAME="Running cargo check"
+echo "$TESTNAME"
+cargo check $@ -q --all-features
+
+TESTNAME="Running cargo fmt check"
+echo "$TESTNAME"
+cargo fmt $@ -q --all -- --check
+
+# Testing all the combinations of clippy.
+# There were many instances where a certain struct
+# differed based on liquid or not(liquid) etc.
+# and the clippy fixes would break the other
+# feature combination.
+#
+# Be prepared to use #[allow(clippy::___)] attributes
+# to "fix" contradictions between feature sets.
+
+TESTNAME="Running cargo clippy check no features"
+echo "$TESTNAME"
+cargo clippy $@ -q
+
+TESTNAME="Running cargo clippy check electrum-discovery"
+echo "$TESTNAME"
+cargo clippy $@ -q -F electrum-discovery
+
+TESTNAME="Running cargo clippy check liquid"
+echo "$TESTNAME"
+cargo clippy $@ -q -F liquid
+
+TESTNAME="Running cargo clippy check electrum-discovery + liquid"
+echo "$TESTNAME"
+cargo clippy $@ -q -F electrum-discovery,liquid
+
+if [ $INCLUDE_TESTS ]; then
+    TESTNAME="Running cargo test with all features"
+    echo "$TESTNAME"
+    cargo test $@ -q --lib --all-features
+fi

src/app.rs

@@ -1,4 +1,4 @@
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
+use bitcoin::hashes::sha256d::Hash as Sha256dHash;
 use std::sync::{Arc, Mutex};
 
 use crate::{daemon, index, signal::Waiter, store};

src/bin/electrs.rs

@@ -1,4 +1,3 @@
-extern crate bitcoin;
 extern crate error_chain;
 #[macro_use]
 extern crate log;
@@ -21,75 +20,123 @@ use electrs::{
     signal::Waiter,
 };
 
+#[cfg(feature = "liquid")]
+use electrs::elements::AssetRegistry;
+
 fn fetch_from(config: &Config, store: &Store) -> FetchFrom {
     let mut jsonrpc_import = config.jsonrpc_import;
     if !jsonrpc_import {
-        jsonrpc_import = !store.is_empty();
+        // switch over to jsonrpc after the initial sync is done
+        jsonrpc_import = store.done_initial_sync();
     }
-    match jsonrpc_import {
-        true => FetchFrom::Bitcoind, // slower, uses JSONRPC (good for incremental updates)
-        false => FetchFrom::BlkFiles, // faster, uses blk*.dat files (good for initial indexing)
-    }
-}
 
-fn finish_verification(daemon: &Daemon, signal: &Waiter) -> Result<()> {
-    loop {
-        let progress = daemon.getblockchaininfo()?.verificationprogress;
-        if progress > 0.9999 {
-            return Ok(());
-        }
-        warn!(
-            "waiting for verification to finish: {:.3}%",
-            progress * 100.0
-        );
-        signal.wait(Duration::from_secs(5))?;
+    if jsonrpc_import {
+        // slower, uses JSONRPC (good for incremental updates)
+        FetchFrom::Bitcoind
+    } else {
+        // faster, uses blk*.dat files (good for initial indexing)
+        FetchFrom::BlkFiles
     }
 }
 
 fn run_server(config: Arc<Config>) -> Result<()> {
-    let signal = Waiter::new();
+    let signal = Waiter::start();
     let metrics = Metrics::new(config.monitoring_addr);
     metrics.start();
 
     let daemon = Arc::new(Daemon::new(
-        &config.daemon_dir,
+        config.daemon_dir.clone(),
+        config.blocks_dir.clone(),
         config.daemon_rpc_addr,
         config.cookie_getter(),
         config.network_type,
+        config.magic,
         signal.clone(),
         &metrics,
     )?);
-    finish_verification(&daemon, &signal)?;
-    let store = Arc::new(Store::open(&config.db_path.join("newindex")));
-    let mut indexer = Indexer::open(Arc::clone(&store), fetch_from(&config, &store), &metrics);
+    let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
+    let mut indexer = Indexer::open(
+        Arc::clone(&store),
+        fetch_from(&config, &store),
+        &config,
+        &metrics,
+    );
     let mut tip = indexer.update(&daemon)?;
 
-    let chain = Arc::new(ChainQuery::new(Arc::clone(&store), &metrics));
+    let chain = Arc::new(ChainQuery::new(
+        Arc::clone(&store),
+        Arc::clone(&daemon),
+        &config,
+        &metrics,
+    ));
 
-    if let Some(ref precache_file) = config.precache_scripts {
-        let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
-            .expect("cannot load scripts to precache");
-        precache::precache(&chain, precache_scripthashes);
+    let mempool = Arc::new(RwLock::new(Mempool::new(
+        Arc::clone(&chain),
+        &metrics,
+        Arc::clone(&config),
+    )));
+    loop {
+        match Mempool::update(&mempool, &daemon) {
+            Ok(_) => break,
+            Err(e) => {
+                warn!(
+                    "Error performing initial mempool update, trying again in 5 seconds: {}",
+                    e.display_chain()
+                );
+                signal.wait(Duration::from_secs(5), false)?;
+            }
+        }
     }
 
-    let mempool = Arc::new(RwLock::new(Mempool::new(Arc::clone(&chain), &metrics)));
-    mempool.write().unwrap().update(&daemon)?;
+    #[cfg(feature = "liquid")]
+    let asset_db = config.asset_db_path.as_ref().map(|db_dir| {
+        let asset_db = Arc::new(RwLock::new(AssetRegistry::new(db_dir.clone())));
+        AssetRegistry::spawn_sync(asset_db.clone());
+        asset_db
+    });
 
     let query = Arc::new(Query::new(
         Arc::clone(&chain),
         Arc::clone(&mempool),
         Arc::clone(&daemon),
+        Arc::clone(&config),
+        #[cfg(feature = "liquid")]
+        asset_db,
     ));
 
     // TODO: configuration for which servers to start
-    let rest_server = rest::run_server(Arc::clone(&config), Arc::clone(&query));
-    let electrum_server =
-        ElectrumRPC::start(config.electrum_rpc_addr, Arc::clone(&query), &metrics);
+    let rest_server = rest::start(Arc::clone(&config), Arc::clone(&query), &metrics);
+    let electrum_server = ElectrumRPC::start(Arc::clone(&config), Arc::clone(&query), &metrics);
+
+    if let Some(ref precache_file) = config.precache_scripts {
+        let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
+            .expect("cannot load scripts to precache");
+        precache::precache(
+            Arc::clone(&chain),
+            precache_scripthashes,
+            config.precache_threads,
+        );
+    }
 
     loop {
-        if let Err(err) = signal.wait(Duration::from_secs(5)) {
+        if let Err(err) = signal.wait(Duration::from_millis(config.main_loop_delay), true) {
             info!("stopping server: {}", err);
+
+            electrs::util::spawn_thread("shutdown-thread-checker", || {
+                let mut counter = 40;
+                let interval_ms = 500;
+
+                while counter > 0 {
+                    electrs::util::with_spawned_threads(|threads| {
+                        debug!("Threads during shutdown: {:?}", threads);
+                    });
+                    std::thread::sleep(std::time::Duration::from_millis(interval_ms));
+                    counter -= 1;
+                }
+            });
+
             rest_server.stop();
+            // the electrum server is stopped when dropped
             break;
         }
 
@@ -101,7 +148,13 @@ fn run_server(config: Arc<Config>) -> Result<()> {
         };
 
         // Update mempool
-        mempool.write().unwrap().update(&daemon)?;
+        if let Err(e) = Mempool::update(&mempool, &daemon) {
+            // Log the error if the result is an Err
+            warn!(
+                "Error updating mempool, skipping mempool update: {}",
+                e.display_chain()
+            );
+        }
 
         // Update subscribed clients
         electrum_server.notify();
@@ -116,4 +169,7 @@ fn main() {
         error!("server failed: {}", e.display_chain());
         process::exit(1);
     }
+    electrs::util::with_spawned_threads(|threads| {
+        debug!("Threads before closing: {:?}", threads);
+    });
 }

src/bin/popular-scripts.rs

@@ -1,48 +1,183 @@
 extern crate electrs;
 
-use bincode;
-use electrs::{
-    config::Config,
-    new_index::{Store, TxHistoryKey},
-};
+use std::{convert::TryInto, thread::ThreadId, time::Instant};
 
+use electrs::{config::Config, new_index::db::open_raw_db};
+use lazy_static::lazy_static;
+
+/*
+// How to run:
+export ELECTRS_DATA=/path/to/electrs
+cargo run \
+  -q --release --bin popular-scripts -- \
+  --db-dir $ELECTRS_DATA/db \
+  > ./contrib/popular-scripts.txt
+*/
+
+type DB = rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>;
+lazy_static! {
+    static ref HISTORY_DB: DB = {
+        let config = Config::from_args();
+        open_raw_db(
+            &config.db_path.join("newindex").join("history"),
+            electrs::new_index::db::OpenMode::ReadOnly,
+        )
+    };
+}
+
+// Dev note:
+// Only use println for file output (lines for output)
+// Use eprintln to print to stderr for dev notifications
 fn main() {
-    let config = Config::from_args();
-    let store = Store::open(&config.db_path.join("newindex"));
+    let high_usage_threshold = std::env::var("HIGH_USAGE_THRESHOLD")
+        .ok()
+        .and_then(|s| s.parse::<u32>().ok())
+        .unwrap_or(4000);
+    let thread_count = std::env::var("JOB_THREAD_COUNT")
+        .ok()
+        .and_then(|s| s.parse::<usize>().ok())
+        .unwrap_or(4);
+    eprintln!(
+        "Seaching for scripts with history rows of {} or more...",
+        high_usage_threshold
+    );
 
-    let mut iter = store.history_db().raw_iterator();
-    iter.seek(b"H");
+    let thread_pool = rayon::ThreadPoolBuilder::new()
+        .num_threads(thread_count)
+        .build()
+        .expect("Built threadpool");
+
+    let (sender, receiver) = crossbeam_channel::unbounded::<[u8; 32]>();
+
+    let increment = 256 / thread_count;
+    let bytes: Vec<u8> = (0u8..=255u8)
+        .filter(|n| *n % increment as u8 == 0)
+        .collect();
+
+    let now = Instant::now();
+    for i in 0..bytes.len() {
+        let sender = sender.clone();
+        let first_byte = bytes[i];
+        let second_byte = bytes.get(i + 1).copied();
+
+        thread_pool.spawn(move || {
+            let id = std::thread::current().id();
+            run_iterator(
+                id,
+                &HISTORY_DB,
+                high_usage_threshold,
+                first_byte,
+                second_byte,
+                sender,
+                now,
+            );
+            eprintln!("{id:?} Finished its job!");
+        })
+    }
+    // If we don't drop this sender
+    // the receiver will hang forever
+    drop(sender);
+
+    while let Ok(script) = receiver.recv() {
+        println!("{}", hex::encode(script));
+    }
+    eprintln!("Finished!!!!");
+}
+
+fn run_iterator(
+    thread_id: ThreadId,
+    db: &DB,
+    high_usage_threshold: u32,
+    first_byte: u8,
+    next_byte: Option<u8>,
+    sender: crossbeam_channel::Sender<[u8; 32]>,
+    now: Instant,
+) {
+    let mut iter = db.raw_iterator();
+    eprintln!(
+        "Thread ({thread_id:?}) Seeking DB to beginning of tx histories for b'H' + {}",
+        hex::encode([first_byte])
+    );
+    let mut compare_vec: Vec<u8> = vec![b'H', first_byte];
+    iter.seek(&compare_vec); // Seek to beginning of our section
+
+    // Insert the byte of the next section for comparing
+    // This will tell us when to stop with a closure
+    type Checker<'a> = Box<dyn Fn(&[u8]) -> bool + 'a>;
+    let is_finished: Checker<'_> = if let Some(next) = next_byte {
+        // Modify the vec to what we're looking for next
+        // to indicate we left our section
+        compare_vec[1] = next;
+        Box::new(|key: &[u8]| -> bool { key.starts_with(&compare_vec) })
+    } else {
+        // Modify the vec to only have H so we know when we left H
+        compare_vec.remove(1);
+        Box::new(|key: &[u8]| -> bool { !key.starts_with(&compare_vec) })
+    };
+
+    eprintln!("Thread ({thread_id:?}) Seeking done");
 
     let mut curr_scripthash = [0u8; 32];
-    let mut total_entries = 0;
+    let mut total_entries: usize = 0;
+    let mut iter_index: usize = 1;
 
     while iter.valid() {
         let key = iter.key().unwrap();
 
-        if !key.starts_with(b"H") {
+        if key.is_empty() || key[0] != b'H' || is_finished(key) {
+            // We have left the txhistory section,
+            // but we need to check the final scripthash
+            send_if_popular(
+                high_usage_threshold,
+                total_entries,
+                curr_scripthash,
+                &sender,
+            );
             break;
         }
 
-        let entry: TxHistoryKey = bincode::config()
-            .big_endian()
-            .deserialize(&key)
-            .expect("failed to deserialize TxHistoryKey");
+        if iter_index % 10_000_000 == 0 {
+            let duration = now.elapsed().as_secs();
+            eprintln!(
+                "Thread ({thread_id:?}) Processing row #{iter_index}... {duration} seconds elapsed"
+            );
+        }
 
-        if curr_scripthash != entry.scripthash {
-            if total_entries > 100 {
-                println!("{} {}", hex::encode(&curr_scripthash), total_entries);
-            }
+        // We know that the TxHistory key is 1 byte "H" followed by
+        // 32 byte scripthash
+        let entry_hash: [u8; 32] = key[1..33].try_into().unwrap();
 
-            curr_scripthash = entry.scripthash;
+        if curr_scripthash != entry_hash {
+            // We have rolled on to a new scripthash
+            // If the last scripthash was popular
+            // Collect for sorting
+            send_if_popular(
+                high_usage_threshold,
+                total_entries,
+                curr_scripthash,
+                &sender,
+            );
+
+            // After collecting, reset values for next scripthash
+            curr_scripthash = entry_hash;
             total_entries = 0;
         }
 
-        total_entries = total_entries + 1;
+        total_entries += 1;
+        iter_index += 1;
 
         iter.next();
     }
+}
 
-    if total_entries > 100 {
-        println!("{} {}", hex::encode(&curr_scripthash), total_entries);
+#[inline]
+fn send_if_popular(
+    high_usage_threshold: u32,
+    total_entries: usize,
+    curr_scripthash: [u8; 32],
+    sender: &crossbeam_channel::Sender<[u8; 32]>,
+) {
+    if total_entries >= high_usage_threshold as usize {
+        sender.send(curr_scripthash).unwrap();
     }
 }

src/bin/tx-fingerprint-stats.rs

@@ -1,4 +1,6 @@
 extern crate electrs;
+
+#[cfg(not(feature = "liquid"))]
 #[macro_use]
 extern crate log;
 
@@ -7,7 +9,7 @@ fn main() {
     use std::collections::HashSet;
     use std::sync::Arc;
 
-    use bitcoin::blockdata::script::Script;
+    use bitcoin::blockdata::script::ScriptBuf;
     use bitcoin::consensus::encode::deserialize;
     use electrs::{
         chain::Transaction,
@@ -19,28 +21,30 @@ fn main() {
         util::has_prevout,
     };
 
-    let signal = Waiter::new();
+    let signal = Waiter::start();
     let config = Config::from_args();
-    let store = Arc::new(Store::open(&config.db_path.join("newindex")));
+    let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
 
     let metrics = Metrics::new(config.monitoring_addr);
     metrics.start();
 
-    let chain = ChainQuery::new(Arc::clone(&store), &metrics);
-
     let daemon = Arc::new(
         Daemon::new(
-            &config.daemon_dir,
+            config.daemon_dir.clone(),
+            config.blocks_dir.clone(),
             config.daemon_rpc_addr,
             config.cookie_getter(),
             config.network_type,
-            signal.clone(),
+            config.magic,
+            signal,
             &metrics,
         )
         .unwrap(),
     );
 
-    let mut indexer = Indexer::open(Arc::clone(&store), FetchFrom::Bitcoind, &metrics);
+    let chain = ChainQuery::new(Arc::clone(&store), Arc::clone(&daemon), &config, &metrics);
+
+    let mut indexer = Indexer::open(Arc::clone(&store), FetchFrom::Bitcoind, &config, &metrics);
     indexer.update(&daemon).unwrap();
 
     let mut iter = store.txstore_db().raw_iterator();
@@ -57,17 +61,17 @@ fn main() {
             break;
         }
 
-        iter.next();
+        let tx: Transaction = deserialize(value).expect("failed to parse Transaction");
+        let txid = tx.compute_txid();
 
-        let tx: Transaction = deserialize(&value).expect("failed to parse Transaction");
-        let txid = tx.txid();
+        iter.next();
 
         // only consider transactions of exactly two outputs
         if tx.output.len() != 2 {
             continue;
         }
         // skip coinbase txs
-        if tx.is_coin_base() {
+        if tx.is_coinbase() {
             continue;
         }
 
@@ -87,12 +91,26 @@ fn main() {
                 .collect(),
         );
 
-        let total_out: u64 = tx.output.iter().map(|out| out.value).sum();
-        let small_out = tx.output.iter().map(|out| out.value).min().unwrap();
-        let large_out = tx.output.iter().map(|out| out.value).max().unwrap();
+        let total_out: u64 = tx.output.iter().map(|out| out.value.to_sat()).sum();
+        let small_out = tx
+            .output
+            .iter()
+            .map(|out| out.value.to_sat())
+            .min()
+            .unwrap();
+        let large_out = tx
+            .output
+            .iter()
+            .map(|out| out.value.to_sat())
+            .max()
+            .unwrap();
 
-        let total_in: u64 = prevouts.values().map(|out| out.value).sum();
-        let smallest_in = prevouts.values().map(|out| out.value).min().unwrap();
+        let total_in: u64 = prevouts.values().map(|out| out.value.to_sat()).sum();
+        let smallest_in = prevouts
+            .values()
+            .map(|out| out.value.to_sat())
+            .min()
+            .unwrap();
 
         let fee = total_in - total_out;
 
@@ -115,7 +133,7 @@ fn main() {
 
         // test for sending back to one of the spent spks
         let has_reuse = {
-            let prev_spks: HashSet<Script> = prevouts
+            let prev_spks: HashSet<ScriptBuf> = prevouts
                 .values()
                 .map(|out| out.script_pubkey.clone())
                 .collect();
@@ -129,8 +147,8 @@ fn main() {
             txid, blockid.height, tx.lock_time, uih, is_multi_spend as u8, has_reuse as u8
         );
 
-        total = total + 1;
-        uih_totals[uih] = uih_totals[uih] + 1;
+        total += 1;
+        uih_totals[uih] += 1;
     }
     info!(
         "processed {} total txs, UIH counts: {:?}",

src/chain.rs

@@ -1,14 +1,65 @@
+use std::str::FromStr;
+
 #[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures
-pub use bitcoin::{Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut};
+pub use bitcoin::{
+    address,
+    block::Header as BlockHeader,
+    blockdata::{opcodes, script},
+    consensus::deserialize,
+    hashes, Block, BlockHash, OutPoint, ScriptBuf as Script, Transaction, TxIn, TxOut, Txid,
+    Witness,
+};
 
 #[cfg(feature = "liquid")]
-pub use elements::{confidential, Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut};
+pub use {
+    crate::elements::asset,
+    elements::{
+        address, bitcoin::bech32::Hrp, confidential, encode::deserialize, hashes, opcodes, script,
+        Address, AssetId, Block, BlockHash, BlockHeader, OutPoint, Script, Transaction, TxIn,
+        TxInWitness as Witness, TxOut, Txid,
+    },
+};
 
 use bitcoin::blockdata::constants::genesis_block;
-use bitcoin::network::constants::Network as BNetwork;
-use bitcoin::util::hash::BitcoinHash;
-use bitcoin_bech32::constants::Network as B32Network;
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
+pub use bitcoin::Network as BNetwork;
+
+// Extension trait for getting txid in a cross-compatible way
+pub trait TxidCompat {
+    fn get_txid(&self) -> Txid;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl TxidCompat for Transaction {
+    fn get_txid(&self) -> Txid {
+        self.compute_txid()
+    }
+}
+
+#[cfg(feature = "liquid")]
+impl TxidCompat for Transaction {
+    fn get_txid(&self) -> Txid {
+        self.txid()
+    }
+}
+
+// Extension trait for getting block size in a cross-compatible way
+pub trait BlockSizeCompat {
+    fn get_block_size(&self) -> usize;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl BlockSizeCompat for Block {
+    fn get_block_size(&self) -> usize {
+        self.total_size()
+    }
+}
+
+#[cfg(feature = "liquid")]
+impl BlockSizeCompat for Block {
+    fn get_block_size(&self) -> usize {
+        self.size()
+    }
+}
 
 #[cfg(not(feature = "liquid"))]
 pub type Value = u64;
@@ -17,32 +68,92 @@ pub use confidential::Value;
 
 #[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize, Ord, PartialOrd, Eq)]
 pub enum Network {
+    #[cfg(not(feature = "liquid"))]
     Bitcoin,
+    #[cfg(not(feature = "liquid"))]
     Testnet,
+    #[cfg(not(feature = "liquid"))]
+    Testnet4,
+    #[cfg(not(feature = "liquid"))]
     Regtest,
+    #[cfg(not(feature = "liquid"))]
+    Signet,
 
     #[cfg(feature = "liquid")]
     Liquid,
     #[cfg(feature = "liquid")]
+    LiquidTestnet,
+    #[cfg(feature = "liquid")]
     LiquidRegtest,
 }
 
+#[cfg(feature = "liquid")]
+pub const LIQUID_TESTNET_PARAMS: address::AddressParams = address::AddressParams {
+    p2pkh_prefix: 36,
+    p2sh_prefix: 19,
+    blinded_prefix: 23,
+    bech_hrp: Hrp::parse_unchecked("tex"),
+    blech_hrp: Hrp::parse_unchecked("tlq"),
+};
+
+/// Magic for testnet4, 0x1c163f28 (from BIP94) with flipped endianness.
+#[cfg(not(feature = "liquid"))]
+const TESTNET4_MAGIC: u32 = 0x283f161c;
+
 impl Network {
-    pub fn genesis_hash(&self) -> Sha256dHash {
-        let block = genesis_block(BNetwork::from(self));
-        block.bitcoin_hash()
+    #[cfg(not(feature = "liquid"))]
+    pub fn magic(self) -> u32 {
+        match self {
+            Self::Testnet4 => TESTNET4_MAGIC,
+            _ => {
+                let magic = BNetwork::from(self).magic();
+                u32::from_le_bytes(magic.to_bytes())
+            }
+        }
     }
 
-    pub fn magic(&self) -> u32 {
+    #[cfg(feature = "liquid")]
+    pub fn magic(self) -> u32 {
         match self {
-            Network::Bitcoin => 0xD9B4BEF9,
-            Network::Testnet => 0x0709110B,
-            Network::Regtest => 0xDAB5BFFA,
+            Network::Liquid | Network::LiquidRegtest => 0xDAB5_BFFA,
+            Network::LiquidTestnet => 0x62DD_0E41,
+        }
+    }
 
+    pub fn is_regtest(self) -> bool {
+        match self {
+            #[cfg(not(feature = "liquid"))]
+            Network::Regtest => true,
             #[cfg(feature = "liquid")]
-            Network::Liquid => 0xDAB5BFFA,
-            #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => 0xDAB5BFFA,
+            Network::LiquidRegtest => true,
+            _ => false,
+        }
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn address_params(self) -> &'static address::AddressParams {
+        // Liquid regtest uses elements's address params
+        match self {
+            Network::Liquid => &address::AddressParams::LIQUID,
+            Network::LiquidRegtest => &address::AddressParams::ELEMENTS,
+            Network::LiquidTestnet => &LIQUID_TESTNET_PARAMS,
+        }
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn native_asset(self) -> &'static AssetId {
+        match self {
+            Network::Liquid => &asset::NATIVE_ASSET_ID,
+            Network::LiquidTestnet => &asset::NATIVE_ASSET_ID_TESTNET,
+            Network::LiquidRegtest => &asset::NATIVE_ASSET_ID_REGTEST,
+        }
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn pegged_asset(self) -> Option<&'static AssetId> {
+        match self {
+            Network::Liquid => Some(&*asset::NATIVE_ASSET_ID),
+            Network::LiquidTestnet | Network::LiquidRegtest => None,
         }
     }
 
@@ -52,29 +163,97 @@ impl Network {
             "mainnet".to_string(),
             "testnet".to_string(),
             "regtest".to_string(),
+            "signet".to_string(),
         ];
 
         #[cfg(feature = "liquid")]
         return vec![
-            "mainnet".to_string(),
-            "testnet".to_string(),
-            "regtest".to_string(),
             "liquid".to_string(),
+            "liquidtestnet".to_string(),
             "liquidregtest".to_string(),
         ];
     }
 }
 
+pub fn genesis_hash(network: Network) -> BlockHash {
+    #[cfg(not(feature = "liquid"))]
+    return bitcoin_genesis_hash(network);
+    #[cfg(feature = "liquid")]
+    return liquid_genesis_hash(network);
+}
+
+pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash {
+    lazy_static! {
+        static ref BITCOIN_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Bitcoin).block_hash();
+        static ref TESTNET_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Testnet).block_hash();
+        static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str(
+            "00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043"
+        )
+        .unwrap();
+        static ref REGTEST_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Regtest).block_hash();
+        static ref SIGNET_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Signet).block_hash();
+    }
+    #[cfg(not(feature = "liquid"))]
+    match network {
+        Network::Bitcoin => *BITCOIN_GENESIS,
+        Network::Testnet => *TESTNET_GENESIS,
+        Network::Testnet4 => *TESTNET4_GENESIS,
+        Network::Regtest => *REGTEST_GENESIS,
+        Network::Signet => *SIGNET_GENESIS,
+    }
+    #[cfg(feature = "liquid")]
+    match network {
+        Network::Liquid => *BITCOIN_GENESIS,
+        Network::LiquidTestnet => *TESTNET_GENESIS,
+        Network::LiquidRegtest => *REGTEST_GENESIS,
+    }
+}
+
+#[cfg(feature = "liquid")]
+pub fn liquid_genesis_hash(network: Network) -> elements::BlockHash {
+    lazy_static! {
+        static ref LIQUID_GENESIS: BlockHash =
+            "1466275836220db2944ca059a3a10ef6fd2ea684b0688d2c379296888a206003"
+                .parse()
+                .unwrap();
+        static ref ZERO_HASH: BlockHash =
+            "0000000000000000000000000000000000000000000000000000000000000000"
+                .parse()
+                .unwrap();
+    }
+
+    match network {
+        Network::Liquid => *LIQUID_GENESIS,
+        // The genesis block for liquid regtest chains varies based on the chain configuration.
+        // This instead uses an all zeroed-out hash, which doesn't matter in practice because its
+        // only used for Electrum server discovery, which isn't active on regtest.
+        _ => *ZERO_HASH,
+    }
+}
+
 impl From<&str> for Network {
     fn from(network_name: &str) -> Self {
         match network_name {
+            #[cfg(not(feature = "liquid"))]
             "mainnet" => Network::Bitcoin,
+            #[cfg(not(feature = "liquid"))]
             "testnet" => Network::Testnet,
+            #[cfg(not(feature = "liquid"))]
+            "testnet4" => Network::Testnet4,
+            #[cfg(not(feature = "liquid"))]
             "regtest" => Network::Regtest,
+            #[cfg(not(feature = "liquid"))]
+            "signet" => Network::Signet,
 
             #[cfg(feature = "liquid")]
             "liquid" => Network::Liquid,
             #[cfg(feature = "liquid")]
+            "liquidtestnet" => Network::LiquidTestnet,
+            #[cfg(feature = "liquid")]
             "liquidregtest" => Network::LiquidRegtest,
 
             _ => panic!("unsupported Bitcoin network: {:?}", network_name),
@@ -82,48 +261,28 @@ impl From<&str> for Network {
     }
 }
 
-impl From<&Network> for BNetwork {
-    fn from(network: &Network) -> Self {
+#[cfg(not(feature = "liquid"))]
+impl From<Network> for BNetwork {
+    fn from(network: Network) -> Self {
         match network {
             Network::Bitcoin => BNetwork::Bitcoin,
             Network::Testnet => BNetwork::Testnet,
+            Network::Testnet4 => BNetwork::Testnet4,
             Network::Regtest => BNetwork::Regtest,
-
-            #[cfg(feature = "liquid")]
-            Network::Liquid => BNetwork::Bitcoin, // @FIXME
-            #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => BNetwork::Regtest, // @FIXME
+            Network::Signet => BNetwork::Signet,
         }
     }
 }
 
-impl From<&Network> for B32Network {
-    fn from(network: &Network) -> Self {
+#[cfg(not(feature = "liquid"))]
+impl From<BNetwork> for Network {
+    fn from(network: BNetwork) -> Self {
         match network {
-            Network::Bitcoin => B32Network::Bitcoin,
-            Network::Testnet => B32Network::Testnet,
-            Network::Regtest => B32Network::Regtest,
-            #[cfg(feature = "liquid")]
-            Network::Liquid => B32Network::Bitcoin, // @FIXME
-            #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => B32Network::Regtest, // @FIXME
-        }
-    }
-}
-
-impl From<&BNetwork> for Network {
-    fn from(network: &BNetwork) -> Self {
-        match network {
-            #[cfg(not(feature = "liquid"))]
             BNetwork::Bitcoin => Network::Bitcoin,
-            #[cfg(not(feature = "liquid"))]
+            BNetwork::Testnet => Network::Testnet,
+            BNetwork::Testnet4 => Network::Testnet4,
             BNetwork::Regtest => Network::Regtest,
-
-            #[cfg(feature = "liquid")]
-            BNetwork::Bitcoin => Network::Liquid, // @FIXME
-            #[cfg(feature = "liquid")]
-            BNetwork::Regtest => Network::LiquidRegtest, // @FIXME
-            BNetwork::Testnet => Network::Testnet, // @FIXME
+            BNetwork::Signet => Network::Signet,
         }
     }
 }

src/config.rs

@@ -1,55 +1,109 @@
 use clap::{App, Arg};
 use dirs::home_dir;
-use num_cpus;
 use std::fs;
 use std::net::SocketAddr;
+use std::net::ToSocketAddrs;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 use stderrlog;
 
-#[cfg(feature = "liquid")]
-use bitcoin_hashes::hex::ToHex;
-
 use crate::chain::Network;
 use crate::daemon::CookieGetter;
-
 use crate::errors::*;
 
+#[cfg(feature = "liquid")]
+use bitcoin::Network as BNetwork;
+
+pub(crate) const APP_NAME: &str = "mempool-electrs";
+pub(crate) const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION");
+pub(crate) const GIT_HASH: Option<&str> = option_env!("GIT_HASH");
+// This will be set only once in the Daemon::new() constructor at startup
+pub(crate) static BITCOIND_SUBVER: OnceLock<String> = OnceLock::new();
+
+lazy_static! {
+    pub(crate) static ref VERSION_STRING: String = {
+        if let Some(hash) = GIT_HASH {
+            format!("{} {}-{}", APP_NAME, ELECTRS_VERSION, hash)
+        } else {
+            format!("{} {}", APP_NAME, ELECTRS_VERSION)
+        }
+    };
+}
+
 #[derive(Debug, Clone)]
 pub struct Config {
     // See below for the documentation of each field:
     pub log: stderrlog::StdErrLog,
     pub network_type: Network,
+    pub magic: Option<u32>,
     pub db_path: PathBuf,
     pub daemon_dir: PathBuf,
+    pub blocks_dir: PathBuf,
     pub daemon_rpc_addr: SocketAddr,
     pub cookie: Option<String>,
     pub electrum_rpc_addr: SocketAddr,
     pub http_addr: SocketAddr,
+    pub http_socket_file: Option<PathBuf>,
+    pub rpc_socket_file: Option<PathBuf>,
     pub monitoring_addr: SocketAddr,
     pub jsonrpc_import: bool,
-    pub index_batch_size: usize,
-    pub bulk_index_threads: usize,
-    pub tx_cache_size: usize,
-    pub prevout_enabled: bool,
+    pub light_mode: bool,
+    pub main_loop_delay: u64,
+    pub address_search: bool,
+    pub index_unspendables: bool,
     pub cors: Option<String>,
     pub precache_scripts: Option<String>,
+    pub precache_threads: usize,
+    pub utxos_limit: usize,
+    pub electrum_txs_limit: usize,
+    pub electrum_banner: String,
+    pub mempool_backlog_stats_ttl: u64,
+    pub mempool_recent_txs_size: usize,
+    pub rest_default_block_limit: usize,
+    pub rest_default_chain_txs_per_page: usize,
+    pub rest_default_max_mempool_txs: usize,
+    pub rest_default_max_address_summary_txs: usize,
+    pub rest_max_mempool_page_size: usize,
+    pub rest_max_mempool_txid_page_size: usize,
+    pub electrum_max_line_size: usize,
+    pub electrum_max_subscriptions: usize,
+    pub electrum_max_clients: usize,
+    pub electrum_idle_timeout: u64,
+    pub electrum_haproxy_depth: usize,
+    pub electrum_connections_per_client: usize,
+    pub electrum_public_hosts: Option<crate::electrum::ServerHosts>,
 
     #[cfg(feature = "liquid")]
-    pub parent_network: Network,
+    pub parent_network: BNetwork,
     #[cfg(feature = "liquid")]
-    pub parent_genesis_hash: String,
+    pub asset_db_path: Option<PathBuf>,
+
+    #[cfg(feature = "electrum-discovery")]
+    pub electrum_announce: bool,
+    #[cfg(feature = "electrum-discovery")]
+    pub tor_proxy: Option<std::net::SocketAddr>,
+}
+
+fn str_to_socketaddr(address: &str, what: &str) -> SocketAddr {
+    address
+        .to_socket_addrs()
+        .unwrap_or_else(|_| panic!("unable to resolve {} address", what))
+        .collect::<Vec<_>>()
+        .pop()
+        .unwrap()
 }
 
 impl Config {
     pub fn from_args() -> Config {
-        let network_help = format!(
-            "Select Bitcoin network type ({})",
-            Network::names().join(", ")
-        );
+        let network_help = format!("Select network type ({})", Network::names().join(", "));
 
-        let args = App::new("Electrum Rust Server")
+        let args = App::new("Mempool Electrum Rust Server")
             .version(crate_version!())
+            .arg(
+                Arg::with_name("version")
+                    .long("version")
+                    .help("Print out the version of this app and quit immediately."),
+            )
             .arg(
                 Arg::with_name("verbosity")
                     .short("v")
@@ -73,6 +127,12 @@ impl Config {
                     .help("Data directory of Bitcoind (default: ~/.bitcoin/)")
                     .takes_value(true),
             )
+            .arg(
+                Arg::with_name("blocks_dir")
+                    .long("blocks-dir")
+                    .help("Analogous to bitcoind's -blocksdir option, this specifies the directory containing the raw blocks files (blk*.dat) (default: ~/.bitcoin/blocks/)")
+                    .takes_value(true),
+            )
             .arg(
                 Arg::with_name("cookie")
                     .long("cookie")
@@ -85,6 +145,12 @@ impl Config {
                     .help(&network_help)
                     .takes_value(true),
             )
+            .arg(
+                Arg::with_name("magic")
+                    .long("magic")
+                    .default_value("")
+                    .takes_value(true),
+            )
             .arg(
                 Arg::with_name("electrum_rpc_addr")
                     .long("electrum-rpc-addr")
@@ -115,27 +181,25 @@ impl Config {
                     .help("Use JSONRPC instead of directly importing blk*.dat files. Useful for remote full node or low memory system"),
             )
             .arg(
-                Arg::with_name("index_batch_size")
-                    .long("index-batch-size")
-                    .help("Number of blocks to get in one JSONRPC request from bitcoind")
-                    .default_value("100"),
+                Arg::with_name("light_mode")
+                    .long("lightmode")
+                    .help("Enable light mode for reduced storage")
             )
             .arg(
-                Arg::with_name("bulk_index_threads")
-                    .long("bulk-index-threads")
-                    .help("Number of threads used for bulk indexing (default: use the # of CPUs)")
-                    .default_value("0")
+                Arg::with_name("main_loop_delay")
+                    .long("main-loop-delay")
+                    .help("The number of milliseconds the main loop will wait between loops. (Can be shortened with SIGUSR1)")
+                    .default_value("500")
             )
             .arg(
-                Arg::with_name("tx_cache_size")
-                    .long("tx-cache-size")
-                    .help("Number of transactions to keep in for query LRU cache")
-                    .default_value("10000")  // should be enough for a small wallet.
+                Arg::with_name("address_search")
+                    .long("address-search")
+                    .help("Enable prefix address search")
             )
             .arg(
-                Arg::with_name("disable_prevout")
-                    .long("disable-prevout")
-                    .help("Don't attach previous output details to inputs")
+                Arg::with_name("index_unspendables")
+                    .long("index-unspendables")
+                    .help("Enable indexing of provably unspendable outputs")
             )
             .arg(
                 Arg::with_name("cors")
@@ -148,110 +212,329 @@ impl Config {
                     .long("precache-scripts")
                     .help("Path to file with list of scripts to pre-cache")
                     .takes_value(true)
+            )
+            .arg(
+                Arg::with_name("precache_threads")
+                    .long("precache-threads")
+                    .help("Non-zero number of threads to use for precache threadpool. [default: 4 * CORE_COUNT]")
+                    .takes_value(true)
+            )
+            .arg(
+                Arg::with_name("utxos_limit")
+                    .long("utxos-limit")
+                    .help("Maximum number of utxos to process per address. Lookups for addresses with more utxos will fail. Applies to the Electrum and HTTP APIs.")
+                    .default_value("500")
+            )
+            .arg(
+                Arg::with_name("mempool_backlog_stats_ttl")
+                    .long("mempool-backlog-stats-ttl")
+                    .help("The number of seconds that need to pass before Mempool::update will update the latency histogram again.")
+                    .default_value("10")
+            )
+            .arg(
+                Arg::with_name("mempool_recent_txs_size")
+                    .long("mempool-recent-txs-size")
+                    .help("The number of transactions that mempool will keep in its recents queue. This is returned by mempool/recent endpoint.")
+                    .default_value("10")
+            )
+            .arg(
+                Arg::with_name("rest_default_block_limit")
+                    .long("rest-default-block-limit")
+                    .help("The default number of blocks returned from the blocks/[start_height] endpoint.")
+                    .default_value("10")
+            )
+            .arg(
+                Arg::with_name("rest_default_chain_txs_per_page")
+                    .long("rest-default-chain-txs-per-page")
+                    .help("The default number of on-chain transactions returned by the txs endpoints.")
+                    .default_value("25")
+            )
+            .arg(
+                Arg::with_name("rest_default_max_mempool_txs")
+                    .long("rest-default-max-mempool-txs")
+                    .help("The default number of mempool transactions returned by the txs endpoints.")
+                    .default_value("50")
+            )
+            .arg(
+                Arg::with_name("rest_default_max_address_summary_txs")
+                    .long("rest-default-max-address-summary-txs")
+                    .help("The default number of transactions returned by the address summary endpoints.")
+                    .default_value("5000")
+            )
+            .arg(
+                Arg::with_name("rest_max_mempool_page_size")
+                    .long("rest-max-mempool-page-size")
+                    .help("The maximum number of transactions returned by the paginated /internal/mempool/txs endpoint.")
+                    .default_value("1000")
+            )
+            .arg(
+                Arg::with_name("rest_max_mempool_txid_page_size")
+                    .long("rest-max-mempool-txid-page-size")
+                    .help("The maximum number of transactions returned by the paginated /mempool/txids/page endpoint.")
+                    .default_value("10000")
+            )
+            .arg(
+                Arg::with_name("electrum_txs_limit")
+                    .long("electrum-txs-limit")
+                    .help("Maximum number of transactions returned by Electrum history queries. Lookups with more results will fail.")
+                    .default_value("500")
+            ).arg(
+                Arg::with_name("electrum_banner")
+                    .long("electrum-banner")
+                    .help("Welcome banner for the Electrum server, shown in the console to clients.")
+                    .takes_value(true)
+            ).arg(
+                Arg::with_name("electrum_max_line_size")
+                    .long("electrum-max-line-size")
+                    .help("Maximum size of a single Electrum request line in bytes (default: 1 MiB).")
+                    .default_value("1048576")
+            ).arg(
+                Arg::with_name("electrum_max_subscriptions")
+                    .long("electrum-max-subscriptions")
+                    .help("Maximum number of scripthash subscriptions per client connection.")
+                    .default_value("100")
+            ).arg(
+                Arg::with_name("electrum_max_clients")
+                    .long("electrum-max-clients")
+                    .help("Maximum number of concurrent Electrum client connections.")
+                    .default_value("10")
+            ).arg(
+                Arg::with_name("electrum_idle_timeout")
+                    .long("electrum-idle-timeout")
+                    .help("Maximum idle time in seconds since the last client request before disconnecting the Electrum connection.")
+                    .default_value("600")
+            ).arg(
+                Arg::with_name("electrum_haproxy_depth")
+                    .long("electrum-haproxy-depth")
+                    .help("Which HAProxy PROXY-protocol header layer identifies the real client IP. 0 disables PROXY-protocol detection; 1 uses the first (outermost) address, 2 the second, and so on. If the requested layer or any PROXY header is absent, no client IP is associated with the connection.")
+                    .default_value("0")
+            ).arg(
+                Arg::with_name("electrum_connections_per_client")
+                    .long("electrum-connections-per-client")
+                    .help("Maximum number of concurrent Electrum connections allowed per client (keyed by the HAProxy-reported address when available, otherwise the peer IP). 0 disables the per-client limit.")
+                    .default_value("10")
+            );
+
+        #[cfg(unix)]
+        let args = args.arg(
+                Arg::with_name("http_socket_file")
+                    .long("http-socket-file")
+                    .help("HTTP server 'unix socket file' to listen on (default disabled, enabling this disables the http server)")
+                    .takes_value(true),
+            );
+
+        #[cfg(unix)]
+        let args = args.arg(
+                Arg::with_name("rpc_socket_file")
+                    .long("rpc-socket-file")
+                    .help("Electrum RPC 'unix socket file' to listen on (default disabled, enabling this ignores the electrum_rpc_addr arg)")
+                    .takes_value(true),
             );
 
         #[cfg(feature = "liquid")]
+        let args = args
+            .arg(
+                Arg::with_name("parent_network")
+                    .long("parent-network")
+                    .help("Select parent network type (mainnet, testnet, regtest)")
+                    .takes_value(true),
+            )
+            .arg(
+                Arg::with_name("asset_db_path")
+                    .long("asset-db-path")
+                    .help("Directory for liquid/elements asset db")
+                    .takes_value(true),
+            );
+
+        #[cfg(feature = "electrum-discovery")]
         let args = args.arg(
-            Arg::with_name("parent_network")
-                .long("parent-network")
-                .help("Select parent network type (mainnet, testnet, regtest)")
+                Arg::with_name("electrum_public_hosts")
+                    .long("electrum-public-hosts")
+                    .help("A dictionary of hosts where the Electrum server can be reached at. Required to enable server discovery. See https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-features")
+                    .takes_value(true)
+            ).arg(
+                Arg::with_name("electrum_announce")
+                    .long("electrum-announce")
+                    .help("Announce the Electrum server to other servers")
+            ).arg(
+            Arg::with_name("tor_proxy")
+                .long("tor-proxy")
+                .help("ip:addr of socks proxy for accessing onion hosts")
                 .takes_value(true),
         );
 
         let m = args.get_matches();
 
+        if m.is_present("version") {
+            eprintln!("{}", *VERSION_STRING);
+            std::process::exit(0);
+        }
+
         let network_name = m.value_of("network").unwrap_or("mainnet");
         let network_type = Network::from(network_name);
+        let magic: Option<u32> = m
+            .value_of("magic")
+            .filter(|s| !s.is_empty())
+            .map(|s| u32::from_str_radix(s, 16).expect("invalid network magic"));
         let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db"));
         let db_path = db_dir.join(network_name);
 
         #[cfg(feature = "liquid")]
-        let parent_network = Network::from(m.value_of("parent_network").unwrap_or("mainnet"));
+        let parent_network = m
+            .value_of("parent_network")
+            .map(|s| s.parse().expect("invalid parent network"))
+            .unwrap_or_else(|| match network_type {
+                Network::Liquid => BNetwork::Bitcoin,
+                // XXX liquid testnet/regtest don't have a parent chain
+                Network::LiquidTestnet | Network::LiquidRegtest => BNetwork::Regtest,
+            });
+
         #[cfg(feature = "liquid")]
-        let parent_genesis_hash = parent_network.genesis_hash().to_hex();
+        let asset_db_path = m.value_of("asset_db_path").map(PathBuf::from);
 
         let default_daemon_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 8332,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 18332,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 18443,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 38332,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 48332,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 7041,
             #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => 7041,
+            Network::LiquidTestnet | Network::LiquidRegtest => 7040,
         };
         let default_electrum_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 50001,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 60001,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 40001,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 60401,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 60601,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 51000,
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => 51301,
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => 51401,
         };
         let default_http_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 3000,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 3001,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 3002,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 3003,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 3004,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 3000,
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => 3001,
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => 3002,
         };
         let default_monitoring_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 4224,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 14224,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 24224,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 44224,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 54224,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 34224,
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => 44324,
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => 44224,
         };
 
-        let daemon_rpc_addr: SocketAddr = m
-            .value_of("daemon_rpc_addr")
-            .unwrap_or(&format!("127.0.0.1:{}", default_daemon_port))
-            .parse()
-            .expect("invalid Bitcoind RPC address");
-        let electrum_rpc_addr: SocketAddr = m
-            .value_of("electrum_rpc_addr")
-            .unwrap_or(&format!("127.0.0.1:{}", default_electrum_port))
-            .parse()
-            .expect("invalid Electrum RPC address");
-        let http_addr: SocketAddr = m
-            .value_of("http_addr")
-            .unwrap_or(&format!("127.0.0.1:{}", default_http_port))
-            .parse()
-            .expect("invalid HTTP server address");
-        let monitoring_addr: SocketAddr = m
-            .value_of("monitoring_addr")
-            .unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port))
-            .parse()
-            .expect("invalid Prometheus monitoring address");
+        let daemon_rpc_addr: SocketAddr = str_to_socketaddr(
+            m.value_of("daemon_rpc_addr")
+                .unwrap_or(&format!("127.0.0.1:{}", default_daemon_port)),
+            "Bitcoin RPC",
+        );
+        let electrum_rpc_addr: SocketAddr = str_to_socketaddr(
+            m.value_of("electrum_rpc_addr")
+                .unwrap_or(&format!("127.0.0.1:{}", default_electrum_port)),
+            "Electrum RPC",
+        );
+        let http_addr: SocketAddr = str_to_socketaddr(
+            m.value_of("http_addr")
+                .unwrap_or(&format!("127.0.0.1:{}", default_http_port)),
+            "HTTP Server",
+        );
+
+        let http_socket_file: Option<PathBuf> = m.value_of("http_socket_file").map(PathBuf::from);
+        let rpc_socket_file: Option<PathBuf> = m.value_of("rpc_socket_file").map(PathBuf::from);
+        let monitoring_addr: SocketAddr = str_to_socketaddr(
+            m.value_of("monitoring_addr")
+                .unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port)),
+            "Prometheus monitoring",
+        );
 
         let mut daemon_dir = m
             .value_of("daemon_dir")
-            .map(|p| PathBuf::from(p))
+            .map(PathBuf::from)
             .unwrap_or_else(|| {
                 let mut default_dir = home_dir().expect("no homedir");
                 default_dir.push(".bitcoin");
                 default_dir
             });
         match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => (),
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => daemon_dir.push("testnet3"),
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => daemon_dir.push("testnet4"),
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => daemon_dir.push("regtest"),
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => daemon_dir.push("signet"),
 
             #[cfg(feature = "liquid")]
             Network::Liquid => daemon_dir.push("liquidv1"),
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => daemon_dir.push("liquidtestnet"),
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => daemon_dir.push("liquidregtest"),
         }
+        let blocks_dir = m
+            .value_of("blocks_dir")
+            .map(PathBuf::from)
+            .unwrap_or_else(|| daemon_dir.join("blocks"));
         let cookie = m.value_of("cookie").map(|s| s.to_owned());
 
+        let electrum_banner = m
+            .value_of("electrum_banner")
+            .map_or_else(|| format!("Welcome to {}", *VERSION_STRING), |s| s.into());
+
+        #[cfg(feature = "electrum-discovery")]
+        let electrum_public_hosts = m
+            .value_of("electrum_public_hosts")
+            .map(|s| serde_json::from_str(s).expect("invalid --electrum-public-hosts"));
+        #[cfg(not(feature = "electrum-discovery"))]
+        let electrum_public_hosts: Option<crate::electrum::ServerHosts> = None;
+
         let mut log = stderrlog::new();
         log.verbosity(m.occurrences_of("verbosity") as usize);
         log.timestamp(if m.is_present("timestamp") {
@@ -260,37 +543,97 @@ impl Config {
             stderrlog::Timestamp::Off
         });
         log.init().expect("logging initialization failed");
-        let mut bulk_index_threads = value_t_or_exit!(m, "bulk_index_threads", usize);
-        if bulk_index_threads == 0 {
-            bulk_index_threads = num_cpus::get();
-        }
         let config = Config {
             log,
             network_type,
+            magic,
             db_path,
             daemon_dir,
+            blocks_dir,
             daemon_rpc_addr,
             cookie,
+            utxos_limit: value_t_or_exit!(m, "utxos_limit", usize),
             electrum_rpc_addr,
+            electrum_txs_limit: value_t_or_exit!(m, "electrum_txs_limit", usize),
+            electrum_banner,
             http_addr,
+            http_socket_file,
+            rpc_socket_file,
             monitoring_addr,
+            mempool_backlog_stats_ttl: value_t_or_exit!(m, "mempool_backlog_stats_ttl", u64),
+            mempool_recent_txs_size: value_t_or_exit!(m, "mempool_recent_txs_size", usize),
+            rest_default_block_limit: value_t_or_exit!(m, "rest_default_block_limit", usize),
+            rest_default_chain_txs_per_page: value_t_or_exit!(
+                m,
+                "rest_default_chain_txs_per_page",
+                usize
+            ),
+            rest_default_max_mempool_txs: value_t_or_exit!(
+                m,
+                "rest_default_max_mempool_txs",
+                usize
+            ),
+            rest_default_max_address_summary_txs: value_t_or_exit!(
+                m,
+                "rest_default_max_address_summary_txs",
+                usize
+            ),
+            rest_max_mempool_page_size: value_t_or_exit!(m, "rest_max_mempool_page_size", usize),
+            rest_max_mempool_txid_page_size: value_t_or_exit!(
+                m,
+                "rest_max_mempool_txid_page_size",
+                usize
+            ),
+            electrum_max_line_size: value_t_or_exit!(m, "electrum_max_line_size", usize),
+            electrum_max_subscriptions: value_t_or_exit!(m, "electrum_max_subscriptions", usize),
+            electrum_max_clients: value_t_or_exit!(m, "electrum_max_clients", usize),
+            electrum_idle_timeout: value_t_or_exit!(m, "electrum_idle_timeout", u64),
+            electrum_haproxy_depth: value_t_or_exit!(m, "electrum_haproxy_depth", usize),
+            electrum_connections_per_client: value_t_or_exit!(
+                m,
+                "electrum_connections_per_client",
+                usize
+            ),
             jsonrpc_import: m.is_present("jsonrpc_import"),
-            index_batch_size: value_t_or_exit!(m, "index_batch_size", usize),
-            bulk_index_threads,
-            tx_cache_size: value_t_or_exit!(m, "tx_cache_size", usize),
-            prevout_enabled: !m.is_present("disable_prevout"),
+            light_mode: m.is_present("light_mode"),
+            main_loop_delay: value_t_or_exit!(m, "main_loop_delay", u64),
+            address_search: m.is_present("address_search"),
+            index_unspendables: m.is_present("index_unspendables"),
             cors: m.value_of("cors").map(|s| s.to_string()),
             precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()),
+            precache_threads: m.value_of("precache_threads").map_or_else(
+                || {
+                    std::thread::available_parallelism()
+                        .expect("Can't get core count")
+                        .get()
+                        * 4
+                },
+                |s| match s.parse::<usize>() {
+                    Ok(v) if v > 0 => v,
+                    _ => clap::Error::value_validation_auto(format!(
+                        "The argument '{}' isn't a valid value",
+                        s
+                    ))
+                    .exit(),
+                },
+            ),
+
             #[cfg(feature = "liquid")]
             parent_network,
             #[cfg(feature = "liquid")]
-            parent_genesis_hash,
+            asset_db_path,
+
+            electrum_public_hosts,
+            #[cfg(feature = "electrum-discovery")]
+            electrum_announce: m.is_present("electrum_announce"),
+            #[cfg(feature = "electrum-discovery")]
+            tor_proxy: m.value_of("tor_proxy").map(|s| s.parse().unwrap()),
         };
         eprintln!("{:?}", config);
         config
     }
 
-    pub fn cookie_getter(&self) -> Arc<CookieGetter> {
+    pub fn cookie_getter(&self) -> Arc<dyn CookieGetter> {
         if let Some(ref value) = self.cookie {
             Arc::new(StaticCookie {
                 value: value.as_bytes().to_vec(),

src/daemon.rs

@@ -1,32 +1,41 @@
-use base64;
-use bitcoin::consensus::encode::{deserialize, serialize};
-use bitcoin::util::hash::BitcoinHash;
-use bitcoin_hashes::hex::{FromHex, ToHex};
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
-use glob;
-use hex;
-use serde_json::{from_str, from_value, Value};
 use std::collections::{HashMap, HashSet};
 use std::io::{BufRead, BufReader, Lines, Write};
 use std::net::{SocketAddr, TcpStream};
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use crate::chain::{Block, BlockHeader, Network, Transaction};
+use base64;
+use bitcoin::hashes::Hash;
+use glob;
+use hex;
+use itertools::Itertools;
+use serde_json::{from_str, from_value, Value};
+
+#[cfg(not(feature = "liquid"))]
+use bitcoin::consensus::encode::{deserialize, serialize};
+#[cfg(feature = "liquid")]
+use elements::encode::{deserialize, serialize};
+
+use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid};
+use crate::config::BITCOIND_SUBVER;
 use crate::metrics::{HistogramOpts, HistogramVec, Metrics};
 use crate::signal::Waiter;
 use crate::util::HeaderList;
 
 use crate::errors::*;
 
-fn parse_hash(value: &Value) -> Result<Sha256dHash> {
-    Ok(Sha256dHash::from_hex(
-        value
-            .as_str()
-            .chain_err(|| format!("non-string value: {}", value))?,
-    )
-    .chain_err(|| format!("non-hex value: {}", value))?)
+fn parse_hash<T>(value: &Value) -> Result<T>
+where
+    T: FromStr,
+    <T as FromStr>::Err: std::fmt::Debug,
+{
+    value
+        .as_str()
+        .chain_err(|| format!("non-string value: {}", value))?
+        .parse::<T>()
+        .map_err(|e| format!("failed to parse hash: {:?}", e).into())
 }
 
 fn header_from_value(value: Value) -> Result<BlockHeader> {
@@ -34,22 +43,19 @@ fn header_from_value(value: Value) -> Result<BlockHeader> {
         .as_str()
         .chain_err(|| format!("non-string header: {}", value))?;
     let header_bytes = hex::decode(header_hex).chain_err(|| "non-hex header")?;
-    Ok(
-        deserialize(&header_bytes)
-            .chain_err(|| format!("failed to parse header {}", header_hex))?,
-    )
+    deserialize(&header_bytes).chain_err(|| format!("failed to parse header {}", header_hex))
 }
 
 fn block_from_value(value: Value) -> Result<Block> {
     let block_hex = value.as_str().chain_err(|| "non-string block")?;
     let block_bytes = hex::decode(block_hex).chain_err(|| "non-hex block")?;
-    Ok(deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))?)
+    deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))
 }
 
 fn tx_from_value(value: Value) -> Result<Transaction> {
     let tx_hex = value.as_str().chain_err(|| "non-string tx")?;
     let tx_bytes = hex::decode(tx_hex).chain_err(|| "non-hex tx")?;
-    Ok(deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))?)
+    deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))
 }
 
 /// Parse JSONRPC error code, if exists.
@@ -61,14 +67,13 @@ fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Resu
     if let Some(reply_obj) = reply.as_object_mut() {
         if let Some(err) = reply_obj.get("error") {
             if !err.is_null() {
-                if let Some(code) = parse_error_code(&err) {
+                if let Some(code) = parse_error_code(err) {
                     match code {
                         // RPC_IN_WARMUP -> retry by later reconnection
                         -28 => bail!(ErrorKind::Connection(err.to_string())),
-                        _ => (),
+                        _ => bail!("{} RPC error: {}", method, err),
                     }
                 }
-                bail!("{} RPC error: {}", method, err);
             }
         }
         let id = reply_obj
@@ -102,38 +107,64 @@ pub struct BlockchainInfo {
     pub initialblockdownload: Option<bool>,
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MempoolInfo {
+    pub loaded: bool,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 struct NetworkInfo {
     version: u64,
     subversion: String,
+    relayfee: f64, // in BTC/kB
 }
 
-pub struct MempoolEntry {
-    fee: u64,   // in satoshis
-    vsize: u32, // in virtual bytes (= weight/4)
-    fee_per_vbyte: f32,
+#[derive(Serialize, Deserialize, Debug)]
+struct MempoolFees {
+    base: f64,
+    #[serde(rename = "effective-feerate")]
+    effective_feerate: f64,
+    #[serde(rename = "effective-includes")]
+    effective_includes: Vec<String>,
 }
 
-impl MempoolEntry {
-    fn new(fee: u64, vsize: u32) -> MempoolEntry {
-        MempoolEntry {
-            fee,
-            vsize,
-            fee_per_vbyte: fee as f32 / vsize as f32,
-        }
-    }
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MempoolAcceptResult {
+    txid: String,
+    wtxid: String,
+    allowed: Option<bool>,
+    vsize: Option<u32>,
+    fees: Option<MempoolFees>,
+    #[serde(rename = "reject-reason")]
+    reject_reason: Option<String>,
+}
 
-    pub fn fee_per_vbyte(&self) -> f32 {
-        self.fee_per_vbyte
-    }
+#[derive(Serialize, Deserialize, Debug)]
+struct MempoolFeesSubmitPackage {
+    base: f64,
+    #[serde(rename = "effective-feerate")]
+    effective_feerate: Option<f64>,
+    #[serde(rename = "effective-includes")]
+    effective_includes: Option<Vec<String>>,
+}
 
-    pub fn fee(&self) -> u64 {
-        self.fee
-    }
+#[derive(Serialize, Deserialize, Debug)]
+pub struct SubmitPackageResult {
+    package_msg: String,
+    #[serde(rename = "tx-results")]
+    tx_results: HashMap<String, TxResult>,
+    #[serde(rename = "replaced-transactions")]
+    replaced_transactions: Option<Vec<String>>,
+}
 
-    pub fn vsize(&self) -> u32 {
-        self.vsize
-    }
+#[derive(Serialize, Deserialize, Debug)]
+pub struct TxResult {
+    txid: String,
+    #[serde(rename = "other-wtxid")]
+    other_wtxid: Option<String>,
+    vsize: Option<u32>,
+    fees: Option<MempoolFeesSubmitPackage>,
+    error: Option<String>,
 }
 
 pub trait CookieGetter: Send + Sync {
@@ -143,7 +174,7 @@ pub trait CookieGetter: Send + Sync {
 struct Connection {
     tx: TcpStream,
     rx: Lines<BufReader<TcpStream>>,
-    cookie_getter: Arc<CookieGetter>,
+    cookie_getter: Arc<dyn CookieGetter>,
     addr: SocketAddr,
     signal: Waiter,
 }
@@ -154,7 +185,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
             Ok(conn) => return Ok(conn),
             Err(err) => {
                 warn!("failed to connect daemon at {}: {}", addr, err);
-                signal.wait(Duration::from_secs(3))?;
+                signal.wait(Duration::from_secs(3), false)?;
                 continue;
             }
         }
@@ -164,7 +195,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
 impl Connection {
     fn new(
         addr: SocketAddr,
-        cookie_getter: Arc<CookieGetter>,
+        cookie_getter: Arc<dyn CookieGetter>,
         signal: Waiter,
     ) -> Result<Connection> {
         let conn = tcp_connect(addr, &signal)?;
@@ -208,7 +239,7 @@ impl Connection {
             .chain_err(|| {
                 ErrorKind::Connection("disconnected from daemon while receiving".to_owned())
             })?
-            .chain_err(|| "failed to read status")?;
+            .chain_err(|| ErrorKind::Connection("failed to read status".to_owned()))?;
         let mut headers = HashMap::new();
         for line in iter {
             let line = line.chain_err(|| ErrorKind::Connection("failed to read".to_owned()))?;
@@ -281,7 +312,9 @@ impl Counter {
 
 pub struct Daemon {
     daemon_dir: PathBuf,
+    blocks_dir: PathBuf,
     network: Network,
+    magic: Option<u32>,
     conn: Mutex<Connection>,
     message_id: Counter, // for monotonic JSONRPC 'id'
     signal: Waiter,
@@ -292,17 +325,22 @@ pub struct Daemon {
 }
 
 impl Daemon {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
-        daemon_dir: &PathBuf,
+        daemon_dir: PathBuf,
+        blocks_dir: PathBuf,
         daemon_rpc_addr: SocketAddr,
-        cookie_getter: Arc<CookieGetter>,
+        cookie_getter: Arc<dyn CookieGetter>,
         network: Network,
+        magic: Option<u32>,
         signal: Waiter,
         metrics: &Metrics,
     ) -> Result<Daemon> {
         let daemon = Daemon {
-            daemon_dir: daemon_dir.clone(),
+            daemon_dir,
+            blocks_dir,
             network,
+            magic,
             conn: Mutex::new(Connection::new(
                 daemon_rpc_addr,
                 cookie_getter,
@@ -321,32 +359,42 @@ impl Daemon {
         };
         let network_info = daemon.getnetworkinfo()?;
         info!("{:?}", network_info);
-        if network_info.version < 00_16_00_00 {
+        if network_info.version < 16_00_00 {
             bail!(
                 "{} is not supported - please use bitcoind 0.16+",
                 network_info.subversion,
             )
         }
+        // Insert the subversion (/Satoshi xx.xx.xx(comment)/) string from bitcoind
+        _ = BITCOIND_SUBVER.set(network_info.subversion);
+
         let blockchain_info = daemon.getblockchaininfo()?;
         info!("{:?}", blockchain_info);
-        if blockchain_info.pruned == true {
+        if blockchain_info.pruned {
             bail!("pruned node is not supported (use '-prune=0' bitcoind flag)".to_owned())
         }
         loop {
             let info = daemon.getblockchaininfo()?;
+            let mempool = daemon.getmempoolinfo()?;
 
-            // initialblockdownload is unavailable on the 0.14-based elements
-            let synced = match info.initialblockdownload {
-                Some(ibd) => !ibd,
-                None => info.verificationprogress == 1.0,
+            let ibd_done = if network.is_regtest() {
+                info.blocks == info.headers
+            } else {
+                !info.initialblockdownload.unwrap_or(false)
             };
 
-            if synced {
+            if mempool.loaded && ibd_done && info.blocks == info.headers {
                 break;
             }
 
-            warn!("wait until bitcoind is synced");
-            signal.wait(Duration::from_secs(3))?;
+            warn!(
+                "waiting for bitcoind sync and mempool load to finish: {}/{} blocks, verification progress: {:.3}%, mempool loaded: {}",
+                info.blocks,
+                info.headers,
+                info.verificationprogress * 100.0,
+                mempool.loaded
+            );
+            signal.wait(Duration::from_secs(5), false)?;
         }
         Ok(daemon)
     }
@@ -354,7 +402,9 @@ impl Daemon {
     pub fn reconnect(&self) -> Result<Daemon> {
         Ok(Daemon {
             daemon_dir: self.daemon_dir.clone(),
+            blocks_dir: self.blocks_dir.clone(),
             network: self.network,
+            magic: self.magic,
             conn: Mutex::new(self.conn.lock().unwrap().reconnect()?),
             message_id: Counter::new(),
             signal: self.signal.clone(),
@@ -364,9 +414,7 @@ impl Daemon {
     }
 
     pub fn list_blk_files(&self) -> Result<Vec<PathBuf>> {
-        let mut path = self.daemon_dir.clone();
-        path.push("blocks");
-        path.push("blk*.dat");
+        let path = self.blocks_dir.join("blk*.dat");
         debug!("listing block files at {:?}", path);
         let mut paths: Vec<PathBuf> = glob::glob(path.to_str().unwrap())
             .chain_err(|| "failed to list blk*.dat files")?
@@ -377,7 +425,7 @@ impl Daemon {
     }
 
     pub fn magic(&self) -> u32 {
-        self.network.magic()
+        self.magic.unwrap_or_else(|| self.network.magic())
     }
 
     fn call_jsonrpc(&self, method: &str, request: &Value) -> Result<Value> {
@@ -397,29 +445,66 @@ impl Daemon {
         Ok(result)
     }
 
-    fn handle_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
+    fn handle_request_batch(
+        &self,
+        method: &str,
+        params_list: &[Value],
+        failure_threshold: f64,
+    ) -> Result<Vec<Value>> {
         let id = self.message_id.next();
-        let reqs = params_list
+        let chunks = params_list
             .iter()
             .map(|params| json!({"method": method, "params": params, "id": id}))
-            .collect();
+            .chunks(50_000); // Max Amount of batched requests
         let mut results = vec![];
-        let mut replies = self.call_jsonrpc(method, &reqs)?;
-        if let Some(replies_vec) = replies.as_array_mut() {
-            for reply in replies_vec {
-                results.push(parse_jsonrpc_reply(reply.take(), method, id)?)
+        let total_requests = params_list.len();
+        let mut failed_requests: u64 = 0;
+        let threshold = (failure_threshold * total_requests as f64).round() as u64;
+        let mut n = 0;
+
+        for chunk in &chunks {
+            let reqs = chunk.collect();
+            let mut replies = self.call_jsonrpc(method, &reqs)?;
+            if let Some(replies_vec) = replies.as_array_mut() {
+                for reply in replies_vec {
+                    n += 1;
+                    match parse_jsonrpc_reply(reply.take(), method, id) {
+                        Ok(parsed_reply) => results.push(parsed_reply),
+                        Err(e) => {
+                            failed_requests += 1;
+                            warn!(
+                                "batch request {} {}/{} failed: {}",
+                                method,
+                                n,
+                                total_requests,
+                                e.to_string()
+                            );
+                            // abort and return the last error once a threshold number of requests have failed
+                            if failed_requests > threshold {
+                                return Err(e);
+                            }
+                        }
+                    }
+                }
+            } else {
+                bail!("non-array replies: {:?}", replies);
             }
-            return Ok(results);
         }
-        bail!("non-array replies: {:?}", replies);
+
+        Ok(results)
     }
 
-    fn retry_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
+    fn retry_request_batch(
+        &self,
+        method: &str,
+        params_list: &[Value],
+        failure_threshold: f64,
+    ) -> Result<Vec<Value>> {
         loop {
-            match self.handle_request_batch(method, params_list) {
+            match self.handle_request_batch(method, params_list, failure_threshold) {
                 Err(Error(ErrorKind::Connection(msg), _)) => {
                     warn!("reconnecting to bitcoind: {}", msg);
-                    self.signal.wait(Duration::from_secs(3))?;
+                    self.signal.wait(Duration::from_secs(3), false)?;
                     let mut conn = self.conn.lock().unwrap();
                     *conn = conn.reconnect()?;
                     continue;
@@ -430,35 +515,40 @@ impl Daemon {
     }
 
     fn request(&self, method: &str, params: Value) -> Result<Value> {
-        let mut values = self.retry_request_batch(method, &[params])?;
+        let mut values = self.retry_request_batch(method, &[params], 0.0)?;
         assert_eq!(values.len(), 1);
         Ok(values.remove(0))
     }
 
     fn requests(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
-        self.retry_request_batch(method, params_list)
+        self.retry_request_batch(method, params_list, 0.0)
     }
 
     // bitcoind JSONRPC API:
 
     pub fn getblockchaininfo(&self) -> Result<BlockchainInfo> {
         let info: Value = self.request("getblockchaininfo", json!([]))?;
-        Ok(from_value(info).chain_err(|| "invalid blockchain info")?)
+        from_value(info).chain_err(|| "invalid blockchain info")
+    }
+
+    fn getmempoolinfo(&self) -> Result<MempoolInfo> {
+        let info: Value = self.request("getmempoolinfo", json!([]))?;
+        from_value(info).chain_err(|| "invalid mempool info")
     }
 
     fn getnetworkinfo(&self) -> Result<NetworkInfo> {
         let info: Value = self.request("getnetworkinfo", json!([]))?;
-        Ok(from_value(info).chain_err(|| "invalid network info")?)
+        from_value(info).chain_err(|| "invalid network info")
     }
 
-    pub fn getbestblockhash(&self) -> Result<Sha256dHash> {
-        parse_hash(&self.request("getbestblockhash", json!([]))?).chain_err(|| "invalid blockhash")
+    pub fn getbestblockhash(&self) -> Result<BlockHash> {
+        parse_hash(&self.request("getbestblockhash", json!([]))?)
     }
 
-    pub fn getblockheader(&self, blockhash: &Sha256dHash) -> Result<BlockHeader> {
+    pub fn getblockheader(&self, blockhash: &BlockHash) -> Result<BlockHeader> {
         header_from_value(self.request(
             "getblockheader",
-            json!([blockhash.to_hex(), /*verbose=*/ false]),
+            json!([blockhash.to_string(), /*verbose=*/ false]),
         )?)
     }
 
@@ -476,22 +566,23 @@ impl Daemon {
         Ok(result)
     }
 
-    pub fn getblock(&self, blockhash: &Sha256dHash) -> Result<Block> {
-        let block = block_from_value(
-            self.request("getblock", json!([blockhash.to_hex(), /*verbose=*/ false]))?,
-        )?;
-        assert_eq!(block.bitcoin_hash(), *blockhash);
+    pub fn getblock(&self, blockhash: &BlockHash) -> Result<Block> {
+        let block = block_from_value(self.request(
+            "getblock",
+            json!([blockhash.to_string(), /*verbose=*/ false]),
+        )?)?;
+        assert_eq!(block.block_hash(), *blockhash);
         Ok(block)
     }
 
-    pub fn getblock_raw(&self, blockhash: &Sha256dHash, verbose: u32) -> Result<Value> {
-        self.request("getblock", json!([blockhash.to_hex(), verbose]))
+    pub fn getblock_raw(&self, blockhash: &BlockHash, verbose: u32) -> Result<Value> {
+        self.request("getblock", json!([blockhash.to_string(), verbose]))
     }
 
-    pub fn getblocks(&self, blockhashes: &[Sha256dHash]) -> Result<Vec<Block>> {
+    pub fn getblocks(&self, blockhashes: &[BlockHash]) -> Result<Vec<Block>> {
         let params_list: Vec<Value> = blockhashes
             .iter()
-            .map(|hash| json!([hash.to_hex(), /*verbose=*/ false]))
+            .map(|hash| json!([hash.to_string(), /*verbose=*/ false]))
             .collect();
         let values = self.requests("getblock", &params_list)?;
         let mut blocks = vec![];
@@ -501,104 +592,145 @@ impl Daemon {
         Ok(blocks)
     }
 
-    pub fn gettransactions(&self, txhashes: &[&Sha256dHash]) -> Result<Vec<Transaction>> {
+    pub fn gettransactions(&self, txhashes: &[&Txid]) -> Result<Vec<Transaction>> {
         let params_list: Vec<Value> = txhashes
             .iter()
-            .map(|txhash| json!([txhash.to_hex(), /*verbose=*/ false]))
+            .map(|txhash| json!([txhash.to_string(), /*verbose=*/ false]))
             .collect();
-
-        let values = self.requests("getrawtransaction", &params_list)?;
+        let values = self.retry_request_batch("getrawtransaction", &params_list, 0.25)?;
         let mut txs = vec![];
         for value in values {
             txs.push(tx_from_value(value)?);
         }
-        assert_eq!(txhashes.len(), txs.len());
+        // missing transactions are skipped, so the number of txs returned may be less than the number of txids requested
         Ok(txs)
     }
 
-    pub fn getmempooltx(&self, txhash: &Sha256dHash) -> Result<Transaction> {
+    pub fn gettransaction_raw(
+        &self,
+        txid: &Txid,
+        blockhash: &BlockHash,
+        verbose: bool,
+    ) -> Result<Value> {
+        self.request(
+            "getrawtransaction",
+            json!([txid.to_string(), verbose, blockhash]),
+        )
+    }
+
+    pub fn getmempooltx(&self, txhash: &Txid) -> Result<Transaction> {
         let value = self.request(
             "getrawtransaction",
-            json!([txhash.to_hex(), /*verbose=*/ false]),
+            json!([txhash.to_string(), /*verbose=*/ false]),
         )?;
         tx_from_value(value)
     }
 
-    pub fn getmempooltxids(&self) -> Result<HashSet<Sha256dHash>> {
-        let txids: Value = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
-        let mut result = HashSet::new();
-        for value in txids.as_array().chain_err(|| "non-array result")? {
-            result.insert(parse_hash(&value).chain_err(|| "invalid txid")?);
-        }
-        Ok(result)
+    pub fn getmempooltxids(&self) -> Result<HashSet<Txid>> {
+        let res = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
+        serde_json::from_value(res).chain_err(|| "invalid getrawmempool reply")
     }
 
-    pub fn getmempoolentry(&self, txid: &Sha256dHash) -> Result<MempoolEntry> {
-        let entry = self.request("getmempoolentry", json!([txid.to_hex()]))?;
-        let fee = (entry
-            .get("fee")
-            .chain_err(|| "missing fee")?
-            .as_f64()
-            .chain_err(|| "non-float fee")?
-            * 100_000_000f64) as u64;
-        let vsize = entry
-            .get("size")
-            .chain_err(|| "missing size")?
-            .as_u64()
-            .chain_err(|| "non-integer size")? as u32;
-        Ok(MempoolEntry::new(fee, vsize))
-    }
-
-    pub fn broadcast(&self, tx: &Transaction) -> Result<Sha256dHash> {
+    pub fn broadcast(&self, tx: &Transaction) -> Result<Txid> {
         self.broadcast_raw(&hex::encode(serialize(tx)))
     }
 
-    pub fn broadcast_raw(&self, txhex: &String) -> Result<Sha256dHash> {
+    pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
         let txid = self.request("sendrawtransaction", json!([txhex]))?;
-        Ok(
-            Sha256dHash::from_hex(txid.as_str().chain_err(|| "non-string txid")?)
-                .chain_err(|| "failed to parse txid")?,
-        )
+        txid.as_str()
+            .chain_err(|| "non-string txid")?
+            .parse::<Txid>()
+            .map_err(|e| format!("failed to parse txid: {:?}", e).into())
     }
 
-    pub fn estimatesmartfee(&self, conf_target: u16) -> Result<f32> {
-        let feerate = self
-            .request("estimatesmartfee", json!([conf_target]))?
-            .get("feerate")
-            .chain_err(|| "missing feerate")?
-            .as_f64()
-            .chain_err(|| "invalid feerate")?;
-
-        if feerate == -1f64 {
-            bail!("not enough data to estimate");
-        }
-
-        // from BTC/kB to sat/b
-        Ok((feerate * 100_000f64) as f32)
+    pub fn test_mempool_accept(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+    ) -> Result<Vec<MempoolAcceptResult>> {
+        let params = match maxfeerate {
+            Some(rate) => json!([txhex, format!("{:.8}", rate)]),
+            None => json!([txhex]),
+        };
+        let result = self.request("testmempoolaccept", params)?;
+        serde_json::from_value::<Vec<MempoolAcceptResult>>(result)
+            .chain_err(|| "invalid testmempoolaccept reply")
     }
 
-    fn get_all_headers(&self, tip: &Sha256dHash) -> Result<Vec<BlockHeader>> {
-        let info: Value = self.request("getblockheader", json!([tip.to_hex()]))?;
+    pub fn submit_package(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+        maxburnamount: Option<f64>,
+    ) -> Result<SubmitPackageResult> {
+        let params = match (maxfeerate, maxburnamount) {
+            (Some(rate), Some(burn)) => {
+                json!([txhex, format!("{:.8}", rate), format!("{:.8}", burn)])
+            }
+            (Some(rate), None) => json!([txhex, format!("{:.8}", rate)]),
+            (None, Some(burn)) => json!([txhex, null, format!("{:.8}", burn)]),
+            (None, None) => json!([txhex]),
+        };
+        let result = self.request("submitpackage", params)?;
+        serde_json::from_value::<SubmitPackageResult>(result)
+            .chain_err(|| "invalid submitpackage reply")
+    }
+
+    // Get estimated feerates for the provided confirmation targets using a batch RPC request
+    // Missing estimates are logged but do not cause a failure, whatever is available is returned
+    #[allow(clippy::float_cmp)]
+    pub fn estimatesmartfee_batch(&self, conf_targets: &[u16]) -> Result<HashMap<u16, f64>> {
+        let params_list: Vec<Value> = conf_targets.iter().map(|t| json!([t])).collect();
+
+        Ok(self
+            .requests("estimatesmartfee", &params_list)?
+            .iter()
+            .zip(conf_targets)
+            .filter_map(|(reply, target)| {
+                if !reply["errors"].is_null() {
+                    warn!(
+                        "failed estimating fee for target {}: {:?}",
+                        target, reply["errors"]
+                    );
+                    return None;
+                }
+
+                let feerate = reply["feerate"]
+                    .as_f64()
+                    .unwrap_or_else(|| panic!("invalid estimatesmartfee response: {:?}", reply));
+
+                if feerate == -1f64 {
+                    warn!("not enough data to estimate fee for target {}", target);
+                    return None;
+                }
+
+                // from BTC/kB to sat/b
+                Some((*target, feerate * 100_000f64))
+            })
+            .collect())
+    }
+
+    fn get_all_headers(&self, tip: &BlockHash) -> Result<Vec<BlockHeader>> {
+        let info: Value = self.request("getblockheader", json!([tip.to_string()]))?;
         let tip_height = info
             .get("height")
             .expect("missing height")
             .as_u64()
             .expect("non-numeric height") as usize;
-        let all_heights: Vec<usize> = (0..tip_height + 1).collect();
+        let all_heights: Vec<usize> = (0..=tip_height).collect();
         let chunk_size = 100_000;
         let mut result = vec![];
-        let null_hash = Sha256dHash::default();
         for heights in all_heights.chunks(chunk_size) {
             trace!("downloading {} block headers", heights.len());
-            let mut headers = self.getblockheaders(&heights)?;
+            let mut headers = self.getblockheaders(heights)?;
             assert!(headers.len() == heights.len());
             result.append(&mut headers);
         }
 
-        let mut blockhash = null_hash;
+        let mut blockhash = BlockHash::all_zeros();
         for header in &result {
             assert_eq!(header.prev_blockhash, blockhash);
-            blockhash = header.bitcoin_hash();
+            blockhash = header.block_hash();
         }
         assert_eq!(blockhash, *tip);
         Ok(result)
@@ -608,10 +740,10 @@ impl Daemon {
     pub fn get_new_headers(
         &self,
         indexed_headers: &HeaderList,
-        bestblockhash: &Sha256dHash,
+        bestblockhash: &BlockHash,
     ) -> Result<Vec<BlockHeader>> {
         // Iterate back over headers until known blockash is found:
-        if indexed_headers.len() == 0 {
+        if indexed_headers.is_empty() {
             debug!("downloading all block headers up to {}", bestblockhash);
             return self.get_all_headers(bestblockhash);
         }
@@ -621,7 +753,7 @@ impl Daemon {
             bestblockhash,
         );
         let mut new_headers = vec![];
-        let null_hash = Sha256dHash::default();
+        let null_hash = BlockHash::all_zeros();
         let mut blockhash = *bestblockhash;
         while blockhash != null_hash {
             if indexed_headers.header_by_blockhash(&blockhash).is_some() {
@@ -637,4 +769,11 @@ impl Daemon {
         new_headers.reverse(); // so the tip is the last vector entry
         Ok(new_headers)
     }
+
+    pub fn get_relayfee(&self) -> Result<f64> {
+        let relayfee = self.getnetworkinfo()?.relayfee;
+
+        // from BTC/kB to sat/b
+        Ok(relayfee * 100_000f64)
+    }
 }

src/electrum.rs

@@ -1,630 +0,0 @@
-use bitcoin::consensus::encode::serialize;
-use bitcoin_hashes::hex::{FromHex, ToHex};
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
-use crypto::digest::Digest;
-use crypto::sha2::Sha256;
-use error_chain::ChainedError;
-use hex;
-use serde_json::{from_str, Value};
-use std::collections::HashMap;
-use std::io::{BufRead, BufReader, Write};
-use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
-use std::sync::mpsc::{Sender, SyncSender, TrySendError};
-use std::sync::{Arc, Mutex};
-use std::thread;
-
-use crate::errors::*;
-use crate::metrics::{Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics};
-use crate::new_index::Query;
-use crate::util::{
-    full_hash, get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof, spawn_thread,
-    BlockId, Channel, FullHash, HeaderEntry, SyncChannel,
-};
-
-// TODO: Sha256dHash should be a generic hash-container (since script hash is single SHA256)
-fn hash_from_value(val: Option<&Value>) -> Result<Sha256dHash> {
-    let script_hash = val.chain_err(|| "missing hash")?;
-    let script_hash = script_hash.as_str().chain_err(|| "non-string hash")?;
-    let script_hash = Sha256dHash::from_hex(script_hash).chain_err(|| "non-hex hash")?;
-    Ok(script_hash)
-}
-
-fn usize_from_value(val: Option<&Value>, name: &str) -> Result<usize> {
-    let val = val.chain_err(|| format!("missing {}", name))?;
-    let val = val.as_u64().chain_err(|| format!("non-integer {}", name))?;
-    Ok(val as usize)
-}
-
-fn usize_from_value_or(val: Option<&Value>, name: &str, default: usize) -> Result<usize> {
-    if val.is_none() {
-        return Ok(default);
-    }
-    usize_from_value(val, name)
-}
-
-fn bool_from_value(val: Option<&Value>, name: &str) -> Result<bool> {
-    let val = val.chain_err(|| format!("missing {}", name))?;
-    let val = val.as_bool().chain_err(|| format!("not a bool {}", name))?;
-    Ok(val)
-}
-
-fn bool_from_value_or(val: Option<&Value>, name: &str, default: bool) -> Result<bool> {
-    if val.is_none() {
-        return Ok(default);
-    }
-    bool_from_value(val, name)
-}
-
-// FIXME: implement caching and delta updates
-// FIXME: ensure stable ordering
-fn get_status_hash(txs: Vec<(Sha256dHash, Option<BlockId>)>) -> Option<FullHash> {
-    if txs.is_empty() {
-        None
-    } else {
-        let mut hash = FullHash::default();
-        let mut sha2 = Sha256::new();
-        for (txid, blockid) in txs {
-            // TODO: use height of 0 to indicate a mempool tx with confirmed inputs
-            let part = format!(
-                "{}:{}:",
-                txid.to_hex(),
-                blockid.map_or(-1, |b| b.height as isize)
-            );
-            sha2.input(part.as_bytes());
-        }
-        sha2.result(&mut hash);
-        Some(hash)
-    }
-}
-
-struct Connection {
-    query: Arc<Query>,
-    last_header_entry: Option<HeaderEntry>,
-    status_hashes: HashMap<Sha256dHash, Value>, // ScriptHash -> StatusHash
-    stream: TcpStream,
-    addr: SocketAddr,
-    chan: SyncChannel<Message>,
-    stats: Arc<Stats>,
-}
-
-impl Connection {
-    pub fn new(
-        query: Arc<Query>,
-        stream: TcpStream,
-        addr: SocketAddr,
-        stats: Arc<Stats>,
-    ) -> Connection {
-        Connection {
-            query,
-            last_header_entry: None, // disable header subscription for now
-            status_hashes: HashMap::new(),
-            stream,
-            addr,
-            chan: SyncChannel::new(10),
-            stats,
-        }
-    }
-
-    fn blockchain_headers_subscribe(&mut self) -> Result<Value> {
-        let entry = self.query.chain().best_header();
-        let hex_header = hex::encode(serialize(entry.header()));
-        let result = json!({"hex": hex_header, "height": entry.height()});
-        self.last_header_entry = Some(entry);
-        Ok(result)
-    }
-
-    fn server_version(&self) -> Result<Value> {
-        // TODO dynamic version
-        Ok(json!(["electrs-esplora", "1.4"]))
-    }
-
-    fn server_banner(&self) -> Result<Value> {
-        // TODO dynamic banner
-        Ok(json!("Welcome to electrs-esplora"))
-    }
-
-    fn server_donation_address(&self) -> Result<Value> {
-        Ok(Value::Null)
-    }
-
-    fn server_peers_subscribe(&self) -> Result<Value> {
-        Ok(json!([]))
-    }
-
-    fn mempool_get_fee_histogram(&self) -> Result<Value> {
-        Ok(json!(&self.query.mempool().backlog_stats().fee_histogram))
-    }
-
-    fn blockchain_block_header(&self, params: &[Value]) -> Result<Value> {
-        let height = usize_from_value(params.get(0), "height")?;
-        let cp_height = usize_from_value_or(params.get(1), "cp_height", 0)?;
-
-        let raw_header_hex: String = self
-            .query
-            .chain()
-            .header_by_height(height)
-            .map(|entry| hex::encode(&serialize(entry.header())))
-            .chain_err(|| "missing header")?;
-
-        if cp_height == 0 {
-            return Ok(json!(raw_header_hex));
-        }
-        let (branch, root) = get_header_merkle_proof(self.query.chain(), height, cp_height)?;
-
-        let branch_vec: Vec<String> = branch.into_iter().map(|b| b.to_hex()).collect();
-
-        return Ok(json!({
-            "header": raw_header_hex,
-            "root": root.to_hex(),
-            "branch": branch_vec
-        }));
-    }
-
-    fn blockchain_block_headers(&self, params: &[Value]) -> Result<Value> {
-        let start_height = usize_from_value(params.get(0), "start_height")?;
-        let count = usize_from_value(params.get(1), "count")?;
-        let cp_height = usize_from_value_or(params.get(2), "cp_height", 0)?;
-        let heights: Vec<usize> = (start_height..(start_height + count)).collect();
-        let headers: Vec<String> = heights
-            .into_iter()
-            .map(|height| {
-                self.query
-                    .chain()
-                    .header_by_height(height)
-                    .map(|entry| hex::encode(&serialize(entry.header())))
-            })
-            .collect::<Option<Vec<String>>>()
-            .chain_err(|| "requested block headers not found")?;
-
-        if count == 0 || cp_height == 0 {
-            return Ok(json!({
-                "count": headers.len(),
-                "hex": headers.join(""),
-                "max": 2016,
-            }));
-        }
-
-        let (branch, root) =
-            get_header_merkle_proof(self.query.chain(), start_height + (count - 1), cp_height)?;
-
-        let branch_vec: Vec<String> = branch.into_iter().map(|b| b.to_hex()).collect();
-
-        Ok(json!({
-            "count": headers.len(),
-            "hex": headers.join(""),
-            "max": 2016,
-            "root": root.to_hex(),
-            "branch" : branch_vec
-        }))
-    }
-
-    fn blockchain_estimatefee(&self, params: &[Value]) -> Result<Value> {
-        let conf_target = usize_from_value(params.get(0), "blocks_count")?;
-        let fee_rate = self
-            .query
-            .estimate_fee(conf_target as u16)
-            .chain_err(|| format!("cannot estimate fee for {} blocks", conf_target))?;
-        // convert from sat/b to BTC/kB, as expected by Electrum clients
-        Ok(json!(fee_rate / 100_000f32))
-    }
-
-    fn blockchain_relayfee(&self) -> Result<Value> {
-        // TODO
-        Ok(json!(0.0)) // allow sending transactions with any fee.
-    }
-
-    fn blockchain_scripthash_subscribe(&mut self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-
-        let history_txids = self.query.history_txids(&script_hash[..]);
-        let status_hash = get_status_hash(history_txids)
-            .map_or(Value::Null, |h| json!(hex::encode(full_hash(&h[..]))));
-
-        self.status_hashes.insert(script_hash, status_hash.clone());
-        Ok(status_hash)
-    }
-
-    #[cfg(not(feature = "liquid"))]
-    fn blockchain_scripthash_get_balance(&self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-        let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]);
-
-        Ok(
-            json!({ "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, "unconfirmed": mempool_stats.funded_txo_sum - mempool_stats.spent_txo_sum}),
-        )
-    }
-
-    fn blockchain_scripthash_get_history(&self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-        let history_txids = self.query.history_txids(&script_hash[..]);
-        Ok(json!(Value::Array(
-            history_txids
-                .into_iter()
-                .map(|(txid, blockid)| json!({"height": blockid.map_or(0, |b| b.height), "tx_hash": txid.to_hex()}))
-                .collect()
-        )))
-    }
-
-    fn blockchain_scripthash_listunspent(&self, params: &[Value]) -> Result<Value> {
-        let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
-        let utxos = self.query.utxo(&script_hash[..]);
-        Ok(json!(Value::Array(
-            utxos
-                .into_iter()
-                .map(|utxo| json!({
-                    "height": utxo.confirmed.map_or(0, |b| b.height),
-                    "tx_pos": utxo.vout,
-                    "tx_hash": utxo.txid.to_hex(),
-                    "value": utxo.value,
-                }))
-                .collect()
-        )))
-    }
-
-    fn blockchain_transaction_broadcast(&self, params: &[Value]) -> Result<Value> {
-        let tx = params.get(0).chain_err(|| "missing tx")?;
-        let tx = tx.as_str().chain_err(|| "non-string tx")?.to_string();
-        let txid = self.query.broadcast_raw(&tx)?;
-        if let Err(e) = self.chan.sender().try_send(Message::PeriodicUpdate) {
-            warn!("failed to issue PeriodicUpdate after broadcast: {}", e);
-        }
-        Ok(json!(txid.to_hex()))
-    }
-
-    fn blockchain_transaction_get(&self, params: &[Value]) -> Result<Value> {
-        let tx_hash = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
-        let verbose = match params.get(1) {
-            Some(value) => value.as_bool().chain_err(|| "non-bool verbose value")?,
-            None => false,
-        };
-
-        // FIXME: implement verbose support
-        if verbose {
-            bail!("verbose transactions are currently unsupported");
-        }
-
-        let tx = self
-            .query
-            .lookup_raw_txn(&tx_hash)
-            .chain_err(|| "missing transaction")?;
-        Ok(json!(hex::encode(tx)))
-    }
-
-    fn blockchain_transaction_get_merkle(&self, params: &[Value]) -> Result<Value> {
-        let txid = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
-        let height = usize_from_value(params.get(1), "height")?;
-        let blockid = self
-            .query
-            .chain()
-            .tx_confirming_block(&txid)
-            .ok_or_else(|| "tx not found or is unconfirmed")?;
-        if blockid.height != height {
-            bail!("invalid confirmation height provided");
-        }
-        let (merkle, pos) = get_tx_merkle_proof(self.query.chain(), &txid, &blockid.hash)
-            .chain_err(|| "cannot create merkle proof")?;
-        let merkle: Vec<String> = merkle.into_iter().map(|txid| txid.to_hex()).collect();
-        Ok(json!({
-                "block_height": blockid.height,
-                "merkle": merkle,
-                "pos": pos}))
-    }
-
-    fn blockchain_transaction_id_from_pos(&self, params: &[Value]) -> Result<Value> {
-        let height = usize_from_value(params.get(0), "height")?;
-        let tx_pos = usize_from_value(params.get(1), "tx_pos")?;
-        let want_merkle = bool_from_value_or(params.get(2), "merkle", false)?;
-
-        let (txid, merkle) = get_id_from_pos(self.query.chain(), height, tx_pos, want_merkle)?;
-
-        if !want_merkle {
-            return Ok(json!(txid.to_hex()));
-        }
-
-        let merkle_vec: Vec<String> = merkle.into_iter().map(|entry| entry.to_hex()).collect();
-
-        Ok(json!({
-            "tx_hash" : txid.to_hex(),
-            "merkle" : merkle_vec}))
-    }
-
-    fn handle_command(&mut self, method: &str, params: &[Value], id: &Value) -> Result<Value> {
-        let timer = self
-            .stats
-            .latency
-            .with_label_values(&[method])
-            .start_timer();
-        let result = match method {
-            "blockchain.block.header" => self.blockchain_block_header(&params),
-            "blockchain.block.headers" => self.blockchain_block_headers(&params),
-            "blockchain.estimatefee" => self.blockchain_estimatefee(&params),
-            "blockchain.headers.subscribe" => self.blockchain_headers_subscribe(),
-            "blockchain.relayfee" => self.blockchain_relayfee(),
-            #[cfg(not(feature = "liquid"))]
-            "blockchain.scripthash.get_balance" => self.blockchain_scripthash_get_balance(&params),
-            "blockchain.scripthash.get_history" => self.blockchain_scripthash_get_history(&params),
-            "blockchain.scripthash.listunspent" => self.blockchain_scripthash_listunspent(&params),
-            "blockchain.scripthash.subscribe" => self.blockchain_scripthash_subscribe(&params),
-            "blockchain.transaction.broadcast" => self.blockchain_transaction_broadcast(&params),
-            "blockchain.transaction.get" => self.blockchain_transaction_get(&params),
-            "blockchain.transaction.get_merkle" => self.blockchain_transaction_get_merkle(&params),
-            "blockchain.transaction.id_from_pos" => {
-                self.blockchain_transaction_id_from_pos(&params)
-            }
-            "mempool.get_fee_histogram" => self.mempool_get_fee_histogram(),
-            "server.banner" => self.server_banner(),
-            "server.donation_address" => self.server_donation_address(),
-            "server.peers.subscribe" => self.server_peers_subscribe(),
-            "server.ping" => Ok(Value::Null),
-            "server.version" => self.server_version(),
-            &_ => bail!("unknown method {} {:?}", method, params),
-        };
-        timer.observe_duration();
-        // TODO: return application errors should be sent to the client
-        Ok(match result {
-            Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}),
-            Err(e) => {
-                warn!(
-                    "rpc #{} {} {:?} failed: {}",
-                    id,
-                    method,
-                    params,
-                    e.display_chain()
-                );
-                json!({"jsonrpc": "2.0", "id": id, "error": format!("{}", e)})
-            }
-        })
-    }
-
-    fn update_subscriptions(&mut self) -> Result<Vec<Value>> {
-        let timer = self
-            .stats
-            .latency
-            .with_label_values(&["periodic_update"])
-            .start_timer();
-        let mut result = vec![];
-        if let Some(ref mut last_entry) = self.last_header_entry {
-            let entry = self.query.chain().best_header();
-            if *last_entry != entry {
-                *last_entry = entry;
-                let hex_header = hex::encode(serialize(last_entry.header()));
-                let header = json!({"hex": hex_header, "height": last_entry.height()});
-                result.push(json!({
-                    "jsonrpc": "2.0",
-                    "method": "blockchain.headers.subscribe",
-                    "params": [header]}));
-            }
-        }
-        for (script_hash, status_hash) in self.status_hashes.iter_mut() {
-            let history_txids = self.query.history_txids(&script_hash[..]);
-            let new_status_hash = get_status_hash(history_txids)
-                .map_or(Value::Null, |h| json!(hex::encode(full_hash(&h[..]))));
-            if new_status_hash == *status_hash {
-                continue;
-            }
-            result.push(json!({
-                "jsonrpc": "2.0",
-                "method": "blockchain.scripthash.subscribe",
-                "params": [script_hash.to_hex(), new_status_hash]}));
-            *status_hash = new_status_hash;
-        }
-        timer.observe_duration();
-        self.stats
-            .subscriptions
-            .set(self.status_hashes.len() as i64);
-        Ok(result)
-    }
-
-    fn send_values(&mut self, values: &[Value]) -> Result<()> {
-        for value in values {
-            let line = value.to_string() + "\n";
-            self.stream
-                .write_all(line.as_bytes())
-                .chain_err(|| format!("failed to send {}", value))?;
-        }
-        Ok(())
-    }
-
-    fn handle_replies(&mut self) -> Result<()> {
-        let empty_params = json!([]);
-        loop {
-            let msg = self.chan.receiver().recv().chain_err(|| "channel closed")?;
-            trace!("RPC {:?}", msg);
-            match msg {
-                Message::Request(line) => {
-                    let cmd: Value = from_str(&line).chain_err(|| "invalid JSON format")?;
-                    let reply = match (
-                        cmd.get("method"),
-                        cmd.get("params").unwrap_or_else(|| &empty_params),
-                        cmd.get("id"),
-                    ) {
-                        (
-                            Some(&Value::String(ref method)),
-                            &Value::Array(ref params),
-                            Some(ref id),
-                        ) => self.handle_command(method, params, id)?,
-                        _ => bail!("invalid command: {}", cmd),
-                    };
-                    self.send_values(&[reply])?
-                }
-                Message::PeriodicUpdate => {
-                    let values = self
-                        .update_subscriptions()
-                        .chain_err(|| "failed to update subscriptions")?;
-                    self.send_values(&values)?
-                }
-                Message::Done => return Ok(()),
-            }
-        }
-    }
-
-    fn handle_requests(mut reader: BufReader<TcpStream>, tx: SyncSender<Message>) -> Result<()> {
-        loop {
-            let mut line = Vec::<u8>::new();
-            reader
-                .read_until(b'\n', &mut line)
-                .chain_err(|| "failed to read a request")?;
-            if line.is_empty() {
-                tx.send(Message::Done).chain_err(|| "channel closed")?;
-                return Ok(());
-            } else {
-                if line.starts_with(&[22, 3, 1]) {
-                    // (very) naive SSL handshake detection
-                    let _ = tx.send(Message::Done);
-                    bail!("invalid request - maybe SSL-encrypted data?: {:?}", line)
-                }
-                match String::from_utf8(line) {
-                    Ok(req) => tx
-                        .send(Message::Request(req))
-                        .chain_err(|| "channel closed")?,
-                    Err(err) => {
-                        let _ = tx.send(Message::Done);
-                        bail!("invalid UTF8: {}", err)
-                    }
-                }
-            }
-        }
-    }
-
-    pub fn run(mut self) {
-        let reader = BufReader::new(self.stream.try_clone().expect("failed to clone TcpStream"));
-        let tx = self.chan.sender();
-        let child = spawn_thread("reader", || Connection::handle_requests(reader, tx));
-        if let Err(e) = self.handle_replies() {
-            error!(
-                "[{}] connection handling failed: {}",
-                self.addr,
-                e.display_chain().to_string()
-            );
-        }
-        debug!("[{}] shutting down connection", self.addr);
-        let _ = self.stream.shutdown(Shutdown::Both);
-        if let Err(err) = child.join().expect("receiver panicked") {
-            error!("[{}] receiver failed: {}", self.addr, err);
-        }
-    }
-}
-
-#[derive(Debug)]
-pub enum Message {
-    Request(String),
-    PeriodicUpdate,
-    Done,
-}
-
-pub enum Notification {
-    Periodic,
-    Exit,
-}
-
-pub struct RPC {
-    notification: Sender<Notification>,
-    server: Option<thread::JoinHandle<()>>, // so we can join the server while dropping this ojbect
-}
-
-struct Stats {
-    latency: HistogramVec,
-    subscriptions: Gauge,
-}
-
-impl RPC {
-    fn start_notifier(
-        notification: Channel<Notification>,
-        senders: Arc<Mutex<Vec<SyncSender<Message>>>>,
-        acceptor: Sender<Option<(TcpStream, SocketAddr)>>,
-    ) {
-        spawn_thread("notification", move || {
-            for msg in notification.receiver().iter() {
-                let mut senders = senders.lock().unwrap();
-                match msg {
-                    Notification::Periodic => {
-                        for sender in senders.split_off(0) {
-                            if let Err(TrySendError::Disconnected(_)) =
-                                sender.try_send(Message::PeriodicUpdate)
-                            {
-                                continue;
-                            }
-                            senders.push(sender);
-                        }
-                    }
-                    Notification::Exit => acceptor.send(None).unwrap(),
-                }
-            }
-        });
-    }
-
-    fn start_acceptor(addr: SocketAddr) -> Channel<Option<(TcpStream, SocketAddr)>> {
-        let chan = Channel::new();
-        let acceptor = chan.sender();
-        spawn_thread("acceptor", move || {
-            let listener = TcpListener::bind(addr).expect(&format!("bind({}) failed", addr));
-            info!("Electrum RPC server running on {}", addr);
-            loop {
-                let (stream, addr) = listener.accept().expect("accept failed");
-                stream
-                    .set_nonblocking(false)
-                    .expect("failed to set connection as blocking");
-                acceptor.send(Some((stream, addr))).expect("send failed");
-            }
-        });
-        chan
-    }
-
-    pub fn start(addr: SocketAddr, query: Arc<Query>, metrics: &Metrics) -> RPC {
-        let stats = Arc::new(Stats {
-            latency: metrics.histogram_vec(
-                HistogramOpts::new("electrum_rpc", "Electrum RPC latency (seconds)"),
-                &["method"],
-            ),
-            subscriptions: metrics.gauge(MetricOpts::new(
-                "electrum_subscriptions",
-                "# of Electrum subscriptions",
-            )),
-        });
-        let notification = Channel::new();
-        let handle = RPC {
-            notification: notification.sender(),
-            server: Some(spawn_thread("rpc", move || {
-                let senders = Arc::new(Mutex::new(Vec::<SyncSender<Message>>::new()));
-                let acceptor = RPC::start_acceptor(addr);
-                RPC::start_notifier(notification, senders.clone(), acceptor.sender());
-                let mut children = vec![];
-                while let Some((stream, addr)) = acceptor.receiver().recv().unwrap() {
-                    let query = query.clone();
-                    let senders = senders.clone();
-                    let stats = stats.clone();
-                    children.push(spawn_thread("peer", move || {
-                        info!("[{}] connected peer", addr);
-                        let conn = Connection::new(query, stream, addr, stats);
-                        senders.lock().unwrap().push(conn.chan.sender());
-                        conn.run();
-                        info!("[{}] disconnected peer", addr);
-                    }));
-                }
-                trace!("closing {} RPC connections", senders.lock().unwrap().len());
-                for sender in senders.lock().unwrap().iter() {
-                    let _ = sender.send(Message::Done);
-                }
-                trace!("waiting for {} RPC handling threads", children.len());
-                for child in children {
-                    let _ = child.join();
-                }
-                trace!("RPC connections are closed");
-            })),
-        };
-        handle
-    }
-
-    pub fn notify(&self) {
-        self.notification.send(Notification::Periodic).unwrap();
-    }
-}
-
-impl Drop for RPC {
-    fn drop(&mut self) {
-        trace!("stop accepting new RPCs");
-        self.notification.send(Notification::Exit).unwrap();
-        self.server.take().map(|t| t.join().unwrap());
-        trace!("RPC server is stopped");
-    }
-}

src/electrum/client.rs

@@ -0,0 +1,41 @@
+use std::collections::HashMap;
+use std::convert::TryFrom;
+
+use bitcoin::hashes::sha256d;
+use bitcoin::hashes::Hash;
+pub use electrum_client::client::Client;
+pub use electrum_client::ServerFeaturesRes;
+
+use crate::chain::BlockHash;
+use crate::electrum::ServerFeatures;
+use crate::errors::{Error, ResultExt};
+
+// Convert from electrum-client's server features struct to ours. We're using a different struct because
+// the electrum-client's one doesn't support the "hosts" key.
+impl TryFrom<ServerFeaturesRes> for ServerFeatures {
+    type Error = Error;
+    fn try_from(mut features: ServerFeaturesRes) -> Result<Self, Self::Error> {
+        features.genesis_hash.reverse();
+
+        Ok(ServerFeatures {
+            // electrum-client doesn't retain the hosts map data, but we already have it from the add_peer request
+            hosts: HashMap::new(),
+            genesis_hash: BlockHash::from_raw_hash(sha256d::Hash::from_byte_array(
+                features.genesis_hash,
+            )),
+            server_version: features.server_version,
+            protocol_min: features
+                .protocol_min
+                .parse()
+                .chain_err(|| "invalid protocol_min")?,
+            protocol_max: features
+                .protocol_max
+                .parse()
+                .chain_err(|| "invalid protocol_max")?,
+            pruning: features.pruning.map(|pruning| pruning as usize),
+            hash_function: features
+                .hash_function
+                .chain_err(|| "missing hash_function")?,
+        })
+    }
+}

src/electrum/discovery.rs

@@ -0,0 +1,603 @@
+use std::cmp::Ordering;
+use std::collections::{hash_map::Entry, BinaryHeap, HashMap, HashSet};
+use std::convert::TryInto;
+use std::fmt;
+use std::net::{IpAddr, SocketAddr, ToSocketAddrs};
+use std::str::FromStr;
+use std::sync::{Arc, RwLock};
+use std::thread;
+use std::time::{Duration, Instant};
+
+use electrum_client::ElectrumApi;
+
+use crate::chain::Network;
+use crate::electrum::{Client, Hostname, Port, ProtocolVersion, ServerFeatures};
+use crate::errors::{Result, ResultExt};
+use crate::util::spawn_thread;
+
+mod default_servers;
+use default_servers::add_default_servers;
+
+const HEALTH_CHECK_FREQ: Duration = Duration::from_secs(3600); // check servers every hour
+const JOB_INTERVAL: Duration = Duration::from_secs(1); // run one health check job every second
+const MAX_CONSECUTIVE_FAILURES: usize = 24; // drop servers after 24 consecutive failing attempts (~24 hours) (~24 hours)
+const MAX_QUEUE_SIZE: usize = 500; // refuse accepting new servers if we have that many health check jobs
+const MAX_SERVERS_PER_REQUEST: usize = 3; // maximum number of server hosts added per server.add_peer call
+const MAX_SERVICES_PER_REQUEST: usize = 6; // maximum number of services added per server.add_peer call
+
+#[derive(Debug)]
+pub struct DiscoveryManager {
+    /// A queue of scheduled health check jobs, including for healthy, unhealthy and untested servers
+    queue: RwLock<BinaryHeap<HealthCheck>>,
+
+    /// A list of servers that were found to be healthy on their last health check
+    healthy: RwLock<HashMap<ServerAddr, Server>>,
+
+    /// Used to test for protocol version compatibility
+    our_version: ProtocolVersion,
+
+    /// So that we don't list ourselves
+    our_addrs: HashSet<ServerAddr>,
+
+    /// For advertising ourself to other servers
+    our_features: ServerFeatures,
+
+    /// Whether we should announce ourselves to the servers we're connecting to
+    announce: bool,
+
+    /// Optional, will not support onion hosts without this
+    tor_proxy: Option<SocketAddr>,
+}
+
+/// A Server corresponds to a single IP address or onion hostname, with one or more services
+/// exposed on different ports.
+#[derive(Debug)]
+struct Server {
+    services: HashSet<Service>,
+    hostname: Hostname,
+    features: ServerFeatures,
+    // the `ServerAddr` isn't kept here directly, but is also available next to `Server` as the key for
+    // the `healthy` field on `DiscoveryManager`
+}
+
+#[derive(Eq, PartialEq, Hash, Clone, Debug)]
+enum ServerAddr {
+    Clearnet(IpAddr),
+    Onion(Hostname),
+}
+
+#[derive(Eq, PartialEq, Hash, Copy, Clone, Debug)]
+pub enum Service {
+    Tcp(Port),
+    Ssl(Port),
+    // unimplemented: Ws and Wss
+}
+
+/// A queued health check job, one per service/port (and not per server)
+#[derive(Eq, Debug)]
+struct HealthCheck {
+    addr: ServerAddr,
+    hostname: Hostname,
+    service: Service,
+    is_default: bool,
+    #[allow(dead_code)]
+    added_by: Option<IpAddr>,
+    last_check: Option<Instant>,
+    last_healthy: Option<Instant>,
+    consecutive_failures: usize,
+}
+
+/// The server entry format returned from server.peers.subscribe
+#[derive(Serialize)]
+pub struct ServerEntry(ServerAddr, Hostname, Vec<String>);
+
+impl DiscoveryManager {
+    pub fn new(
+        our_network: Network,
+        our_features: ServerFeatures,
+        our_version: ProtocolVersion,
+        announce: bool,
+        tor_proxy: Option<SocketAddr>,
+    ) -> Self {
+        let our_addrs = our_features
+            .hosts
+            .keys()
+            .filter_map(|hostname| {
+                ServerAddr::resolve(hostname)
+                    .map_err(|e| warn!("failed resolving own hostname {}: {:?}", hostname, e))
+                    .ok()
+            })
+            .collect();
+        let discovery = Self {
+            our_addrs,
+            our_version,
+            our_features,
+            announce,
+            tor_proxy,
+            healthy: Default::default(),
+            queue: Default::default(),
+        };
+        add_default_servers(&discovery, our_network);
+        discovery
+    }
+
+    /// Add a server requested via `server.add_peer`
+    pub fn add_server_request(&self, added_by: IpAddr, features: ServerFeatures) -> Result<()> {
+        self.verify_compatibility(&features)?;
+
+        let mut queue = self.queue.write().unwrap();
+        ensure!(queue.len() < MAX_QUEUE_SIZE, "queue size exceeded");
+
+        // TODO optimize
+        let mut existing_services: HashMap<ServerAddr, HashSet<Service>> = HashMap::new();
+        for job in queue.iter() {
+            existing_services
+                .entry(job.addr.clone())
+                .or_default()
+                .insert(job.service);
+        }
+
+        // collect HealthChecks for candidate services
+        let jobs = features
+            .hosts
+            .iter()
+            .take(MAX_SERVERS_PER_REQUEST)
+            .filter_map(|(hostname, ports)| {
+                let hostname = hostname.to_lowercase();
+
+                if hostname.len() > 100 {
+                    warn!("skipping invalid hostname");
+                    return None;
+                }
+                let addr = match ServerAddr::resolve(&hostname) {
+                    Ok(addr) => addr,
+                    Err(e) => {
+                        warn!("failed resolving {}: {:?}", hostname, e);
+                        return None;
+                    }
+                };
+                if !is_remote_addr(&addr) || self.our_addrs.contains(&addr) {
+                    warn!("skipping own or non-remote server addr");
+                    return None;
+                }
+                // ensure the server address matches the ip that advertised it to us.
+                // onion hosts are exempt.
+                if let ServerAddr::Clearnet(ip) = addr {
+                    if ip != added_by {
+                        warn!(
+                            "server ip does not match source ip ({}, {} != {})",
+                            hostname, ip, added_by
+                        );
+                        return None;
+                    }
+                }
+                Some((addr, hostname, ports))
+            })
+            .flat_map(|(addr, hostname, ports)| {
+                let tcp_service = ports.tcp_port.into_iter().map(Service::Tcp);
+                let ssl_service = ports.ssl_port.into_iter().map(Service::Ssl);
+                let services = tcp_service.chain(ssl_service).collect::<HashSet<Service>>();
+
+                services
+                    .into_iter()
+                    .filter(|service| {
+                        existing_services
+                            .get(&addr)
+                            .is_none_or(|s| !s.contains(service))
+                    })
+                    .map(|service| {
+                        HealthCheck::new(addr.clone(), hostname.clone(), service, Some(added_by))
+                    })
+                    .collect::<Vec<_>>()
+            })
+            .take(MAX_SERVICES_PER_REQUEST)
+            .collect::<Vec<_>>();
+
+        ensure!(
+            queue.len() + jobs.len() <= MAX_QUEUE_SIZE,
+            "queue size exceeded"
+        );
+
+        queue.extend(jobs);
+        Ok(())
+    }
+
+    /// Add a default server. Default servers are exempt from limits and given more leniency
+    /// before being removed due to unavailability.
+    pub fn add_default_server(&self, hostname: Hostname, services: Vec<Service>) -> Result<()> {
+        let addr = ServerAddr::resolve(&hostname)?;
+        let mut queue = self.queue.write().unwrap();
+        queue.extend(
+            services
+                .into_iter()
+                .map(|service| HealthCheck::new(addr.clone(), hostname.clone(), service, None)),
+        );
+        Ok(())
+    }
+
+    /// Get the list of healthy servers formatted for `servers.peers.subscribe`
+    pub fn get_servers(&self) -> Vec<ServerEntry> {
+        // XXX return a random sample instead of everything?
+        self.healthy
+            .read()
+            .unwrap()
+            .iter()
+            .map(|(addr, server)| {
+                ServerEntry(addr.clone(), server.hostname.clone(), server.feature_strs())
+            })
+            .collect()
+    }
+
+    pub fn our_features(&self) -> &ServerFeatures {
+        &self.our_features
+    }
+
+    /// Run the next health check in the queue (a single one)
+    fn run_health_check(&self) -> Result<()> {
+        // abort if there are no entries in the queue, or its still too early for the next one up
+        if self.queue.read().unwrap().peek().is_none_or(|next| {
+            next.last_check
+                .is_some_and(|t| t.elapsed() < HEALTH_CHECK_FREQ)
+        }) {
+            return Ok(());
+        }
+
+        let mut job = self.queue.write().unwrap().pop().unwrap();
+        debug!("processing {:?}", job);
+
+        let was_healthy = job.is_healthy();
+
+        match self.check_server(&job.addr, &job.hostname, job.service) {
+            Ok(features) => {
+                debug!("{} {:?} is available", job.hostname, job.service);
+
+                if !was_healthy {
+                    self.save_healthy_service(&job, features);
+                }
+                // XXX update features?
+
+                job.last_check = Some(Instant::now());
+                job.last_healthy = job.last_check;
+                job.consecutive_failures = 0;
+                // schedule the next health check
+                self.queue.write().unwrap().push(job);
+
+                Ok(())
+            }
+            Err(e) => {
+                debug!("{} {:?} is unavailable: {:?}", job.hostname, job.service, e);
+
+                if was_healthy {
+                    // XXX should we assume the server's other services are down too?
+                    self.remove_unhealthy_service(&job);
+                }
+
+                job.last_check = Some(Instant::now());
+                job.consecutive_failures += 1;
+
+                if job.should_retry() {
+                    self.queue.write().unwrap().push(job);
+                } else {
+                    debug!("giving up on {:?}", job);
+                }
+
+                Err(e)
+            }
+        }
+    }
+
+    /// Upsert the server/service into the healthy set
+    fn save_healthy_service(&self, job: &HealthCheck, features: ServerFeatures) {
+        let addr = job.addr.clone();
+        let mut healthy = self.healthy.write().unwrap();
+        healthy
+            .entry(addr)
+            .or_insert_with(|| Server::new(job.hostname.clone(), features))
+            .services
+            .insert(job.service);
+    }
+
+    /// Remove the service, and remove the server entirely if it has no other reamining healthy services
+    fn remove_unhealthy_service(&self, job: &HealthCheck) {
+        let addr = job.addr.clone();
+        let mut healthy = self.healthy.write().unwrap();
+        if let Entry::Occupied(mut entry) = healthy.entry(addr) {
+            let server = entry.get_mut();
+            assert!(server.services.remove(&job.service));
+            if server.services.is_empty() {
+                entry.remove_entry();
+            }
+        } else {
+            unreachable!("missing expected server, corrupted state");
+        }
+    }
+
+    fn check_server(
+        &self,
+        addr: &ServerAddr,
+        hostname: &Hostname,
+        service: Service,
+    ) -> Result<ServerFeatures> {
+        debug!("checking service {:?} {:?}", addr, service);
+
+        let server_url = match (addr, service) {
+            (ServerAddr::Clearnet(ip), Service::Tcp(port)) => format!("tcp://{}:{}", ip, port),
+            (ServerAddr::Clearnet(_), Service::Ssl(port)) => format!("ssl://{}:{}", hostname, port),
+            (ServerAddr::Onion(onion_host), Service::Tcp(port)) => {
+                format!("tcp://{}:{}", onion_host, port)
+            }
+            (ServerAddr::Onion(onion_host), Service::Ssl(port)) => {
+                format!("ssl://{}:{}", onion_host, port)
+            }
+        };
+
+        let mut config = electrum_client::ConfigBuilder::new();
+        if let ServerAddr::Onion(_) = addr {
+            let socks = electrum_client::Socks5Config::new(
+                self.tor_proxy
+                    .chain_err(|| "no tor proxy configured, onion hosts are unsupported")?,
+            );
+            config = config.socks5(Some(socks))
+        }
+
+        let client = Client::from_config(&server_url, config.build())?;
+
+        let features = client.server_features()?.try_into()?;
+        self.verify_compatibility(&features)?;
+
+        if self.announce {
+            // XXX should we require the other side to reciprocate?
+            ensure!(
+                client.server_add_peer(&self.our_features)?,
+                "server does not reciprocate"
+            );
+        }
+
+        Ok(features)
+    }
+
+    fn verify_compatibility(&self, features: &ServerFeatures) -> Result<()> {
+        ensure!(
+            features.genesis_hash == self.our_features.genesis_hash,
+            "incompatible networks"
+        );
+
+        ensure!(
+            features.protocol_min <= self.our_version && features.protocol_max >= self.our_version,
+            "incompatible protocol versions"
+        );
+
+        ensure!(
+            features.hash_function == "sha256",
+            "incompatible hash function"
+        );
+
+        Ok(())
+    }
+
+    pub fn spawn_jobs_thread(manager: Arc<DiscoveryManager>) {
+        spawn_thread("discovery-jobs", move || loop {
+            if let Err(e) = manager.run_health_check() {
+                debug!("health check failed: {:?}", e);
+            }
+            // XXX use a dynamic JOB_INTERVAL, adjusted according to the queue size and HEALTH_CHECK_FREQ?
+            thread::sleep(JOB_INTERVAL);
+        });
+    }
+}
+
+impl Server {
+    fn new(hostname: Hostname, features: ServerFeatures) -> Self {
+        Server {
+            hostname,
+            features,
+            services: HashSet::new(),
+        }
+    }
+
+    /// Get server features and services in the compact string array format used for `servers.peers.subscribe`
+    fn feature_strs(&self) -> Vec<String> {
+        let mut strs = Vec::with_capacity(self.services.len() + 1);
+        strs.push(format!("v{}", self.features.protocol_max));
+        if let Some(pruning) = self.features.pruning {
+            strs.push(format!("p{}", pruning));
+        }
+        strs.extend(self.services.iter().map(|s| s.to_string()));
+        strs
+    }
+}
+
+impl ServerAddr {
+    fn resolve(host: &str) -> Result<Self> {
+        Ok(if host.ends_with(".onion") {
+            ServerAddr::Onion(host.into())
+        } else if let Ok(ip) = IpAddr::from_str(host) {
+            ServerAddr::Clearnet(ip)
+        } else {
+            let ip = format!("{}:1", host)
+                .to_socket_addrs()
+                .chain_err(|| "hostname resolution failed")?
+                .next()
+                .chain_err(|| "hostname resolution failed")?
+                .ip();
+            ServerAddr::Clearnet(ip)
+        })
+    }
+}
+
+impl fmt::Display for ServerAddr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ServerAddr::Clearnet(ip) => write!(f, "{}", ip),
+            ServerAddr::Onion(hostname) => write!(f, "{}", hostname),
+        }
+    }
+}
+
+impl serde::Serialize for ServerAddr {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl HealthCheck {
+    fn new(
+        addr: ServerAddr,
+        hostname: Hostname,
+        service: Service,
+        added_by: Option<IpAddr>,
+    ) -> Self {
+        HealthCheck {
+            addr,
+            hostname,
+            service,
+            is_default: added_by.is_none(),
+            added_by,
+            last_check: None,
+            last_healthy: None,
+            consecutive_failures: 0,
+        }
+    }
+
+    fn is_healthy(&self) -> bool {
+        match (self.last_check, self.last_healthy) {
+            (Some(last_check), Some(last_healthy)) => last_check == last_healthy,
+            _ => false,
+        }
+    }
+
+    // allow the server to fail up to MAX_CONSECTIVE_FAILURES time before giving up on it.
+    // if its a non-default server and the very first attempt fails, give up immediatly.
+    fn should_retry(&self) -> bool {
+        (self.last_healthy.is_some() || self.is_default)
+            && self.consecutive_failures < MAX_CONSECUTIVE_FAILURES
+    }
+}
+
+impl PartialEq for HealthCheck {
+    fn eq(&self, other: &Self) -> bool {
+        self.hostname == other.hostname && self.service == other.service
+    }
+}
+
+impl Ord for HealthCheck {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.last_check.cmp(&other.last_check).reverse()
+    }
+}
+
+impl PartialOrd for HealthCheck {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl fmt::Display for Service {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Service::Tcp(port) => write!(f, "t{}", port),
+            Service::Ssl(port) => write!(f, "s{}", port),
+        }
+    }
+}
+
+fn is_remote_addr(addr: &ServerAddr) -> bool {
+    match addr {
+        ServerAddr::Onion(_) => true,
+        ServerAddr::Clearnet(ip) => {
+            !ip.is_loopback()
+                && !ip.is_unspecified()
+                && !ip.is_multicast()
+                && !match ip {
+                    IpAddr::V4(ipv4) => ipv4.is_private(),
+                    IpAddr::V6(_) => false,
+                }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chain::genesis_hash;
+    use crate::chain::Network;
+    use std::time;
+
+    use crate::config::VERSION_STRING;
+
+    const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::new(1, 4);
+
+    #[test]
+    #[ignore = "This test requires external connection to server that no longer exists"]
+    fn test() -> Result<()> {
+        stderrlog::new().verbosity(4).init().unwrap();
+
+        #[cfg(feature = "liquid")]
+        let testnet = Network::LiquidTestnet;
+        #[cfg(not(feature = "liquid"))]
+        let testnet = Network::Testnet;
+
+        let features = ServerFeatures {
+            hosts: serde_json::from_str("{\"test.foobar.example\":{\"tcp_port\":60002}}").unwrap(),
+            server_version: VERSION_STRING.clone(),
+            genesis_hash: genesis_hash(testnet),
+            protocol_min: PROTOCOL_VERSION,
+            protocol_max: PROTOCOL_VERSION,
+            hash_function: "sha256".into(),
+            pruning: None,
+        };
+        let discovery = Arc::new(DiscoveryManager::new(
+            testnet,
+            features,
+            PROTOCOL_VERSION,
+            false,
+            None,
+        ));
+        discovery
+            .add_default_server(
+                "electrum.blockstream.info".into(),
+                vec![Service::Tcp(60001)],
+            )
+            .unwrap();
+        discovery
+            .add_default_server("testnet.hsmiths.com".into(), vec![Service::Ssl(53012)])
+            .unwrap();
+        discovery
+            .add_default_server(
+                "tn.not.fyi".into(),
+                vec![Service::Tcp(55001), Service::Ssl(55002)],
+            )
+            .unwrap();
+        discovery
+            .add_default_server(
+                "electrum.blockstream.info".into(),
+                vec![Service::Tcp(60001), Service::Ssl(60002)],
+            )
+            .unwrap();
+        discovery
+            .add_default_server(
+                "explorerzydxu5ecjrkwceayqybizmpjjznk5izmitf2modhcusuqlid.onion".into(),
+                vec![Service::Tcp(143)],
+            )
+            .unwrap();
+
+        debug!("{:#?}", discovery);
+
+        for _ in 0..12 {
+            discovery
+                .run_health_check()
+                .map_err(|e| warn!("{:?}", e))
+                .ok();
+            thread::sleep(time::Duration::from_secs(1));
+        }
+
+        debug!("{:#?}", discovery);
+
+        info!("{}", json!(discovery.get_servers()));
+
+        Ok(())
+    }
+}

src/electrum/discovery/default_servers.rs

@@ -0,0 +1,449 @@
+use crate::chain::Network;
+#[allow(unused_imports)]
+use crate::electrum::discovery::{DiscoveryManager, Service};
+
+#[allow(unused_variables)]
+pub fn add_default_servers(discovery: &DiscoveryManager, network: Network) {
+    match network {
+        #[cfg(not(feature = "liquid"))]
+        Network::Bitcoin => {
+            discovery
+                .add_default_server(
+                    "3smoooajg7qqac2y.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "81-7-10-251.blue.kundencontroller.de".into(),
+                    vec![Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "E-X.not.fyi".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "VPS.hsmiths.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "b.ooze.cc".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "bauerjda5hnedjam.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "bauerjhejlv6di7s.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "bitcoin.corgi.party".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "bitcoin3nqy3db7c.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "bitcoins.sk".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "btc.cihar.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "btc.xskyx.net".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "currentlane.lovebitco.in".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "daedalus.bauerj.eu".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.jochen-hoenicke.de".into(),
+                    vec![Service::Tcp(50003), Service::Ssl(50005)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "dragon085.startdedicated.de".into(),
+                    vec![Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "e-1.claudioboxx.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "e.keff.org".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum-server.ninja".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum-unlimited.criptolayer.net".into(),
+                    vec![Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.eff.ro".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.festivaldelhumor.org".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.hsmiths.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.leblancnet.us".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("electrum.mindspot.org".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.qtornado.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("electrum.taborsky.cz".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.villocq.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum2.eff.ro".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum2.villocq.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrumx.bot.nu".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrumx.ddns.net".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("electrumx.ftp.sh".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrumx.ml".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrumx.soon.it".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("electrumxhqdsmlu.onion".into(), vec![Service::Tcp(50001)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "elx01.knas.systems".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "enode.duckdns.org".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "fedaykin.goip.de".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "fn.48.org".into(),
+                    vec![Service::Tcp(50003), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "helicarrier.bauerj.eu".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "hsmiths4fyqlw5xw.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "hsmiths5mjk6uijs.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "icarus.tetradrachm.net".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrum.emzy.de".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "ndnd.selfhost.eu".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("ndndword5lpb7eex.onion".into(), vec![Service::Tcp(50001)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "orannis.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "ozahtqwp25chjdjd.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "qtornadoklbgdyww.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("rbx.curalle.ovh".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server("s7clinmo4cazmhul.onion".into(), vec![Service::Tcp(50001)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "tardis.bauerj.eu".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("technetium.network".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "tomscryptos.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "ulrichard.ch".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "vmd27610.contaboserver.net".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "vmd30612.contaboserver.net".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "wsw6tua3xl24gsmi264zaep6seppjyrkyucpsmuxnjzyt3f3j6swshad.onion".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "xray587.startdedicated.de".into(),
+                    vec![Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "yuio.top".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "bitcoin.dragon.zone".into(),
+                    vec![Service::Tcp(50003), Service::Ssl(50004)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "ecdsa.net".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(110)],
+                )
+                .ok();
+            discovery
+                .add_default_server("btc.usebsv.com".into(), vec![Service::Ssl(50006)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "e2.keff.org".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server("electrum.hodlister.co".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server("electrum3.hodlister.co".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server("electrum5.hodlister.co".into(), vec![Service::Ssl(50002)])
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrumx.electricnewyear.net".into(),
+                    vec![Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "fortress.qtornado.com".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(443)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "green-gold.westeurope.cloudapp.azure.com".into(),
+                    vec![Service::Tcp(56001), Service::Ssl(56002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "electrumx.erbium.eu".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+        }
+        #[cfg(not(feature = "liquid"))]
+        Network::Testnet => {
+            discovery
+                .add_default_server(
+                    "hsmithsxurybd7uh.onion".into(),
+                    vec![Service::Tcp(53011), Service::Ssl(53012)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "testnet.hsmiths.com".into(),
+                    vec![Service::Tcp(53011), Service::Ssl(53012)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "testnet.qtornado.com".into(),
+                    vec![Service::Tcp(51001), Service::Ssl(51002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "testnet1.bauerj.eu".into(),
+                    vec![Service::Tcp(50001), Service::Ssl(50002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "tn.not.fyi".into(),
+                    vec![Service::Tcp(55001), Service::Ssl(55002)],
+                )
+                .ok();
+            discovery
+                .add_default_server(
+                    "bitcoin.cluelessperson.com".into(),
+                    vec![Service::Tcp(51001), Service::Ssl(51002)],
+                )
+                .ok();
+        }
+
+        _ => (),
+    }
+}

src/electrum/mod.rs

@@ -0,0 +1,119 @@
+mod server;
+pub use server::RPC;
+
+#[cfg(feature = "electrum-discovery")]
+mod client;
+#[cfg(feature = "electrum-discovery")]
+mod discovery;
+#[cfg(feature = "electrum-discovery")]
+pub use {client::Client, discovery::DiscoveryManager};
+
+use std::cmp::Ordering;
+use std::collections::HashMap;
+use std::str::FromStr;
+
+use serde::{de, Deserialize, Deserializer, Serialize};
+
+use crate::chain::BlockHash;
+use crate::errors::ResultExt;
+use crate::util::BlockId;
+
+pub fn get_electrum_height(blockid: Option<BlockId>, has_unconfirmed_parents: bool) -> isize {
+    match (blockid, has_unconfirmed_parents) {
+        (Some(blockid), _) => blockid.height as isize,
+        (None, false) => 0,
+        (None, true) => -1,
+    }
+}
+
+pub type Port = u16;
+pub type Hostname = String;
+
+pub type ServerHosts = HashMap<Hostname, ServerPorts>;
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ServerFeatures {
+    pub hosts: ServerHosts,
+    pub genesis_hash: BlockHash,
+    pub server_version: String,
+    pub protocol_min: ProtocolVersion,
+    pub protocol_max: ProtocolVersion,
+    pub pruning: Option<usize>,
+    pub hash_function: String,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ServerPorts {
+    tcp_port: Option<Port>,
+    ssl_port: Option<Port>,
+}
+
+#[derive(Eq, PartialEq, Debug, Clone, Default)]
+pub struct ProtocolVersion {
+    major: usize,
+    minor: usize,
+}
+
+impl ProtocolVersion {
+    pub const fn new(major: usize, minor: usize) -> Self {
+        Self { major, minor }
+    }
+}
+
+impl Ord for ProtocolVersion {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.major
+            .cmp(&other.major)
+            .then_with(|| self.minor.cmp(&other.minor))
+    }
+}
+
+impl PartialOrd for ProtocolVersion {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl FromStr for ProtocolVersion {
+    type Err = crate::errors::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut iter = s.split('.');
+        Ok(Self {
+            major: iter
+                .next()
+                .chain_err(|| "missing major")?
+                .parse()
+                .chain_err(|| "invalid major")?,
+            minor: iter
+                .next()
+                .chain_err(|| "missing minor")?
+                .parse()
+                .chain_err(|| "invalid minor")?,
+        })
+    }
+}
+
+impl std::fmt::Display for ProtocolVersion {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}.{}", self.major, self.minor)
+    }
+}
+
+impl Serialize for ProtocolVersion {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.collect_str(&self)
+    }
+}
+
+impl<'de> Deserialize<'de> for ProtocolVersion {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        FromStr::from_str(&s).map_err(de::Error::custom)
+    }
+}

src/elements/asset.rs

@@ -0,0 +1,662 @@
+use std::collections::{HashMap, HashSet};
+use std::sync::{Arc, RwLock, RwLockReadGuard};
+
+use bitcoin::hashes::{sha256, Hash};
+use elements::confidential::{Asset, Value};
+use elements::encode::{deserialize, serialize};
+use elements::secp256k1_zkp::ZERO_TWEAK;
+use elements::{issuance::ContractHash, AssetId, AssetIssuance, OutPoint, Transaction, TxIn};
+
+use crate::chain::{BNetwork, BlockHash, Network, Txid};
+use crate::elements::peg::{get_pegin_data, get_pegout_data, PeginInfo, PegoutInfo};
+use crate::elements::registry::{AssetMeta, AssetRegistry};
+use crate::errors::*;
+use crate::new_index::schema::{Operation, TxHistoryInfo, TxHistoryKey, TxHistoryRow};
+use crate::new_index::{db::DBFlush, ChainQuery, DBRow, Mempool, Query};
+use crate::util::{
+    bincode_util, full_hash, Bytes, FullHash, IsProvablyUnspendable, TransactionStatus, TxInput,
+};
+
+lazy_static! {
+    pub static ref NATIVE_ASSET_ID: AssetId =
+        "6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d"
+            .parse()
+            .unwrap();
+    pub static ref NATIVE_ASSET_ID_TESTNET: AssetId =
+        "144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49"
+            .parse()
+            .unwrap();
+    pub static ref NATIVE_ASSET_ID_REGTEST: AssetId =
+        "5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225"
+            .parse()
+            .unwrap();
+}
+
+fn parse_asset_id(sl: &[u8]) -> AssetId {
+    AssetId::from_slice(sl).expect("failed to parse AssetId")
+}
+
+#[derive(Serialize)]
+#[serde(untagged)]
+#[allow(clippy::large_enum_variant)]
+pub enum LiquidAsset {
+    Issued(IssuedAsset),
+    Native(PeggedAsset),
+}
+
+#[derive(Serialize)]
+pub struct PeggedAsset {
+    pub asset_id: AssetId,
+    pub chain_stats: PeggedAssetStats,
+    pub mempool_stats: PeggedAssetStats,
+}
+
+#[derive(Serialize)]
+pub struct IssuedAsset {
+    pub asset_id: AssetId,
+    pub issuance_txin: TxInput,
+    #[serde(serialize_with = "crate::util::serialize_outpoint")]
+    pub issuance_prevout: OutPoint,
+    pub reissuance_token: AssetId,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub contract_hash: Option<ContractHash>,
+
+    // the confirmation status of the initial issuance transaction
+    pub status: TransactionStatus,
+
+    pub chain_stats: IssuedAssetStats,
+    pub mempool_stats: IssuedAssetStats,
+
+    // optional metadata from registry
+    #[serde(flatten)]
+    pub meta: Option<AssetMeta>,
+}
+
+// DB representation (issued assets only)
+#[derive(Serialize, Deserialize, Debug)]
+pub struct AssetRow {
+    pub issuance_txid: FullHash,
+    pub issuance_vin: u32,
+    pub prev_txid: FullHash,
+    pub prev_vout: u32,
+    pub issuance: Bytes, // bincode does not like dealing with AssetIssuance, deserialization fails with "invalid type: sequence, expected a struct"
+    pub reissuance_token: FullHash,
+}
+
+impl IssuedAsset {
+    pub fn new(
+        asset_id: &AssetId,
+        asset: &AssetRow,
+        (chain_stats, mempool_stats): (IssuedAssetStats, IssuedAssetStats),
+        meta: Option<AssetMeta>,
+        status: TransactionStatus,
+    ) -> Self {
+        let issuance: AssetIssuance =
+            deserialize(&asset.issuance).expect("failed parsing AssetIssuance");
+
+        let reissuance_token = parse_asset_id(&asset.reissuance_token);
+
+        let contract_hash = if issuance.asset_entropy != [0u8; 32] {
+            Some(ContractHash::from_byte_array(issuance.asset_entropy))
+        } else {
+            None
+        };
+
+        Self {
+            asset_id: *asset_id,
+            issuance_txin: TxInput {
+                txid: deserialize(&asset.issuance_txid).unwrap(),
+                vin: asset.issuance_vin,
+            },
+            issuance_prevout: OutPoint {
+                txid: deserialize(&asset.prev_txid).unwrap(),
+                vout: asset.prev_vout,
+            },
+            contract_hash,
+            reissuance_token,
+            status,
+            chain_stats,
+            mempool_stats,
+            meta,
+        }
+    }
+}
+
+impl LiquidAsset {
+    pub fn supply(&self) -> Option<u64> {
+        match self {
+            LiquidAsset::Native(asset) => Some(
+                asset.chain_stats.peg_in_amount
+                    - asset.chain_stats.peg_out_amount
+                    - asset.chain_stats.burned_amount
+                    + asset.mempool_stats.peg_in_amount
+                    - asset.mempool_stats.peg_out_amount
+                    - asset.mempool_stats.burned_amount,
+            ),
+            LiquidAsset::Issued(asset) => {
+                if asset.chain_stats.has_blinded_issuances
+                    || asset.mempool_stats.has_blinded_issuances
+                {
+                    None
+                } else {
+                    Some(
+                        asset.chain_stats.issued_amount - asset.chain_stats.burned_amount
+                            + asset.mempool_stats.issued_amount
+                            - asset.mempool_stats.burned_amount,
+                    )
+                }
+            }
+        }
+    }
+    pub fn precision(&self) -> u8 {
+        match self {
+            LiquidAsset::Native(_) => 8,
+            LiquidAsset::Issued(asset) => asset.meta.as_ref().map_or(0, |m| m.precision),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
+pub struct IssuingInfo {
+    pub txid: FullHash,
+    pub vin: u32,
+    pub is_reissuance: bool,
+    // None for blinded issuances
+    pub issued_amount: Option<u64>,
+    pub token_amount: Option<u64>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
+pub struct BurningInfo {
+    pub txid: FullHash,
+    pub vout: u32,
+    pub value: u64,
+}
+
+// Index confirmed transaction issuances and save as db rows
+pub fn index_confirmed_tx_assets(
+    tx: &Transaction,
+    confirmed_height: u32,
+    tx_position: u16,
+    network: Network,
+    parent_network: BNetwork,
+    rows: &mut Vec<DBRow>,
+    op: &Operation,
+) {
+    let (history, issuances) = index_tx_assets(tx, network, parent_network);
+
+    rows.extend(history.into_iter().map(|(asset_id, info)| {
+        let history_row = asset_history_row(&asset_id, confirmed_height, tx_position, info);
+        if let Operation::DeleteBlocksWithHistory(tx) = op {
+            tx.send(history_row.key.hash)
+                .expect("unbounded channel won't fail");
+        }
+        history_row.into_row()
+    }));
+
+    // the initial issuance is kept twice: once in the history index under I<asset><height><txid:vin>,
+    // and once separately under i<asset> for asset lookup with some more associated metadata.
+    // reissuances are only kept under the history index.
+    rows.extend(issuances.into_iter().map(|(asset_id, asset_row)| DBRow {
+        key: [b"i", &asset_id.into_inner()[..]].concat(),
+        value: bincode_util::serialize_little(&asset_row).unwrap(),
+    }));
+}
+
+// Index mempool transaction issuances and save to in-memory store
+pub fn index_mempool_tx_assets(
+    tx: &Transaction,
+    network: Network,
+    parent_network: BNetwork,
+    asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
+    asset_issuance: &mut HashMap<AssetId, AssetRow>,
+) {
+    let (history, issuances) = index_tx_assets(tx, network, parent_network);
+    for (asset_id, info) in history {
+        asset_history.entry(asset_id).or_default().push(info);
+    }
+    for (asset_id, issuance) in issuances {
+        asset_issuance.insert(asset_id, issuance);
+    }
+}
+
+// Remove mempool transaction issuances from in-memory store
+pub fn remove_mempool_tx_assets(
+    to_remove: &HashSet<&Txid>,
+    asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
+    asset_issuance: &mut HashMap<AssetId, AssetRow>,
+) {
+    // TODO optimize
+    asset_history.retain(|_assethash, entries| {
+        entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
+        !entries.is_empty()
+    });
+
+    asset_issuance.retain(|_assethash, issuance| {
+        let txid: Txid = deserialize(&issuance.issuance_txid).unwrap();
+        !to_remove.contains(&txid)
+    });
+}
+
+type HistoryAndIssuances = (Vec<(AssetId, TxHistoryInfo)>, Vec<(AssetId, AssetRow)>);
+// Internal utility function, index a transaction and return its history entries and issuances
+fn index_tx_assets(
+    tx: &Transaction,
+    network: Network,
+    parent_network: BNetwork,
+) -> HistoryAndIssuances {
+    let mut history = vec![];
+    let mut issuances = vec![];
+
+    let txid = full_hash(&tx.txid()[..]);
+
+    for (txo_index, txo) in tx.output.iter().enumerate() {
+        if let Some(pegout) = get_pegout_data(txo, network, parent_network) {
+            history.push((
+                pegout.asset.explicit().unwrap(),
+                TxHistoryInfo::Pegout(PegoutInfo {
+                    txid,
+                    vout: txo_index as u32,
+                    value: pegout.value,
+                }),
+            ));
+        } else if txo.script_pubkey.is_provably_unspendable_() && !txo.is_fee() {
+            if let (Asset::Explicit(asset_id), Value::Explicit(value)) = (txo.asset, txo.value) {
+                if value > 0 {
+                    history.push((
+                        asset_id,
+                        TxHistoryInfo::Burning(BurningInfo {
+                            txid,
+                            vout: txo_index as u32,
+                            value,
+                        }),
+                    ));
+                }
+            }
+        }
+    }
+
+    for (txi_index, txi) in tx.input.iter().enumerate() {
+        if let Some(pegin) = get_pegin_data(txi, network) {
+            history.push((
+                pegin.asset,
+                TxHistoryInfo::Pegin(PeginInfo {
+                    txid,
+                    vin: txi_index as u32,
+                    value: pegin.value,
+                }),
+            ));
+        } else if txi.has_issuance() {
+            let is_reissuance = txi.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
+
+            let asset_entropy = get_issuance_entropy(txi).expect("invalid issuance");
+            let asset_id = AssetId::from_entropy(asset_entropy);
+
+            let issued_amount = match txi.asset_issuance.amount {
+                Value::Explicit(amount) => Some(amount),
+                Value::Null => Some(0),
+                _ => None,
+            };
+            let token_amount = match txi.asset_issuance.inflation_keys {
+                Value::Explicit(amount) => Some(amount),
+                Value::Null => Some(0),
+                _ => None,
+            };
+
+            history.push((
+                asset_id,
+                TxHistoryInfo::Issuing(IssuingInfo {
+                    txid,
+                    vin: txi_index as u32,
+                    is_reissuance,
+                    issued_amount,
+                    token_amount,
+                }),
+            ));
+
+            if !is_reissuance {
+                let is_confidential =
+                    matches!(txi.asset_issuance.inflation_keys, Value::Confidential(..));
+                let reissuance_token =
+                    AssetId::reissuance_token_from_entropy(asset_entropy, is_confidential);
+
+                issuances.push((
+                    asset_id,
+                    AssetRow {
+                        issuance_txid: txid,
+                        issuance_vin: txi_index as u32,
+                        prev_txid: full_hash(&txi.previous_output.txid[..]),
+                        prev_vout: txi.previous_output.vout,
+                        issuance: serialize(&txi.asset_issuance),
+                        reissuance_token: full_hash(&reissuance_token.into_inner()[..]),
+                    },
+                ));
+            }
+        }
+    }
+
+    (history, issuances)
+}
+
+fn asset_history_row(
+    asset_id: &AssetId,
+    confirmed_height: u32,
+    tx_position: u16,
+    txinfo: TxHistoryInfo,
+) -> TxHistoryRow {
+    let key = TxHistoryKey {
+        code: b'I',
+        hash: full_hash(&asset_id.into_inner()[..]),
+        confirmed_height,
+        tx_position,
+        txinfo,
+    };
+    TxHistoryRow { key }
+}
+
+pub enum AssetRegistryLock<'a> {
+    RwLock(&'a Arc<RwLock<AssetRegistry>>),
+    RwLockReadGuard(&'a RwLockReadGuard<'a, AssetRegistry>),
+}
+
+pub fn lookup_asset(
+    query: &Query,
+    registry: Option<AssetRegistryLock>,
+    asset_id: &AssetId,
+    meta: Option<&AssetMeta>, // may optionally be provided if already known
+) -> Result<Option<LiquidAsset>> {
+    if query.network().pegged_asset() == Some(asset_id) {
+        let (chain_stats, mempool_stats) = pegged_asset_stats(query, asset_id);
+
+        return Ok(Some(LiquidAsset::Native(PeggedAsset {
+            asset_id: *asset_id,
+            chain_stats,
+            mempool_stats,
+        })));
+    }
+
+    let history_db = query.chain().store().history_db();
+    let mempool = query.mempool();
+    let mempool_issuances = &mempool.asset_issuance;
+
+    let chain_row = history_db
+        .get(&[b"i", &asset_id.into_inner()[..]].concat())
+        .map(|row| {
+            bincode_util::deserialize_little::<AssetRow>(&row).expect("failed parsing AssetRow")
+        });
+
+    let row = chain_row
+        .as_ref()
+        .or_else(|| mempool_issuances.get(asset_id));
+
+    Ok(if let Some(row) = row {
+        let reissuance_token = parse_asset_id(&row.reissuance_token);
+
+        let meta = meta.cloned().or_else(|| match registry {
+            Some(AssetRegistryLock::RwLock(rwlock)) => {
+                rwlock.read().unwrap().get(asset_id).cloned()
+            }
+            Some(AssetRegistryLock::RwLockReadGuard(guard)) => guard.get(asset_id).cloned(),
+            None => None,
+        });
+        let stats = issued_asset_stats(query.chain(), &mempool, asset_id, &reissuance_token);
+        let status = query.get_tx_status(&deserialize(&row.issuance_txid).unwrap());
+
+        let asset = IssuedAsset::new(asset_id, row, stats, meta, status);
+
+        Some(LiquidAsset::Issued(asset))
+    } else {
+        None
+    })
+}
+
+pub fn get_issuance_entropy(txin: &TxIn) -> Result<sha256::Midstate> {
+    if !txin.has_issuance() {
+        bail!("input has no issuance");
+    }
+
+    let is_reissuance = txin.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
+
+    Ok(if !is_reissuance {
+        let contract_hash = ContractHash::from_slice(&txin.asset_issuance.asset_entropy)
+            .chain_err(|| "invalid entropy (contract hash)")?;
+        AssetId::generate_asset_entropy(txin.previous_output, contract_hash)
+    } else {
+        sha256::Midstate::from_slice(&txin.asset_issuance.asset_entropy)
+            .chain_err(|| "invalid entropy (reissuance)")?
+    })
+}
+
+//
+// Asset stats
+//
+
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct IssuedAssetStats {
+    pub tx_count: usize,
+    pub issuance_count: usize,
+    pub issued_amount: u64,
+    pub burned_amount: u64,
+    pub has_blinded_issuances: bool,
+    pub reissuance_tokens: Option<u64>, // none if confidential
+    pub burned_reissuance_tokens: u64,
+}
+
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct PeggedAssetStats {
+    pub tx_count: usize,
+    pub peg_in_count: usize,
+    pub peg_in_amount: u64,
+    pub peg_out_count: usize,
+    pub peg_out_amount: u64,
+    pub burn_count: usize,
+    pub burned_amount: u64,
+}
+
+type AssetStatApplyFn<T> = fn(&TxHistoryInfo, &mut T, &mut HashSet<Txid>);
+
+fn asset_cache_key(asset_id: &AssetId) -> Bytes {
+    [b"z", &asset_id.into_inner()[..]].concat()
+}
+
+fn asset_cache_row<T>(asset_id: &AssetId, stats: &T, blockhash: &BlockHash) -> DBRow
+where
+    T: serde::Serialize,
+{
+    DBRow {
+        key: asset_cache_key(asset_id),
+        value: bincode_util::serialize_little(&(stats, blockhash)).unwrap(),
+    }
+}
+
+// Get stats for the network's pegged asset
+fn pegged_asset_stats(query: &Query, asset_id: &AssetId) -> (PeggedAssetStats, PeggedAssetStats) {
+    (
+        chain_asset_stats(query.chain(), asset_id, apply_pegged_asset_stats),
+        mempool_asset_stats(&query.mempool(), asset_id, apply_pegged_asset_stats),
+    )
+}
+
+// Get stats for issued assets
+fn issued_asset_stats(
+    chain: &ChainQuery,
+    mempool: &Mempool,
+    asset_id: &AssetId,
+    reissuance_token: &AssetId,
+) -> (IssuedAssetStats, IssuedAssetStats) {
+    let afn = apply_issued_asset_stats;
+
+    let mut chain_stats = chain_asset_stats(chain, asset_id, afn);
+    chain_stats.burned_reissuance_tokens =
+        chain_asset_stats(chain, reissuance_token, afn).burned_amount;
+
+    let mut mempool_stats = mempool_asset_stats(mempool, asset_id, afn);
+    mempool_stats.burned_reissuance_tokens =
+        mempool_asset_stats(mempool, reissuance_token, afn).burned_amount;
+
+    (chain_stats, mempool_stats)
+}
+
+// Get on-chain confirmed asset stats (issued or the pegged asset)
+fn chain_asset_stats<T>(chain: &ChainQuery, asset_id: &AssetId, apply_fn: AssetStatApplyFn<T>) -> T
+where
+    T: Default + serde::Serialize + serde::de::DeserializeOwned,
+{
+    // get the last known stats and the blockhash they are updated for.
+    // invalidates the cache if the block was orphaned.
+    let cache: Option<(T, usize)> = chain
+        .store()
+        .cache_db()
+        .get(&asset_cache_key(asset_id))
+        .map(|c| bincode_util::deserialize_little(&c).unwrap())
+        .and_then(|(stats, blockhash)| {
+            chain
+                .height_by_hash(&blockhash)
+                .map(|height| (stats, height))
+        });
+
+    // update stats with new transactions since
+    let (newstats, lastblock) = cache.map_or_else(
+        || chain_asset_stats_delta(chain, asset_id, T::default(), 0, apply_fn),
+        |(oldstats, blockheight)| {
+            chain_asset_stats_delta(chain, asset_id, oldstats, blockheight + 1, apply_fn)
+        },
+    );
+
+    // save updated stats to cache
+    if let Some(lastblock) = lastblock {
+        chain.store().cache_db().write(
+            vec![asset_cache_row(asset_id, &newstats, &lastblock)],
+            DBFlush::Enable,
+        );
+    }
+
+    newstats
+}
+
+// Update the asset stats with the delta of confirmed txs since start_height
+fn chain_asset_stats_delta<T>(
+    chain: &ChainQuery,
+    asset_id: &AssetId,
+    init_stats: T,
+    start_height: usize,
+    apply_fn: AssetStatApplyFn<T>,
+) -> (T, Option<BlockHash>) {
+    let history_iter = chain
+        .history_iter_scan(b'I', &asset_id.into_inner()[..], start_height)
+        .map(TxHistoryRow::from_row)
+        .filter_map(|history| {
+            chain
+                .tx_confirming_block(&history.get_txid())
+                .map(|blockid| (history, blockid))
+        });
+
+    let mut stats = init_stats;
+    let mut seen_txids = HashSet::new();
+    let mut lastblock = None;
+
+    for (row, blockid) in history_iter {
+        if lastblock != Some(blockid.hash) {
+            seen_txids.clear();
+        }
+        apply_fn(&row.key.txinfo, &mut stats, &mut seen_txids);
+        lastblock = Some(blockid.hash);
+    }
+
+    (stats, lastblock)
+}
+
+// Get mempool asset stats (issued or the pegged asset)
+pub fn mempool_asset_stats<T>(
+    mempool: &Mempool,
+    asset_id: &AssetId,
+    apply_fn: AssetStatApplyFn<T>,
+) -> T
+where
+    T: Default,
+{
+    let mut stats = T::default();
+
+    if let Some(history) = mempool.asset_history.get(asset_id) {
+        let mut seen_txids = HashSet::new();
+        for info in history {
+            apply_fn(info, &mut stats, &mut seen_txids)
+        }
+    }
+
+    stats
+}
+
+fn apply_issued_asset_stats(
+    info: &TxHistoryInfo,
+    stats: &mut IssuedAssetStats,
+    seen_txids: &mut HashSet<Txid>,
+) {
+    if seen_txids.insert(info.get_txid()) {
+        stats.tx_count += 1;
+    }
+
+    match info {
+        TxHistoryInfo::Issuing(issuance) => {
+            stats.issuance_count += 1;
+
+            match issuance.issued_amount {
+                Some(amount) => stats.issued_amount += amount,
+                None => stats.has_blinded_issuances = true,
+            }
+
+            if !issuance.is_reissuance {
+                stats.reissuance_tokens = issuance.token_amount;
+            }
+        }
+
+        TxHistoryInfo::Burning(info) => {
+            stats.burned_amount += info.value;
+        }
+
+        TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
+            // we don't keep funding/spending entries for assets
+            unreachable!();
+        }
+
+        TxHistoryInfo::Pegin(_) | TxHistoryInfo::Pegout(_) => {
+            // issued assets cannot have pegins/pegouts
+            unreachable!();
+        }
+    }
+}
+
+fn apply_pegged_asset_stats(
+    info: &TxHistoryInfo,
+    stats: &mut PeggedAssetStats,
+    seen_txids: &mut HashSet<Txid>,
+) {
+    if seen_txids.insert(info.get_txid()) {
+        stats.tx_count += 1;
+    }
+
+    match info {
+        TxHistoryInfo::Pegin(info) => {
+            stats.peg_in_count += 1;
+            stats.peg_in_amount += info.value;
+        }
+        TxHistoryInfo::Pegout(info) => {
+            stats.peg_out_count += 1;
+            stats.peg_out_amount += info.value;
+        }
+        TxHistoryInfo::Burning(info) => {
+            stats.burn_count += 1;
+            stats.burned_amount += info.value;
+        }
+        TxHistoryInfo::Issuing(_) => {
+            warn!("encountered issuance of native asset, ignoring (possibly freeinitialcoins?)");
+        }
+        TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
+            // these history entries variants are never kept for native assets
+            unreachable!();
+        }
+    }
+}

src/elements/mod.rs

@@ -0,0 +1,77 @@
+use bitcoin::hashes::Hash;
+use elements::hex::ToHex;
+use elements::secp256k1_zkp::ZERO_TWEAK;
+use elements::{confidential::Value, encode::serialize, issuance::ContractHash, AssetId, TxIn};
+
+pub mod asset;
+pub mod peg;
+mod registry;
+
+use asset::get_issuance_entropy;
+pub use asset::{lookup_asset, LiquidAsset};
+pub use registry::{AssetMeta, AssetRegistry, AssetSorting};
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct IssuanceValue {
+    pub asset_id: String,
+    pub is_reissuance: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub asset_blinding_nonce: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub contract_hash: Option<String>,
+    pub asset_entropy: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub assetamount: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub assetamountcommitment: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tokenamount: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tokenamountcommitment: Option<String>,
+}
+
+impl From<&TxIn> for IssuanceValue {
+    fn from(txin: &TxIn) -> Self {
+        let issuance = &txin.asset_issuance;
+        let is_reissuance = issuance.asset_blinding_nonce != ZERO_TWEAK;
+
+        let asset_entropy = get_issuance_entropy(txin).expect("invalid issuance");
+        let asset_id = AssetId::from_entropy(asset_entropy);
+
+        let contract_hash = if !is_reissuance {
+            Some(ContractHash::from_slice(&issuance.asset_entropy).expect("invalid asset entropy"))
+        } else {
+            None
+        };
+
+        IssuanceValue {
+            asset_id: asset_id.to_hex(),
+            asset_entropy: asset_entropy.to_hex(),
+            contract_hash: contract_hash.map(|h| h.to_hex()),
+            is_reissuance,
+            asset_blinding_nonce: if is_reissuance {
+                Some(hex::encode(issuance.asset_blinding_nonce.as_ref()))
+            } else {
+                None
+            },
+            assetamount: match issuance.amount {
+                Value::Explicit(value) => Some(value),
+                Value::Null => Some(0),
+                Value::Confidential(..) => None,
+            },
+            assetamountcommitment: match issuance.amount {
+                Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
+                _ => None,
+            },
+            tokenamount: match issuance.inflation_keys {
+                Value::Explicit(value) => Some(value),
+                Value::Null => Some(0),
+                Value::Confidential(..) => None,
+            },
+            tokenamountcommitment: match issuance.inflation_keys {
+                Value::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys))),
+                _ => None,
+            },
+        }
+    }
+}

src/elements/peg.rs

@@ -0,0 +1,84 @@
+use elements::hex::ToHex;
+use elements::{confidential::Asset, PeginData, PegoutData, TxIn, TxOut};
+
+use crate::chain::{bitcoin_genesis_hash, BNetwork, Network};
+use crate::util::FullHash;
+
+pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option<PeginData<'_>> {
+    let pegged_asset_id = network.pegged_asset()?;
+    txout.pegin_data().and_then(|pegin| {
+        if pegin.asset == *pegged_asset_id {
+            Some(pegin)
+        } else {
+            None
+        }
+    })
+}
+
+pub fn get_pegout_data(
+    txout: &TxOut,
+    network: Network,
+    parent_network: BNetwork,
+) -> Option<PegoutData<'_>> {
+    let pegged_asset_id = network.pegged_asset()?;
+    txout.pegout_data().and_then(|pegout| {
+        if pegout.asset == Asset::Explicit(*pegged_asset_id)
+            && pegout.genesis_hash
+                == bitcoin_genesis_hash(match parent_network {
+                    BNetwork::Bitcoin => Network::Liquid,
+                    BNetwork::Testnet | BNetwork::Testnet4 => Network::LiquidTestnet,
+                    BNetwork::Signet => return None,
+                    BNetwork::Regtest => Network::LiquidRegtest,
+                })
+        {
+            Some(pegout)
+        } else {
+            None
+        }
+    })
+}
+
+// API representation of pegout data assocaited with an output
+#[derive(Serialize, Deserialize, Clone)]
+pub struct PegoutValue {
+    pub genesis_hash: String,
+    pub scriptpubkey: bitcoin::ScriptBuf,
+    pub scriptpubkey_asm: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scriptpubkey_address: Option<String>,
+}
+
+impl PegoutValue {
+    pub fn from_txout(txout: &TxOut, network: Network, parent_network: BNetwork) -> Option<Self> {
+        let pegoutdata = get_pegout_data(txout, network, parent_network)?;
+
+        // pending https://github.com/ElementsProject/rust-elements/pull/69 is merged
+        let scriptpubkey = bitcoin::ScriptBuf::from(pegoutdata.script_pubkey.into_bytes());
+        let address = bitcoin::Address::from_script(&scriptpubkey, parent_network).ok();
+
+        Some(PegoutValue {
+            genesis_hash: pegoutdata.genesis_hash.to_hex(),
+            scriptpubkey_asm: scriptpubkey.to_asm_string(),
+            scriptpubkey_address: address.map(|s| s.to_string()),
+            scriptpubkey,
+        })
+    }
+}
+
+// Inner type for the indexer TxHistoryInfo::Pegin variant
+#[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
+pub struct PeginInfo {
+    pub txid: FullHash,
+    pub vin: u32,
+    pub value: u64,
+}
+
+// Inner type for the indexer TxHistoryInfo::Pegout variant
+#[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
+pub struct PegoutInfo {
+    pub txid: FullHash,
+    pub vout: u32,
+    pub value: u64,
+}

src/elements/registry.rs

@@ -0,0 +1,298 @@
+use std::collections::HashMap;
+use std::str::FromStr;
+use std::sync::{Arc, RwLock};
+use std::time::{Duration, SystemTime};
+use std::{cmp, fs, path, thread};
+
+use serde_json::Value as JsonValue;
+
+use elements::AssetId;
+
+use crate::errors::*;
+
+// length of asset id prefix to use for sub-directory partitioning
+// (in number of hex characters, not bytes)
+
+const DIR_PARTITION_LEN: usize = 2;
+const SEARCH_SORT_CANDIDATE_LIMIT: usize = 2000;
+
+pub struct AssetRegistry {
+    directory: path::PathBuf,
+    assets_cache: HashMap<AssetId, (SystemTime, AssetMeta)>,
+}
+
+pub type AssetEntry<'a> = (&'a AssetId, &'a AssetMeta);
+
+impl AssetRegistry {
+    pub fn new(directory: path::PathBuf) -> Self {
+        Self {
+            directory,
+            assets_cache: Default::default(),
+        }
+    }
+
+    pub fn get(&self, asset_id: &AssetId) -> Option<&AssetMeta> {
+        self.assets_cache
+            .get(asset_id)
+            .map(|(_, metadata)| metadata)
+    }
+
+    pub fn list(
+        &self,
+        start_index: usize,
+        limit: usize,
+        sorting: AssetSorting,
+    ) -> (usize, Vec<AssetEntry<'_>>) {
+        let mut assets: Vec<AssetEntry> = self
+            .assets_cache
+            .iter()
+            .map(|(asset_id, (_, metadata))| (asset_id, metadata))
+            .collect();
+        assets.sort_by(sorting.as_comparator());
+        (
+            assets.len(),
+            assets.into_iter().skip(start_index).take(limit).collect(),
+        )
+    }
+
+    pub fn search(&self, query: &str, limit: usize) -> Vec<AssetEntry<'_>> {
+        let query = query.trim();
+        if query.is_empty() || limit == 0 {
+            return vec![];
+        }
+
+        let (mut results, candidates) = search_by(
+            self.assets_cache
+                .iter()
+                .map(|(asset_id, (_, metadata))| (asset_id, metadata)),
+            query,
+            limit,
+            |metadata| metadata.ticker.as_deref(),
+        );
+
+        if results.len() < limit {
+            let (name_matches, candidates) =
+                search_by(candidates, query, limit - results.len(), |metadata| {
+                    Some(&metadata.name)
+                });
+            results.extend(name_matches);
+
+            if results.len() < limit {
+                let (domain_matches, _) =
+                    search_by(candidates, query, limit - results.len(), AssetMeta::domain);
+                results.extend(domain_matches);
+            }
+        }
+
+        results.truncate(limit);
+        results
+    }
+
+    pub fn fs_sync(&mut self) -> Result<()> {
+        for entry in fs::read_dir(&self.directory).chain_err(|| "failed reading asset dir")? {
+            let entry = entry.chain_err(|| "invalid fh")?;
+            let filetype = entry.file_type().chain_err(|| "failed getting file type")?;
+            if !filetype.is_dir() || entry.file_name().len() != DIR_PARTITION_LEN {
+                continue;
+            }
+
+            for file_entry in
+                fs::read_dir(entry.path()).chain_err(|| "failed reading asset subdir")?
+            {
+                let file_entry = file_entry.chain_err(|| "invalid fh")?;
+                let path = file_entry.path();
+                if path.extension().and_then(|e| e.to_str()) != Some("json") {
+                    continue;
+                }
+
+                let asset_id = AssetId::from_str(
+                    path.file_stem()
+                        .unwrap() // cannot fail if extension() succeeded
+                        .to_str()
+                        .chain_err(|| "invalid filename")?,
+                )
+                .chain_err(|| "invalid filename")?;
+
+                let modified = file_entry
+                    .metadata()
+                    .chain_err(|| "failed reading metadata")?
+                    .modified()
+                    .chain_err(|| "metadata modified failed")?;
+
+                if let Some((last_update, _)) = self.assets_cache.get(&asset_id) {
+                    if *last_update == modified {
+                        continue;
+                    }
+                }
+
+                let metadata: AssetMeta = serde_json::from_str(
+                    &fs::read_to_string(path).chain_err(|| "failed reading file")?,
+                )
+                .chain_err(|| "failed parsing file")?;
+
+                self.assets_cache.insert(asset_id, (modified, metadata));
+            }
+        }
+        Ok(())
+    }
+
+    pub fn spawn_sync(asset_db: Arc<RwLock<AssetRegistry>>) -> thread::JoinHandle<()> {
+        crate::util::spawn_thread("asset-registry", move || loop {
+            if let Err(e) = asset_db.write().unwrap().fs_sync() {
+                error!("registry fs_sync failed: {:?}", e);
+            }
+
+            thread::sleep(Duration::from_secs(15));
+            // TODO handle shutdowm
+        })
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct AssetMeta {
+    #[serde(skip_serializing_if = "JsonValue::is_null")]
+    pub contract: JsonValue,
+    #[serde(skip_serializing_if = "JsonValue::is_null")]
+    pub entity: JsonValue,
+    pub precision: u8,
+    pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ticker: Option<String>,
+}
+
+impl AssetMeta {
+    pub(crate) fn domain(&self) -> Option<&str> {
+        self.entity["domain"].as_str()
+    }
+}
+
+pub struct AssetSorting(AssetSortField, AssetSortDir);
+
+pub enum AssetSortField {
+    Name,
+    Domain,
+    Ticker,
+}
+pub enum AssetSortDir {
+    Descending,
+    Ascending,
+}
+
+type Comparator = Box<dyn Fn(&AssetEntry, &AssetEntry) -> cmp::Ordering>;
+
+impl AssetSorting {
+    #[allow(clippy::wrong_self_convention)]
+    fn as_comparator(self) -> Comparator {
+        let sort_fn: Comparator = match self.0 {
+            AssetSortField::Name => {
+                // Order by name first, use asset id as a tie breaker. the other sorting fields
+                // don't require this because they're guaranteed to be unique.
+                Box::new(|a, b| lc_cmp(&a.1.name, &b.1.name).then_with(|| a.0.cmp(b.0)))
+            }
+            AssetSortField::Domain => Box::new(|a, b| a.1.domain().cmp(&b.1.domain())),
+            AssetSortField::Ticker => Box::new(|a, b| lc_cmp_opt(&a.1.ticker, &b.1.ticker)),
+        };
+
+        match self.1 {
+            AssetSortDir::Ascending => sort_fn,
+            AssetSortDir::Descending => Box::new(move |a, b| sort_fn(a, b).reverse()),
+        }
+    }
+
+    pub fn from_query_params(query: &HashMap<String, String>) -> Result<Self> {
+        let field = match query.get("sort_field").map(String::as_str) {
+            None => AssetSortField::Ticker,
+            Some("name") => AssetSortField::Name,
+            Some("domain") => AssetSortField::Domain,
+            Some("ticker") => AssetSortField::Ticker,
+            _ => bail!("invalid sort field"),
+        };
+
+        let dir = match query.get("sort_dir").map(String::as_str) {
+            None => AssetSortDir::Ascending,
+            Some("asc") => AssetSortDir::Ascending,
+            Some("desc") => AssetSortDir::Descending,
+            _ => bail!("invalid sort direction"),
+        };
+
+        Ok(Self(field, dir))
+    }
+}
+
+fn lc_cmp(a: &str, b: &str) -> cmp::Ordering {
+    a.to_lowercase().cmp(&b.to_lowercase())
+}
+fn lc_cmp_opt(a: &Option<String>, b: &Option<String>) -> cmp::Ordering {
+    a.as_ref()
+        .map(|a| a.to_lowercase())
+        .cmp(&b.as_ref().map(|b| b.to_lowercase()))
+}
+
+fn search_by<'a, I, F>(
+    candidates: I,
+    query: &str,
+    limit: usize,
+    field: F,
+) -> (Vec<AssetEntry<'a>>, Vec<AssetEntry<'a>>)
+where
+    I: IntoIterator<Item = AssetEntry<'a>>,
+    F: Fn(&AssetMeta) -> Option<&str>,
+{
+    let mut matches = vec![];
+    let mut remaining = vec![];
+
+    for (asset_id, metadata) in candidates {
+        let position = field(metadata).and_then(|field| {
+            // registry fields are ascii, so we don't need full unicode case-folding
+            ascii_ci_find(field, query).map(|position| (position, field))
+        });
+
+        if let Some((position, field)) = position {
+            if matches.len() >= SEARCH_SORT_CANDIDATE_LIMIT {
+                continue;
+            }
+            matches.push((position, field, asset_id, metadata));
+        } else {
+            remaining.push((asset_id, metadata));
+        }
+    }
+
+    matches.sort_unstable_by(|a, b| {
+        a.0.cmp(&b.0)
+            .then_with(|| ascii_ci_cmp(a.1, b.1))
+            .then_with(|| a.2.cmp(b.2))
+    });
+
+    (
+        matches
+            .into_iter()
+            .take(limit)
+            .map(|(_, _, asset_id, metadata)| (asset_id, metadata))
+            .collect(),
+        remaining,
+    )
+}
+
+// zero-allocation case-insensitive ASCII substring search
+// returns the byte offset of the first match
+fn ascii_ci_find(haystack: &str, needle: &str) -> Option<usize> {
+    let (haystack, needle) = (haystack.as_bytes(), needle.as_bytes());
+    if needle.is_empty() {
+        return Some(0);
+    }
+    haystack
+        .windows(needle.len())
+        .position(|window| window.eq_ignore_ascii_case(needle))
+}
+
+// zero-allocation case-insensitive ASCII string comparison
+fn ascii_ci_cmp(a: &str, b: &str) -> cmp::Ordering {
+    let (a, b) = (a.as_bytes(), b.as_bytes());
+    for i in 0..a.len().min(b.len()) {
+        match a[i].to_ascii_lowercase().cmp(&b[i].to_ascii_lowercase()) {
+            cmp::Ordering::Equal => continue,
+            ord => return ord,
+        }
+    }
+    a.len().cmp(&b.len())
+}

src/errors.rs

@@ -1,5 +1,4 @@
-use chan_signal::Signal;
-
+#![allow(unexpected_cfgs)]
 error_chain! {
     types {
         Error, ErrorKind, ResultExt, Result;
@@ -11,9 +10,33 @@ error_chain! {
             display("Connection error: {}", msg)
         }
 
-        Interrupt(signal: Signal) {
+        Interrupt(sig: i32) {
             description("Interruption by external signal")
-            display("Iterrupted by SIG{:?}", signal)
+            display("Iterrupted by signal {}", sig)
         }
+
+        TooManyUtxos(limit: usize) {
+            description("Too many unspent transaction outputs. Contact support to raise limits.")
+            display("Too many unspent transaction outputs (>{}). Contact support to raise limits.", limit)
+        }
+
+        TooManyTxs(limit: usize) {
+            description("Too many history transactions. Contact support to raise limits.")
+            display("Too many history transactions (>{}). Contact support to raise limits.", limit)
+        }
+
+        #[cfg(feature = "electrum-discovery")]
+        ElectrumClient(e: electrum_client::Error) {
+            description("Electrum client error")
+            display("Electrum client error: {:?}", e)
+        }
+
+    }
+}
+
+#[cfg(feature = "electrum-discovery")]
+impl From<electrum_client::Error> for Error {
+    fn from(e: electrum_client::Error) -> Self {
+        Error::from(ErrorKind::ElectrumClient(e))
     }
 }

src/lib.rs

@@ -1,38 +1,7 @@
 #![recursion_limit = "1024"]
 
-extern crate base64;
-extern crate bincode;
-extern crate bitcoin;
-extern crate bitcoin_bech32;
-extern crate bitcoin_hashes;
-extern crate chan_signal;
-extern crate crypto;
-extern crate dirs;
-extern crate glob;
-extern crate hex;
-extern crate hyper;
-extern crate itertools;
-extern crate libc;
-extern crate lru;
-extern crate lru_cache;
-extern crate num_cpus;
-extern crate page_size;
-extern crate prometheus;
-extern crate rayon;
 extern crate rocksdb;
-extern crate secp256k1;
-extern crate serde;
-extern crate stderrlog;
-extern crate sysconf;
-extern crate time;
-extern crate tiny_http;
-extern crate url;
 
-#[cfg(feature = "liquid")]
-extern crate elements;
-
-#[macro_use]
-extern crate chan;
 #[macro_use]
 extern crate clap;
 #[macro_use]
@@ -46,14 +15,19 @@ extern crate serde_derive;
 #[macro_use]
 extern crate serde_json;
 
+#[macro_use]
+extern crate lazy_static;
+
 pub mod chain;
 pub mod config;
 pub mod daemon;
 pub mod electrum;
 pub mod errors;
 pub mod metrics;
+pub mod new_index;
 pub mod rest;
 pub mod signal;
 pub mod util;
 
-pub mod new_index;
+#[cfg(feature = "liquid")]
+pub mod elements;

src/metrics.rs

@@ -5,7 +5,6 @@ use std::io;
 use std::net::SocketAddr;
 use std::thread;
 use std::time::Duration;
-use sysconf;
 use tiny_http;
 
 pub use prometheus::{
@@ -67,11 +66,9 @@ impl Metrics {
     }
 
     pub fn start(&self) {
-        let server = tiny_http::Server::http(self.addr).expect(&format!(
-            "failed to start monitoring HTTP server at {}",
-            self.addr
-        ));
-        start_process_exporter(&self);
+        let server = tiny_http::Server::http(self.addr)
+            .unwrap_or_else(|_| panic!("failed to start monitoring HTTP server at {}", self.addr));
+        start_process_exporter(self);
         let reg = self.reg.clone();
         spawn_thread("metrics", move || loop {
             if let Err(e) = handle_request(&reg, server.recv()) {
@@ -100,6 +97,14 @@ struct Stats {
     fds: usize,
 }
 
+fn get_ticks_per_second() -> Result<f64> {
+    // Safety: This code is taken directly from sysconf
+    match unsafe { libc::sysconf(libc::_SC_CLK_TCK) } {
+        -1 => Err("Clock Tick unsupported".into()),
+        ret => Ok(ret as f64),
+    }
+}
+
 fn parse_stats() -> Result<Stats> {
     if cfg!(target_os = "macos") {
         return Ok(Stats {
@@ -111,15 +116,14 @@ fn parse_stats() -> Result<Stats> {
     let value = fs::read_to_string("/proc/self/stat").chain_err(|| "failed to read stats")?;
     let parts: Vec<&str> = value.split_whitespace().collect();
     let page_size = page_size::get() as u64;
-    let ticks_per_second = sysconf::raw::sysconf(sysconf::raw::SysconfVariable::ScClkTck)
-        .expect("failed to get _SC_CLK_TCK") as f64;
+    let ticks_per_second = get_ticks_per_second().expect("failed to get _SC_CLK_TCK");
 
     let parse_part = |index: usize, name: &str| -> Result<u64> {
-        Ok(parts
+        parts
             .get(index)
             .chain_err(|| format!("missing {}: {:?}", name, parts))?
             .parse::<u64>()
-            .chain_err(|| format!("invalid {}: {:?}", name, parts))?)
+            .chain_err(|| format!("invalid {}: {:?}", name, parts))
     };
 
     // For details, see '/proc/[pid]/stat' section at `man 5 proc`:
@@ -144,7 +148,7 @@ fn start_process_exporter(metrics: &Metrics) {
     spawn_thread("exporter", move || loop {
         match parse_stats() {
             Ok(stats) => {
-                cpu.with_label_values(&["utime"]).set(stats.utime as f64);
+                cpu.with_label_values(&["utime"]).set(stats.utime);
                 rss.set(stats.rss as i64);
                 fds.set(stats.fds as i64);
             }

src/new_index/db.rs

@@ -2,9 +2,14 @@ use rocksdb;
 
 use std::path::Path;
 
-use crate::util::Bytes;
+use crate::config::Config;
+use crate::util::{bincode_util, Bytes};
 
-static DB_VERSION: u32 = 1;
+/// Each version will break any running instance with a DB that has a differing version.
+/// It will also break if light mode is enabled or disabled.
+// 1 = Original DB (since fork from Blockstream)
+// 2 = Add tx position to TxHistory rows and place Spending before Funding
+static DB_VERSION: u32 = 2;
 
 #[derive(Debug, Eq, PartialEq)]
 pub struct DBRow {
@@ -18,14 +23,14 @@ pub struct ScanIterator<'a> {
     done: bool,
 }
 
-impl<'a> Iterator for ScanIterator<'a> {
+impl Iterator for ScanIterator<'_> {
     type Item = DBRow;
 
     fn next(&mut self) -> Option<DBRow> {
         if self.done {
             return None;
         }
-        let (key, value) = self.iter.next()?;
+        let (key, value) = self.iter.next().map(Result::ok)??;
         if !key.starts_with(&self.prefix) {
             self.done = true;
             return None;
@@ -43,7 +48,7 @@ pub struct ReverseScanIterator<'a> {
     done: bool,
 }
 
-impl<'a> Iterator for ReverseScanIterator<'a> {
+impl Iterator for ReverseScanIterator<'_> {
     type Item = DBRow;
 
     fn next(&mut self) -> Option<DBRow> {
@@ -56,11 +61,76 @@ impl<'a> Iterator for ReverseScanIterator<'a> {
             self.done = true;
             return None;
         }
-        let value = self.iter.value().unwrap();
+
+        let row = DBRow {
+            key: key.into(),
+            value: self.iter.value().unwrap().into(),
+        };
 
         self.iter.prev();
 
-        Some(DBRow { key, value })
+        Some(row)
+    }
+}
+
+pub struct ReverseScanGroupIterator<'a> {
+    iters: Vec<ReverseScanIterator<'a>>,
+    next_rows: Vec<Option<DBRow>>,
+    value_offset: usize,
+    done: bool,
+}
+
+impl<'a> ReverseScanGroupIterator<'a> {
+    pub fn new(
+        mut iters: Vec<ReverseScanIterator<'a>>,
+        value_offset: usize,
+    ) -> ReverseScanGroupIterator<'a> {
+        let mut next_rows: Vec<Option<DBRow>> = Vec::with_capacity(iters.len());
+        for iter in &mut iters {
+            let next = iter.next();
+            next_rows.push(next);
+        }
+        let done = next_rows.iter().all(|row| row.is_none());
+        ReverseScanGroupIterator {
+            iters,
+            next_rows,
+            value_offset,
+            done,
+        }
+    }
+}
+
+impl Iterator for ReverseScanGroupIterator<'_> {
+    type Item = DBRow;
+
+    fn next(&mut self) -> Option<DBRow> {
+        if self.done {
+            return None;
+        }
+
+        let best_index = self
+            .next_rows
+            .iter()
+            .enumerate()
+            .max_by(|(a_index, a_opt), (b_index, b_opt)| match (a_opt, b_opt) {
+                (None, None) => a_index.cmp(b_index),
+
+                (Some(_), None) => std::cmp::Ordering::Greater,
+
+                (None, Some(_)) => std::cmp::Ordering::Less,
+
+                (Some(a), Some(b)) => a.key[self.value_offset..].cmp(&(b.key[self.value_offset..])),
+            })
+            .map(|(index, _)| index)
+            .unwrap_or(0);
+
+        let best = self.next_rows[best_index].take();
+        self.next_rows[best_index] = self.iters.get_mut(best_index)?.next();
+        if self.next_rows.iter().all(|row| row.is_none()) {
+            self.done = true;
+        }
+
+        best
     }
 }
 
@@ -76,28 +146,11 @@ pub enum DBFlush {
 }
 
 impl DB {
-    pub fn open(path: &Path) -> DB {
-        debug!("opening DB at {:?}", path);
-        let mut db_opts = rocksdb::Options::default();
-        db_opts.create_if_missing(true);
-        db_opts.set_max_open_files(-1); // TODO: make sure to `ulimit -n` this process correctly
-        db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
-        db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
-        db_opts.set_target_file_size_base(256 << 20);
-        db_opts.set_write_buffer_size(256 << 20);
-        db_opts.set_disable_auto_compactions(true); // for initial bulk load
-
-        // db_opts.set_advise_random_on_open(???);
-        db_opts.set_compaction_readahead_size(1 << 20);
-        db_opts.increase_parallelism(2);
-
-        // let mut block_opts = rocksdb::BlockBasedOptions::default();
-        // block_opts.set_block_size(???);
-
+    pub fn open(path: &Path, config: &Config) -> DB {
         let db = DB {
-            db: rocksdb::DB::open(&db_opts, path).expect("failed to open RocksDB"),
+            db: open_raw_db(path, OpenMode::ReadWrite),
         };
-        db.verify_compatibility();
+        db.verify_compatibility(config);
         db
     }
 
@@ -113,11 +166,11 @@ impl DB {
         self.db.set_options(&opts).unwrap();
     }
 
-    pub fn raw_iterator(&self) -> rocksdb::DBRawIterator {
+    pub fn raw_iterator(&self) -> rocksdb::DBRawIterator<'_> {
         self.db.raw_iterator()
     }
 
-    pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator {
+    pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator<'_> {
         ScanIterator {
             prefix: prefix.to_vec(),
             iter: self.db.prefix_iterator(prefix),
@@ -125,7 +178,7 @@ impl DB {
         }
     }
 
-    pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator {
+    pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator<'_> {
         let iter = self.db.iterator(rocksdb::IteratorMode::From(
             start_at,
             rocksdb::Direction::Forward,
@@ -137,7 +190,7 @@ impl DB {
         }
     }
 
-    pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator {
+    pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator<'_> {
         let mut iter = self.db.raw_iterator();
         iter.seek_for_prev(prefix_max);
 
@@ -148,6 +201,25 @@ impl DB {
         }
     }
 
+    pub fn iter_scan_group_reverse(
+        &self,
+        prefixes: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
+        value_offset: usize,
+    ) -> ReverseScanGroupIterator<'_> {
+        let iters = prefixes
+            .map(|(prefix, prefix_max)| {
+                let mut iter = self.db.raw_iterator();
+                iter.seek_for_prev(prefix_max);
+                ReverseScanIterator {
+                    prefix: prefix.to_vec(),
+                    iter,
+                    done: false,
+                }
+            })
+            .collect();
+        ReverseScanGroupIterator::new(iters, value_offset)
+    }
+
     pub fn write(&self, mut rows: Vec<DBRow>, flush: DBFlush) {
         debug!(
             "writing {} rows to {:?}, flush={:?}",
@@ -158,7 +230,7 @@ impl DB {
         rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
         let mut batch = rocksdb::WriteBatch::default();
         for row in rows {
-            batch.put(&row.key, &row.value).unwrap();
+            batch.put(&row.key, &row.value);
         }
         let do_flush = match flush {
             DBFlush::Enable => true,
@@ -170,16 +242,43 @@ impl DB {
         self.db.write_opt(batch, &opts).unwrap();
     }
 
+    pub fn delete(&self, keys: Vec<Vec<u8>>) {
+        debug!("deleting {} rows from {:?}", keys.len(), self.db);
+        for key in keys {
+            let _ = self.db.delete(key).inspect_err(|err| {
+                warn!("Error while deleting DB row: {err}");
+            });
+        }
+    }
+
+    pub fn flush(&self) {
+        self.db.flush().unwrap();
+    }
+
     pub fn put(&self, key: &[u8], value: &[u8]) {
         self.db.put(key, value).unwrap();
     }
 
+    pub fn put_sync(&self, key: &[u8], value: &[u8]) {
+        let mut opts = rocksdb::WriteOptions::new();
+        opts.set_sync(true);
+        self.db.put_opt(key, value, &opts).unwrap();
+    }
+
     pub fn get(&self, key: &[u8]) -> Option<Bytes> {
         self.db.get(key).unwrap().map(|v| v.to_vec())
     }
 
-    fn verify_compatibility(&self) {
-        let compatibility_bytes = bincode::serialize(&DB_VERSION).unwrap();
+    fn verify_compatibility(&self, config: &Config) {
+        let mut compatibility_bytes = bincode_util::serialize_little(&DB_VERSION).unwrap();
+
+        if config.light_mode {
+            // append a byte to indicate light_mode is enabled.
+            // we're not letting bincode serialize this so that the compatiblity bytes won't change
+            // (and require a reindex) when light_mode is disabled. this should be chagned the next
+            // time we bump DB_VERSION and require a re-index anyway.
+            compatibility_bytes.push(1);
+        }
 
         match self.get(b"V") {
             None => self.put(b"V", &compatibility_bytes),
@@ -191,34 +290,41 @@ impl DB {
     }
 }
 
-#[cfg(test)]
-mod tests {
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+#[repr(u8)]
+pub enum OpenMode {
+    ReadOnly,
+    ReadWrite,
+}
 
-    use crate::new_index::db::{DBRow, DB};
-    use tempfile;
+pub fn open_raw_db<T: rocksdb::ThreadMode>(
+    path: &Path,
+    read_mode: OpenMode,
+) -> rocksdb::DBWithThreadMode<T> {
+    debug!("opening DB at {:?}", path);
+    let mut db_opts = rocksdb::Options::default();
+    db_opts.create_if_missing(true);
+    db_opts.set_max_open_files(100_000); // TODO: make sure to `ulimit -n` this process correctly
+    db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
+    db_opts.set_compression_type(rocksdb::DBCompressionType::None);
+    db_opts.set_target_file_size_base(1_073_741_824);
+    db_opts.set_write_buffer_size(256 << 20);
+    db_opts.set_disable_auto_compactions(true); // for initial bulk load
 
-    #[test]
-    fn test_db_iterator() {
-        let dir = tempfile::tempdir().unwrap();
-        let db = DB::open(dir.path());
-        let rows = vec![
-            DBRow {
-                key: b"X11".to_vec(),
-                value: b"a".to_vec(),
-            },
-            DBRow {
-                key: b"X22".to_vec(),
-                value: b"b".to_vec(),
-            },
-            DBRow {
-                key: b"X33".to_vec(),
-                value: b"c".to_vec(),
-            },
-        ];
-        for row in &rows {
-            db.put(&row.key, &row.value);
+    // db_opts.set_advise_random_on_open(???);
+    db_opts.set_compaction_readahead_size(1 << 20);
+    db_opts.increase_parallelism(2);
+
+    // let mut block_opts = rocksdb::BlockBasedOptions::default();
+    // block_opts.set_block_size(???);
+
+    match read_mode {
+        OpenMode::ReadOnly => {
+            rocksdb::DBWithThreadMode::<T>::open_for_read_only(&db_opts, path, false)
+                .expect("failed to open RocksDB (READ ONLY)")
+        }
+        OpenMode::ReadWrite => {
+            rocksdb::DBWithThreadMode::<T>::open(&db_opts, path).expect("failed to open RocksDB")
         }
-        let actual: Vec<DBRow> = db.iter_scan(b"X").collect();
-        assert_eq!(rows, actual);
     }
 }

src/new_index/fetch.rs

@@ -1,17 +1,17 @@
-use crate::chain::Block;
-
-use bitcoin::consensus::encode::{deserialize, serialize, Decodable};
-use bitcoin::util::hash::BitcoinHash;
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
 use rayon::prelude::*;
 
+#[cfg(not(feature = "liquid"))]
+use bitcoin::consensus::encode::{deserialize, Decodable};
+#[cfg(feature = "liquid")]
+use elements::encode::{deserialize, Decodable};
+
 use std::collections::HashMap;
 use std::fs;
-use std::io::{Cursor, Seek, SeekFrom};
+use std::io::Cursor;
 use std::path::PathBuf;
-use std::sync::mpsc::Receiver;
 use std::thread;
 
+use crate::chain::{Block, BlockHash, BlockSizeCompat};
 use crate::daemon::Daemon;
 use crate::errors::*;
 use crate::util::{spawn_thread, HeaderEntry, SyncChannel};
@@ -42,19 +42,70 @@ pub struct BlockEntry {
 
 type SizedBlock = (Block, u32);
 
+pub struct SequentialFetcher<T> {
+    fetcher: Box<dyn FnOnce() -> Vec<Vec<T>>>,
+}
+
+impl<T> SequentialFetcher<T> {
+    fn from<F: FnOnce() -> Vec<Vec<T>> + 'static>(pre_func: F) -> Self {
+        SequentialFetcher {
+            fetcher: Box::new(pre_func),
+        }
+    }
+
+    pub fn map<FN>(self, mut func: FN)
+    where
+        FN: FnMut(Vec<T>),
+    {
+        for item in (self.fetcher)() {
+            func(item);
+        }
+    }
+}
+
+pub fn bitcoind_sequential_fetcher(
+    daemon: &Daemon,
+    new_headers: Vec<HeaderEntry>,
+) -> Result<SequentialFetcher<BlockEntry>> {
+    let daemon = daemon.reconnect()?;
+    Ok(SequentialFetcher::from(move || {
+        new_headers
+            .chunks(100)
+            .map(|entries| {
+                let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
+                let blocks = daemon
+                    .getblocks(&blockhashes)
+                    .expect("failed to get blocks from bitcoind");
+                assert_eq!(blocks.len(), entries.len());
+                let block_entries: Vec<BlockEntry> = blocks
+                    .into_iter()
+                    .zip(entries)
+                    .map(|(block, entry)| BlockEntry {
+                        entry: entry.clone(), // TODO: remove this clone()
+                        size: block.get_block_size() as u32,
+                        block,
+                    })
+                    .collect();
+                assert_eq!(block_entries.len(), entries.len());
+                block_entries
+            })
+            .collect()
+    }))
+}
+
 pub struct Fetcher<T> {
-    receiver: Receiver<T>,
+    receiver: crossbeam_channel::Receiver<T>,
     thread: thread::JoinHandle<()>,
 }
 
 impl<T> Fetcher<T> {
-    fn from(receiver: Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
+    fn from(receiver: crossbeam_channel::Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
         Fetcher { receiver, thread }
     }
 
     pub fn map<F>(self, mut func: F)
     where
-        F: FnMut(T) -> (),
+        F: FnMut(T),
     {
         for item in self.receiver {
             func(item);
@@ -67,17 +118,17 @@ fn bitcoind_fetcher(
     daemon: &Daemon,
     new_headers: Vec<HeaderEntry>,
 ) -> Result<Fetcher<Vec<BlockEntry>>> {
-    new_headers.last().map(|tip| {
-        debug!("{:?} ({} new blocks to fetch)", tip, new_headers.len());
-    });
+    if let Some(tip) = new_headers.last() {
+        debug!("{:?} ({} left to index)", tip, new_headers.len());
+    };
     let daemon = daemon.reconnect()?;
     let chan = SyncChannel::new(1);
     let sender = chan.sender();
     Ok(Fetcher::from(
         chan.into_receiver(),
-        spawn_thread("bitcoind_fetcher", move || -> () {
+        spawn_thread("bitcoind_fetcher", move || {
             for entries in new_headers.chunks(100) {
-                let blockhashes: Vec<Sha256dHash> = entries.iter().map(|e| *e.hash()).collect();
+                let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
                 let blocks = daemon
                     .getblocks(&blockhashes)
                     .expect("failed to get blocks from bitcoind");
@@ -86,8 +137,8 @@ fn bitcoind_fetcher(
                     .into_iter()
                     .zip(entries)
                     .map(|(block, entry)| BlockEntry {
-                        entry: entry.clone(),                 // TODO: remove this clone()
-                        size: serialize(&block).len() as u32, // TODO: avoid re-serializing
+                        entry: entry.clone(), // TODO: remove this clone()
+                        size: block.get_block_size() as u32,
                         block,
                     })
                     .collect();
@@ -110,18 +161,18 @@ fn blkfiles_fetcher(
     let chan = SyncChannel::new(1);
     let sender = chan.sender();
 
-    let mut entry_map: HashMap<Sha256dHash, HeaderEntry> =
+    let mut entry_map: HashMap<BlockHash, HeaderEntry> =
         new_headers.into_iter().map(|h| (*h.hash(), h)).collect();
 
     let parser = blkfiles_parser(blkfiles_reader(blk_files), magic);
     Ok(Fetcher::from(
         chan.into_receiver(),
-        spawn_thread("blkfiles_fetcher", move || -> () {
+        spawn_thread("blkfiles_fetcher", move || {
             parser.map(|sizedblocks| {
                 let block_entries: Vec<BlockEntry> = sizedblocks
                     .into_iter()
                     .filter_map(|(block, size)| {
-                        let blockhash = block.bitcoin_hash();
+                        let blockhash = block.block_hash();
                         entry_map
                             .remove(&blockhash)
                             .map(|entry| BlockEntry { block, entry, size })
@@ -149,16 +200,36 @@ fn blkfiles_fetcher(
 fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
     let chan = SyncChannel::new(1);
     let sender = chan.sender();
+    let xor_key = blk_files.first().and_then(|p| {
+        let xor_file = p
+            .parent()
+            .expect("blk.dat files must exist in a directory")
+            .join("xor.dat");
+        if xor_file.exists() {
+            Some(fs::read(xor_file).expect("xor.dat exists"))
+        } else {
+            None
+        }
+    });
 
     Fetcher::from(
         chan.into_receiver(),
-        spawn_thread("blkfiles_reader", move || -> () {
+        spawn_thread("blkfiles_reader", move || {
             for path in blk_files {
                 trace!("reading {:?}", path);
-                let blob = fs::read(&path).expect(&format!("failed to read {:?}", path));
+                let mut blob = fs::read(&path)
+                    .unwrap_or_else(|e| panic!("failed to read {:?}: {:?}", path, e));
+
+                // If the xor.dat exists. Use it to decrypt the block files.
+                if let Some(xor_key) = &xor_key {
+                    for (&key, byte) in xor_key.iter().cycle().zip(blob.iter_mut()) {
+                        *byte ^= key;
+                    }
+                }
+
                 sender
                     .send(blob)
-                    .expect(&format!("failed to send {:?} contents", path));
+                    .unwrap_or_else(|_| panic!("failed to send {:?} contents", path));
             }
         }),
     )
@@ -170,7 +241,7 @@ fn blkfiles_parser(blobs: Fetcher<Vec<u8>>, magic: u32) -> Fetcher<Vec<SizedBloc
 
     Fetcher::from(
         chan.into_receiver(),
-        spawn_thread("blkfiles_parser", move || -> () {
+        spawn_thread("blkfiles_parser", move || {
             blobs.map(|blob| {
                 trace!("parsing {} bytes", blob.len());
                 let blocks = parse_blocks(blob, magic).expect("failed to parse blk*.dat file");
@@ -186,34 +257,40 @@ fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<SizedBlock>> {
     let mut cursor = Cursor::new(&blob);
     let mut slices = vec![];
     let max_pos = blob.len() as u64;
+
     while cursor.position() < max_pos {
+        let offset = cursor.position();
         match u32::consensus_decode(&mut cursor) {
             Ok(value) => {
                 if magic != value {
-                    cursor
-                        .seek(SeekFrom::Current(-3))
-                        .expect("failed to seek back");
+                    cursor.set_position(offset + 1);
                     continue;
                 }
             }
             Err(_) => break, // EOF
         };
         let block_size = u32::consensus_decode(&mut cursor).chain_err(|| "no block size")?;
-        let start = cursor.position() as usize;
-        cursor
-            .seek(SeekFrom::Current(block_size as i64))
-            .chain_err(|| format!("seek {} failed", block_size))?;
-        let end = cursor.position() as usize;
+        let start = cursor.position();
+        let end = start + block_size as u64;
 
-        slices.push((&blob[start..end], block_size));
+        // If Core's WriteBlockToDisk ftell fails, only the magic bytes and size will be written
+        // and the block body won't be written to the blk*.dat file.
+        // Since the first 4 bytes should contain the block's version, we can skip such blocks
+        // by peeking the cursor (and skipping previous `magic` and `block_size`).
+        match u32::consensus_decode(&mut cursor) {
+            Ok(value) => {
+                if magic == value {
+                    cursor.set_position(start);
+                    continue;
+                }
+            }
+            Err(_) => break, // EOF
+        }
+        slices.push((&blob[start as usize..end as usize], block_size));
+        cursor.set_position(end);
     }
 
-    let pool = rayon::ThreadPoolBuilder::new()
-        .num_threads(0) // CPU-bound
-        .thread_name(|i| format!("parse-blocks-{}", i))
-        .build()
-        .unwrap();
-    Ok(pool.install(|| {
+    Ok(super::THREAD_POOL.install(|| {
         slices
             .into_par_iter()
             .map(|(slice, size)| (deserialize(slice).expect("failed to parse Block"), size))

src/new_index/mempool.rs

@@ -1,46 +1,58 @@
-use arraydeque::{ArrayDeque, Wrapping};
-use bitcoin::consensus::encode::serialize;
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
+use bounded_vec_deque::BoundedVecDeque;
 use itertools::Itertools;
 
-use std::collections::{BTreeSet, HashMap, HashSet};
+#[cfg(not(feature = "liquid"))]
+use bitcoin::consensus::encode::serialize;
+#[cfg(feature = "liquid")]
+use elements::{encode::serialize, AssetId};
+
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::iter::FromIterator;
-use std::sync::Arc;
+use std::ops::Bound::{Excluded, Unbounded};
+use std::sync::{Arc, RwLock};
 use std::time::{Duration, Instant};
 
-use crate::chain::{OutPoint, Transaction, TxOut};
+use crate::chain::{deserialize, Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
+use crate::config::Config;
 use crate::daemon::Daemon;
 use crate::errors::*;
 use crate::metrics::{GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics};
 use crate::new_index::{
-    compute_script_hash, parse_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats,
-    SpendingInfo, SpendingInput, TxHistoryInfo, Utxo,
+    compute_script_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats, SpendingInfo,
+    SpendingInput, TxHistoryInfo, Utxo,
 };
 use crate::util::fees::{make_fee_histogram, TxFeeInfo};
-use crate::util::{full_hash, has_prevout, is_spendable, Bytes};
+use crate::util::{extract_tx_prevouts, full_hash, has_prevout, is_spendable, Bytes};
 
-const RECENT_TXS_SIZE: usize = 10;
-const BACKLOG_STATS_TTL: u64 = 10;
+#[cfg(feature = "liquid")]
+use crate::elements::asset;
 
 pub struct Mempool {
     chain: Arc<ChainQuery>,
-    txstore: HashMap<Sha256dHash, Transaction>,
-    feeinfo: HashMap<Sha256dHash, TxFeeInfo>,
+    config: Arc<Config>,
+    txstore: BTreeMap<Txid, Transaction>,
+    feeinfo: HashMap<Txid, TxFeeInfo>,
     history: HashMap<FullHash, Vec<TxHistoryInfo>>, // ScriptHash -> {history_entries}
-    edges: HashMap<OutPoint, (Sha256dHash, u32)>,   // OutPoint -> (spending_txid, spending_vin)
-    recent: ArrayDeque<[TxOverview; RECENT_TXS_SIZE], Wrapping>, // The N most recent txs to enter the mempool
+    edges: HashMap<OutPoint, (Txid, u32)>,          // OutPoint -> (spending_txid, spending_vin)
+    recent: BoundedVecDeque<TxOverview>,            // The N most recent txs to enter the mempool
     backlog_stats: (BacklogStats, Instant),
 
     // monitoring
     latency: HistogramVec, // mempool requests latency
     delta: HistogramVec,   // # of added/removed txs
     count: GaugeVec,       // current state of the mempool
+
+    // elements only
+    #[cfg(feature = "liquid")]
+    pub asset_history: HashMap<AssetId, Vec<TxHistoryInfo>>,
+    #[cfg(feature = "liquid")]
+    pub asset_issuance: HashMap<AssetId, asset::AssetRow>,
 }
 
 // A simplified transaction view used for the list of most recent transactions
 #[derive(Serialize)]
 pub struct TxOverview {
-    txid: Sha256dHash,
+    txid: Txid,
     fee: u64,
     vsize: u32,
     #[cfg(not(feature = "liquid"))]
@@ -48,17 +60,17 @@ pub struct TxOverview {
 }
 
 impl Mempool {
-    pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics) -> Self {
+    pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics, config: Arc<Config>) -> Self {
         Mempool {
             chain,
-            txstore: HashMap::new(),
+            txstore: BTreeMap::new(),
             feeinfo: HashMap::new(),
             history: HashMap::new(),
             edges: HashMap::new(),
-            recent: ArrayDeque::new(),
+            recent: BoundedVecDeque::new(config.mempool_recent_txs_size),
             backlog_stats: (
                 BacklogStats::default(),
-                Instant::now() - Duration::from_secs(BACKLOG_STATS_TTL),
+                Instant::now() - Duration::from_secs(config.mempool_backlog_stats_ttl),
             ),
             latency: metrics.histogram_vec(
                 HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"),
@@ -72,14 +84,24 @@ impl Mempool {
                 MetricOpts::new("mempool_count", "# of elements currently at the mempool"),
                 &["type"],
             ),
+
+            #[cfg(feature = "liquid")]
+            asset_history: HashMap::new(),
+            #[cfg(feature = "liquid")]
+            asset_issuance: HashMap::new(),
+            config,
         }
     }
 
-    pub fn lookup_txn(&self, txid: &Sha256dHash) -> Option<Transaction> {
-        self.txstore.get(txid).map(|item| item.clone())
+    pub fn network(&self) -> Network {
+        self.config.network_type
     }
 
-    pub fn lookup_raw_txn(&self, txid: &Sha256dHash) -> Option<Bytes> {
+    pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
+        self.txstore.get(txid).cloned()
+    }
+
+    pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
         self.txstore.get(txid).map(serialize)
     }
 
@@ -95,30 +117,122 @@ impl Mempool {
         self.edges.contains_key(outpoint)
     }
 
-    // TODO: return as Vec<(Transaction,Option<BlockId>)>?
-    pub fn history(&self, scripthash: &[u8], limit: usize) -> Vec<Transaction> {
-        let _timer = self.latency.with_label_values(&["history"]).start_timer();
-        match self.history.get(scripthash) {
-            None => return vec![],
-            Some(entries) => entries
-                .iter()
-                .map(get_entry_txid)
-                .unique()
-                .take(limit)
-                .map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
-                .cloned()
-                .collect(),
-        }
+    pub fn get_tx_fee(&self, txid: &Txid) -> Option<u64> {
+        Some(self.feeinfo.get(txid)?.fee)
     }
 
-    pub fn history_txids(&self, scripthash: &[u8]) -> Vec<Sha256dHash> {
+    pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
+        let tx = match self.txstore.get(txid) {
+            Some(tx) => tx,
+            None => return false,
+        };
+        tx.input
+            .iter()
+            .any(|txin| self.txstore.contains_key(&txin.previous_output.txid))
+    }
+
+    pub fn history(
+        &self,
+        scripthash: &[u8],
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<Transaction> {
+        let _timer = self.latency.with_label_values(&["history"]).start_timer();
+        self.history
+            .get(scripthash)
+            .map_or_else(std::vec::Vec::new, |entries| {
+                self._history(entries, last_seen_txid, limit)
+            })
+    }
+
+    pub fn history_txids_iter<'a>(&'a self, scripthash: &[u8]) -> impl Iterator<Item = Txid> + 'a {
+        self.history
+            .get(scripthash)
+            .into_iter()
+            .flat_map(|v| v.iter().map(|e| e.get_txid()).unique())
+    }
+
+    fn _history(
+        &self,
+        entries: &[TxHistoryInfo],
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<Transaction> {
+        entries
+            .iter()
+            .map(|e| e.get_txid())
+            .unique()
+            // TODO seek directly to last seen tx without reading earlier rows
+            .skip_while(|txid| {
+                // skip until we reach the last_seen_txid
+                last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
+            })
+            .skip(match last_seen_txid {
+                Some(_) => 1, // skip the last_seen_txid itself
+                None => 0,
+            })
+            .take(limit)
+            .map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
+            .cloned()
+            .collect()
+    }
+
+    pub fn history_group(
+        &self,
+        scripthashes: &[[u8; 32]],
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<Transaction> {
+        let _timer = self
+            .latency
+            .with_label_values(&["history_group"])
+            .start_timer();
+        scripthashes
+            .iter()
+            .filter_map(|scripthash| self.history.get(&scripthash[..]))
+            .flat_map(|entries| entries.iter())
+            .map(|e| e.get_txid())
+            .unique()
+            // TODO seek directly to last seen tx without reading earlier rows
+            .skip_while(|txid| {
+                // skip until we reach the last_seen_txid
+                last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
+            })
+            .skip(match last_seen_txid {
+                Some(_) => 1, // skip the last_seen_txid itself
+                None => 0,
+            })
+            .take(limit)
+            .map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
+            .cloned()
+            .collect()
+    }
+
+    pub fn history_txids_iter_group<'a>(
+        &'a self,
+        scripthashes: &'a [[u8; 32]],
+    ) -> impl Iterator<Item = Txid> + 'a {
+        scripthashes
+            .iter()
+            .filter_map(move |scripthash| self.history.get(&scripthash[..]))
+            .flat_map(|entries| entries.iter())
+            .map(|entry| entry.get_txid())
+            .unique()
+    }
+
+    pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<Txid> {
         let _timer = self
             .latency
             .with_label_values(&["history_txids"])
             .start_timer();
         match self.history.get(scripthash) {
-            None => return vec![],
-            Some(entries) => entries.iter().map(get_entry_txid).unique().collect(),
+            None => vec![],
+            Some(entries) => entries
+                .iter()
+                .map(|e| e.get_txid())
+                .unique()
+                .take(limit)
+                .collect(),
         }
     }
 
@@ -130,15 +244,33 @@ impl Mempool {
         };
 
         entries
-            .into_iter()
+            .iter()
             .filter_map(|entry| match entry {
-                TxHistoryInfo::Funding(info) => Some(Utxo {
-                    txid: parse_hash(&info.txid),
-                    vout: info.vout as u32,
-                    value: info.value,
-                    confirmed: None,
-                }),
-                TxHistoryInfo::Spending(..) => None,
+                TxHistoryInfo::Funding(info) => {
+                    // Liquid requires some additional information from the txo that's not available in the TxHistoryInfo index.
+                    #[cfg(feature = "liquid")]
+                    let txo = self.lookup_txo(&entry.get_funded_outpoint())?;
+
+                    Some(Utxo {
+                        txid: deserialize(&info.txid).expect("invalid txid"),
+                        vout: info.vout,
+                        value: info.value,
+                        confirmed: None,
+
+                        #[cfg(feature = "liquid")]
+                        asset: txo.asset,
+                        #[cfg(feature = "liquid")]
+                        nonce: txo.nonce,
+                        #[cfg(feature = "liquid")]
+                        witness: txo.witness,
+                    })
+                }
+                TxHistoryInfo::Spending(_) => None,
+                #[cfg(feature = "liquid")]
+                TxHistoryInfo::Issuing(_)
+                | TxHistoryInfo::Burning(_)
+                | TxHistoryInfo::Pegin(_)
+                | TxHistoryInfo::Pegout(_) => unreachable!(),
             })
             .filter(|utxo| !self.has_spend(&OutPoint::from(utxo)))
             .collect()
@@ -156,7 +288,7 @@ impl Mempool {
         };
 
         for entry in entries {
-            if seen_txids.insert(get_entry_txid(entry)) {
+            if seen_txids.insert(entry.get_txid()) {
                 stats.tx_count += 1;
             }
 
@@ -166,20 +298,27 @@ impl Mempool {
                     stats.funded_txo_count += 1;
                     stats.funded_txo_sum += info.value;
                 }
-                #[cfg(feature = "liquid")]
-                TxHistoryInfo::Funding(_) => {
-                    stats.funded_txo_count += 1;
-                }
 
                 #[cfg(not(feature = "liquid"))]
                 TxHistoryInfo::Spending(info) => {
                     stats.spent_txo_count += 1;
                     stats.spent_txo_sum += info.value;
                 }
+
+                // Elements
+                #[cfg(feature = "liquid")]
+                TxHistoryInfo::Funding(_) => {
+                    stats.funded_txo_count += 1;
+                }
                 #[cfg(feature = "liquid")]
                 TxHistoryInfo::Spending(_) => {
                     stats.spent_txo_count += 1;
                 }
+                #[cfg(feature = "liquid")]
+                TxHistoryInfo::Issuing(_)
+                | TxHistoryInfo::Burning(_)
+                | TxHistoryInfo::Pegin(_)
+                | TxHistoryInfo::Pegout(_) => unreachable!(),
             };
         }
 
@@ -187,11 +326,54 @@ impl Mempool {
     }
 
     // Get all txids in the mempool
-    pub fn txids(&self) -> Vec<&Sha256dHash> {
+    pub fn txids(&self) -> Vec<&Txid> {
         let _timer = self.latency.with_label_values(&["txids"]).start_timer();
         self.txstore.keys().collect()
     }
 
+    // Get n txids after the given txid in the mempool
+    pub fn txids_page(&self, n: usize, start: Option<Txid>) -> Vec<&Txid> {
+        let _timer = self
+            .latency
+            .with_label_values(&["txids_page"])
+            .start_timer();
+        let start_bound = match start {
+            Some(txid) => Excluded(txid),
+            None => Unbounded,
+        };
+
+        self.txstore
+            .range((start_bound, Unbounded))
+            .take(n)
+            .map(|(k, _v)| k)
+            .collect()
+    }
+
+    // Get all txs in the mempool
+    pub fn txs(&self) -> Vec<Transaction> {
+        let _timer = self.latency.with_label_values(&["txs"]).start_timer();
+        self.txstore.values().cloned().collect()
+    }
+
+    // Get n txs after the given txid in the mempool
+    pub fn txs_page(&self, n: usize, start: Option<Txid>) -> Vec<Transaction> {
+        let _timer = self.latency.with_label_values(&["txs_page"]).start_timer();
+        let mut page = Vec::with_capacity(n);
+        let start_bound = match start {
+            Some(txid) => Excluded(txid),
+            None => Unbounded,
+        };
+
+        self.txstore
+            .range((start_bound, Unbounded))
+            .take(n)
+            .for_each(|(_, value)| {
+                page.push(value.clone());
+            });
+
+        page
+    }
+
     // Get an overview of the most recent transactions
     pub fn recent_txs_overview(&self) -> Vec<&TxOverview> {
         // We don't bother ever deleting elements from the recent list.
@@ -204,100 +386,150 @@ impl Mempool {
         &self.backlog_stats.0
     }
 
-    pub fn update(&mut self, daemon: &Daemon) -> Result<()> {
-        let _timer = self.latency.with_label_values(&["update"]).start_timer();
-        let new_txids = daemon
+    pub fn unique_txids(&self) -> HashSet<Txid> {
+        HashSet::from_iter(self.txstore.keys().cloned())
+    }
+
+    pub fn update(mempool: &RwLock<Mempool>, daemon: &Daemon) -> Result<()> {
+        // 1. Start the metrics timer and get the current mempool txids
+        // [LOCK] Takes read lock for whole scope.
+        let (_timer, old_txids) = {
+            let mempool = mempool.read().unwrap();
+            (
+                mempool.latency.with_label_values(&["update"]).start_timer(),
+                mempool.unique_txids(),
+            )
+        };
+
+        // 2. Get all the mempool txids from the RPC.
+        // [LOCK] No lock taken. Wait for RPC request. Get lists of remove/add txes.
+        let all_txids = daemon
             .getmempooltxids()
             .chain_err(|| "failed to update mempool from daemon")?;
-        let old_txids = HashSet::from_iter(self.txstore.keys().cloned());
-        let to_remove: HashSet<&Sha256dHash> = old_txids.difference(&new_txids).collect();
+        let txids_to_remove: HashSet<&Txid> = old_txids.difference(&all_txids).collect();
+        let txids_to_add: Vec<&Txid> = all_txids.difference(&old_txids).collect();
 
-        // Download and add new transactions from bitcoind's mempool
-        let txids: Vec<&Sha256dHash> = new_txids.difference(&old_txids).collect();
-        let to_add = match daemon.gettransactions(&txids) {
-            Ok(txs) => txs,
-            Err(err) => {
-                warn!("failed to get transactions {:?}: {}", txids, err); // e.g. new block or RBF
-                return Ok(()); // keep the mempool until next update()
+        // 3. Remove missing transactions. Even if we are unable to download new transactions from
+        // the daemon, we still want to remove the transactions that are no longer in the mempool.
+        // [LOCK] Write lock is released at the end of the call to remove().
+        mempool.write().unwrap().remove(txids_to_remove);
+
+        // 4. Download the new transactions from the daemon's mempool
+        // [LOCK] No lock taken, waiting for RPC response.
+        let txs_to_add = daemon
+            .gettransactions(&txids_to_add)
+            .chain_err(|| format!("failed to get {} transactions", txids_to_add.len()))?;
+
+        // 4. Update local mempool to match daemon's state
+        // [LOCK] Takes Write lock for whole scope.
+        {
+            let mut mempool = mempool.write().unwrap();
+            // Add new transactions
+            if txs_to_add.len() > mempool.add(txs_to_add) {
+                debug!("Mempool update added less transactions than expected");
             }
-        };
-        // Add new transactions
-        self.add(to_add);
-        // Remove missing transactions
-        self.remove(to_remove);
 
-        self.count
-            .with_label_values(&["txs"])
-            .set(self.txstore.len() as f64);
+            mempool
+                .count
+                .with_label_values(&["txs"])
+                .set(mempool.txstore.len() as f64);
 
-        // Update cached backlog stats (if expired)
-        if self.backlog_stats.1.elapsed() > Duration::from_secs(BACKLOG_STATS_TTL) {
-            let _timer = self
-                .latency
-                .with_label_values(&["update_backlog_stats"])
-                .start_timer();
-            self.backlog_stats = (BacklogStats::new(&self.feeinfo), Instant::now());
+            // Update cached backlog stats (if expired)
+            if mempool.backlog_stats.1.elapsed()
+                > Duration::from_secs(mempool.config.mempool_backlog_stats_ttl)
+            {
+                let _timer = mempool
+                    .latency
+                    .with_label_values(&["update_backlog_stats"])
+                    .start_timer();
+                mempool.backlog_stats = (BacklogStats::new(&mempool.feeinfo), Instant::now());
+            }
+
+            Ok(())
         }
+    }
 
+    pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) -> Result<()> {
+        if !self.txstore.contains_key(txid) {
+            if let Ok(tx) = daemon.getmempooltx(txid) {
+                if self.add(vec![tx]) == 0 {
+                    return Err(format!(
+                        "Unable to add {txid} to mempool likely due to missing parents."
+                    )
+                    .into());
+                }
+            }
+        }
         Ok(())
     }
 
-    pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Sha256dHash) {
-        if let Ok(tx) = daemon.getmempooltx(&txid) {
-            self.add(vec![tx])
-        }
-    }
-
-    fn add(&mut self, txs: Vec<Transaction>) {
+    /// Add transactions to the mempool.
+    ///
+    /// The return value is the number of transactions processed.
+    #[must_use = "Must deal with [[input vec's length]] > [[result]]."]
+    fn add(&mut self, txs: Vec<Transaction>) -> usize {
         self.delta
             .with_label_values(&["add"])
             .observe(txs.len() as f64);
         let _timer = self.latency.with_label_values(&["add"]).start_timer();
+        let txlen = txs.len();
+        if txlen == 0 {
+            return 0;
+        }
+        debug!("Adding {} transactions to Mempool", txlen);
 
-        let mut txids = vec![];
+        let mut txids = Vec::with_capacity(txs.len());
         // Phase 1: add to txstore
         for tx in txs {
-            let txid = tx.txid();
-            txids.push(txid);
-            self.txstore.insert(txid, tx);
-        }
-        // Phase 2: index history and spend edges (can fail if some txos cannot be found)
-        let txos = match self.lookup_txos(&self.get_prevouts(&txids)) {
-            Ok(txos) => txos,
-            Err(err) => {
-                warn!("lookup txouts failed: {}", err);
-                // TODO: should we remove txids from txstore?
-                return;
+            let txid = tx.get_txid();
+            // Only push if it doesn't already exist.
+            // This is important now that update doesn't lock during
+            // the entire function body.
+            if self.txstore.insert(txid, tx).is_none() {
+                txids.push(txid);
             }
-        };
+        }
+
+        // Phase 2: index history and spend edges (some txos can be missing)
+        let txos = self.lookup_txos(&self.get_prevouts(&txids));
+
+        // Count how many transactions were actually processed.
+        let mut processed_count = 0;
+
+        // Phase 3: Iterate over the transactions and do the following:
+        // 1. Find all of the TxOuts of each input parent using `txos`
+        // 2. If any parent wasn't found, skip parsing this transaction
+        // 3. Insert TxFeeInfo into info.
+        // 4. Push TxOverview into recent tx queue.
+        // 5. Create the Spend and Fund TxHistory structs for inputs + outputs
+        // 6. Insert all TxHistory into history.
+        // 7. Insert the tx edges into edges (HashMap of (Outpoint, (Txid, vin)))
+        // 8. (Liquid only) Parse assets of tx.
         for txid in txids {
-            let tx = self.txstore.get(&txid).expect("missing mempool tx");
+            let tx = self.txstore.get(&txid).expect("missing tx from txstore");
+
+            let prevouts = match extract_tx_prevouts(tx, &txos) {
+                Ok(v) => v,
+                Err(e) => {
+                    warn!("Skipping tx {txid} missing parent error: {e}");
+                    continue;
+                }
+            };
             let txid_bytes = full_hash(&txid[..]);
 
-            let prevouts: HashMap<u32, &TxOut> = tx
-                .input
-                .iter()
-                .enumerate()
-                .filter(|(_, txi)| has_prevout(txi))
-                .map(|(index, txi)| {
-                    (
-                        index as u32,
-                        txos.get(&txi.previous_output)
-                            .expect(&format!("missing outpoint {:?}", txi.previous_output)),
-                    )
-                })
-                .collect();
-
             // Get feeinfo for caching and recent tx overview
-            let feeinfo = TxFeeInfo::new(&tx, &prevouts);
+            let feeinfo = TxFeeInfo::new(tx, &prevouts, self.config.network_type);
 
-            // recent is an ArrayDeque that automatically evicts the oldest elements
+            // recent is an BoundedVecDeque that automatically evicts the oldest elements
             self.recent.push_front(TxOverview {
-                txid: txid,
+                txid,
                 fee: feeinfo.fee,
                 vsize: feeinfo.vsize,
                 #[cfg(not(feature = "liquid"))]
-                value: prevouts.values().map(|prevout| prevout.value).sum(),
+                value: prevouts
+                    .values()
+                    .map(|prevout| prevout.value.to_sat())
+                    .sum(),
             });
 
             self.feeinfo.insert(txid, feeinfo);
@@ -305,49 +537,82 @@ impl Mempool {
             // An iterator over (ScriptHash, TxHistoryInfo)
             let spending = prevouts.into_iter().map(|(input_index, prevout)| {
                 let txi = tx.input.get(input_index as usize).unwrap();
+                #[cfg(not(feature = "liquid"))]
+                let value = prevout.value.to_sat();
+                #[cfg(feature = "liquid")]
+                let value = prevout.value;
                 (
                     compute_script_hash(&prevout.script_pubkey),
                     TxHistoryInfo::Spending(SpendingInfo {
                         txid: txid_bytes,
-                        vin: input_index as u16,
+                        vin: input_index,
                         prev_txid: full_hash(&txi.previous_output.txid[..]),
-                        prev_vout: txi.previous_output.vout as u16,
-                        value: prevout.value,
+                        prev_vout: txi.previous_output.vout,
+                        value,
                     }),
                 )
             });
 
+            let config = &self.config;
+
             // An iterator over (ScriptHash, TxHistoryInfo)
             let funding = tx
                 .output
                 .iter()
                 .enumerate()
-                .filter(|(_, txo)| is_spendable(txo))
+                .filter(|(_, txo)| is_spendable(txo) || config.index_unspendables)
                 .map(|(index, txo)| {
+                    #[cfg(not(feature = "liquid"))]
+                    let value = txo.value.to_sat();
+                    #[cfg(feature = "liquid")]
+                    let value = txo.value;
                     (
                         compute_script_hash(&txo.script_pubkey),
                         TxHistoryInfo::Funding(FundingInfo {
                             txid: txid_bytes,
-                            vout: index as u16,
-                            value: txo.value,
+                            vout: index as u32,
+                            value,
                         }),
                     )
                 });
 
             // Index funding/spending history entries and spend edges
             for (scripthash, entry) in funding.chain(spending) {
-                self.history
-                    .entry(scripthash)
-                    .or_insert_with(|| Vec::new())
-                    .push(entry);
+                self.history.entry(scripthash).or_default().push(entry);
             }
             for (i, txi) in tx.input.iter().enumerate() {
                 self.edges.insert(txi.previous_output, (txid, i as u32));
             }
+
+            // Index issued assets & native asset pegins/pegouts/burns
+            #[cfg(feature = "liquid")]
+            asset::index_mempool_tx_assets(
+                tx,
+                self.config.network_type,
+                self.config.parent_network,
+                &mut self.asset_history,
+                &mut self.asset_issuance,
+            );
+
+            processed_count += 1;
         }
+
+        processed_count
     }
 
-    pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
+    /// Returns None if the lookup fails (mempool transaction RBF-ed etc.)
+    pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option<TxOut> {
+        let mut outpoints = BTreeSet::new();
+        outpoints.insert(*outpoint);
+        // This can possibly be None now
+        self.lookup_txos(&outpoints).remove(outpoint)
+    }
+
+    /// For a given set of OutPoints, return a HashMap<OutPoint, TxOut>
+    ///
+    /// Not all OutPoints from mempool transactions are guaranteed to be there.
+    /// Ensure you deal with the None case in your logic.
+    pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
         let _timer = self
             .latency
             .with_label_values(&["lookup_txos"])
@@ -358,21 +623,29 @@ impl Mempool {
         let mempool_txos = outpoints
             .iter()
             .filter(|outpoint| !confirmed_txos.contains_key(outpoint))
-            .map(|outpoint| {
+            .flat_map(|outpoint| {
                 self.txstore
                     .get(&outpoint.txid)
                     .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned())
                     .map(|txout| (*outpoint, txout))
-                    .chain_err(|| format!("missing outpoint {:?}", outpoint))
+                    .or_else(|| {
+                        warn!("missing outpoint {:?}", outpoint);
+                        None
+                    })
             })
-            .collect::<Result<HashMap<OutPoint, TxOut>>>()?;
+            .collect::<HashMap<OutPoint, TxOut>>();
 
         let mut txos = confirmed_txos;
         txos.extend(mempool_txos);
-        Ok(txos)
+        txos
     }
 
-    fn get_prevouts(&self, txids: &[Sha256dHash]) -> BTreeSet<OutPoint> {
+    fn get_prevouts(&self, txids: &[Txid]) -> BTreeSet<OutPoint> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_prevouts"])
+            .start_timer();
+
         txids
             .iter()
             .map(|txid| self.txstore.get(txid).expect("missing mempool tx"))
@@ -385,7 +658,7 @@ impl Mempool {
             .collect()
     }
 
-    fn remove(&mut self, to_remove: HashSet<&Sha256dHash>) {
+    fn remove(&mut self, to_remove: HashSet<&Txid>) {
         self.delta
             .with_label_values(&["remove"])
             .observe(to_remove.len() as f64);
@@ -394,20 +667,43 @@ impl Mempool {
         for txid in &to_remove {
             self.txstore
                 .remove(*txid)
-                .expect(&format!("missing mempool tx {}", txid));
-            self.feeinfo
-                .remove(*txid)
-                .expect(&format!("missing mempool tx feeinfo {}", txid));
+                .unwrap_or_else(|| panic!("missing mempool tx {}", txid));
+
+            self.feeinfo.remove(*txid).or_else(|| {
+                warn!("missing mempool tx feeinfo {}", txid);
+                None
+            });
         }
+
         // TODO: make it more efficient (currently it takes O(|mempool|) time)
         self.history.retain(|_scripthash, entries| {
-            entries.retain(|entry| !to_remove.contains(&get_entry_txid(entry)));
+            entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
             !entries.is_empty()
         });
 
+        #[cfg(feature = "liquid")]
+        asset::remove_mempool_tx_assets(
+            &to_remove,
+            &mut self.asset_history,
+            &mut self.asset_issuance,
+        );
+
         self.edges
             .retain(|_outpoint, (txid, _vin)| !to_remove.contains(txid));
     }
+
+    #[cfg(feature = "liquid")]
+    pub fn asset_history(&self, asset_id: &AssetId, limit: usize) -> Vec<Transaction> {
+        let _timer = self
+            .latency
+            .with_label_values(&["asset_history"])
+            .start_timer();
+        self.asset_history
+            .get(asset_id)
+            .map_or_else(std::vec::Vec::new, |entries| {
+                self._history(entries, None, limit)
+            })
+    }
 }
 
 #[derive(Serialize)]
@@ -428,7 +724,7 @@ impl BacklogStats {
         }
     }
 
-    fn new(feeinfo: &HashMap<Sha256dHash, TxFeeInfo>) -> Self {
+    fn new(feeinfo: &HashMap<Txid, TxFeeInfo>) -> Self {
         let (count, vsize, total_fee) = feeinfo
             .values()
             .fold((0, 0, 0), |(count, vsize, fee), feeinfo| {
@@ -443,10 +739,3 @@ impl BacklogStats {
         }
     }
 }
-
-fn get_entry_txid(entry: &TxHistoryInfo) -> Sha256dHash {
-    match entry {
-        TxHistoryInfo::Funding(info) => parse_hash(&info.txid),
-        TxHistoryInfo::Spending(info) => parse_hash(&info.txid),
-    }
-}

src/new_index/mod.rs

@@ -1,14 +1,25 @@
-mod db;
+pub mod db;
 mod fetch;
 mod mempool;
 pub mod precache;
 mod query;
-mod schema;
+pub mod schema;
 
+use std::sync::LazyLock;
+
+pub(crate) static THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(0) // 0 = use number of logical CPUs
+        .thread_name(|i| format!("electrs-worker-{}", i))
+        .build()
+        .expect("failed to create global rayon thread pool")
+});
+
+pub use self::db::{DBRow, DB};
 pub use self::fetch::{BlockEntry, FetchFrom};
 pub use self::mempool::Mempool;
 pub use self::query::Query;
 pub use self::schema::{
     compute_script_hash, parse_hash, ChainQuery, FundingInfo, Indexer, ScriptStats, SpendingInfo,
-    SpendingInput, Store, TxHistoryInfo, TxHistoryKey, Utxo,
+    SpendingInput, Store, TxHistoryInfo, TxHistoryKey, TxHistoryRow, Utxo,
 };

src/new_index/precache.rs

@@ -1,37 +1,59 @@
 use crate::errors::*;
 use crate::new_index::ChainQuery;
-use crate::util::{full_hash, Address, FullHash};
+use crate::util::{full_hash, FullHash};
 
-use crypto::digest::Digest;
-use crypto::sha2::Sha256;
 use rayon::prelude::*;
 
 use hex;
 use std::fs::File;
 use std::io;
 use std::io::prelude::*;
-use std::str::FromStr;
+use std::sync::{atomic::AtomicUsize, Arc};
+use std::time::Instant;
 
-pub fn precache(chain: &ChainQuery, scripthashes: Vec<FullHash>) {
+pub fn precache(chain: Arc<ChainQuery>, scripthashes: Vec<FullHash>, threads: usize) {
     let total = scripthashes.len();
-    info!("Pre-caching stats and utxo set for {} scripthashes", total);
+    info!(
+        "Pre-caching stats and utxo set on {} threads for {} scripthashes",
+        threads, total
+    );
 
     let pool = rayon::ThreadPoolBuilder::new()
-        .num_threads(16)
+        .num_threads(threads)
         .thread_name(|i| format!("precache-{}", i))
         .build()
         .unwrap();
-    pool.install(|| {
-        scripthashes
-            .par_iter()
-            .enumerate()
-            .for_each(|(i, scripthash)| {
-                if i % 5 == 0 {
-                    debug!("Starting pre-cache for scripthash {}/{}", i + 1, total);
-                }
-                chain.stats(&scripthash[..]);
-                //chain.utxo(&scripthash[..]);
-            })
+    let now = Instant::now();
+    let counter = AtomicUsize::new(0);
+    std::thread::spawn(move || {
+        pool.install(|| {
+            scripthashes
+                .par_iter()
+                .for_each(|scripthash| {
+                    // First, cache
+                    chain.stats(&scripthash[..], crate::new_index::db::DBFlush::Disable);
+                    let _ = chain.utxo(&scripthash[..], usize::MAX, crate::new_index::db::DBFlush::Disable);
+
+                    // Then, increment the counter
+                    let pre_increment = counter.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                    let post_increment_counter = pre_increment + 1;
+
+                    // Then, log
+                    if post_increment_counter % 500 == 0 {
+                        let now_millis = now.elapsed().as_millis();
+                        info!("{post_increment_counter}/{total} Processed in {now_millis} ms running pre-cache for scripthash");
+                    }
+
+                    // Every 10k counts, flush the DB to disk
+                    if post_increment_counter % 10000 == 0 {
+                        info!("Flushing cache_db... {post_increment_counter}");
+                        chain.store().cache_db().flush();
+                        info!("Done Flushing cache_db!!! {post_increment_counter}");
+                    }
+                })
+        });
+        // After everything is done, flush the cache
+        chain.store().cache_db().flush();
     });
 }
 
@@ -42,34 +64,7 @@ pub fn scripthashes_from_file(path: String) -> Result<Vec<FullHash>> {
         .lines()
         .map(|line| {
             let line = line.chain_err(|| "cannot read scripthash line")?;
-            let cols: Vec<&str> = line.split(",").collect();
-            to_scripthash(cols[0], cols[1])
+            Ok(full_hash(&hex::decode(line).chain_err(|| "invalid hex")?))
         })
         .collect()
 }
-
-fn to_scripthash(script_type: &str, script_str: &str) -> Result<FullHash> {
-    match script_type {
-        "address" => address_to_scripthash(script_str),
-        "scripthash" => Ok(full_hash(
-            &hex::decode(script_str).chain_err(|| "invalid hex")?,
-        )),
-        "scriptpubkey" => Ok(compute_script_hash(
-            &hex::decode(script_str).chain_err(|| "invalid hex")?,
-        )),
-        _ => bail!("Invalid script type".to_string()),
-    }
-}
-
-fn address_to_scripthash(addr: &str) -> Result<FullHash> {
-    let addr = Address::from_str(addr).chain_err(|| "invalid address")?;
-    Ok(compute_script_hash(&addr.script_pubkey().as_bytes()))
-}
-
-pub fn compute_script_hash(data: &[u8]) -> FullHash {
-    let mut hash = FullHash::default();
-    let mut sha2 = Sha256::new();
-    sha2.input(data);
-    sha2.result(&mut hash);
-    hash
-}

src/new_index/query.rs

@@ -1,31 +1,55 @@
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
 use rayon::prelude::*;
 
 use std::collections::{BTreeSet, HashMap};
 use std::sync::{Arc, RwLock, RwLockReadGuard};
+use std::time::{Duration, Instant};
 
-use crate::chain::{OutPoint, Transaction, TxOut};
-use crate::daemon::Daemon;
+use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
+use crate::config::Config;
+use crate::daemon::{Daemon, MempoolAcceptResult, SubmitPackageResult};
 use crate::errors::*;
 use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo};
 use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus};
 
-const CONF_TARGETS: [u16; 9] = [
-    2u16, 3u16, 4u16, 6u16, 10u16, 20u16, 144u16, 504u16, 1008u16,
+#[cfg(feature = "liquid")]
+use crate::{
+    chain::{asset::AssetRegistryLock, AssetId},
+    elements::{lookup_asset, AssetMeta, AssetRegistry, AssetSorting, LiquidAsset},
+};
+
+const FEE_ESTIMATES_TTL: u64 = 60; // seconds
+
+const CONF_TARGETS: [u16; 28] = [
+    1u16, 2u16, 3u16, 4u16, 5u16, 6u16, 7u16, 8u16, 9u16, 10u16, 11u16, 12u16, 13u16, 14u16, 15u16,
+    16u16, 17u16, 18u16, 19u16, 20u16, 21u16, 22u16, 23u16, 24u16, 25u16, 144u16, 504u16, 1008u16,
 ];
 
 pub struct Query {
     chain: Arc<ChainQuery>, // TODO: should be used as read-only
     mempool: Arc<RwLock<Mempool>>,
     daemon: Arc<Daemon>,
+    config: Arc<Config>,
+    cached_estimates: RwLock<(HashMap<u16, f64>, Option<Instant>)>,
+    cached_relayfee: RwLock<Option<f64>>,
+    #[cfg(feature = "liquid")]
+    asset_db: Option<Arc<RwLock<AssetRegistry>>>,
 }
 
 impl Query {
-    pub fn new(chain: Arc<ChainQuery>, mempool: Arc<RwLock<Mempool>>, daemon: Arc<Daemon>) -> Self {
+    #[cfg(not(feature = "liquid"))]
+    pub fn new(
+        chain: Arc<ChainQuery>,
+        mempool: Arc<RwLock<Mempool>>,
+        daemon: Arc<Daemon>,
+        config: Arc<Config>,
+    ) -> Self {
         Query {
             chain,
             mempool,
             daemon,
+            config,
+            cached_estimates: RwLock::new((HashMap::new(), None)),
+            cached_relayfee: RwLock::new(None),
         }
     }
 
@@ -33,37 +57,73 @@ impl Query {
         &self.chain
     }
 
-    pub fn mempool(&self) -> RwLockReadGuard<Mempool> {
+    pub fn config(&self) -> &Config {
+        &self.config
+    }
+
+    pub fn network(&self) -> Network {
+        self.config.network_type
+    }
+
+    pub fn mempool(&self) -> RwLockReadGuard<'_, Mempool> {
         self.mempool.read().unwrap()
     }
 
-    pub fn broadcast_raw(&self, txhex: &String) -> Result<Sha256dHash> {
-        let txid = self.daemon.broadcast_raw(&txhex)?;
-        self.mempool
+    pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
+        let txid = self.daemon.broadcast_raw(txhex)?;
+        // The important part is whether we succeeded in broadcasting.
+        // Ignore errors in adding to the cache and show an internal warning.
+        if let Err(e) = self
+            .mempool
             .write()
             .unwrap()
-            .add_by_txid(&self.daemon, &txid);
+            .add_by_txid(&self.daemon, &txid)
+        {
+            warn!(
+                "broadcast_raw of {txid} succeeded to broadcast \
+                but failed to add to mempool-electrs Mempool cache: {e}"
+            );
+        }
         Ok(txid)
     }
 
-    pub fn utxo(&self, scripthash: &[u8]) -> Vec<Utxo> {
-        let mut utxos = self.chain.utxo(scripthash);
+    pub fn test_mempool_accept(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+    ) -> Result<Vec<MempoolAcceptResult>> {
+        self.daemon.test_mempool_accept(txhex, maxfeerate)
+    }
+
+    pub fn submit_package(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+        maxburnamount: Option<f64>,
+    ) -> Result<SubmitPackageResult> {
+        self.daemon.submit_package(txhex, maxfeerate, maxburnamount)
+    }
+
+    pub fn utxo(&self, scripthash: &[u8]) -> Result<Vec<Utxo>> {
+        let mut utxos = self.chain.utxo(
+            scripthash,
+            self.config.utxos_limit,
+            super::db::DBFlush::Enable,
+        )?;
         let mempool = self.mempool();
         utxos.retain(|utxo| !mempool.has_spend(&OutPoint::from(utxo)));
         utxos.extend(mempool.utxo(scripthash));
-        utxos
+        Ok(utxos)
     }
 
-    pub fn history_txids(&self, scripthash: &[u8]) -> Vec<(Sha256dHash, Option<BlockId>)> {
-        let confirmed_txids = self
-            .chain
-            .history_txids(scripthash)
-            .into_iter()
-            .map(|(tx, b)| (tx, Some(b)));
+    pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<(Txid, Option<BlockId>)> {
+        let confirmed_txids = self.chain.history_txids(scripthash, limit);
+        let confirmed_len = confirmed_txids.len();
+        let confirmed_txids = confirmed_txids.into_iter().map(|(tx, b)| (tx, Some(b)));
 
         let mempool_txids = self
             .mempool()
-            .history_txids(scripthash)
+            .history_txids(scripthash, limit - confirmed_len)
             .into_iter()
             .map(|tx| (tx, None));
 
@@ -72,27 +132,26 @@ impl Query {
 
     pub fn stats(&self, scripthash: &[u8]) -> (ScriptStats, ScriptStats) {
         (
-            self.chain.stats(scripthash),
+            self.chain.stats(scripthash, super::db::DBFlush::Enable),
             self.mempool().stats(scripthash),
         )
     }
 
-    pub fn lookup_txn(&self, txid: &Sha256dHash) -> Option<Transaction> {
+    pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
         self.chain
-            .lookup_txn(txid)
+            .lookup_txn(txid, None)
             .or_else(|| self.mempool().lookup_txn(txid))
     }
-    pub fn lookup_raw_txn(&self, txid: &Sha256dHash) -> Option<Bytes> {
+    pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
         self.chain
-            .lookup_raw_txn(txid)
+            .lookup_raw_txn(txid, None)
             .or_else(|| self.mempool().lookup_raw_txn(txid))
     }
 
+    /// Not all OutPoints from mempool transactions are guaranteed to be included in the result
     pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
         // the mempool lookup_txos() internally looks up confirmed txos as well
-        self.mempool()
-            .lookup_txos(outpoints)
-            .expect("failed loading txos")
+        self.mempool().lookup_txos(outpoints)
     }
 
     pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option<SpendingInput> {
@@ -102,7 +161,7 @@ impl Query {
     }
 
     pub fn lookup_tx_spends(&self, tx: Transaction) -> Vec<Option<SpendingInput>> {
-        let txid = tx.txid();
+        let txid = tx.get_txid();
 
         tx.output
             .par_iter()
@@ -120,23 +179,155 @@ impl Query {
             .collect()
     }
 
-    pub fn get_tx_status(&self, txid: &Sha256dHash) -> TransactionStatus {
+    pub fn get_tx_status(&self, txid: &Txid) -> TransactionStatus {
         TransactionStatus::from(self.chain.tx_confirming_block(txid))
     }
 
-    // TODO cache, only allow getting estimatess for cached items
-    pub fn estimate_fee(&self, conf_target: u16) -> Option<f32> {
-        self.daemon.estimatesmartfee(conf_target).ok()
+    pub fn get_mempool_tx_fee(&self, txid: &Txid) -> Option<u64> {
+        self.mempool().get_tx_fee(txid)
     }
 
-    // TODO cache
-    pub fn estimate_fee_targets(&self) -> HashMap<u16, f32> {
-        CONF_TARGETS
-            .iter()
-            .filter_map(|conf_target| {
-                self.estimate_fee(*conf_target)
-                    .map(|feerate| (*conf_target, feerate))
+    pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
+        self.mempool().has_unconfirmed_parents(txid)
+    }
+
+    pub fn estimate_fee(&self, conf_target: u16) -> Option<f64> {
+        if self.config.network_type.is_regtest() {
+            return self.get_relayfee().ok();
+        }
+        if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
+            if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
+                return cache.get(&conf_target).copied();
+            }
+        }
+
+        self.update_fee_estimates();
+        self.cached_estimates
+            .read()
+            .unwrap()
+            .0
+            .get(&conf_target)
+            .copied()
+    }
+
+    pub fn estimate_fee_map(&self) -> HashMap<u16, f64> {
+        if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
+            if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
+                return cache.clone();
+            }
+        }
+
+        self.update_fee_estimates();
+        self.cached_estimates.read().unwrap().0.clone()
+    }
+
+    fn update_fee_estimates(&self) {
+        match self.daemon.estimatesmartfee_batch(&CONF_TARGETS) {
+            Ok(estimates) => {
+                *self.cached_estimates.write().unwrap() = (estimates, Some(Instant::now()));
+            }
+            Err(err) => {
+                warn!("failed estimating feerates: {:?}", err);
+            }
+        }
+    }
+
+    pub fn get_relayfee(&self) -> Result<f64> {
+        if let Some(cached) = *self.cached_relayfee.read().unwrap() {
+            return Ok(cached);
+        }
+
+        let relayfee = self.daemon.get_relayfee()?;
+        self.cached_relayfee.write().unwrap().replace(relayfee);
+        Ok(relayfee)
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn new(
+        chain: Arc<ChainQuery>,
+        mempool: Arc<RwLock<Mempool>>,
+        daemon: Arc<Daemon>,
+        config: Arc<Config>,
+        asset_db: Option<Arc<RwLock<AssetRegistry>>>,
+    ) -> Self {
+        Query {
+            chain,
+            mempool,
+            daemon,
+            config,
+            asset_db,
+            cached_estimates: RwLock::new((HashMap::new(), None)),
+            cached_relayfee: RwLock::new(None),
+        }
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn lookup_asset(&self, asset_id: &AssetId) -> Result<Option<LiquidAsset>> {
+        lookup_asset(
+            self,
+            self.asset_db.as_ref().map(AssetRegistryLock::RwLock),
+            asset_id,
+            None,
+        )
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn lookup_registry_asset(&self, asset_id: &AssetId) -> Result<Option<AssetMeta>> {
+        let asset_db = self
+            .asset_db
+            .as_ref()
+            .chain_err(|| "asset registry unavailable")?;
+        Ok(asset_db.read().unwrap().get(asset_id).cloned())
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn list_registry_assets(
+        &self,
+        start_index: usize,
+        limit: usize,
+        sorting: AssetSorting,
+    ) -> Result<(usize, Vec<LiquidAsset>)> {
+        let asset_db = match &self.asset_db {
+            None => return Ok((0, vec![])),
+            Some(db) => db.read().unwrap(),
+        };
+        let (total_num, results) = asset_db.list(start_index, limit, sorting);
+        // Attach on-chain information alongside the registry metadata
+        let results = results
+            .into_iter()
+            .map(|(asset_id, metadata)| {
+                lookup_asset(
+                    self,
+                    Some(AssetRegistryLock::RwLockReadGuard(&asset_db)),
+                    asset_id,
+                    Some(metadata),
+                )?
+                .chain_err(|| "missing registered asset")
             })
-            .collect()
+            .collect::<Result<Vec<_>>>()?;
+        Ok((total_num, results))
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn search_registry_assets<T, F>(
+        &self,
+        search: &str,
+        limit: usize,
+        mut map: F,
+    ) -> Result<Vec<T>>
+    where
+        F: FnMut(&AssetId, &AssetMeta) -> T,
+    {
+        let asset_db = self
+            .asset_db
+            .as_ref()
+            .chain_err(|| "asset registry unavailable")?;
+        Ok(asset_db
+            .read()
+            .unwrap()
+            .search(search, limit)
+            .into_iter()
+            .map(|(asset_id, metadata)| map(asset_id, metadata))
+            .collect())
     }
 }

src/signal.rs

@@ -1,34 +1,58 @@
-use chan;
-use chan_signal;
-use std::time::Duration;
+use crossbeam_channel as channel;
+use crossbeam_channel::RecvTimeoutError;
+use std::time::{Duration, Instant};
+
+use signal_hook::consts::{SIGINT, SIGTERM, SIGUSR1};
 
 use crate::errors::*;
 
 #[derive(Clone)] // so multiple threads could wait on signals
 pub struct Waiter {
-    signal: chan::Receiver<chan_signal::Signal>,
+    receiver: channel::Receiver<i32>,
+}
+
+fn notify(signals: &[i32]) -> channel::Receiver<i32> {
+    let (s, r) = channel::bounded(1);
+    let mut signals =
+        signal_hook::iterator::Signals::new(signals).expect("failed to register signal hook");
+    crate::util::spawn_thread("signal-notifier", move || {
+        for signal in signals.forever() {
+            s.send(signal)
+                .unwrap_or_else(|_| panic!("failed to send signal {}", signal));
+        }
+    });
+    r
 }
 
 impl Waiter {
-    pub fn new() -> Waiter {
+    pub fn start() -> Waiter {
         Waiter {
-            signal: chan_signal::notify(&[chan_signal::Signal::INT, chan_signal::Signal::TERM]),
+            receiver: notify(&[
+                SIGINT, SIGTERM,
+                SIGUSR1, // allow external triggering (e.g. via bitcoind `blocknotify`)
+            ]),
         }
     }
-    pub fn wait(&self, duration: Duration) -> Result<()> {
-        let signal = &self.signal;
-        let timeout = chan::after(duration);
-        chan_select! {
-            signal.recv() -> s => {
-                if let Some(sig) = s {
-                    bail!(ErrorKind::Interrupt(sig));
+
+    pub fn wait(&self, duration: Duration, accept_sigusr: bool) -> Result<()> {
+        // Determine the deadline time based on the duration, so that it doesn't
+        // get pushed back when wait_deadline() recurses
+        self.wait_deadline(Instant::now() + duration, accept_sigusr)
+    }
+
+    fn wait_deadline(&self, deadline: Instant, accept_sigusr: bool) -> Result<()> {
+        match self.receiver.recv_deadline(deadline) {
+            Ok(sig) if sig == SIGUSR1 => {
+                trace!("notified via SIGUSR1");
+                if accept_sigusr {
+                    Ok(())
+                } else {
+                    self.wait_deadline(deadline, accept_sigusr)
                 }
-            },
-            timeout.recv() => {},
+            }
+            Ok(sig) => bail!(ErrorKind::Interrupt(sig)),
+            Err(RecvTimeoutError::Timeout) => Ok(()),
+            Err(RecvTimeoutError::Disconnected) => bail!("signal hook channel disconnected"),
         }
-        Ok(())
-    }
-    pub fn poll(&self) -> Result<()> {
-        self.wait(Duration::from_secs(0))
     }
 }

src/util/address.rs

@@ -1,320 +0,0 @@
-// Rust Bitcoin Library
-// Written in 2014 by
-//     Andrew Poelstra <apoelstra@wpsoftware.net>
-// To the extent possible under law, the author(s) have dedicated all
-// copyright and related and neighboring rights to this software to
-// the public domain worldwide. This software is distributed without
-// any warranty.
-//
-// You should have received a copy of the CC0 Public Domain Dedication
-// along with this software.
-// If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-//
-
-//! Addresses
-//!
-//! Support for ordinary base58 Bitcoin addresses and private keys
-//!
-
-use std::fmt::{self, Display, Formatter};
-use std::str::FromStr;
-
-use bitcoin_bech32::{self, u5, WitnessProgram};
-use bitcoin_hashes::{hash160, Hash};
-
-use crate::chain::Network;
-use bitcoin::blockdata::opcodes;
-use bitcoin::blockdata::script;
-use bitcoin::consensus::encode;
-use bitcoin::util::base58;
-use bitcoin::util::key;
-
-/// The method used to produce an address
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum Payload {
-    /// pay-to-pkhash address
-    PubkeyHash(hash160::Hash),
-    /// P2SH address
-    ScriptHash(hash160::Hash),
-    /// Segwit address
-    WitnessProgram(WitnessProgram),
-}
-
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-/// A Bitcoin address
-pub struct Address {
-    /// The type of the address
-    pub payload: Payload,
-    /// The network on which this address is usable
-    pub network: Network,
-}
-
-impl Address {
-    /// Creates a pay to (compressed) public key hash address from a public key
-    /// This is the preferred non-witness type address
-    #[inline]
-    pub fn p2pkh(pk: &key::PublicKey, network: Network) -> Address {
-        let mut hash_engine = hash160::Hash::engine();
-        pk.write_into(&mut hash_engine);
-
-        Address {
-            network: network,
-            payload: Payload::PubkeyHash(hash160::Hash::from_engine(hash_engine)),
-        }
-    }
-
-    /// Creates a pay to script hash P2SH address from a script
-    /// This address type was introduced with BIP16 and is the popular type to implement multi-sig these days.
-    #[inline]
-    pub fn p2sh(script: &script::Script, network: Network) -> Address {
-        Address {
-            network: network,
-            payload: Payload::ScriptHash(hash160::Hash::hash(&script[..])),
-        }
-    }
-
-    /// Create a witness pay to public key address from a public key
-    /// This is the native segwit address type for an output redeemable with a single signature
-    pub fn p2wpkh(pk: &key::PublicKey, network: Network) -> Address {
-        let mut hash_engine = hash160::Hash::engine();
-        pk.write_into(&mut hash_engine);
-
-        Address {
-            network: network,
-            payload: Payload::WitnessProgram(
-                // unwrap is safe as witness program is known to be correct as above
-                WitnessProgram::new(
-                    u5::try_from_u8(0).expect("0<32"),
-                    hash160::Hash::from_engine(hash_engine)[..].to_vec(),
-                    Address::bech_network(network),
-                )
-                .unwrap(),
-            ),
-        }
-    }
-
-    /// Create a pay to script address that embeds a witness pay to public key
-    /// This is a segwit address type that looks familiar (as p2sh) to legacy clients
-    pub fn p2shwpkh(pk: &key::PublicKey, network: Network) -> Address {
-        let mut hash_engine = hash160::Hash::engine();
-        pk.write_into(&mut hash_engine);
-
-        let builder = script::Builder::new()
-            .push_int(0)
-            .push_slice(&hash160::Hash::from_engine(hash_engine)[..]);
-
-        Address {
-            network: network,
-            payload: Payload::ScriptHash(hash160::Hash::hash(builder.into_script().as_bytes())),
-        }
-    }
-
-    /// Create a witness pay to script hash address
-    pub fn p2wsh(script: &script::Script, network: Network) -> Address {
-        use bitcoin_hashes::sha256;
-        use bitcoin_hashes::Hash;
-
-        Address {
-            network: network,
-            payload: Payload::WitnessProgram(
-                // unwrap is safe as witness program is known to be correct as above
-                WitnessProgram::new(
-                    u5::try_from_u8(0).expect("0<32"),
-                    sha256::Hash::hash(&script[..])[..].to_vec(),
-                    Address::bech_network(network),
-                )
-                .unwrap(),
-            ),
-        }
-    }
-
-    /// Create a pay to script address that embeds a witness pay to script hash address
-    /// This is a segwit address type that looks familiar (as p2sh) to legacy clients
-    pub fn p2shwsh(script: &script::Script, network: Network) -> Address {
-        use bitcoin_hashes::hash160;
-        use bitcoin_hashes::sha256;
-        use bitcoin_hashes::Hash;
-
-        let ws = script::Builder::new()
-            .push_int(0)
-            .push_slice(&sha256::Hash::hash(&script[..])[..])
-            .into_script();
-
-        Address {
-            network: network,
-            payload: Payload::ScriptHash(hash160::Hash::hash(&ws[..])),
-        }
-    }
-
-    #[inline]
-    /// convert Network to bech32 network (this should go away soon)
-    fn bech_network(network: Network) -> bitcoin_bech32::constants::Network {
-        match network {
-            Network::Bitcoin => bitcoin_bech32::constants::Network::Bitcoin,
-            Network::Testnet => bitcoin_bech32::constants::Network::Testnet,
-            Network::Regtest => bitcoin_bech32::constants::Network::Regtest,
-
-            // this should never actually happen, Liquid does not have bech32 addresses
-            #[cfg(feature = "liquid")]
-            Network::Liquid | Network::LiquidRegtest => bitcoin_bech32::constants::Network::Bitcoin,
-        }
-    }
-
-    /// Generates a script pubkey spending to this address
-    pub fn script_pubkey(&self) -> script::Script {
-        match self.payload {
-            Payload::PubkeyHash(ref hash) => script::Builder::new()
-                .push_opcode(opcodes::all::OP_DUP)
-                .push_opcode(opcodes::all::OP_HASH160)
-                .push_slice(&hash[..])
-                .push_opcode(opcodes::all::OP_EQUALVERIFY)
-                .push_opcode(opcodes::all::OP_CHECKSIG),
-            Payload::ScriptHash(ref hash) => script::Builder::new()
-                .push_opcode(opcodes::all::OP_HASH160)
-                .push_slice(&hash[..])
-                .push_opcode(opcodes::all::OP_EQUAL),
-            Payload::WitnessProgram(ref witprog) => script::Builder::new()
-                .push_int(witprog.version().to_u8() as i64)
-                .push_slice(witprog.program()),
-        }
-        .into_script()
-    }
-}
-
-impl Display for Address {
-    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
-        match self.payload {
-            Payload::PubkeyHash(ref hash) => {
-                let mut prefixed = [0; 21];
-                prefixed[0] = match self.network {
-                    Network::Bitcoin => 0,
-                    Network::Testnet | Network::Regtest => 111,
-
-                    #[cfg(feature = "liquid")]
-                    Network::Liquid => 57,
-                    #[cfg(feature = "liquid")]
-                    Network::LiquidRegtest => 235,
-                };
-                prefixed[1..].copy_from_slice(&hash[..]);
-                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
-            }
-            Payload::ScriptHash(ref hash) => {
-                let mut prefixed = [0; 21];
-                prefixed[0] = match self.network {
-                    Network::Bitcoin => 5,
-                    Network::Testnet | Network::Regtest => 196,
-
-                    #[cfg(feature = "liquid")]
-                    Network::Liquid => 39,
-                    #[cfg(feature = "liquid")]
-                    Network::LiquidRegtest => 75,
-                };
-                prefixed[1..].copy_from_slice(&hash[..]);
-                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
-            }
-            Payload::WitnessProgram(ref witprog) => fmt.write_str(&witprog.to_address()),
-        }
-    }
-}
-
-impl FromStr for Address {
-    type Err = encode::Error;
-
-    fn from_str(s: &str) -> Result<Address, encode::Error> {
-        // bech32 (note that upper or lowercase is allowed but NOT mixed case)
-        if s.starts_with("bc1")
-            || s.starts_with("BC1")
-            || s.starts_with("tb1")
-            || s.starts_with("TB1")
-            || s.starts_with("bcrt1")
-            || s.starts_with("BCRT1")
-        {
-            let witprog = WitnessProgram::from_address(s)?;
-            let network = match witprog.network() {
-                bitcoin_bech32::constants::Network::Bitcoin => Network::Bitcoin,
-                bitcoin_bech32::constants::Network::Testnet => Network::Testnet,
-                bitcoin_bech32::constants::Network::Regtest => Network::Regtest,
-                _ => panic!("unknown network"),
-            };
-            if witprog.version().to_u8() != 0 {
-                return Err(encode::Error::UnsupportedWitnessVersion(
-                    witprog.version().to_u8(),
-                ));
-            }
-            return Ok(Address {
-                network: network,
-                payload: Payload::WitnessProgram(witprog),
-            });
-        }
-
-        if s.len() > 50 {
-            return Err(encode::Error::Base58(base58::Error::InvalidLength(
-                s.len() * 11 / 15,
-            )));
-        }
-
-        // Base 58
-        let data = base58::from_check(s)?;
-
-        if data.len() != 21 {
-            return Err(encode::Error::Base58(base58::Error::InvalidLength(
-                data.len(),
-            )));
-        }
-
-        let (network, payload) = match data[0] {
-            0 => (
-                Network::Bitcoin,
-                Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            5 => (
-                Network::Bitcoin,
-                Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            111 => (
-                Network::Testnet,
-                Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            196 => (
-                Network::Testnet,
-                Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            #[cfg(feature = "liquid")]
-            57 => (
-                Network::Liquid,
-                Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            #[cfg(feature = "liquid")]
-            39 => (
-                Network::Liquid,
-                Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            #[cfg(feature = "liquid")]
-            235 => (
-                Network::LiquidRegtest,
-                Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            #[cfg(feature = "liquid")]
-            75 => (
-                Network::LiquidRegtest,
-                Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
-            ),
-            x => {
-                return Err(encode::Error::Base58(base58::Error::InvalidVersion(vec![
-                    x,
-                ])));
-            }
-        };
-
-        Ok(Address {
-            network: network,
-            payload: payload,
-        })
-    }
-}
-
-impl ::std::fmt::Debug for Address {
-    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
-        write!(f, "{}", self.to_string())
-    }
-}

src/util/bincode_tests.rs

@@ -0,0 +1,112 @@
+/*
+
+The tests below show us the following defaults for each method of using bincode.
+
+1. Using bincode::[de]serialize() directly: "function"
+2. Using bincode::config().[de]serialize(): "Config" (deprecated)
+3. Using bincode::options().[de]serialize(): "Options" (currently recommended for v1.3.3)
+
+```
++----------+--------+------------+----------------+------------+
+|          | Endian | Int Length | Allow Trailing | Byte Limit |
++----------+--------+------------+----------------+------------+
+| function | little | fixed      | allow          | unlimited  |
+| Config   | little | fixed      | allow          | unlimited  |
+| Options  | little | variable * | reject *       | unlimited  |
++----------+--------+------------+----------------+------------+
+```
+
+Thus we only need to change the int length from variable to fixed,
+and allow trailing to allow in order to match the previous behavior.
+(note: TxHistory was using Big Endian by explicitly setting it to big.)
+
+*/
+
+use bincode_do_not_use_directly as bincode;
+
+#[test]
+fn bincode_settings() {
+    let value = TestStruct::new();
+    let mut large = [0_u8; 4096];
+    let decoded = [
+        8_u8, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 8, 7, 6, 5, 4, 3, 2, 1, 1,
+        2, 3, 4, 5, 6, 7, 8, 12, 0, 0, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114,
+        108, 100, 33,
+    ];
+    large[0..56].copy_from_slice(&decoded);
+
+    // Using functions: Little endian, Fixint, Allow trailing, Unlimited
+    assert_eq!(bincode::serialize(&value).unwrap(), &decoded);
+    assert_eq!(bincode::deserialize::<TestStruct>(&large).unwrap(), value);
+
+    // Using Config (deprecated)
+    // Little endian, fixint, Allow trailing, Unlimited
+    #[allow(deprecated)]
+    {
+        assert_eq!(bincode::config().serialize(&value).unwrap(), &decoded);
+        assert_eq!(
+            bincode::config().deserialize::<TestStruct>(&large).unwrap(),
+            value
+        );
+    }
+
+    // Using Options
+    // Little endian, VARINT (different), Reject trailing (different), unlimited
+    use bincode::Options;
+    assert_eq!(
+        bincode::options()
+            .with_fixint_encoding()
+            .allow_trailing_bytes()
+            .serialize(&value)
+            .unwrap(),
+        &decoded
+    );
+    assert_eq!(
+        bincode::options()
+            .with_fixint_encoding()
+            .allow_trailing_bytes()
+            .deserialize::<TestStruct>(&large)
+            .unwrap(),
+        value
+    );
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct TestStruct {
+    a: u64,
+    b: [u8; 8],
+    c: TestData,
+    d: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+enum TestData {
+    Foo(FooStruct),
+    Bar(BarStruct),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct FooStruct {
+    a: u64,
+    b: [u8; 8],
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct BarStruct {
+    a: u64,
+    b: [u8; 8],
+}
+
+impl TestStruct {
+    fn new() -> Self {
+        Self {
+            a: 0x0102030405060708,
+            b: [1, 2, 3, 4, 5, 6, 7, 8],
+            c: TestData::Foo(FooStruct {
+                a: 0x0102030405060708,
+                b: [1, 2, 3, 4, 5, 6, 7, 8],
+            }),
+            d: String::from("Hello World!"),
+        }
+    }
+}

src/util/bincode_util.rs

@@ -0,0 +1,70 @@
+//! This module creates two sets of serialize and deserialize for bincode.
+//! They explicitly spell out the bincode settings so that switching to
+//! new versions in the future is less error prone.
+//!
+//! This is a list of all the row types and their settings for bincode.
+//! +--------------+--------+------------+----------------+------------+
+//! |              | Endian | Int Length | Allow Trailing | Byte Limit |
+//! +--------------+--------+------------+----------------+------------+
+//! | TxHistoryRow | big    | fixed      | allow          | unlimited  |
+//! | All others   | little | fixed      | allow          | unlimited  |
+//! +--------------+--------+------------+----------------+------------+
+
+// We only want people to use bincode_util
+use bincode::Options;
+use bincode_do_not_use_directly as bincode;
+
+pub fn serialize_big<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
+where
+    T: ?Sized + serde::Serialize,
+{
+    big_endian().serialize(value)
+}
+
+pub fn deserialize_big<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
+where
+    T: serde::Deserialize<'a>,
+{
+    big_endian().deserialize(bytes)
+}
+
+pub fn serialize_little<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
+where
+    T: ?Sized + serde::Serialize,
+{
+    little_endian().serialize(value)
+}
+
+pub fn deserialize_little<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
+where
+    T: serde::Deserialize<'a>,
+{
+    little_endian().deserialize(bytes)
+}
+
+/// This is the default settings for Options,
+/// but all explicitly spelled out, except for endianness.
+/// The following functions will add endianness.
+#[inline]
+fn options() -> impl Options {
+    bincode::options()
+        .with_fixint_encoding()
+        .with_no_limit()
+        .allow_trailing_bytes()
+}
+
+/// Adding the endian flag for big endian
+#[inline]
+fn big_endian() -> impl Options {
+    options().with_big_endian()
+}
+
+/// Adding the endian flag for little endian
+#[inline]
+fn little_endian() -> impl Options {
+    options().with_little_endian()
+}
+
+#[cfg(test)]
+#[path = "./bincode_tests.rs"]
+mod bincode_tests;

src/util/block.rs

@@ -1,21 +1,22 @@
-use crate::chain::{Block, BlockHeader};
+use bitcoin::hashes::Hash;
+
+use crate::chain::{BlockHash, BlockHeader};
 use crate::errors::*;
 use crate::new_index::BlockEntry;
 
-use bitcoin::consensus::encode::serialize;
-use bitcoin::util::hash::BitcoinHash;
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
-
 use std::collections::HashMap;
 use std::fmt;
 use std::iter::FromIterator;
 use std::slice;
-use time;
+use time::format_description::well_known::Rfc3339;
+use time::OffsetDateTime as DateTime;
+
+const MTP_SPAN: usize = 11;
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct BlockId {
     pub height: usize,
-    pub hash: Sha256dHash,
+    pub hash: BlockHash,
     pub time: u32,
 }
 
@@ -32,12 +33,12 @@ impl From<&HeaderEntry> for BlockId {
 #[derive(Eq, PartialEq, Clone)]
 pub struct HeaderEntry {
     height: usize,
-    hash: Sha256dHash,
+    hash: BlockHash,
     header: BlockHeader,
 }
 
 impl HeaderEntry {
-    pub fn hash(&self) -> &Sha256dHash {
+    pub fn hash(&self) -> &BlockHash {
         &self.hash
     }
 
@@ -52,23 +53,21 @@ impl HeaderEntry {
 
 impl fmt::Debug for HeaderEntry {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let last_block_time = time::at_utc(time::Timespec::new(self.header().time as i64, 0))
-            .rfc3339()
-            .to_string();
+        let last_block_time = DateTime::from_unix_timestamp(self.header().time as i64).unwrap();
         write!(
             f,
-            "best={} height={} @ {}",
+            "hash={} height={} @ {}",
             self.hash(),
             self.height(),
-            last_block_time,
+            last_block_time.format(&Rfc3339).unwrap(),
         )
     }
 }
 
 pub struct HeaderList {
     headers: Vec<HeaderEntry>,
-    heights: HashMap<Sha256dHash, usize>,
-    tip: Sha256dHash,
+    heights: HashMap<BlockHash, usize>,
+    tip: BlockHash,
 }
 
 impl HeaderList {
@@ -76,19 +75,57 @@ impl HeaderList {
         HeaderList {
             headers: vec![],
             heights: HashMap::new(),
-            tip: Sha256dHash::default(),
+            tip: BlockHash::all_zeros(),
         }
     }
 
+    pub fn new(
+        mut headers_map: HashMap<BlockHash, BlockHeader>,
+        tip_hash: BlockHash,
+    ) -> HeaderList {
+        trace!(
+            "processing {} headers, tip at {:?}",
+            headers_map.len(),
+            tip_hash
+        );
+
+        let mut blockhash = tip_hash;
+        let mut headers_chain: Vec<BlockHeader> = vec![];
+        let null_hash = BlockHash::all_zeros();
+
+        while blockhash != null_hash {
+            let header = headers_map.remove(&blockhash).unwrap_or_else(|| {
+                panic!(
+                    "missing expected blockhash in headers map: {:?}, pointed from: {:?}",
+                    blockhash,
+                    headers_chain.last().map(|h| h.block_hash())
+                )
+            });
+            blockhash = header.prev_blockhash;
+            headers_chain.push(header);
+        }
+        headers_chain.reverse();
+
+        trace!(
+            "{} chained headers ({} orphan blocks left)",
+            headers_chain.len(),
+            headers_map.len()
+        );
+
+        let mut headers = HeaderList::empty();
+        headers.apply(headers.order(headers_chain));
+        headers
+    }
+
     pub fn order(&self, new_headers: Vec<BlockHeader>) -> Vec<HeaderEntry> {
         // header[i] -> header[i-1] (i.e. header.last() is the tip)
         struct HashedHeader {
-            blockhash: Sha256dHash,
+            blockhash: BlockHash,
             header: BlockHeader,
         }
         let hashed_headers =
             Vec::<HashedHeader>::from_iter(new_headers.into_iter().map(|header| HashedHeader {
-                blockhash: header.bitcoin_hash(),
+                blockhash: header.block_hash(),
                 header,
             }));
         for i in 1..hashed_headers.len() {
@@ -101,26 +138,31 @@ impl HeaderList {
             Some(h) => h.header.prev_blockhash,
             None => return vec![], // hashed_headers is empty
         };
-        let null_hash = Sha256dHash::default();
+        let null_hash = BlockHash::all_zeros();
         let new_height: usize = if prev_blockhash == null_hash {
             0
         } else {
             self.header_by_blockhash(&prev_blockhash)
-                .expect(&format!("{} is not part of the blockchain", prev_blockhash))
+                .unwrap_or_else(|| panic!("{} is not part of the blockchain", prev_blockhash))
                 .height()
                 + 1
         };
         (new_height..)
-            .zip(hashed_headers.into_iter())
+            .zip(hashed_headers)
             .map(|(height, hashed_header)| HeaderEntry {
-                height: height,
+                height,
                 hash: hashed_header.blockhash,
                 header: hashed_header.header,
             })
             .collect()
     }
 
-    pub fn apply(&mut self, new_headers: Vec<HeaderEntry>) {
+    /// Returns any rolled back blocks in order from old tip first and first block in the fork is last
+    /// It also returns the blockhash of the post-rollback tip.
+    pub fn apply(
+        &mut self,
+        new_headers: Vec<HeaderEntry>,
+    ) -> (Vec<HeaderEntry>, Option<BlockHash>) {
         // new_headers[i] -> new_headers[i - 1] (i.e. new_headers.last() is the tip)
         for i in 1..new_headers.len() {
             assert_eq!(new_headers[i - 1].height() + 1, new_headers[i].height());
@@ -135,19 +177,27 @@ impl HeaderList {
                 let expected_prev_blockhash = if height > 0 {
                     *self.headers[height - 1].hash()
                 } else {
-                    Sha256dHash::default()
+                    BlockHash::all_zeros()
                 };
                 assert_eq!(entry.header().prev_blockhash, expected_prev_blockhash);
                 height
             }
-            None => return,
+            None => return (vec![], None),
         };
         debug!(
             "applying {} new headers from height {}",
             new_headers.len(),
             new_height
         );
-        self.headers.split_off(new_height); // keep [0..new_height) entries
+        let mut removed = self.headers.split_off(new_height); // keep [0..new_height) entries
+
+        // If we reorged, we should return the last blockhash before adding the new chain's blockheaders.
+        let reorged_tip = if !removed.is_empty() {
+            self.headers.last().map(|be| be.hash()).cloned()
+        } else {
+            None
+        };
+
         for new_header in new_headers {
             let height = new_header.height();
             assert_eq!(height, self.headers.len());
@@ -155,9 +205,11 @@ impl HeaderList {
             self.headers.push(new_header);
             self.heights.insert(self.tip, height);
         }
+        removed.reverse();
+        (removed, reorged_tip)
     }
 
-    pub fn header_by_blockhash(&self, blockhash: &Sha256dHash) -> Option<&HeaderEntry> {
+    pub fn header_by_blockhash(&self, blockhash: &BlockHash) -> Option<&HeaderEntry> {
         let height = self.heights.get(blockhash)?;
         let header = self.headers.get(*height)?;
         if *blockhash == *header.hash() {
@@ -168,9 +220,8 @@ impl HeaderList {
     }
 
     pub fn header_by_height(&self, height: usize) -> Option<&HeaderEntry> {
-        self.headers.get(height).map(|entry| {
+        self.headers.get(height).inspect(|entry| {
             assert_eq!(entry.height(), height);
-            entry
         })
     }
 
@@ -178,13 +229,13 @@ impl HeaderList {
         self.headers.last() == other.headers.last()
     }
 
-    pub fn tip(&self) -> &Sha256dHash {
+    pub fn tip(&self) -> &BlockHash {
         assert_eq!(
             self.tip,
             self.headers
                 .last()
                 .map(|h| *h.hash())
-                .unwrap_or(Sha256dHash::default())
+                .unwrap_or(BlockHash::all_zeros())
         );
         &self.tip
     }
@@ -193,20 +244,41 @@ impl HeaderList {
         self.headers.len()
     }
 
-    pub fn iter(&self) -> slice::Iter<HeaderEntry> {
+    pub fn is_empty(&self) -> bool {
+        self.headers.is_empty()
+    }
+
+    pub fn iter(&self) -> slice::Iter<'_, HeaderEntry> {
         self.headers.iter()
     }
+
+    /// Get the Median Time Past
+    pub fn get_mtp(&self, height: usize) -> u32 {
+        // Use the timestamp as the mtp of the genesis block.
+        // Matches bitcoind's behaviour: bitcoin-cli getblock `bitcoin-cli getblockhash 0` | jq '.time == .mediantime'
+        if height == 0 {
+            self.headers.first().unwrap().header.time
+        } else if height > self.len() - 1 {
+            0
+        } else {
+            let mut timestamps = (height.saturating_sub(MTP_SPAN - 1)..=height)
+                .map(|p_height| self.headers.get(p_height).unwrap().header.time)
+                .collect::<Vec<_>>();
+            timestamps.sort_unstable();
+            timestamps[timestamps.len() / 2]
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize)]
 pub struct BlockStatus {
     pub in_best_chain: bool,
     pub height: Option<usize>,
-    pub next_best: Option<Sha256dHash>,
+    pub next_best: Option<BlockHash>,
 }
 
 impl BlockStatus {
-    pub fn confirmed(height: usize, next_best: Option<Sha256dHash>) -> BlockStatus {
+    pub fn confirmed(height: usize, next_best: Option<BlockHash>) -> BlockStatus {
         BlockStatus {
             in_best_chain: true,
             height: Some(height),
@@ -225,6 +297,7 @@ impl BlockStatus {
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct BlockMeta {
+    #[serde(alias = "nTx")]
     pub tx_count: u32,
     pub size: u32,
     pub weight: u32,
@@ -233,23 +306,18 @@ pub struct BlockMeta {
 pub struct BlockHeaderMeta {
     pub header_entry: HeaderEntry,
     pub meta: BlockMeta,
-}
-
-impl From<&Block> for BlockMeta {
-    fn from(block: &Block) -> BlockMeta {
-        BlockMeta {
-            tx_count: block.txdata.len() as u32,
-            weight: block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
-            size: serialize(block).len() as u32,
-        }
-    }
+    pub mtp: u32,
 }
 
 impl From<&BlockEntry> for BlockMeta {
     fn from(b: &BlockEntry) -> BlockMeta {
+        #[cfg(not(feature = "liquid"))]
+        let weight = b.block.weight().to_wu() as u32;
+        #[cfg(feature = "liquid")]
+        let weight = b.block.weight() as u32;
         BlockMeta {
             tx_count: b.block.txdata.len() as u32,
-            weight: b.block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
+            weight,
             size: b.size,
         }
     }

src/util/electrum_merkle.rs

@@ -1,20 +1,22 @@
-use bitcoin_hashes::{sha256d::Hash as Sha256dHash, Hash};
-
+use crate::chain::{BlockHash, Txid};
 use crate::errors::*;
 use crate::new_index::ChainQuery;
+use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash};
 
 pub fn get_tx_merkle_proof(
     chain: &ChainQuery,
-    tx_hash: &Sha256dHash,
-    block_hash: &Sha256dHash,
+    tx_hash: &Txid,
+    block_hash: &BlockHash,
 ) -> Result<(Vec<Sha256dHash>, usize)> {
     let txids = chain
-        .get_block_txids(&block_hash)
+        .get_block_txids(block_hash)
         .chain_err(|| format!("missing block txids for #{}", block_hash))?;
     let pos = txids
         .iter()
         .position(|txid| txid == tx_hash)
         .chain_err(|| format!("missing txid {}", tx_hash))?;
+    let txids = txids.into_iter().map(Sha256dHash::from).collect();
+
     let (branch, _root) = create_merkle_branch_and_root(txids, pos);
     Ok((branch, pos))
 }
@@ -37,12 +39,14 @@ pub fn get_header_merkle_proof(
         );
     }
 
-    let heights: Vec<usize> = (0..cp_height + 1).collect();
-    let header_hashes: Vec<Sha256dHash> = heights
+    let heights: Vec<usize> = (0..=cp_height).collect();
+    let header_hashes: Vec<BlockHash> = heights
         .into_iter()
         .map(|height| chain.hash_by_height(height))
-        .collect::<Option<Vec<Sha256dHash>>>()
+        .collect::<Option<Vec<BlockHash>>>()
         .chain_err(|| "missing block headers")?;
+
+    let header_hashes = header_hashes.into_iter().map(Sha256dHash::from).collect();
     Ok(create_merkle_branch_and_root(header_hashes, height))
 }
 
@@ -51,7 +55,7 @@ pub fn get_id_from_pos(
     height: usize,
     tx_pos: usize,
     want_merkle: bool,
-) -> Result<(Sha256dHash, Vec<Sha256dHash>)> {
+) -> Result<(Txid, Vec<Sha256dHash>)> {
     let header_hash = chain
         .hash_by_height(height)
         .chain_err(|| format!("missing block #{}", height))?;
@@ -64,6 +68,8 @@ pub fn get_id_from_pos(
         .get(tx_pos)
         .chain_err(|| format!("No tx in position #{} in block #{}", tx_pos, height))?;
 
+    let txids = txids.into_iter().map(Sha256dHash::from).collect();
+
     let branch = if want_merkle {
         create_merkle_branch_and_root(txids, tx_pos).0
     } else {
@@ -84,12 +90,12 @@ fn create_merkle_branch_and_root(
     let mut merkle = vec![];
     while hashes.len() > 1 {
         if hashes.len() % 2 != 0 {
-            let last = hashes.last().unwrap().clone();
+            let last = *hashes.last().unwrap();
             hashes.push(last);
         }
         index = if index % 2 == 0 { index + 1 } else { index - 1 };
         merkle.push(hashes[index]);
-        index = index / 2;
+        index /= 2;
         hashes = hashes
             .chunks(2)
             .map(|pair| merklize(pair[0], pair[1]))

src/util/elements.rs

@@ -1,130 +0,0 @@
-use bitcoin::blockdata::script::Instruction::PushBytes;
-use bitcoin::consensus::encode::serialize;
-use bitcoin::Script;
-use elements::confidential::Value;
-use elements::{AssetIssuance, Proof};
-
-use hex;
-
-use crate::chain::Network;
-use crate::util::{get_script_asm, script_to_address};
-
-#[derive(Serialize, Deserialize)]
-pub struct BlockProofValue {
-    challenge: Script,
-    challenge_asm: String,
-    solution: Script,
-    solution_asm: String,
-}
-
-impl From<&Proof> for BlockProofValue {
-    fn from(proof: &Proof) -> Self {
-        BlockProofValue {
-            challenge_asm: get_script_asm(&proof.challenge),
-            challenge: proof.challenge.clone(),
-            solution_asm: get_script_asm(&proof.solution),
-            solution: proof.solution.clone(),
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone)]
-pub struct IssuanceValue {
-    pub is_reissuance: bool,
-    pub asset_blinding_nonce: Option<String>,
-    pub asset_entropy: Option<String>,
-    pub assetamount: Option<u64>,
-    pub assetamountcommitment: Option<String>,
-    pub tokenamount: Option<u64>,
-    pub tokenamountcommitment: Option<String>,
-}
-
-impl From<&AssetIssuance> for IssuanceValue {
-    fn from(issuance: &AssetIssuance) -> Self {
-        let zero = [0u8; 32];
-        let is_reissuance = issuance.asset_blinding_nonce != zero;
-
-        IssuanceValue {
-            is_reissuance,
-            asset_blinding_nonce: if is_reissuance {
-                Some(hex::encode(issuance.asset_blinding_nonce))
-            } else {
-                None
-            },
-            asset_entropy: if issuance.asset_entropy != zero {
-                Some(hex::encode(issuance.asset_entropy))
-            } else {
-                None
-            },
-            assetamount: match issuance.amount {
-                Value::Explicit(value) => Some(value),
-                _ => None,
-            },
-            assetamountcommitment: match issuance.amount {
-                Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
-                _ => None,
-            },
-            tokenamount: match issuance.inflation_keys {
-                Value::Explicit(value) => Some(value / 100000000), // https://github.com/ElementsProject/rust-elements/issues/7
-                _ => None,
-            },
-            tokenamountcommitment: match issuance.inflation_keys {
-                Value::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys))),
-                _ => None,
-            },
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone)]
-pub struct PegOutRequest {
-    pub genesis_hash: String,
-    pub scriptpubkey: Script,
-    pub scriptpubkey_asm: String,
-    pub scriptpubkey_address: Option<String>,
-}
-
-impl PegOutRequest {
-    pub fn parse(
-        script: &Script,
-        parent_network: &Network,
-        parent_genesis_hash: &str,
-    ) -> Option<PegOutRequest> {
-        if !script.is_op_return() {
-            return None;
-        }
-
-        let nulldata: Vec<_> = script.iter(true).skip(1).collect();
-        if nulldata.len() < 2 {
-            return None;
-        }
-
-        let genesis_hash = if let PushBytes(data) = nulldata[0] {
-            let mut data = data.to_vec();
-            data.reverse();
-            hex::encode(data)
-        } else {
-            return None;
-        };
-
-        let scriptpubkey = if let PushBytes(data) = nulldata[1] {
-            Script::from(data.to_vec())
-        } else {
-            return None;
-        };
-
-        if genesis_hash != parent_genesis_hash {
-            return None;
-        }
-
-        let scriptpubkey_asm = get_script_asm(&scriptpubkey);
-        let scriptpubkey_address = script_to_address(&scriptpubkey, parent_network);
-
-        Some(PegOutRequest {
-            genesis_hash,
-            scriptpubkey,
-            scriptpubkey_asm,
-            scriptpubkey_address,
-        })
-    }
-}

src/util/fees.rs

@@ -1,6 +1,4 @@
-#[cfg(feature = "liquid")]
-use crate::chain::Value;
-use crate::chain::{Transaction, TxOut};
+use crate::chain::{Network, Transaction, TxOut};
 use std::collections::HashMap;
 
 const VSIZE_BIN_WIDTH: u32 = 50_000; // in vbytes
@@ -12,12 +10,12 @@ pub struct TxFeeInfo {
 }
 
 impl TxFeeInfo {
-    #[cfg(not(feature = "liquid"))]
-    pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>) -> Self {
-        let total_in: u64 = prevouts.values().map(|prevout| prevout.value).sum();
-        let total_out: u64 = tx.output.iter().map(|vout| vout.value).sum();
-        let fee = total_in - total_out;
-        let vsize = tx.get_weight() / 4;
+    pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, network: Network) -> Self {
+        let fee = get_tx_fee(tx, prevouts, network);
+        #[cfg(not(feature = "liquid"))]
+        let vsize = tx.weight().to_wu() / 4;
+        #[cfg(feature = "liquid")]
+        let vsize = tx.weight() / 4;
 
         TxFeeInfo {
             fee,
@@ -25,26 +23,25 @@ impl TxFeeInfo {
             fee_per_vbyte: fee as f32 / vsize as f32,
         }
     }
+}
 
-    #[cfg(feature = "liquid")]
-    pub fn new(tx: &Transaction, _prevouts: &HashMap<u32, &TxOut>) -> Self {
-        let fee = tx
-            .output
-            .iter()
-            .find(|vout| vout.is_fee())
-            .map_or(0, |vout| match vout.value {
-                Value::Explicit(value) => value,
-                _ => 0u64,
-            });
-
-        let vsize = tx.get_weight() / 4;
-
-        TxFeeInfo {
-            fee,
-            vsize: vsize as u32,
-            fee_per_vbyte: fee as f32 / vsize as f32,
-        }
+#[cfg(not(feature = "liquid"))]
+pub fn get_tx_fee(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, _network: Network) -> u64 {
+    if tx.is_coinbase() {
+        return 0;
     }
+
+    let total_in: u64 = prevouts
+        .values()
+        .map(|prevout| prevout.value.to_sat())
+        .sum();
+    let total_out: u64 = tx.output.iter().map(|vout| vout.value.to_sat()).sum();
+    total_in - total_out
+}
+
+#[cfg(feature = "liquid")]
+pub fn get_tx_fee(tx: &Transaction, _prevouts: &HashMap<u32, &TxOut>, network: Network) -> u64 {
+    tx.fee_in(*network.native_asset())
 }
 
 pub fn make_fee_histogram(mut entries: Vec<&TxFeeInfo>) -> Vec<(f32, u32)> {
@@ -52,18 +49,18 @@ pub fn make_fee_histogram(mut entries: Vec<&TxFeeInfo>) -> Vec<(f32, u32)> {
 
     let mut histogram = vec![];
     let mut bin_size = 0;
-    let mut last_fee_rate = None;
+    let mut last_fee_rate = 0.0;
     for e in entries.iter().rev() {
-        last_fee_rate = Some(e.fee_per_vbyte);
-        bin_size += e.vsize;
-        if bin_size > VSIZE_BIN_WIDTH {
-            // vsize of transactions paying >= e.fee_per_vbyte
-            histogram.push((e.fee_per_vbyte, bin_size));
+        if bin_size > VSIZE_BIN_WIDTH && last_fee_rate != e.fee_per_vbyte {
+            // vsize of transactions paying >= last_fee_rate
+            histogram.push((last_fee_rate, bin_size));
             bin_size = 0;
         }
+        last_fee_rate = e.fee_per_vbyte;
+        bin_size += e.vsize;
     }
-    if let Some(fee_rate) = last_fee_rate {
-        histogram.push((fee_rate, bin_size));
+    if bin_size > 0 {
+        histogram.push((last_fee_rate, bin_size));
     }
     histogram
 }

src/util/mod.rs

@@ -1,33 +1,31 @@
-mod address;
 mod block;
-mod merkle;
 mod script;
 mod transaction;
 
+pub mod bincode_util;
+pub mod electrum_merkle;
 pub mod fees;
 
-#[cfg(feature = "liquid")]
-mod elements;
-
-pub use self::address::{Address, Payload};
 pub use self::block::{BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, HeaderEntry, HeaderList};
-pub use self::merkle::{get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof};
-pub use self::script::{get_script_asm, script_to_address};
-pub use self::transaction::{has_prevout, is_coinbase, is_spendable, TransactionStatus};
-
-#[cfg(feature = "liquid")]
-pub use self::elements::{BlockProofValue, IssuanceValue, PegOutRequest};
-
-#[cfg(feature = "liquid")]
-pub const REGTEST_INITIAL_ISSUANCE_PREVOUT: &str =
-    "cb4953dfe64aba4687d0de08e5ce17d8f7640deeef63be4ac3bac3afdefdfd78";
+pub use self::fees::get_tx_fee;
+pub use self::script::{
+    get_innerscripts, IsProvablyUnspendable, ScriptToAddr, ScriptToAsm, SegwitDetection,
+};
+pub use self::transaction::{
+    extract_tx_prevouts, has_prevout, is_coinbase, is_spendable, serialize_outpoint,
+    sigops::transaction_sigop_count, TransactionStatus, TxInput,
+};
 
 use std::collections::HashMap;
-use std::sync::mpsc::{channel, sync_channel, Receiver, Sender, SyncSender};
-use std::thread;
+use std::sync::atomic::AtomicUsize;
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::Mutex;
+use std::thread::{self, ThreadId};
 
 use crate::chain::BlockHeader;
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
+use bitcoin::hashes::sha256d::Hash as Sha256dHash;
+use socket2::{Domain, Protocol, Socket, Type};
+use std::net::SocketAddr;
 
 pub type Bytes = Vec<u8>;
 pub type HeaderMap = HashMap<Sha256dHash, BlockHeader>;
@@ -38,30 +36,42 @@ const HASH_LEN: usize = 32;
 pub type FullHash = [u8; HASH_LEN];
 
 pub fn full_hash(hash: &[u8]) -> FullHash {
-    array_ref![hash, 0, HASH_LEN].clone()
+    *array_ref![hash, 0, HASH_LEN]
 }
 
 pub struct SyncChannel<T> {
-    tx: SyncSender<T>,
-    rx: Receiver<T>,
+    tx: Option<crossbeam_channel::Sender<T>>,
+    rx: Option<crossbeam_channel::Receiver<T>>,
 }
 
 impl<T> SyncChannel<T> {
     pub fn new(size: usize) -> SyncChannel<T> {
-        let (tx, rx) = sync_channel(size);
-        SyncChannel { tx, rx }
+        let (tx, rx) = crossbeam_channel::bounded(size);
+        SyncChannel {
+            tx: Some(tx),
+            rx: Some(rx),
+        }
     }
 
-    pub fn sender(&self) -> SyncSender<T> {
-        self.tx.clone()
+    pub fn sender(&self) -> crossbeam_channel::Sender<T> {
+        self.tx.as_ref().expect("No Sender").clone()
     }
 
-    pub fn receiver(&self) -> &Receiver<T> {
-        &self.rx
+    pub fn receiver(&self) -> &crossbeam_channel::Receiver<T> {
+        self.rx.as_ref().expect("No Receiver")
     }
 
-    pub fn into_receiver(self) -> Receiver<T> {
-        self.rx
+    pub fn into_receiver(self) -> crossbeam_channel::Receiver<T> {
+        self.rx.expect("No Receiver")
+    }
+
+    /// This drops the sender and receiver, causing all other methods to panic.
+    ///
+    /// Use only when you know that the channel will no longer be used.
+    /// ie. shutdown.
+    pub fn close(&mut self) -> Option<crossbeam_channel::Receiver<T>> {
+        self.tx.take();
+        self.rx.take()
     }
 }
 
@@ -71,7 +81,7 @@ pub struct Channel<T> {
 }
 
 impl<T> Channel<T> {
-    pub fn new() -> Channel<T> {
+    pub fn unbounded() -> Self {
         let (tx, rx) = channel();
         Channel { tx, rx }
     }
@@ -89,14 +99,131 @@ impl<T> Channel<T> {
     }
 }
 
-pub fn spawn_thread<F, T>(name: &str, f: F) -> thread::JoinHandle<T>
+/// This static HashMap contains all the threads spawned with [`spawn_thread`] with their name
+#[inline]
+pub fn with_spawned_threads<F>(f: F)
+where
+    F: FnOnce(&mut HashMap<ThreadId, String>),
+{
+    lazy_static! {
+        static ref SPAWNED_THREADS: Mutex<HashMap<ThreadId, String>> = Mutex::new(HashMap::new());
+    }
+    let mut lock = match SPAWNED_THREADS.lock() {
+        Ok(threads) => threads,
+        // There's no possible broken state
+        Err(threads) => {
+            warn!("SPAWNED_THREADS is in a poisoned state! Be wary of incorrect logs!");
+            threads.into_inner()
+        }
+    };
+    f(&mut lock)
+}
+
+pub fn spawn_thread<F, T>(prefix: &str, do_work: F) -> thread::JoinHandle<T>
 where
     F: FnOnce() -> T,
     F: Send + 'static,
     T: Send + 'static,
 {
+    static THREAD_COUNTER: AtomicUsize = AtomicUsize::new(0);
+    let counter = THREAD_COUNTER.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
     thread::Builder::new()
-        .name(name.to_owned())
-        .spawn(f)
+        .name(format!("{}-{}", prefix, counter))
+        .spawn(move || {
+            let thread = std::thread::current();
+            let name = thread.name().unwrap();
+            let id = thread.id();
+
+            trace!("[THREAD] GETHASHMAP INSERT | {name} {id:?}");
+            with_spawned_threads(|threads| {
+                threads.insert(id, name.to_owned());
+            });
+            trace!("[THREAD] START WORK        | {name} {id:?}");
+
+            let result = do_work();
+
+            trace!("[THREAD] FINISHED WORK     | {name} {id:?}");
+            trace!("[THREAD] GETHASHMAP REMOVE | {name} {id:?}");
+            with_spawned_threads(|threads| {
+                threads.remove(&id);
+            });
+            trace!("[THREAD] HASHMAP REMOVED   | {name} {id:?}");
+
+            result
+        })
         .unwrap()
 }
+
+// Similar to https://doc.rust-lang.org/std/primitive.bool.html#method.then (nightly only),
+// but with a function that returns an `Option<T>` instead of `T`. Adding something like
+// this to std is being discussed: https://github.com/rust-lang/rust/issues/64260
+
+pub trait BoolThen {
+    fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T>;
+}
+
+impl BoolThen for bool {
+    fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T> {
+        if self {
+            f()
+        } else {
+            None
+        }
+    }
+}
+
+pub fn create_socket(addr: &SocketAddr) -> Socket {
+    let domain = match &addr {
+        SocketAddr::V4(_) => Domain::IPV4,
+        SocketAddr::V6(_) => Domain::IPV6,
+    };
+    let socket =
+        Socket::new(domain, Type::STREAM, Some(Protocol::TCP)).expect("creating socket failed");
+
+    #[cfg(unix)]
+    socket
+        .set_reuse_port(true)
+        .expect("cannot enable SO_REUSEPORT");
+
+    socket.bind(&(*addr).into()).expect("cannot bind");
+
+    socket
+}
+
+/// A module used for serde serialization of bytes in hexadecimal format.
+///
+/// The module is compatible with the serde attribute.
+///
+/// Copied from https://github.com/rust-bitcoin/rust-bitcoincore-rpc/blob/master/json/src/lib.rs
+pub mod serde_hex {
+    use bitcoin::hashes::hex::FromHex;
+    use serde::de::Error;
+    use serde::{Deserializer, Serializer};
+
+    pub fn serialize<S: Serializer>(b: &[u8], s: S) -> Result<S::Ok, S::Error> {
+        s.serialize_str(&hex::encode(b))
+    }
+
+    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
+        let hex_str: String = ::serde::Deserialize::deserialize(d)?;
+        FromHex::from_hex(&hex_str).map_err(D::Error::custom)
+    }
+
+    pub mod opt {
+        use bitcoin::hashes::hex::FromHex;
+        use serde::de::Error;
+        use serde::{Deserializer, Serializer};
+
+        pub fn serialize<S: Serializer>(b: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
+            match *b {
+                None => s.serialize_none(),
+                Some(ref b) => s.serialize_str(&hex::encode(b)),
+            }
+        }
+
+        pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
+            let hex_str: String = ::serde::Deserialize::deserialize(d)?;
+            Ok(Some(FromHex::from_hex(&hex_str).map_err(D::Error::custom)?))
+        }
+    }
+}

src/util/script.rs

@@ -1,54 +1,196 @@
-use crate::chain::Network;
-use crate::util::{Address, Payload};
-use bitcoin::Script;
-use bitcoin_bech32::constants::Network as B32Network;
-use bitcoin_bech32::{u5, WitnessProgram};
-use bitcoin_hashes::{hash160::Hash as Hash160, Hash};
+#[cfg(feature = "liquid")]
+use elements::address as elements_address;
 
-// @XXX we can't use any of the Address:p2{...}h utility methods, since they expect the pre-image data, which we don't have.
-// we must instead create the Payload manually, which results in code duplication with the p2{...}h methods, especially for witness programs.
-// ideally, this should be implemented as part of the rust-bitcoin lib.
-pub fn script_to_address(script: &Script, network: &Network) -> Option<String> {
-    let payload = if script.is_p2pkh() {
-        Some(Payload::PubkeyHash(
-            Hash160::from_slice(&script[3..23]).ok()?,
-        ))
-    } else if script.is_p2sh() {
-        Some(Payload::ScriptHash(
-            Hash160::from_slice(&script[2..22]).ok()?,
-        ))
-    } else if script.is_v0_p2wpkh() {
-        Some(Payload::WitnessProgram(
-            WitnessProgram::new(
-                u5::try_from_u8(0).expect("0<32"),
-                script[2..22].to_vec(),
-                B32Network::from(network),
-            )
-            .unwrap(),
-        ))
-    } else if script.is_v0_p2wsh() {
-        Some(Payload::WitnessProgram(
-            WitnessProgram::new(
-                u5::try_from_u8(0).expect("0<32"),
-                script[2..34].to_vec(),
-                B32Network::from(network),
-            )
-            .unwrap(),
-        ))
+use crate::chain::{script, Network, Script, TxIn, TxOut};
+use script::Instruction::PushBytes;
+
+pub struct InnerScripts {
+    pub redeem_script: Option<Script>,
+    pub witness_script: Option<Script>,
+}
+
+pub trait IsProvablyUnspendable {
+    fn is_provably_unspendable_(&self) -> bool;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl IsProvablyUnspendable for bitcoin::Script {
+    // is_provably_unspendable() is deprecated in rust-bitcoin
+    // so we re-implement it here. Copy pasted.
+    fn is_provably_unspendable_(&self) -> bool {
+        use bitcoin::blockdata::opcodes::{
+            Class::{IllegalOp, ReturnOp},
+            ClassifyContext, Opcode,
+        };
+
+        match self.as_bytes().first() {
+            Some(b) => {
+                let first = Opcode::from(*b);
+                let class = first.classify(ClassifyContext::Legacy);
+
+                class == ReturnOp || class == IllegalOp
+            }
+            None => false,
+        }
+    }
+}
+
+#[cfg(feature = "liquid")]
+impl IsProvablyUnspendable for elements::Script {
+    #[inline(always)]
+    fn is_provably_unspendable_(&self) -> bool {
+        // Not deprecated yet
+        self.is_provably_unspendable()
+    }
+}
+
+// Extension trait for segwit script detection that works across bitcoin and elements
+pub trait SegwitDetection {
+    fn segwit_is_p2wpkh(&self) -> bool;
+    fn segwit_is_p2wsh(&self) -> bool;
+    fn segwit_is_p2tr(&self) -> bool;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl SegwitDetection for bitcoin::Script {
+    fn segwit_is_p2wpkh(&self) -> bool {
+        self.is_p2wpkh()
+    }
+    fn segwit_is_p2wsh(&self) -> bool {
+        self.is_p2wsh()
+    }
+    fn segwit_is_p2tr(&self) -> bool {
+        self.is_p2tr()
+    }
+}
+
+#[cfg(not(feature = "liquid"))]
+impl SegwitDetection for bitcoin::ScriptBuf {
+    fn segwit_is_p2wpkh(&self) -> bool {
+        self.is_p2wpkh()
+    }
+    fn segwit_is_p2wsh(&self) -> bool {
+        self.is_p2wsh()
+    }
+    fn segwit_is_p2tr(&self) -> bool {
+        self.is_p2tr()
+    }
+}
+
+#[cfg(feature = "liquid")]
+impl SegwitDetection for elements::Script {
+    fn segwit_is_p2wpkh(&self) -> bool {
+        self.is_v0_p2wpkh()
+    }
+    fn segwit_is_p2wsh(&self) -> bool {
+        self.is_v0_p2wsh()
+    }
+    fn segwit_is_p2tr(&self) -> bool {
+        self.is_v1_p2tr()
+    }
+}
+
+pub trait ScriptToAsm: std::fmt::Debug {
+    fn to_asm(&self) -> String {
+        let asm = format!("{:?}", self);
+        asm[7..asm.len() - 1].to_string()
+    }
+}
+impl ScriptToAsm for bitcoin::Script {}
+impl ScriptToAsm for bitcoin::ScriptBuf {}
+#[cfg(feature = "liquid")]
+impl ScriptToAsm for elements::Script {}
+
+pub trait ScriptToAddr {
+    fn to_address_str(&self, network: Network) -> Option<String>;
+}
+#[cfg(not(feature = "liquid"))]
+impl ScriptToAddr for bitcoin::Script {
+    fn to_address_str(&self, network: Network) -> Option<String> {
+        bitcoin::Address::from_script(self, bitcoin::Network::from(network))
+            .ok()
+            .map(|s| s.to_string())
+    }
+}
+#[cfg(feature = "liquid")]
+impl ScriptToAddr for elements::Script {
+    fn to_address_str(&self, network: Network) -> Option<String> {
+        elements_address::Address::from_script(self, None, network.address_params())
+            .map(|a| a.to_string())
+    }
+}
+
+// Returns the witnessScript in the case of p2wsh, or the redeemScript in the case of p2sh.
+pub fn get_innerscripts(txin: &TxIn, prevout: &TxOut) -> InnerScripts {
+    // Wrapped redeemScript for P2SH spends
+    let redeem_script = if prevout.script_pubkey.is_p2sh() {
+        if let Some(Ok(PushBytes(redeemscript))) = txin.script_sig.instructions().last() {
+            #[cfg(not(feature = "liquid"))]
+            let bytes = redeemscript.as_bytes().to_vec();
+            #[cfg(feature = "liquid")]
+            let bytes = redeemscript.to_vec();
+            Some(Script::from(bytes))
+        } else {
+            None
+        }
     } else {
         None
     };
 
-    Some(
-        Address {
-            payload: payload?,
-            network: *network,
-        }
-        .to_string(),
-    )
-}
+    // Wrapped witnessScript for P2WSH or P2SH-P2WSH spends
+    let witness_script = if prevout.script_pubkey.segwit_is_p2wsh()
+        || prevout.script_pubkey.segwit_is_p2tr()
+        || redeem_script.as_ref().is_some_and(|s| s.segwit_is_p2wsh())
+    {
+        let witness = &txin.witness;
+        #[cfg(feature = "liquid")]
+        let witness = &witness.script_witness;
 
-pub fn get_script_asm(script: &Script) -> String {
-    let asm = format!("{:?}", script);
-    (&asm[7..asm.len() - 1]).to_string()
+        // rust-bitcoin returns witness items as a [u8] slice, while rust-elements returns a Vec<u8>
+        #[cfg(not(feature = "liquid"))]
+        let wit_to_vec = Vec::from;
+        #[cfg(feature = "liquid")]
+        let wit_to_vec = Clone::clone;
+
+        let inner_script_slice = if prevout.script_pubkey.segwit_is_p2tr() {
+            // Witness stack is potentially very large
+            // so we avoid to_vec() or iter().collect() for performance
+            let w_len = witness.len();
+            witness
+                .last()
+                // Get the position of the script spend script (if it exists)
+                .map(|last_elem| {
+                    // From BIP341:
+                    // If there are at least two witness elements, and the first byte of
+                    // the last element is 0x50, this last element is called annex a
+                    // and is removed from the witness stack.
+                    if w_len >= 2 && last_elem.first().filter(|&&v| v == 0x50).is_some() {
+                        // account for the extra item removed from the end
+                        3
+                    } else {
+                        // otherwise script is 2nd from last
+                        2
+                    }
+                })
+                // Convert to None if not script spend
+                // Note: Option doesn't have filter_map() method
+                .filter(|&script_pos_from_last| w_len >= script_pos_from_last)
+                .and_then(|script_pos_from_last| {
+                    // Can't use second_to_last() since it might be 3rd to last
+                    #[allow(clippy::iter_nth)]
+                    witness.iter().nth(w_len - script_pos_from_last)
+                })
+        } else {
+            witness.last()
+        };
+
+        inner_script_slice.map(wit_to_vec).map(Script::from)
+    } else {
+        None
+    };
+
+    InnerScripts {
+        redeem_script,
+        witness_script,
+    }
 }

src/util/transaction.rs

@@ -1,18 +1,29 @@
-use bitcoin_hashes::sha256d::Hash as Sha256dHash;
+use crate::chain::{BlockHash, OutPoint, Transaction, TxIn, TxOut, Txid};
+use crate::errors;
+use crate::util::{BlockId, IsProvablyUnspendable};
 
-use crate::chain::{TxIn, TxOut};
-use crate::util::BlockId;
+use std::collections::HashMap;
 
 #[cfg(feature = "liquid")]
-use crate::util::REGTEST_INITIAL_ISSUANCE_PREVOUT;
-#[cfg(feature = "liquid")]
-use bitcoin_hashes::hex::ToHex;
+lazy_static! {
+    static ref REGTEST_INITIAL_ISSUANCE_PREVOUT: Txid =
+        "50cdc410c9d0d61eeacc531f52d2c70af741da33af127c364e52ac1ee7c030a5"
+            .parse()
+            .unwrap();
+    static ref TESTNET_INITIAL_ISSUANCE_PREVOUT: Txid =
+        "0c52d2526a5c9f00e9fb74afd15dd3caaf17c823159a514f929ae25193a43a52"
+            .parse()
+            .unwrap();
+}
 
 #[derive(Serialize, Deserialize)]
 pub struct TransactionStatus {
     pub confirmed: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub block_height: Option<usize>,
-    pub block_hash: Option<Sha256dHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub block_hash: Option<BlockHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub block_time: Option<u32>,
 }
 
@@ -21,7 +32,7 @@ impl From<Option<BlockId>> for TransactionStatus {
         match blockid {
             Some(b) => TransactionStatus {
                 confirmed: true,
-                block_height: Some(b.height as usize),
+                block_height: Some(b.height),
                 block_hash: Some(b.hash),
                 block_time: Some(b.time),
             },
@@ -35,6 +46,12 @@ impl From<Option<BlockId>> for TransactionStatus {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct TxInput {
+    pub txid: Txid,
+    pub vin: u32,
+}
+
 pub fn is_coinbase(txin: &TxIn) -> bool {
     #[cfg(not(feature = "liquid"))]
     return txin.previous_output.is_null();
@@ -48,12 +65,328 @@ pub fn has_prevout(txin: &TxIn) -> bool {
     #[cfg(feature = "liquid")]
     return !txin.is_coinbase()
         && !txin.is_pegin
-        && txin.previous_output.txid.to_hex() != REGTEST_INITIAL_ISSUANCE_PREVOUT;
+        && txin.previous_output.txid != *REGTEST_INITIAL_ISSUANCE_PREVOUT
+        && txin.previous_output.txid != *TESTNET_INITIAL_ISSUANCE_PREVOUT;
 }
 
 pub fn is_spendable(txout: &TxOut) -> bool {
     #[cfg(not(feature = "liquid"))]
-    return !txout.script_pubkey.is_provably_unspendable();
+    return !txout.script_pubkey.is_provably_unspendable_();
     #[cfg(feature = "liquid")]
-    return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable();
+    return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable_();
+}
+
+/// Extract the previous TxOuts of a Transaction's TxIns
+///
+/// # Errors
+///
+/// This function MUST NOT return an error variant when allow_missing is true.
+/// If allow_missing is false, it will return an error when any Outpoint is
+/// missing from the keys of the txos argument's HashMap.
+pub fn extract_tx_prevouts<'a>(
+    tx: &Transaction,
+    txos: &'a HashMap<OutPoint, TxOut>,
+) -> Result<HashMap<u32, &'a TxOut>, errors::Error> {
+    tx.input
+        .iter()
+        .enumerate()
+        .filter(|(_, txi)| has_prevout(txi))
+        .map(|(index, txi)| {
+            Ok((
+                index as u32,
+                match txos.get(&txi.previous_output) {
+                    Some(txo) => txo,
+                    None => {
+                        return Err(format!("missing outpoint {:?}", txi.previous_output).into());
+                    }
+                },
+            ))
+        })
+        .collect()
+}
+
+pub fn serialize_outpoint<S>(outpoint: &OutPoint, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::ser::Serializer,
+{
+    use serde::ser::SerializeStruct;
+    let mut s = serializer.serialize_struct("OutPoint", 2)?;
+    s.serialize_field("txid", &outpoint.txid)?;
+    s.serialize_field("vout", &outpoint.vout)?;
+    s.end()
+}
+
+pub(super) mod sigops {
+    use crate::chain::{
+        opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
+        script::{self, Instruction},
+        Transaction, TxOut, Witness,
+    };
+    #[cfg(not(feature = "liquid"))]
+    use bitcoin::opcodes::Opcode;
+    #[cfg(feature = "liquid")]
+    use elements::opcodes::All as Opcode;
+    use std::collections::HashMap;
+
+    /// Get sigop count for transaction. prevout_map must have all the prevouts.
+    pub fn transaction_sigop_count(
+        tx: &Transaction,
+        prevout_map: &HashMap<u32, &TxOut>,
+    ) -> Result<usize, script::Error> {
+        let input_count = tx.input.len();
+        let mut prevouts = Vec::with_capacity(input_count);
+
+        #[cfg(not(feature = "liquid"))]
+        let is_coinbase_or_pegin = tx.is_coinbase();
+        #[cfg(feature = "liquid")]
+        let is_coinbase_or_pegin = tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin);
+
+        if !is_coinbase_or_pegin {
+            for idx in 0..input_count {
+                prevouts.push(
+                    *prevout_map
+                        .get(&(idx as u32))
+                        .ok_or(script::Error::EarlyEndOfScript)?,
+                );
+            }
+        }
+
+        // coinbase tx won't use prevouts so it can be empty.
+        get_sigop_cost(tx, &prevouts, true, true)
+    }
+
+    fn decode_pushnum(op: &Opcode) -> Option<u8> {
+        // 81 = OP_1, 96 = OP_16
+        // 81 -> 1, so... 81 - 80 -> 1
+        #[cfg(not(feature = "liquid"))]
+        let self_u8 = op.to_u8();
+        #[cfg(feature = "liquid")]
+        let self_u8 = op.into_u8();
+        match self_u8 {
+            81..=96 => Some(self_u8 - 80),
+            _ => None,
+        }
+    }
+
+    fn count_sigops(script: &script::Script, accurate: bool) -> usize {
+        let mut n = 0;
+        let mut pushnum_cache = None;
+        for inst in script.instructions() {
+            match inst {
+                Ok(Instruction::Op(opcode)) => {
+                    match opcode {
+                        OP_CHECKSIG | OP_CHECKSIGVERIFY => {
+                            n += 1;
+                        }
+                        OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => {
+                            match (accurate, pushnum_cache) {
+                                (true, Some(pushnum)) => {
+                                    // Add the number of pubkeys in the multisig as sigop count
+                                    n += usize::from(pushnum);
+                                }
+                                _ => {
+                                    // MAX_PUBKEYS_PER_MULTISIG from Bitcoin Core
+                                    // https://github.com/bitcoin/bitcoin/blob/v25.0/src/script/script.h#L29-L30
+                                    n += 20;
+                                }
+                            }
+                        }
+                        _ => {
+                            pushnum_cache = decode_pushnum(&opcode);
+                        }
+                    }
+                }
+                // We ignore errors as well as pushdatas
+                _ => {
+                    pushnum_cache = None;
+                }
+            }
+        }
+
+        n
+    }
+
+    /// Get the sigop count for legacy transactions
+    fn get_legacy_sigop_count(tx: &Transaction) -> usize {
+        let mut n = 0;
+        for input in &tx.input {
+            n += count_sigops(&input.script_sig, false);
+        }
+        for output in &tx.output {
+            n += count_sigops(&output.script_pubkey, false);
+        }
+        n
+    }
+
+    fn get_p2sh_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
+        #[cfg(not(feature = "liquid"))]
+        if tx.is_coinbase() {
+            return 0;
+        }
+        #[cfg(feature = "liquid")]
+        if tx.is_coinbase() {
+            return 0;
+        }
+        let mut n = 0;
+        for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
+            if prevout.script_pubkey.is_p2sh() {
+                if let Some(Ok(script::Instruction::PushBytes(redeem))) =
+                    input.script_sig.instructions().last()
+                {
+                    #[cfg(not(feature = "liquid"))]
+                    let script = script::Script::from_bytes(redeem.as_bytes());
+                    #[cfg(feature = "liquid")]
+                    let script = script::Script::from(redeem.to_vec());
+                    #[allow(clippy::needless_borrow)]
+                    {
+                        n += count_sigops(&script, true);
+                    }
+                }
+            }
+        }
+        n
+    }
+
+    fn get_witness_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
+        let mut n = 0;
+
+        #[inline]
+        fn is_push_only(script: &script::Script) -> bool {
+            for inst in script.instructions() {
+                match inst {
+                    Err(_) => return false,
+                    Ok(Instruction::Op(_)) => return false,
+                    Ok(Instruction::PushBytes(_)) => {}
+                }
+            }
+            true
+        }
+
+        #[inline]
+        fn last_pushdata(script: &script::Script) -> Option<&[u8]> {
+            match script.instructions().last() {
+                #[cfg(not(feature = "liquid"))]
+                Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes.as_bytes()),
+                #[cfg(feature = "liquid")]
+                Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes),
+                _ => None,
+            }
+        }
+
+        #[inline]
+        fn count_with_prevout(
+            prevout: &TxOut,
+            script_sig: &script::Script,
+            witness: &Witness,
+        ) -> usize {
+            let mut n = 0;
+
+            let script_owned;
+            let script: &script::Script = if prevout.script_pubkey.is_witness_program() {
+                &prevout.script_pubkey
+            } else if prevout.script_pubkey.is_p2sh()
+                && is_push_only(script_sig)
+                && !script_sig.is_empty()
+            {
+                #[cfg(not(feature = "liquid"))]
+                {
+                    script_owned =
+                        script::ScriptBuf::from(last_pushdata(script_sig).unwrap().to_vec());
+                }
+                #[cfg(feature = "liquid")]
+                {
+                    script_owned =
+                        script::Script::from(last_pushdata(script_sig).unwrap().to_vec());
+                }
+                &script_owned
+            } else {
+                return 0;
+            };
+
+            #[cfg(not(feature = "liquid"))]
+            if script.is_p2wsh() {
+                let bytes = script.as_bytes();
+                n += sig_ops(witness, bytes[0], &bytes[2..]);
+            } else if script.is_p2wpkh() {
+                n += 1;
+            }
+            #[cfg(feature = "liquid")]
+            if script.is_v0_p2wsh() {
+                let bytes = script.as_bytes();
+                n += sig_ops(witness, bytes[0], &bytes[2..]);
+            } else if script.is_v0_p2wpkh() {
+                n += 1;
+            }
+            n
+        }
+
+        for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
+            n += count_with_prevout(prevout, &input.script_sig, &input.witness);
+        }
+        n
+    }
+
+    /// Get the sigop cost for this transaction.
+    fn get_sigop_cost(
+        tx: &Transaction,
+        previous_outputs: &[&TxOut],
+        verify_p2sh: bool,
+        verify_witness: bool,
+    ) -> Result<usize, script::Error> {
+        let mut n_sigop_cost = get_legacy_sigop_count(tx) * 4;
+        #[cfg(not(feature = "liquid"))]
+        if tx.is_coinbase() {
+            return Ok(n_sigop_cost);
+        }
+        #[cfg(feature = "liquid")]
+        if tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin) {
+            return Ok(n_sigop_cost);
+        }
+        if tx.input.len() != previous_outputs.len() {
+            return Err(script::Error::EarlyEndOfScript);
+        }
+        if verify_witness && !verify_p2sh {
+            return Err(script::Error::EarlyEndOfScript);
+        }
+        if verify_p2sh {
+            n_sigop_cost += get_p2sh_sigop_count(tx, previous_outputs) * 4;
+        }
+        if verify_witness {
+            n_sigop_cost += get_witness_sigop_count(tx, previous_outputs);
+        }
+
+        Ok(n_sigop_cost)
+    }
+
+    /// Get sigops for the Witness
+    ///
+    /// witness_version is the raw opcode. OP_0 is 0, OP_1 is 81, etc.
+    #[allow(clippy::redundant_closure)]
+    fn sig_ops(witness: &Witness, witness_version: u8, witness_program: &[u8]) -> usize {
+        #[cfg(feature = "liquid")]
+        let last_witness = witness.script_witness.last();
+        #[cfg(not(feature = "liquid"))]
+        let last_witness = witness.last();
+        match (witness_version, witness_program.len()) {
+            (0, 20) => 1,
+            (0, 32) => {
+                #[cfg(not(feature = "liquid"))]
+                {
+                    #[allow(clippy::needless_borrow)]
+                    last_witness
+                        .map(|sl| script::Script::from_bytes(sl))
+                        .map(|s| count_sigops(s, true))
+                        .unwrap_or_default()
+                }
+                #[cfg(feature = "liquid")]
+                {
+                    last_witness
+                        .map(|sl| script::Script::from(sl.clone()))
+                        .map(|s| count_sigops(&s, true))
+                        .unwrap_or_default()
+                }
+            }
+            _ => 0,
+        }
+    }
 }

start

@@ -0,0 +1,274 @@
+#!/usr/bin/env zsh
+
+# initialize variables
+DAEMON=bitcoin
+NETWORK=mainnet
+FEATURES=default
+DB_FOLDER=/electrs
+ASSET_DB_ARGS=()
+NODENAME=$(hostname|cut -d . -f1)
+LOCATION=$(hostname|cut -d . -f2)
+USAGE="Usage: $0 (mainnet|testnet|signet|liquid|liquidtestnet) [popular-scripts]"
+
+# load rust if necessary
+if [ -e "${HOME}/.cargo/env" ];then
+	source "${HOME}/.cargo/env"
+	export PATH="${HOME}/.cargo/bin:${PATH}"
+fi
+
+# which OS?
+case "$(uname -s)" in
+	FreeBSD)
+		OS=FreeBSD
+		NPROC=$(sysctl -n hw.ncpu)
+		export CC=/usr/local/bin/clang17
+		export CXX=/usr/local/bin/clang++17
+		export CPP=/usr/local/bin/clang-cpp17
+		export RUSTFLAGS="-C linker=clang17"
+	;;
+	Darwin)
+		OS=Darwin
+		NPROC=$(sysctl -n hw.ncpu)
+	;;
+	Linux)
+		OS=Linux
+		NPROC=$(grep -c proc /proc/cpuinfo)
+	;;
+	*)
+		OS=Unknown
+		NPROC=4
+	;;
+esac
+
+# which network?
+case "${1}" in
+	mainnet)
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="20 4 * * *"
+	;;
+	testnet)
+		NETWORK=testnet
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="2 4 * * *"
+	;;
+	testnet4)
+		NETWORK=testnet4
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="17 4 * * *"
+	;;
+	signet)
+		NETWORK=signet
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="9 4 * * *"
+	;;
+	liquid)
+		DAEMON=elements
+		NETWORK=liquid
+		FEATURES=liquid
+		ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_db)
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="12 4 * * *"
+	;;
+	liquidtestnet)
+		DAEMON=elements
+		NETWORK=liquidtestnet
+		FEATURES=liquid
+		ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_testnet_db)
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="17 4 * * *"
+	;;
+	*)
+		echo "${USAGE}"
+		exit 1
+	;;
+esac
+
+# Run the popular address txt file generator before each run
+POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}"
+POPULAR_SCRIPTS_FILE_RAW="${POPULAR_SCRIPTS_FOLDER}/popular-scripts-raw.txt"
+POPULAR_SCRIPTS_FILE="${POPULAR_SCRIPTS_FOLDER}/popular-scripts.txt"
+
+# This function runs the job for generating the popular scripts text file for the precache arg
+generate_popular_scripts() {
+	mkdir -p "${POPULAR_SCRIPTS_FOLDER}"
+
+	## Use nproc * 4 threads to generate the txt file (lots of iowait, so 2x~4x core count is ok)
+	## Only pick up addresses with 101 history events or more
+	## (Without lowering MIN_HISTORY_ITEMS_TO_CACHE this is the lowest we can go)
+	## It prints out progress to STDERR
+	echo "[*] Generating popular-scripts using ${THREADS} threads..."
+	cd "${HOME}/electrs"
+	HIGH_USAGE_THRESHOLD=101 \
+	JOB_THREAD_COUNT=${THREADS} \
+	nice cargo run \
+		--release \
+		--bin popular-scripts \
+		--features "${FEATURES}" \
+		-- \
+		--network "${NETWORK}" \
+		--db-dir "${DB_FOLDER}" \
+		> "${POPULAR_SCRIPTS_FILE_RAW}"
+
+	## Only overwrite the existing file if the popular-scripts cargo run succeeded
+	if [ "$?" = "0" ];then
+		## Sorted and deduplicated just in case
+		echo "Sorting popular scripts for final results..."
+		sort "${POPULAR_SCRIPTS_FILE_RAW}" | uniq > "${POPULAR_SCRIPTS_FILE}"
+	fi
+
+	rm "${POPULAR_SCRIPTS_FILE_RAW}"
+}
+
+# This function is for inserting the cronjob for generating the popular scripts
+CRONJOB_CMD="\"${HOME}/electrs/start\" \"${NETWORK}\" popular-scripts"
+echo "${CRONJOB_TIMING} ${CRONJOB_CMD}"
+
+case "${2}" in
+	popular-scripts)
+		echo "[*] Only generate popular-scripts, then exit"
+		generate_popular_scripts
+		exit 0
+	;;
+	version)
+		echo "[*] Only print versions, then exit"
+		cargo run --bin electrs --release -- --version
+		cargo run --bin popular-scripts --release -- --version
+		exit 0
+	;;
+	"")
+		# If the 2nd arg isn't passed, just run the normal electrs script as-is
+	;;
+	*)
+		echo "${USAGE}"
+		exit 1
+	;;
+esac
+
+# run in loop in case of crash
+until false
+do
+	# reset CWD
+	cd "${HOME}/electrs"
+
+	# disable making electrs.core files
+	ulimit -c 0
+
+	# prepare run-time variables
+	UTXOS_LIMIT=500
+	ELECTRUM_TXS_LIMIT=500
+	ELECTRUM_MAX_LINE_SIZE=1048576 # 1 MiB
+	ELECTRUM_MAX_SUBSCRIPTIONS=100
+	ELECTRUM_MAX_CLIENTS=1000
+	MAIN_LOOP_DELAY=500
+	DAEMON_CONF="${HOME}/${DAEMON}.conf"
+	HTTP_SOCKET_FILE="${HOME}/socket/esplora-${DAEMON}-${NETWORK}"
+	RPC_SOCKET_FILE="${HOME}/socket/electrum-${DAEMON}-${NETWORK}"
+
+	# get RPC credentials from bitcoin.conf or elements.conf directly
+	echo "[*] Getting RPC credentials from ${DAEMON_CONF}"
+	RPC_USER=$(grep 'rpcuser=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
+	RPC_PASS=$(grep 'rpcpassword=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
+
+	# override limits based on hostname
+	if [ "${NODENAME}" = "node201" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+		MAIN_LOOP_DELAY=14000
+	fi
+	if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "sg1" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "hnl" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node206" ] && [ "${LOCATION}" = "tk7" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node211" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node212" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node213" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node214" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NETWORK}" = "testnet4" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${LOCATION}" = "fmt" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+
+	if [ ! -e "${POPULAR_SCRIPTS_FILE}" ];then
+		generate_popular_scripts
+	fi
+
+	# Run the electrs process (Note: db-dir is used in both commands)
+	nice cargo run \
+		--release \
+		--bin electrs \
+		--features "${FEATURES}" \
+		-- \
+		--network "${NETWORK}" \
+		--daemon-dir "${HOME}" \
+		--db-dir "${DB_FOLDER}" \
+		"${ASSET_DB_ARGS[@]}" \
+		--main-loop-delay "${MAIN_LOOP_DELAY}" \
+		--rpc-socket-file "${RPC_SOCKET_FILE}" \
+		--http-socket-file "${HTTP_SOCKET_FILE}" \
+		--precache-scripts "${POPULAR_SCRIPTS_FILE}" \
+		--precache-threads "${THREADS}" \
+		--cookie "${RPC_USER}:${RPC_PASS}" \
+		--cors '*' \
+		--address-search \
+		--utxos-limit "${UTXOS_LIMIT}" \
+		--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
+		--electrum-max-line-size "${ELECTRUM_MAX_LINE_SIZE}" \
+		--electrum-max-subscriptions "${ELECTRUM_MAX_SUBSCRIPTIONS}" \
+		--electrum-max-clients "${ELECTRUM_MAX_CLIENTS}" \
+		-vv
+	sleep 1
+done